diff --git a/.github/workflows/model-compat.yml b/.github/workflows/model-compat.yml new file mode 100644 index 0000000000..1db03a8315 --- /dev/null +++ b/.github/workflows/model-compat.yml @@ -0,0 +1,261 @@ +name: Model compatibility + +# Does every model the picker offers still answer through the clients OpenBot ships? The picker +# lists each provider's models live, but every harness image carries its model client at a fixed +# version (the Claude Agent SDK harness bundles a Claude Code CLI, the LangGraph harness sends the +# ChatGPT plan's backend a fixed Codex client version, the others carry the provider SDKs they were +# built with), and a provider can refuse a new model to an older client. scripts/model-compat.ts +# starts each harness image, one job per harness, and sends it one request per listed model on +# every provider that has a key here. See docs/releasing.md, "Model compatibility". +# +# Keys are repository secrets, one per provider. A provider without one is reported as skipped, +# not failed. On any failure the run fails and opens (or updates) one issue titled "Model +# compatibility check failing"; a passing run closes it. +# +# What answered on the `source` image is the model lock (shared/model-lock.json), the models the +# picker may offer for each harness. When a model newly answers or stops answering, the run opens +# or updates one pull request that refreshes the lock. See docs/releasing.md, "Model lock". +# +# Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever +# runs with these keys. +on: + schedule: + - cron: "41 6 * * *" + workflow_dispatch: + inputs: + tags: + description: Harness images to check, comma-separated. `latest` is the published release people run; `source` is built from this commit, which is what the next release ships. + type: string + default: latest,source + required: false + +permissions: {} + +concurrency: + group: model-compat + cancel-in-progress: false + +jobs: + # The published harnesses, and every engine the model lock names whether it is published yet or + # not (an engine new Bots run on is locked, and checked, from before its first release). + harnesses: + name: List the harnesses to check + if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + images: ${{ steps.list.outputs.images }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - id: list + run: echo "images=$(bun scripts/model-compat.ts harnesses)" >> "$GITHUB_OUTPUT" + + check: + name: ${{ matrix.harness }} + needs: harnesses + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + harness: ${{ fromJSON(needs.harnesses.outputs.images) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + # Writes its results and exits 0 whatever they say: the report job below decides. + - name: One request per model through this harness + env: + HARNESS: ${{ matrix.harness }} + TAGS: ${{ inputs.tags || 'latest,source' }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + run: >- + bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero + --out "$RUNNER_TEMP/model-compat-$HARNESS.json" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: model-compat-${{ matrix.harness }} + path: ${{ runner.temp }}/model-compat-${{ matrix.harness }}.json + if-no-files-found: error + retention-days: 14 + + report: + name: Report + needs: [harnesses, check] + if: always() && needs.harnesses.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write # open, update or close the one "Model compatibility check failing" issue + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TITLE: Model compatibility check failing + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: model-compat-* + path: ${{ runner.temp }}/results + merge-multiple: true + - name: Summarize + id: summary + run: | + set +e + shopt -s nullglob + bun scripts/model-compat.ts report --summary "$RUNNER_TEMP/summary.md" \ + --issue "$RUNNER_TEMP/issue-table.md" "$RUNNER_TEMP"/results/*.json + code=$? + echo "code=$code" >> "$GITHUB_OUTPUT" + if [ -f "$RUNNER_TEMP/summary.md" ]; then cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY"; fi + exit 0 + - name: Open or update the issue + if: steps.summary.outputs.code != '0' + run: | + set -euo pipefail + body="$RUNNER_TEMP/issue.md" + { + echo "The model compatibility check failed on $(date -u +%Y-%m-%d): $RUN_URL" + echo + echo "A model refused by a pinned client means that client needs bumping: see docs/releasing.md, \"Model compatibility\". This issue is updated by every failing run and closed by the first passing one." + echo + if [ -f "$RUNNER_TEMP/issue-table.md" ]; then + cat "$RUNNER_TEMP/issue-table.md" + else + echo "No harness job wrote results. See the run." + fi + } > "$body" + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$body" + fi + - name: Close the issue when everything passes + if: steps.summary.outputs.code == '0' + run: | + set -euo pipefail + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Passing again: $RUN_URL" + fi + - name: Fail when a model failed + if: steps.summary.outputs.code != '0' + run: | + echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"." + exit 1 + + lock: + name: Refresh the model lock + needs: [harnesses, check] + if: always() && needs.harnesses.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # push the one refresh branch + pull-requests: write # open or update its pull request + env: + BRANCH: model-lock/refresh + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: model-compat-* + path: ${{ runner.temp }}/results + merge-multiple: true + - name: Write the lock from the source images' results + id: lock + run: | + set -euo pipefail + shopt -s nullglob + files=("$RUNNER_TEMP"/results/*.json) + if [ "${#files[@]}" -eq 0 ]; then + echo "No harness job wrote results; the lock stays as it is." + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + bun scripts/model-compat.ts lock --changes "$RUNNER_TEMP/lock-changes.md" "${files[@]}" + if git diff --quiet -- shared/model-lock.json; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + { echo "## Model lock changes"; echo; cat "$RUNNER_TEMP/lock-changes.md"; } >> "$GITHUB_STEP_SUMMARY" + fi + # GITHUB_TOKEN's pull requests get CI runs that wait for a maintainer to approve them. A + # GitHub App installation token, when one is configured, has no such limit. + - id: app-token + if: steps.lock.outputs.changed == 'true' && vars.OPENBOT_UPGRADE_APP_CLIENT_ID != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ vars.OPENBOT_UPGRADE_APP_CLIENT_ID }} + private-key: ${{ secrets.OPENBOT_UPGRADE_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + - name: Open or update the refresh pull request + if: steps.lock.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }} + TOKEN_KIND: ${{ steps.app-token.outputs.token && 'app' || 'github-token' }} + # Credentials for this one push, through gh, and no hook can run. + GIT_CONFIG_COUNT: "3" + GIT_CONFIG_KEY_0: credential.helper + GIT_CONFIG_VALUE_0: "" + GIT_CONFIG_KEY_1: credential.helper + GIT_CONFIG_VALUE_1: "!gh auth git-credential" + GIT_CONFIG_KEY_2: core.hooksPath + GIT_CONFIG_VALUE_2: /dev/null + run: | + set -euo pipefail + git switch -C "$BRANCH" + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -m "Refresh the model lock from the daily model check" -- shared/model-lock.json + git push --force "https://github.com/$GITHUB_REPOSITORY.git" "HEAD:refs/heads/$BRANCH" + body="$RUNNER_TEMP/lock-pr.md" + { + echo "The daily model check found models that started or stopped answering through the clients the harnesses ship. This refreshes \`shared/model-lock.json\`, the models the picker may offer for each harness." + echo + cat "$RUNNER_TEMP/lock-changes.md" + echo + echo "From $RUN_URL. Every run that finds a change updates this pull request. See docs/releasing.md, \"Model lock\"." + if [ "$TOKEN_KIND" != app ]; then + echo + echo "**CI must be started by hand.** This pull request was opened with \`GITHUB_TOKEN\`, so its CI runs wait for a maintainer to approve them." + fi + } > "$body" + number=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq ".[0].number // empty") + if [ -n "$number" ]; then + gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" + else + gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$BRANCH" \ + --title "Refresh the model lock" --body-file "$body" + fi diff --git a/.github/workflows/model-lock.yml b/.github/workflows/model-lock.yml new file mode 100644 index 0000000000..12f568d454 --- /dev/null +++ b/.github/workflows/model-lock.yml @@ -0,0 +1,121 @@ +name: Model lock + +# A pull request that changes a harness's model client (its requirements, its package.json or +# lockfile, its Dockerfile, or the ChatGPT plan's Codex client version) must not break a model the +# model lock (shared/model-lock.json) says that harness runs: a person set to it would get the +# provider's refusal on every message. This builds the changed harnesses from the pull request, +# sends each listed model one request through them (scripts/model-compat.ts), and fails when a +# model the lock verifies no longer answers. See docs/releasing.md, "Model lock". +# +# Only for pull requests from this repository: a fork's pull request is never given the provider +# keys, so it is skipped and the daily check covers it once it merges. +on: + pull_request: + paths: + - "agent-*/requirements.txt" + - "agent-*/Dockerfile" + - "agent-*/package.json" + - "agent-*/bun.lock" + - "agent-*/package-lock.json" + - "agent-langgraph-agui/src/chatgpt_models.py" + - "shared/model-lock.json" + - "scripts/model-compat.ts" + - ".github/workflows/model-lock.yml" + +permissions: {} + +concurrency: + group: model-lock-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + changed: + name: Harnesses whose client changed + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + harnesses: ${{ steps.list.outputs.harnesses }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - id: list + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA") + # The harnesses the check runs: the published ones, and every engine the lock names, + # published yet or not, that has a row in shared/harnesses.json. + checked=$(bun scripts/model-compat.ts harnesses) + # The check itself, or the lock, changed: every harness the lock names is checked. + if grep -qxE 'shared/model-lock.json|scripts/model-compat.ts|\.github/workflows/model-lock.yml' <<<"$changed"; then + harnesses=$(jq -c --argjson checked "$checked" \ + '[.engines | keys[] | select(. as $h | $checked | index($h))]' shared/model-lock.json) + else + harnesses=$(grep -oE '^agent-[a-z0-9-]+/' <<<"$changed" | tr -d / | sort -u \ + | jq -R . | jq -sc --argjson checked "$checked" \ + 'map(select(. as $h | $checked | index($h)))') + fi + echo "harnesses=$harnesses" >> "$GITHUB_OUTPUT" + echo "Harnesses to check: $harnesses" + + check: + name: ${{ matrix.harness }} + needs: changed + if: needs.changed.outputs.harnesses != '[]' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + harness: ${{ fromJSON(needs.changed.outputs.harnesses) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - name: One request per model through this pull request's harness + env: + HARNESS: ${{ matrix.harness }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # The published image too, with the same keys in the same run: a verified model that fails + # on both is the provider's or the key's today, not this change's (`brokenVerified`). + run: >- + bun scripts/model-compat.ts --harness "$HARNESS" --tags latest,source --exit-zero + --out "$RUNNER_TEMP/model-compat.json" + # The run's results, with the repository's own keys: `bun scripts/model-compat.ts lock` writes + # the lock from them when a pin moved (docs/releasing.md, "Model lock"). + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: model-lock-${{ matrix.harness }} + path: ${{ runner.temp }}/model-compat.json + if-no-files-found: error + retention-days: 14 + - name: Every model the lock verifies still answers + env: + HARNESS: ${{ matrix.harness }} + run: | + set +e + bun scripts/model-compat.ts guard --summary "$RUNNER_TEMP/guard.md" "$RUNNER_TEMP/model-compat.json" + code=$? + cat "$RUNNER_TEMP/guard.md" >> "$GITHUB_STEP_SUMMARY" + if [ "$code" -ne 0 ]; then + echo "::error::A model the lock verifies for $HARNESS no longer answers through this change's client. See the summary." + fi + exit "$code" diff --git a/api/client-boundary.allow.ts b/api/client-boundary.allow.ts index 7dd7aa1964..cadc9bbe89 100644 --- a/api/client-boundary.allow.ts +++ b/api/client-boundary.allow.ts @@ -398,6 +398,12 @@ export const ALLOWED: Allowed[] = [ rule: "third-party", why: "image registries, never OpenBot's server", }, + { + file: "scripts/model-compat.ts", + what: /^(fetch\(\)|\/health|\/api\/agents\/openbot\/generate)$/, + rule: "third-party", + why: "model providers' listings and the harness images the model check starts on this machine, never OpenBot's server", + }, // Scripts that run and stop a server on this machine. { file: "scripts/update-local.ts", diff --git a/api/openapi.yaml b/api/openapi.yaml index 33761be811..77f1f33921 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -23721,6 +23721,8 @@ components: type: - string - 'null' + unverified: + $ref: '#/components/schemas/UnverifiedModel' ModelChoiceInput: type: object additionalProperties: false @@ -24857,6 +24859,19 @@ components: text: type: string description: What was said. + UnverifiedModel: + type: object + description: The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason. + required: + - model + - label + properties: + model: + type: string + description: The chosen model's id. + label: + type: string + description: Its name, as a model picker shows it. AppSource: type: string enum: diff --git a/api/openapi/model-connections.yaml b/api/openapi/model-connections.yaml index e42d9df16a..590bb25e74 100644 --- a/api/openapi/model-connections.yaml +++ b/api/openapi/model-connections.yaml @@ -587,6 +587,23 @@ components: type: [string, 'null'] effort: type: [string, 'null'] + unverified: + $ref: '#/components/schemas/UnverifiedModel' + UnverifiedModel: + type: object + description: >- + The model a person chose for this conversation, set aside because it + hasn't been checked to work with the client this Bot's harness + ships. Runs answer on the default model until it has been. Absent when + the choice is in use, or set aside for another reason. + required: [model, label] + properties: + model: + type: string + description: The chosen model's id. + label: + type: string + description: Its name, as a model picker shows it. ModelChoiceInput: type: object additionalProperties: false diff --git a/app/src/lib/api/generated/model/index.ts b/app/src/lib/api/generated/model/index.ts index f4c9e71d9c..f23cd5219d 100644 --- a/app/src/lib/api/generated/model/index.ts +++ b/app/src/lib/api/generated/model/index.ts @@ -1440,6 +1440,7 @@ export * from "./unpinBotHeaders"; export * from "./unpublishComponentHeaders"; export * from "./unpublishTeamBotHeaders"; export * from "./unsupportedFileResponse"; +export * from "./unverifiedModel"; export * from "./updateAppHeaders"; export * from "./updateAppRequest"; export * from "./updateApprovalPreferencesHeaders"; diff --git a/app/src/lib/api/generated/model/modelChoice.ts b/app/src/lib/api/generated/model/modelChoice.ts index 3b7cddda63..6d84c877ce 100644 --- a/app/src/lib/api/generated/model/modelChoice.ts +++ b/app/src/lib/api/generated/model/modelChoice.ts @@ -1,6 +1,7 @@ /** * Generated by Orval from api/openapi.yaml. Do not edit: change the spec and run `bun run api:generate`. */ +import type { UnverifiedModel } from "./unverifiedModel"; /** * The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default. @@ -10,4 +11,5 @@ export interface ModelChoice { model: string | null; /** @nullable */ effort: string | null; + unverified?: UnverifiedModel; } diff --git a/app/src/lib/api/generated/model/unverifiedModel.ts b/app/src/lib/api/generated/model/unverifiedModel.ts new file mode 100644 index 0000000000..c2cedf19df --- /dev/null +++ b/app/src/lib/api/generated/model/unverifiedModel.ts @@ -0,0 +1,13 @@ +/** + * Generated by Orval from api/openapi.yaml. Do not edit: change the spec and run `bun run api:generate`. + */ + +/** + * The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason. + */ +export interface UnverifiedModel { + /** The chosen model's id. */ + model: string; + /** Its name, as a model picker shows it. */ + label: string; +} diff --git a/app/src/lib/api/generated/models/models.zod.ts b/app/src/lib/api/generated/models/models.zod.ts index 296e895d13..ea9d04eb29 100644 --- a/app/src/lib/api/generated/models/models.zod.ts +++ b/app/src/lib/api/generated/models/models.zod.ts @@ -134,6 +134,15 @@ export const GetModelChoiceResponse = zod .object({ model: zod.string().nullable(), effort: zod.string().nullable(), + unverified: zod + .object({ + model: zod.string().describe("The chosen model's id."), + label: zod.string().describe("Its name, as a model picker shows it."), + }) + .optional() + .describe( + "The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason.", + ), }) .describe( "The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default.", @@ -216,6 +225,15 @@ export const ReplaceModelChoiceResponse = zod .object({ model: zod.string().nullable(), effort: zod.string().nullable(), + unverified: zod + .object({ + model: zod.string().describe("The chosen model's id."), + label: zod.string().describe("Its name, as a model picker shows it."), + }) + .optional() + .describe( + "The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason.", + ), }) .describe( "The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default.", diff --git a/app/src/lib/models/queries.ts b/app/src/lib/models/queries.ts index 9b126f7d76..f0a4612e24 100644 --- a/app/src/lib/models/queries.ts +++ b/app/src/lib/models/queries.ts @@ -80,6 +80,12 @@ export type AvailableModels = { export type ConversationModel = { model: string | null; effort: string | null; + /** + * The model the person chose here, set aside because it hasn't been checked to work with this + * Bot's harness (shared/model-lock.ts). Runs use the default until it has been. Only ever on a + * read from the server. + */ + unverified?: { model: string; label: string }; }; export const modelKeys = { diff --git a/app/src/lib/models/run-model.ts b/app/src/lib/models/run-model.ts index c90768375c..7937dda6bf 100644 --- a/app/src/lib/models/run-model.ts +++ b/app/src/lib/models/run-model.ts @@ -28,6 +28,15 @@ export function withdrawnModelNotice( return `${picked} ${gone} any more, so this conversation now uses ${now}. Pick another model to change it.`; } +/** + * What the composer says when the model a person chose here hasn't been proven to work with this + * Bot (shared/model-lock.ts): the provider would turn down every message, so runs use the default + * until a check proves it. + */ +export function unverifiedModelNotice(picked: string, now: string): string { + return `${picked} doesn't work with this Bot yet, so this conversation uses ${now}. Pick another model to change it.`; +} + /** * The model and thinking effort one conversation runs on, as its picker shows them, and the way * to change them. @@ -133,8 +142,19 @@ export function useConversationModel( ? undefined : shownPick(picked, catalogue), saving: save.isPending, - /** What the composer says about the pick, when the picked model stopped being offered. */ - notice: withdrawn?.channelId === channelId ? withdrawn.notice : null, + /** + * What the composer says about the pick: the chosen model doesn't work with this Bot yet, or + * it stopped being offered. + */ + notice: + (!save.isPending && saved.data?.unverified && catalogue + ? unverifiedModelNotice( + saved.data.unverified.label, + catalogue.models.find((option) => option.id === catalogue.default) + ?.label ?? catalogue.default, + ) + : null) ?? + (withdrawn?.channelId === channelId ? withdrawn.notice : null), // Read the ref at submission time, including a choice made before React renders again. beforeSend: async () => { await pendingSave.current; diff --git a/docs/releasing.md b/docs/releasing.md index 8b844feb17..0713855b23 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1080,6 +1080,108 @@ it when it is not. Signed desktop builds are the exception: they read the `macos `windows-signing` environments described above and in [Windows signing](windows-signing.md). Downloads sync signs in to AWS with GitHub's OIDC token rather than a stored key. +## Model compatibility + +The model picker lists each provider's models live, but every harness image carries its model +client at a fixed version: the Claude Agent SDK harness bundles a Claude Code CLI (through +`claude-agent-sdk` in `agent-claude-sdk/requirements.txt`), the LangGraph harness sends the ChatGPT +plan's backend a fixed Codex client version (`DEFAULT_CLIENT_VERSION` in +`agent-langgraph-agui/src/chatgpt_models.py`), and the others carry the provider SDKs they were built +with. A provider can refuse a new model to an older client while the picker keeps offering it: +"Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required". + +**Model compatibility** (`.github/workflows/model-compat.yml`) checks for that daily, and by hand +from the Actions tab. One job per harness in `.github/published-images.json` runs +[`scripts/model-compat.ts`](../scripts/model-compat.ts), which: + +- lists each provider's models from the same source, with the same filter, as the picker (see + [The model list](configuration.md#the-model-list)); +- starts the harness image on each provider the way the Desktop configures it, and sends one AG-UI + run per model, so the harness's own client makes the call. It checks `latest`, the published + image people run, and `source`, the image built from this commit; +- compares the LangGraph harness's Codex client version with the minimum each ChatGPT plan model + needs, from openai/codex's published model catalogue. A plan sign-in cannot run in CI. A Claude + plan runs through the same Claude Code CLI as an Anthropic key, so the key's rows cover it. + +It writes a pass, fail or skipped row per harness, image, provider and model, with the provider's +own words, to the run's summary. On any failure the run fails and opens or updates one issue, +**Model compatibility check failing**, and the first passing run closes it. It runs only on a +schedule or by hand on `main`, never on a pull request. + +Each provider needs its key as a repository secret: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, +`GEMINI_API_KEY` (Google, also used for Antigravity) and `XAI_API_KEY`. A provider without one is +reported as skipped, not failed, and the summary names the missing secrets. Unbiased AI is not +checked: it is a gateway serving its own model, not a provider's listing. + +**When it fails**, read the provider's words in the table: + +- A version refusal (a model "not supported" by the client's version, or a ChatGPT plan model that + needs a newer Codex) means bumping the pinned client: `claude-agent-sdk` in + `agent-claude-sdk/requirements.txt` (each release bundles a newer Claude Code CLI), or + `DEFAULT_CLIENT_VERSION` in `agent-langgraph-agui/src/chatgpt_models.py`, or the harness's + provider SDK in its `requirements.txt` or `package.json`. Run the check by hand: the `source` rows + show whether the bump fixed it before a release ships it, and the `latest` rows stay red until + one does. +- A refused parameter or endpoint (a model that takes tools only through `/v1/responses`, or + refuses a `temperature`) means that harness builds its request in a way the model no longer + accepts: fix how the harness sets up its model, or bump the framework if a newer one does. +- Billing, rate limits or a refused key are the key's problem, not a client's: fix the key. + +To run it locally (needs Docker), with keys in the environment or an env file: + +```sh +bun scripts/model-compat.ts --env-file .env --harness agent-claude-sdk --provider anthropic +``` + +`--model ` narrows the models, `--tags latest` skips building from source. + +### Model lock + +The picker offers a model only when this check has proven it on the Bot's harness. +`shared/model-lock.json` holds, per harness, the model clients its `source` image carried and, per +provider, the models that answered and the ones refused, in the provider's words. The server's model +catalogue reads it (`shared/model-lock.ts` has the rule): a live model the lock doesn't verify for +the harness is hidden, and a conversation already set to one runs on the default and says so. + +- **Which harness.** A Claude plan runs every Bot through `agent-claude-sdk`, a ChatGPT plan through + `agent-langgraph-agui`, and with a key, the harness picked in setup when it takes OpenBot's model + choice. +- **What is locked.** Only an engine and provider pair that has been checked. A pair never checked (a + provider with no key here, such as xAI today) stays open, and a gateway is never locked. +- **A new engine** can be locked before its first release, by giving it empty pairs for the + providers it runs on. Nothing is offered on it until a run proves a model, including on a + provider the check has no key for. The check runs every engine the lock names, published or not + yet, from its `source` image (`bun scripts/model-compat.ts harnesses` lists them). An engine new + Bots run on by default (`DEFAULT_ENGINES` in `shared/model-lock.ts`, today the LangGraph + (TypeScript) engine, `agent-langgraph-ts`) must verify at least one model on every provider it is + locked for, or `scripts/model-lock.test.ts` fails: write its entry from a run, never empty. The + lock can't reach main before an engine it names has its row in `shared/harnesses.json`. +- **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull + request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that + started and stopped answering and the clients that moved. Merging it lets the picker offer them. + Every failure is asked once more before it counts. A failure that says nothing about the client + keeps what the lock said before: the harness never started, a timeout, a rate limit, or a bare + 404 for a model the check's key can't see. A model the provider retires ("no longer available to + new users") is refused, so a person set to it is told, and it never fails a pull request. +- **A client bump** is held to the lock: a pull request from this repository that changes a + harness's `requirements.txt`, `package.json`, `bun.lock`, `Dockerfile` or the Codex client version + runs the **Model lock** workflow, which builds that harness, sends every listed model a request + through it and through the published image, and fails when a model the lock verifies no longer + answers through the change's image but still does through the published one. Models the bump + newly fixes are added by the next daily refresh. +- **The lock matches the pins.** The lock records the clients it was proven on, and + `scripts/model-lock.test.ts` fails when a harness pins a different version (an exact + `requirements.txt` pin, `agent-mastra/bun.lock`, or the Codex client version): a lock proven on + other clients may hide models that now work. Write it again from the change's own **Model lock** + run, whose results each harness uploads as `model-lock-`: download them and run `lock`. + +To write the lock from results by hand: + +```sh +bun scripts/model-compat.ts lock --changes changes.md results/*.json +bun scripts/model-compat.ts guard results/*.json +``` + ## The one thing CI cannot do The smoke journey in `tests/smoke` is the only check that proves the parts are wired to each other: diff --git a/scripts/model-compat-langgraph-ts.test.ts b/scripts/model-compat-langgraph-ts.test.ts new file mode 100644 index 0000000000..1f0f15b5f2 --- /dev/null +++ b/scripts/model-compat-langgraph-ts.test.ts @@ -0,0 +1,351 @@ +import { expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import anthropicListing from "../shared/fixtures/model-listings/anthropic-models.json"; +import openaiListing from "../shared/fixtures/model-listings/openai-models.json"; +import { lockAllows, MODEL_LOCK, type ModelLock } from "../shared/model-lock"; +import { + buildLock, + buildMatrix, + type Harness, + NODE_VERSIONS, + probedHarnesses, + providersFor, + type Result, + type Results, + usableModels, +} from "./model-compat"; + +/* + * The LangGraph (TypeScript) engine, `agent-langgraph-ts`, is locked before its first release: + * it becomes the engine new Bots run on, so a model is offered on it only once the model check has + * sent that model a real request through the engine's own client and it answered. Until then, + * nothing is offered on it, for any provider, even one CI has no key for. + */ + +const ENGINE = "agent-langgraph-ts"; +const ROOT = join(import.meta.dir, ".."); +const NOW = Date.parse("2026-11-17T00:00:00Z"); + +/** Its row in shared/harnesses.json, as the shared contract names it. */ +const LANGGRAPH_TS: Harness & { id: string } = { + id: "langgraph-ts", + image: ENGINE, + port: 4215, + runPath: "", + credential: "any-provider", +}; + +const anthropicIds = usableModels( + "anthropic", + anthropicListing, + new Map(), + NOW, +); +const openaiIds = usableModels("openai", openaiListing, new Map(), NOW); +const geminiIds = ["gemini-3.8-flash", "gemini-4-argon", "gemini-3.5-pro"]; + +const UNRUN: ModelLock = { + checkedAt: "2026-11-16", + engines: { + [ENGINE]: { + clients: {}, + providers: { + anthropic: { verified: [], refused: {} }, + openai: { verified: [], refused: {} }, + google: { verified: [], refused: {} }, + }, + }, + }, +}; + +test("before its first run, nothing is offered on langgraph-ts, on any provider", () => { + expect(anthropicIds.length).toBeGreaterThan(0); + expect(openaiIds.length).toBeGreaterThan(0); + for (const [provider, ids] of [ + ["anthropic", anthropicIds], + ["openai", openaiIds], + ["google", geminiIds], + ] as const) + for (const model of ids) + expect( + lockAllows(UNRUN, ENGINE, provider, model), + `${provider} ${model}`, + ).toBe(false); +}); + +const row = ( + providerId: "anthropic" | "openai" | "google", + model: string, + status: Result["status"], + detail = "OK", + tag = "source", +): Result => ({ + harness: ENGINE, + tag, + provider: { anthropic: "Anthropic", openai: "OpenAI", google: "Google" }[ + providerId + ], + providerId, + model, + status, + detail, +}); + +test("after a run, exactly the models that answered are offered on langgraph-ts", () => { + const run: Results = { + results: [ + row("anthropic", "claude-sonnet-5-5", "pass"), + row("anthropic", "claude-opus-5-5", "pass"), + row( + "anthropic", + "claude-fable-5-2", + "fail", + "400 invalid_request_error: model: claude-fable-5-2", + ), + // The runner's bad day: says nothing about the model, and verifies nothing. + row( + "anthropic", + "claude-sonnet-4-5-20250929", + "fail", + "no answer: TimeoutError: The operation timed out.", + ), + row("openai", "gpt-6.1-terra", "pass"), + row( + "openai", + "gpt-5.5", + "fail", + "the stream ended without finishing the run", + ), + // The published image is what people ran before; the lock follows `source` alone. + row("openai", "gpt-6.2-sol", "pass", "OK", "latest"), + { + harness: ENGINE, + tag: "latest, source", + provider: "Google", + providerId: "google", + model: "(all)", + status: "skipped", + detail: "skipped: no key (GEMINI_API_KEY)", + }, + ], + clients: { + [ENGINE]: { + source: { + "@langchain/langgraph": "1.7.5", + "@langchain/anthropic": "1.3.1", + }, + }, + }, + }; + const lock = buildLock(UNRUN, [run], "2026-11-17"); + const allowed = ( + provider: "anthropic" | "openai" | "google", + ids: string[], + ) => ids.filter((model) => lockAllows(lock, ENGINE, provider, model)); + + expect(allowed("anthropic", anthropicIds).sort()).toEqual([ + "claude-opus-5-5", + "claude-sonnet-5-5", + ]); + expect(openaiIds).toContain("gpt-6.1-terra"); + expect(allowed("openai", openaiIds)).toEqual(["gpt-6.1-terra"]); + // No key for Gemini today: the pair stays locked, and nothing is offered on it. + expect(allowed("google", geminiIds)).toEqual([]); + expect(lock.engines[ENGINE]?.providers.google).toEqual({ + verified: [], + refused: {}, + }); + expect(lock.engines[ENGINE]?.providers.anthropic?.refused).toEqual({ + "claude-fable-5-2": "400 invalid_request_error: model: claude-fable-5-2", + }); + expect(lock.engines[ENGINE]?.clients).toEqual({ + "@langchain/anthropic": "1.3.1", + "@langchain/langgraph": "1.7.5", + }); +}); + +test("an inconclusive failure never verifies a model langgraph-ts has not answered", () => { + const before: ModelLock = { + checkedAt: "2026-11-16", + engines: { + [ENGINE]: { + clients: { "@langchain/langgraph": "1.7.5" }, + providers: { + anthropic: { verified: ["claude-sonnet-5-5"], refused: {} }, + openai: { verified: [], refused: {} }, + google: { verified: [], refused: {} }, + }, + }, + }, + }; + const inconclusive = [ + "no answer: TimeoutError: The operation timed out.", + "the harness did not start: port is already allocated", + "the harness never became healthy: Error: Cannot find module", + `could not get ghcr.io/copilotkit/openbot-${ENGINE}:latest: manifest unknown`, + "the stream ended without finishing the run", + ]; + const run: Results = { + results: inconclusive.flatMap((detail, index) => [ + row("anthropic", `claude-model-${index}`, "fail", detail), + row("openai", `gpt-model-${index}`, "fail", detail), + // One it had answered before keeps its place: a bad day takes nothing away. + row("anthropic", "claude-sonnet-5-5", "fail", detail), + ]), + }; + const lock = buildLock(before, [run], "2026-11-17"); + expect(lock.engines[ENGINE]?.providers.anthropic?.verified).toEqual([ + "claude-sonnet-5-5", + ]); + expect(lock.engines[ENGINE]?.providers.openai?.verified).toEqual([]); + for (let index = 0; index < inconclusive.length; index++) { + expect(lockAllows(lock, ENGINE, "anthropic", `claude-model-${index}`)).toBe( + false, + ); + expect(lockAllows(lock, ENGINE, "openai", `gpt-model-${index}`)).toBe( + false, + ); + } +}); + +test("the lock names langgraph-ts, with every provider a key reaches locked", () => { + const entry = MODEL_LOCK.engines[ENGINE]; + expect(entry).toBeDefined(); + for (const provider of ["anthropic", "openai", "google"] as const) + expect(entry?.providers[provider], provider).toBeDefined(); +}); + +test("an engine the lock names is checked before it is published", () => { + const published = ["agent-claude-sdk", "agent-langgraph-agui"]; + const known: Harness[] = [ + { + image: "agent-claude-sdk", + credential: "anthropic", + port: 4212, + runPath: "", + }, + { + image: "agent-langgraph-agui", + credential: "any-provider", + port: 4201, + runPath: "", + }, + LANGGRAPH_TS, + { + image: "agent-unlisted", + credential: "any-provider", + port: 4299, + runPath: "", + }, + ]; + const lock: ModelLock = { + checkedAt: "2026-11-17", + engines: { + [ENGINE]: { clients: {}, providers: {} }, + // A lock entry with no harness row has nothing to start, so it is not run. + "agent-gone": { clients: {}, providers: {} }, + }, + }; + expect( + probedHarnesses(known, published, lock).map((harness) => harness.image), + ).toEqual(["agent-claude-sdk", "agent-langgraph-agui", ENGINE]); + expect( + probedHarnesses(known, published, lock, [ENGINE]).map( + (harness) => harness.image, + ), + ).toEqual([ENGINE]); +}); + +test("langgraph-ts is run on Anthropic, OpenAI and Gemini, one row per listed model", () => { + expect(providersFor(LANGGRAPH_TS)).toEqual( + expect.arrayContaining(["anthropic", "openai", "google"]), + ); + const rows = buildMatrix({ + harnesses: [LANGGRAPH_TS], + tags: ["source"], + listings: new Map([ + ["anthropic", { models: anthropicIds }], + ["openai", { models: openaiIds }], + ["google", { models: geminiIds }], + ]), + }); + const runs = rows.filter((matrixRow) => matrixRow.kind === "run"); + expect(runs.length).toBe( + anthropicIds.length + openaiIds.length + geminiIds.length, + ); + for (const [provider, ids] of [ + ["anthropic", anthropicIds], + ["openai", openaiIds], + ["google", geminiIds], + ] as const) + for (const model of ids) + expect( + runs.some( + (run) => + run.kind === "run" && + run.provider === provider && + run.model === model, + ), + `${provider} ${model}`, + ).toBe(true); +}); + +test("the probe reads an engine's LangChain and AG-UI clients under plain Node", async () => { + // A Node 22 image's working directory, with its packages installed beside it (and one hoisted + // to the directory above, as a workspace install leaves it). + const image = mkdtempSync(join(tmpdir(), "model-compat-node-")); + const app = join(image, "app"); + const installed: Record = { + "@langchain/langgraph": "1.7.5", + "@langchain/core": "1.2.0", + "@langchain/anthropic": "1.3.1", + "@langchain/openai": "1.4.0", + "@langchain/google-genai": "2.1.0", + // The engine's Gemini client, the one its package.json names. + "@langchain/google": "0.2.15", + "@ag-ui/client": "0.0.46", + }; + for (const [name, version] of Object.entries(installed)) { + const dir = join(app, "node_modules", name); + mkdirSync(dir, { recursive: true }); + // An `exports` map without `./package.json`, as some packages ship: a plain require of the + // package's manifest is refused, so the probe reads the file. + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name, version, exports: { ".": "./index.js" } }), + ); + } + const hoisted = join(image, "node_modules", "@ag-ui", "core"); + mkdirSync(hoisted, { recursive: true }); + writeFileSync( + join(hoisted, "package.json"), + JSON.stringify({ name: "@ag-ui/core", version: "0.0.46" }), + ); + + const child = Bun.spawn(["node", "-e", NODE_VERSIONS], { + cwd: app, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, code] = await Promise.all([ + new Response(child.stdout).text(), + child.exited, + ]); + expect(code).toBe(0); + expect(JSON.parse(stdout.trim())).toEqual({ + ...installed, + "@ag-ui/core": "0.0.46", + }); +}); + +test("both workflows check every engine the lock names, not only the published ones", () => { + for (const workflow of ["model-compat.yml", "model-lock.yml"]) { + const text = readFileSync( + join(ROOT, ".github/workflows", workflow), + "utf8", + ); + expect(text, workflow).toContain("bun scripts/model-compat.ts harnesses"); + } +}); diff --git a/scripts/model-compat.test.ts b/scripts/model-compat.test.ts new file mode 100644 index 0000000000..9fcc62e67e --- /dev/null +++ b/scripts/model-compat.test.ts @@ -0,0 +1,403 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + buildMatrix, + classifyMastra, + classifyRun, + codexGate, + type Harness, + type MatrixRow, + NOT_CHECKED, + PROVIDERS, + parseCapabilities, + providersFor, + renderTable, + usableModels, +} from "./model-compat"; + +const ROOT = join(import.meta.dir, ".."); +const NOW = Date.parse("2026-11-17T12:00:00Z"); + +// The listings below are cut from what each provider answered on 2026-11-17. +const openaiListing = { + data: [ + { id: "gpt-6.1-terra", created: 1792000000 }, + { id: "gpt-6.2-sol", created: 1793000000 }, + { id: "gpt-5.5", created: 1776900000 }, + { id: "gpt-5.5-2026-04-23", created: 1776900000 }, + { id: "gpt-5.3-codex", created: 1770000000 }, + { id: "gpt-5-search-api", created: 1760000000 }, + { id: "chat-latest", created: 1760000000 }, + { id: "gpt-4o-mini-tts", created: 1740000000 }, + { id: "gpt-realtime-2", created: 1780000000 }, + { id: "gpt-image-2", created: 1776000000 }, + { id: "text-embedding-3-large", created: 1700000000 }, + { id: "whisper-1", created: 1680000000 }, + { id: "gpt-live-1", created: 1790000000 }, + { id: "gpt-4-turbo", created: 1700000000 }, + { id: "o1", created: 1720000000 }, + { id: "o3", created: 1744000000, shutdown_date: "2026-11-01" }, + { id: "o3-pro", created: 1749000000, shutdown_date: "2026-12-11" }, + ], +}; + +const anthropicListing = { + data: [ + { + id: "claude-fable-5-2", + created_at: "2026-11-09T18:30:00Z", + lifecycle: "active", + }, + { + id: "claude-opus-5-5", + created_at: "2026-09-21T16:24:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-5-5", + created_at: "2026-09-28T00:00:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-4-5-20250929", + created_at: "2025-09-29T00:00:00Z", + retires_at: "2026-11-30T17:00:00Z", + lifecycle: "deprecated", + }, + { + id: "claude-3-opus-20240229", + created_at: "2024-02-29T00:00:00Z", + retires_at: "2026-01-05T00:00:00Z", + lifecycle: "retired", + }, + ], +}; + +const geminiListing = { + object: "list", + data: [ + { id: "models/gemini-3.8-flash", object: "model", owned_by: "google" }, + { id: "models/gemini-4-argon", object: "model", owned_by: "google" }, + { id: "models/gemini-embedding-001", object: "model", owned_by: "google" }, + { + id: "models/imagen-4.0-generate-001", + object: "model", + owned_by: "google", + }, + { id: "models/veo-3.0-generate-001", object: "model", owned_by: "google" }, + { id: "models/gemini-3.8-flash-live", object: "model", owned_by: "google" }, + { + id: "models/gemini-robotics-er-1.5", + object: "model", + owned_by: "google", + }, + { id: "models/aqa", object: "model", owned_by: "google" }, + { + id: "models/gemini-2.5-flash-preview-tts", + object: "model", + owned_by: "google", + }, + ], +}; + +test("an OpenAI key is offered only the chat models a Bot can run, newest first", () => { + expect(usableModels("openai", openaiListing, new Map(), NOW)).toEqual([ + "gpt-6.2-sol", + "gpt-6.1-terra", + "gpt-5.5", + "o3-pro", + ]); +}); + +test("models.dev saying a model takes no tools drops it, as the picker does", () => { + const capabilities = parseCapabilities({ + openai: { models: { "gpt-6.1-terra": { tool_call: false } } }, + }); + expect( + usableModels("openai", openaiListing, capabilities, NOW), + ).not.toContain("gpt-6.1-terra"); +}); + +test("an Anthropic key is offered every current Claude, retired ones dropped", () => { + expect(usableModels("anthropic", anthropicListing, new Map(), NOW)).toEqual([ + "claude-fable-5-2", + "claude-sonnet-5-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + ]); +}); + +test("a Google key keeps the Gemini chat models and drops media, live, robots and aqa", () => { + expect(usableModels("google", geminiListing, new Map(), NOW).sort()).toEqual([ + "gemini-3.8-flash", + "gemini-4-argon", + ]); +}); + +const harnesses: Harness[] = [ + { + image: "agent-claude-sdk", + credential: "anthropic", + port: 4212, + runPath: "", + }, + { image: "agent-antigravity", credential: "google", port: 4214, runPath: "" }, + { + image: "agent-agno", + credential: "any-provider", + port: 4203, + runPath: "/agui", + }, +]; + +test("each harness is paired with the providers it can run on", () => { + expect(providersFor(harnesses[0]!)).toEqual(["anthropic"]); + expect(providersFor(harnesses[1]!)).toEqual(["google"]); + expect(providersFor(harnesses[2]!)).toEqual( + PROVIDERS.map((provider) => provider.id), + ); +}); + +test("the matrix has one row per harness, image tag and model, and skips a provider with no key", () => { + const rows = buildMatrix({ + harnesses, + tags: ["latest", "main"], + listings: new Map([ + ["anthropic", { models: ["claude-opus-5-5", "claude-sonnet-5-5"] }], + ["openai", { error: "401 Incorrect API key provided" }], + ]), + }); + const runs = rows.filter( + (row): row is Extract => row.kind === "run", + ); + // claude-sdk and agno on Anthropic: 2 harnesses x 2 tags x 2 models. + expect(runs).toHaveLength(8); + expect( + runs + .filter((row) => row.harness.image === "agent-claude-sdk") + .map((row) => `${row.tag} ${row.model}`), + ).toEqual([ + "latest claude-opus-5-5", + "latest claude-sonnet-5-5", + "main claude-opus-5-5", + "main claude-sonnet-5-5", + ]); + const listingFailures = rows.filter((row) => row.kind === "listing-failed"); + expect( + listingFailures.map((row) => [row.harness.image, row.provider]), + ).toEqual([["agent-agno", "openai"]]); + const skipped = rows.filter((row) => row.kind === "no-key"); + expect( + skipped.map((row) => [row.harness.image, row.provider]).sort(), + ).toEqual([ + ["agent-agno", "google"], + ["agent-agno", "xai"], + ["agent-antigravity", "google"], + ]); +}); + +// What the Claude Agent SDK harness at claude-agent-sdk==0.2.152 streamed for claude-opus-5-5. +const claudeSdkRefusal = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"CUSTOM","name":"system:status","value":{"status":"requesting"}}', + `data: {"type":"RUN_ERROR","message":"API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.","code":"OPENBOT_MODEL_REFUSED"}`, + "", +].join("\n\n"); + +// What the Agno harness streamed for claude-sonnet-5-5, and for a model Anthropic does not have. +const agnoReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"TEXT_MESSAGE_START","messageId":"m","role":"assistant"}', + 'data: {"type":"TEXT_MESSAGE_CONTENT","messageId":"m","delta":"OK"}', + 'data: {"type":"TEXT_MESSAGE_END","messageId":"m"}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', +].join("\n\n"); +const agnoNotFound = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + `data: {"type":"RUN_ERROR","rawEvent":{"event":"RunError","content":"litellm.NotFoundError: AnthropicException - b'{\\"type\\":\\"error\\",\\"error\\":{\\"type\\":\\"not_found_error\\",\\"message\\":\\"model: claude-nonexistent-9\\"}}'"}}`, +].join("\n\n"); + +test("a run the provider refused fails with the provider's own words", () => { + const result = classifyRun({ status: 200, body: claudeSdkRefusal }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain( + "Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required", + ); +}); + +test("a run that answered passes", () => { + expect(classifyRun({ status: 200, body: agnoReply })).toEqual({ + status: "pass", + detail: "OK", + }); +}); + +test("a reply that carries the model's reasoning fails, though it says OK", () => { + // What gemma-4-31b-it answered through the LangGraph (TypeScript) engine. + const leaked = agnoReply.replace( + '"delta":"OK"', + '"delta":"The user wants one word.OK"', + ); + const outcome = classifyRun({ status: 200, body: leaked }); + expect(outcome.status).toBe("fail"); + expect(outcome.detail).toContain("reasoning in the reply text"); +}); + +test("an error a framework puts in its raw event is still the failure's text", () => { + const result = classifyRun({ status: 200, body: agnoNotFound }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("not_found_error"); +}); + +test("a run that finished without a reply, or answered with an error as text, fails", () => { + expect( + classifyRun({ + status: 200, + body: 'data: {"type":"RUN_STARTED"}\n\ndata: {"type":"RUN_FINISHED"}', + }).status, + ).toBe("fail"); + const asText = classifyRun({ + status: 200, + body: [ + 'data: {"type":"TEXT_MESSAGE_CONTENT","delta":"Error code: 400 - model not supported"}', + 'data: {"type":"RUN_FINISHED"}', + ].join("\n\n"), + }); + expect(asText.status).toBe("fail"); + expect(asText.detail).toContain("model not supported"); +}); + +test("a reply that arrives only as a messages snapshot, as CrewAI's bridge sends it, passes", () => { + const crewaiReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"MESSAGES_SNAPSHOT","messages":[{"id":"u","role":"user","content":"Reply with the single word OK."},{"id":"a","role":"assistant","content":"OK"}]}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', + ].join("\n\n"); + expect(classifyRun({ status: 200, body: crewaiReply })).toEqual({ + status: "pass", + detail: "OK", + }); + // The person's own message in the snapshot is not a reply. + const onlyThePrompt = crewaiReply.replace( + ',{"id":"a","role":"assistant","content":"OK"}', + "", + ); + expect(classifyRun({ status: 200, body: onlyThePrompt }).status).toBe("fail"); +}); + +test("Mastra's own API answer is read as a reply or a refusal", () => { + expect( + classifyMastra({ + status: 200, + body: '{"text":"OK","finishReason":"stop"}', + }), + ).toEqual({ status: "pass", detail: "OK" }); + expect( + classifyMastra({ + status: 500, + body: '{"error":"This model is only supported in v1/responses and not in v1/chat/completions."}', + }), + ).toEqual({ + status: "fail", + detail: + "HTTP 500: This model is only supported in v1/responses and not in v1/chat/completions.", + }); +}); + +test("a refused request fails with its status and body", () => { + expect( + classifyRun({ status: 401, body: '{"error":"unauthorised"}' }), + ).toEqual({ + status: "fail", + detail: 'HTTP 401: {"error":"unauthorised"}', + }); +}); + +// Shaped as openai/codex publishes codex-rs/models-manager/models.json. +const codexCatalogue = { + models: [ + { + slug: "gpt-6.1-terra", + visibility: "list", + minimal_client_version: "0.153.0", + }, + { + slug: "gpt-6.3-sol", + visibility: "list", + minimal_client_version: "0.170.0", + }, + { + slug: "gpt-daybreak-blue-latest", + visibility: "hide", + minimal_client_version: "0.142.2", + }, + ], +}; + +test("a ChatGPT plan model newer than the harness's Codex client version fails the gate", () => { + expect(codexGate("0.167.0", codexCatalogue)).toEqual([ + { + model: "gpt-6.1-terra", + status: "pass", + detail: "needs Codex 0.153.0, harness sends 0.167.0", + }, + { + model: "gpt-6.3-sol", + status: "fail", + detail: "needs Codex 0.170.0, harness sends 0.167.0", + }, + ]); + // Compared as numbers, not text. + expect( + codexGate("0.99.0", { + models: [ + { slug: "m", visibility: "list", minimal_client_version: "0.100.0" }, + ], + })[0]!.status, + ).toBe("fail"); +}); + +test("the table escapes what would break a Markdown row", () => { + const table = renderTable([ + { + harness: "agent-agno", + tag: "latest", + provider: "openai", + model: "gpt-6.1-terra", + status: "fail", + detail: "a | b\nc", + }, + ]); + expect(table).toContain("a \\| b c"); +}); + +test("every provider the Desktop offers by address is checked", () => { + const picker = readFileSync( + join(ROOT, "desktop/src/ProviderPicker.tsx"), + "utf8", + ); + const presets = [...picker.matchAll(/baseUrl: "(https:[^"]+)"/g)].map( + (match) => match[1]!.replace(/\/+$/, ""), + ); + expect(presets.length).toBeGreaterThan(0); + const checked = [ + ...PROVIDERS.map((provider) => provider.baseUrl), + ...NOT_CHECKED.map((left) => left.baseUrl), + ].map((url) => url?.replace(/\/+$/, "")); + for (const preset of presets) expect(checked).toContain(preset); +}); + +test("every published harness is one the matrix can run", () => { + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const { harnesses: known } = JSON.parse( + readFileSync(join(ROOT, "shared/harnesses.json"), "utf8"), + ) as { harnesses: Harness[] }; + for (const image of published) { + const harness = known.find((row) => row.image === image); + expect(harness, image).toBeDefined(); + expect(providersFor(harness!).length, image).toBeGreaterThan(0); + } +}); diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts new file mode 100644 index 0000000000..2a0b97742c --- /dev/null +++ b/scripts/model-compat.ts @@ -0,0 +1,1748 @@ +/** + * Does every model the picker offers still answer through the clients OpenBot ships? + * + * A harness image carries its own model client at a fixed version: the Claude Agent SDK harness + * bundles a Claude Code CLI, the LangGraph harness sends the ChatGPT plan's Codex backend a fixed + * client version, and every other harness carries the provider SDKs (LiteLLM, LangChain, the AI + * SDK, ...) it was built with. A provider can start refusing a model to an older client ("Claude + * Code 2.1.259 does not support this model; version 2.1.290 or newer is required") while the picker, + * which lists models live, keeps offering it. Nothing else calls today's models through the + * published images, so this does, once per model: + * + * 1. Lists each provider's models from the same source, with the same filter, as the picker + * (docs/configuration.md, "The model list"). + * 2. Starts each harness image on that provider, as the Desktop's settings would, and sends it + * one AG-UI run per model. The harness's own client makes the call, at the version that image + * ships. `latest` is the published image people run (ghcr.io/copilotkit/openbot-); + * `source` is the image built from this checkout, which shows whether a bumped pin fixed it. + * 3. Checks the ChatGPT plan's Codex client version in each image against the minimum each + * listed Codex model needs, from openai/codex's published model catalogue. A plan sign-in + * cannot run here, and the plan's version gate is a fact about the client version. + * + * A Claude plan runs through the same bundled CLI as an Anthropic key, so the key's rows cover it. + * + * Keys come from the environment (or --env-file), one per provider: ANTHROPIC_API_KEY, + * OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY. A provider with no key is reported as skipped, + * not failed. Keys are never printed: every detail is scrubbed of them. + * + * What passed on the `source` image becomes the model lock (shared/model-lock.ts): the models + * the picker may offer for each harness. `lock` writes it from the results; `guard` fails when a + * model the lock verifies no longer answers, for a pull request that changes a harness's client. + * + * bun scripts/model-compat.ts [--env-file .env] [--harness agent-claude-sdk,...] + * [--provider anthropic,...] [--model ] [--tags latest,source] [--concurrency 6] + * [--out results.json] [--exit-zero] + * bun scripts/model-compat.ts report --summary [--issue ] results/*.json + * bun scripts/model-compat.ts lock [--lock shared/model-lock.json] [--date YYYY-MM-DD] results/*.json + * bun scripts/model-compat.ts guard [--lock shared/model-lock.json] [--summary ] results/*.json + * bun scripts/model-compat.ts harnesses + * + * The harnesses it runs are the published ones and every engine the lock names, published or not + * yet (`harnesses` prints them, for the workflows' matrix). An engine is put in the lock with its + * providers' pairs empty before its first release, so nothing is offered on it until it has + * passed here: the LangGraph (TypeScript) engine, agent-langgraph-ts, starts that way. + * + * Needs Docker. Exits 1 when anything failed (unless --exit-zero); `report` merges the results of + * several runs, writes the Markdown summary, and exits 1 when any of them failed. `lock` exits 0, + * and says whether the lock changed. `guard` exits 1 when a verified model failed. + */ + +import { randomBytes } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { + LockedEngine, + LockProvider, + ModelLock, +} from "../shared/model-lock"; + +const ROOT = join(import.meta.dir, ".."); +const REGISTRY = "ghcr.io/copilotkit/openbot-"; +const SOURCE = "source"; +const MODELS_DEV = "https://models.dev/api.json"; +const CODEX_CATALOGUE = + "https://raw.githubusercontent.com/openai/codex/main/codex-rs/models-manager/models.json"; +const PROMPT = "Reply with the single word OK."; +/** One tool the prompt has no use for, so the request takes the client's tool path. */ +const PROBE_TOOL = { + name: "look_up_order", + description: "Look up the status of an order by its number.", + parameters: { + type: "object", + properties: { order: { type: "string", description: "The order number." } }, + required: ["order"], + }, +}; +const RUN_TIMEOUT_MS = 300_000; +const START_TIMEOUT_MS = 120_000; + +// --------------------------------------------------------------------------------------------- +// The picker's list. These rules are the ones the server's model catalogue applies, so this offers +// what a person can pick: the provider's own listing, minus what no Bot can chat on. +// --------------------------------------------------------------------------------------------- + +export type Source = "openai" | "anthropic" | "google" | "xai" | "gateway"; + +/** Models no chat can run on, by any provider or gateway's naming. */ +const NEVER_CHAT = + /(embed|tts|whisper|transcribe|dall-e|image|moderation|realtime|audio|sora|translate|video|lyria|imagen|veo-|(^|[-/])live([-.]|$))/i; +/** Chat models in OpenAI's listing a Bot is still not put on. */ +const OPENAI_NOT_FOR_BOTS = + /(search|deep-research|computer-use|chat-latest|codex|instruct|^gpt-live|^gpt-(3\.5|4)(-|$)|^o1(-|$)|^(babbage|davinci))/i; +/** The same, for the other providers a deployment names directly. */ +const VENDOR_NOT_FOR_BOTS = + /(deep-research|computer-use|robotics|antigravity|nano-banana|multi-agent|(^|-)aqa$)/i; +/** OpenAI's dated snapshots, which repeat a model already listed under its own name. */ +const DATED_SNAPSHOT = /-(\d{4}-\d{2}-\d{2}|\d{4})$/; + +const VENDOR_FAMILIES: readonly [RegExp, string][] = [ + [/^(gpt-|o\d|chatgpt-)/i, "openai"], + [/^claude/i, "anthropic"], + [/^(gemini|gemma)/i, "google"], + [/^grok/i, "xai"], +]; + +function modelVendor(id: string): string | undefined { + const slash = id.indexOf("/"); + if (slash > 0) return id.slice(0, slash).toLowerCase().replace(/^~/, ""); + return VENDOR_FAMILIES.find(([family]) => family.test(id))?.[1]; +} + +function chatModel(id: string, officialOpenAI: boolean): boolean { + if (NEVER_CHAT.test(id)) return false; + if (!officialOpenAI) return true; + return ( + /^(gpt-|o\d)/i.test(id) && + !OPENAI_NOT_FOR_BOTS.test(id) && + !DATED_SNAPSHOT.test(id) + ); +} + +function seconds(value: unknown): number { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed / 1000 : 0; + } + return 0; +} + +function pastDate(value: unknown, now: number): boolean { + if (typeof value !== "string" || !value) return false; + const at = Date.parse(value); + return Number.isFinite(at) && at <= now; +} + +/** What models.dev says a model can do, by `vendor/id`. */ +export type Capabilities = Map< + string, + { tools?: boolean; text?: boolean; released?: number } +>; + +export function parseCapabilities(body: unknown): Capabilities { + const found: Capabilities = new Map(); + if (!body || typeof body !== "object") return found; + for (const vendor of ["openai", "anthropic", "google", "xai"]) { + const models = (body as Record)[vendor] + ?.models; + if (!models || typeof models !== "object") continue; + for (const [id, raw] of Object.entries(models as Record)) { + if (!raw || typeof raw !== "object") continue; + const model = raw as Record; + const outputs = (model.modalities as { output?: unknown } | undefined) + ?.output; + found.set(`${vendor}/${id}`, { + ...(typeof model.tool_call === "boolean" + ? { tools: model.tool_call } + : {}), + ...(Array.isArray(outputs) ? { text: outputs.includes("text") } : {}), + ...(typeof model.release_date === "string" + ? { released: seconds(model.release_date) } + : {}), + }); + } + } + return found; +} + +/** The models the picker offers from one listing, newest first. */ +export function usableModels( + source: Source, + body: unknown, + capabilities: Capabilities, + now: number, +): string[] { + const record = + body && typeof body === "object" ? (body as Record) : {}; + const data = Array.isArray(record.data) + ? record.data + : Array.isArray(record.models) + ? record.models + : []; + const vendorOfSource = source === "gateway" ? undefined : source; + const usable: { id: string; created: number }[] = []; + for (const raw of data) { + if (!raw || typeof raw !== "object") continue; + const item = raw as Record; + const named = + typeof item.id === "string" + ? item.id + : typeof item.name === "string" + ? item.name + : ""; + const id = named.replace(/^models\//, ""); + if (!id) continue; + const outputs = item.output_modalities; + const methods = item.supportedGenerationMethods; + const notChat = + (Array.isArray(outputs) && !outputs.includes("text")) || + (Array.isArray(methods) && !methods.includes("generateContent")); + const retired = + pastDate(item.shutdown_date, now) || + pastDate(item.retires_at, now) || + item.lifecycle === "retired"; + const bare = id.split("/").at(-1) ?? id; + const vendor = vendorOfSource ?? modelVendor(id); + const known = vendor ? capabilities.get(`${vendor}/${bare}`) : undefined; + const chat = !notChat && known?.tools !== false && known?.text !== false; + const offered = + !retired && + chat && + (source === "openai" + ? chatModel(id, true) + : source === "gateway" + ? chatModel(id, false) + : !NEVER_CHAT.test(id) && !VENDOR_NOT_FOR_BOTS.test(id)); + if (offered) + usable.push({ + id, + created: + seconds(item.created ?? item.created_at) || known?.released || 0, + }); + } + return usable + .sort((a, b) => b.created - a.created || a.id.localeCompare(b.id)) + .map((entry) => entry.id); +} + +// --------------------------------------------------------------------------------------------- +// Providers and harnesses. +// --------------------------------------------------------------------------------------------- + +/** A row of shared/harnesses.json, the fields this reads. */ +export type Harness = { + image: string | null; + credential: string; + port: number | null; + runPath: string; +}; + +export type ProviderId = "openai" | "anthropic" | "google" | "xai"; + +export type Provider = { + id: ProviderId; + label: string; + /** The repository secret, and the environment variable, that holds this provider's key. */ + secret: string; + source: Source; + /** The OpenAI-compatible address the Desktop writes as OPENAI_BASE_URL for this provider. */ + baseUrl?: string; + listing: (key: string) => { url: string; headers: Record }; + /** The environment a harness is started with on this provider, as the Desktop writes it. */ + harnessEnv: (key: string, harness: Harness) => Record; +}; + +function compatible(baseUrl: string): Pick { + return { + baseUrl, + harnessEnv: (key) => ({ + OPENAI_API_KEY: key, + OPENAI_BASE_URL: baseUrl, + BOT_PROVIDER: "openai", + }), + }; +} + +function listedAt(url: string) { + return (key: string) => ({ + url, + headers: { authorization: `Bearer ${key}` }, + }); +} + +export const PROVIDERS: readonly Provider[] = [ + { + id: "openai", + label: "OpenAI", + secret: "OPENAI_API_KEY", + source: "openai", + listing: listedAt("https://api.openai.com/v1/models"), + harnessEnv: (key) => ({ OPENAI_API_KEY: key, BOT_PROVIDER: "openai" }), + }, + { + id: "anthropic", + label: "Anthropic", + secret: "ANTHROPIC_API_KEY", + source: "anthropic", + listing: (key) => ({ + url: "https://api.anthropic.com/v1/models?limit=1000", + headers: { "x-api-key": key, "anthropic-version": "2023-06-01" }, + }), + harnessEnv: (key) => ({ + ANTHROPIC_API_KEY: key, + BOT_PROVIDER: "anthropic", + }), + }, + { + id: "google", + label: "Google", + secret: "GEMINI_API_KEY", + source: "google", + listing: listedAt( + "https://generativelanguage.googleapis.com/v1beta/openai/models", + ), + ...compatible("https://generativelanguage.googleapis.com/v1beta/openai/"), + // Antigravity is pinned to Gemini's own API and reads the key under Google's name. + harnessEnv: (key, harness) => + harness.credential === "google" + ? { GOOGLE_API_KEY: key } + : compatible( + "https://generativelanguage.googleapis.com/v1beta/openai/", + ).harnessEnv(key, harness), + }, + { + id: "xai", + label: "xAI", + secret: "XAI_API_KEY", + source: "xai", + listing: listedAt("https://api.x.ai/v1/language-models"), + ...compatible("https://api.x.ai/v1"), + }, +]; + +/** + * Providers the Desktop offers by address that this check leaves out on purpose, with why. A + * gateway serves its own aliases, not a provider's models, so the lock has nothing to say about + * it and leaves its models open. + */ +export const NOT_CHECKED: readonly { baseUrl: string; why: string }[] = [ + { + baseUrl: "https://api.unbiased.ai/v1", + why: "Unbiased AI is a gateway serving its own `pareto` model, not a provider's listing", + }, +]; + +function provider(id: string): Provider { + const found = PROVIDERS.find((row) => row.id === id); + if (!found) throw new Error(`No provider ${id}`); + return found; +} + +/** The providers a harness can run on, by the credential shared/harnesses.json gives it. */ +export function providersFor(harness: Harness): ProviderId[] { + if (!harness.image || !harness.port) return []; + switch (harness.credential) { + case "anthropic": + return ["anthropic"]; + case "google": + return ["google"]; + case "any-provider": + return PROVIDERS.map((row) => row.id); + default: + return []; + } +} + +/** + * The harnesses the check runs: every published harness, and every engine the model lock names + * whether it is published yet or not. An engine the lock names offers a model only once it has + * passed here, so it is checked from before its first release (its `source` image, built from this + * checkout; a `latest` that isn't published yet is an inconclusive row). A lock entry with no row + * in shared/harnesses.json has nothing to start, so it is not run. `only` narrows to the named + * image directories. + */ +export function probedHarnesses( + known: readonly Harness[], + published: readonly string[], + lock: ModelLock | undefined, + only?: readonly string[], +): Harness[] { + const locked = Object.keys(lock?.engines ?? {}); + return known.filter( + (row) => + !!row.image && + (published.includes(row.image) || locked.includes(row.image)) && + (!only || only.includes(row.image)), + ); +} + +function readProbed(only?: readonly string[]): Harness[] { + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const { harnesses } = JSON.parse( + readFileSync(join(ROOT, "shared/harnesses.json"), "utf8"), + ) as { harnesses: Harness[] }; + const lockPath = join(ROOT, "shared/model-lock.json"); + const lock = existsSync(lockPath) + ? (JSON.parse(readFileSync(lockPath, "utf8")) as ModelLock) + : undefined; + return probedHarnesses(harnesses, published, lock, only); +} + +export type Listing = { models: string[] } | { error: string }; + +export type MatrixRow = + | { + kind: "run"; + harness: Harness; + tag: string; + provider: ProviderId; + model: string; + } + | { kind: "no-key"; harness: Harness; provider: ProviderId } + | { + kind: "listing-failed"; + harness: Harness; + provider: ProviderId; + detail: string; + }; + +/** + * One run per harness, image tag and listed model. A provider with no key is one skipped row per + * harness, and a listing that failed one failed row, since there are no models to name. + */ +export function buildMatrix(options: { + harnesses: readonly Harness[]; + tags: readonly string[]; + listings: ReadonlyMap; +}): MatrixRow[] { + const rows: MatrixRow[] = []; + for (const harness of options.harnesses) { + for (const id of providersFor(harness)) { + const listing = options.listings.get(id); + if (!listing) rows.push({ kind: "no-key", harness, provider: id }); + else if ("error" in listing) + rows.push({ + kind: "listing-failed", + harness, + provider: id, + detail: listing.error, + }); + else + for (const tag of options.tags) + for (const model of listing.models) + rows.push({ kind: "run", harness, tag, provider: id, model }); + } + } + return rows; +} + +// --------------------------------------------------------------------------------------------- +// Reading a run. +// --------------------------------------------------------------------------------------------- + +export type Outcome = { status: "pass" | "fail"; detail: string }; + +function oneLine(text: string, limit = 400): string { + const flat = text.replace(/\s+/g, " ").trim(); + return flat.length > limit ? `${flat.slice(0, limit)}...` : flat; +} + +function errorText(event: Record): string { + if (typeof event.message === "string" && event.message) return event.message; + const raw = event.rawEvent as Record | undefined; + for (const field of ["content", "error", "message"]) { + const value = raw?.[field]; + if (typeof value === "string" && value) return value; + } + return JSON.stringify(event); +} + +/** Pass when the harness streamed a reply that says OK; otherwise fail, in the provider's words. */ +export function classifyRun(response: { + status: number; + body: string; +}): Outcome { + if (response.status !== 200) + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(response.body)}`, + }; + let streamed = ""; + // Some bridges (CrewAI's) send the reply only as the conversation's new state. + let snapshot = ""; + let finished = false; + let calledTool = false; + for (const line of response.body.split("\n")) { + if (!line.startsWith("data:")) continue; + let event: Record; + try { + event = JSON.parse(line.slice(5)); + } catch { + continue; + } + if (event.type === "RUN_ERROR") + return { status: "fail", detail: oneLine(errorText(event)) }; + if ( + (event.type === "TEXT_MESSAGE_CONTENT" || + event.type === "TEXT_MESSAGE_CHUNK") && + typeof event.delta === "string" + ) + streamed += event.delta; + if (event.type === "MESSAGES_SNAPSHOT" && Array.isArray(event.messages)) { + const last = event.messages.at(-1) as + | { role?: unknown; content?: unknown } + | undefined; + snapshot = + last?.role === "assistant" && typeof last.content === "string" + ? last.content + : ""; + } + if (event.type === "TOOL_CALL_START") calledTool = true; + if (event.type === "RUN_FINISHED") finished = true; + } + return judgeReply( + streamed || snapshot, + calledTool + ? "the model called the probe's tool instead of replying" + : finished + ? "the run finished without a reply" + : "the stream ended without finishing the run", + ); +} + +/** + * A model's own reasoning left in its reply, which a person would read as the Bot's answer: the + * Gemma 4 models, through the LangGraph (TypeScript) engine, answered "…OK". + */ +const REASONING_IN_REPLY = /<\/?(thought|think|thinking|reasoning)>/i; + +export function judgeReply(text: string, silence: string): Outcome { + if (REASONING_IN_REPLY.test(text)) + return { + status: "fail", + detail: `answered with its reasoning in the reply text: ${oneLine(text, 160)}`, + }; + if (/\bok(ay)?\b/i.test(text)) + return { status: "pass", detail: oneLine(text, 80) }; + if (text) + return { status: "fail", detail: `answered without OK: ${oneLine(text)}` }; + return { status: "fail", detail: silence }; +} + +/** + * Mastra serves no AG-UI route: OpenBot reaches it through Mastra's own agent API, so its run is a + * `generate` call on that API, answered with the reply's `text` or an `error`. + */ +export function classifyMastra(response: { + status: number; + body: string; +}): Outcome { + let parsed: { text?: unknown; error?: unknown; message?: unknown } = {}; + try { + parsed = JSON.parse(response.body); + } catch { + // Not JSON: reported as it is, below. + } + if (response.status !== 200) { + const said = [parsed.error, parsed.message].find( + (value): value is string => typeof value === "string" && !!value, + ); + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(said ?? response.body)}`, + }; + } + return judgeReply( + typeof parsed.text === "string" ? parsed.text : "", + "the agent answered without text", + ); +} + +// --------------------------------------------------------------------------------------------- +// The ChatGPT plan's version gate. +// --------------------------------------------------------------------------------------------- + +function versionParts(version: string): number[] { + return version.split(/[.-]/).map((part) => Number.parseInt(part, 10) || 0); +} + +export function compareVersions(a: string, b: string): number { + const left = versionParts(a); + const right = versionParts(b); + for (let i = 0; i < Math.max(left.length, right.length); i++) { + const difference = (left[i] ?? 0) - (right[i] ?? 0); + if (difference) return Math.sign(difference); + } + return 0; +} + +/** Each model the Codex picker lists, against the client version the harness sends the plan. */ +export function codexGate( + clientVersion: string, + catalogue: unknown, +): { model: string; status: "pass" | "fail"; detail: string }[] { + const models = (catalogue as { models?: unknown })?.models; + if (!Array.isArray(models)) return []; + return models + .filter( + (row): row is { slug: string; minimal_client_version?: string } => + !!row && row.visibility === "list" && typeof row.slug === "string", + ) + .map((row) => { + const needs = row.minimal_client_version ?? "0.0.0"; + return { + model: row.slug, + status: compareVersions(clientVersion, needs) >= 0 ? "pass" : "fail", + detail: `needs Codex ${needs}, harness sends ${clientVersion}`, + }; + }); +} + +// --------------------------------------------------------------------------------------------- +// Results. +// --------------------------------------------------------------------------------------------- + +export type Result = { + harness: string; + tag: string; + /** What a person reads: "Anthropic (key or Claude plan)", "ChatGPT plan". */ + provider: string; + /** What the lock files it under. */ + providerId: LockProvider; + model: string; + status: "pass" | "fail" | "skipped"; + detail: string; +}; + +/** One results file: the rows, and the model clients each image carried. */ +export type Results = { + results: Result[]; + /** By harness, then image tag: package name to version. */ + clients?: Record>>; +}; + +function cell(text: string): string { + return oneLine(text).replace(/\|/g, "\\|"); +} + +export function renderTable(results: readonly Result[]): string { + const lines = [ + "| Harness | Image | Provider | Model | Result | Provider's answer |", + "| --- | --- | --- | --- | --- | --- |", + ]; + for (const row of results) + lines.push( + `| ${cell(row.harness)} | ${cell(row.tag)} | ${cell(row.provider)} | ${cell(row.model)} | ${row.status} | ${cell(row.detail)} |`, + ); + return lines.join("\n"); +} + +const ORDER = { fail: 0, skipped: 1, pass: 2 } as const; + +/** + * The summary: what failed first, then what was skipped for want of a key, then what passed. The + * issue's copy leaves out what passed and keeps at most `maxFailed` failures, since an issue body + * holds 65,536 characters and the run's own summary has the rest. + */ +export function renderSummary( + results: readonly Result[], + options: { passed?: boolean; maxFailed?: number } = {}, +): string { + const { passed: showPassed = true, maxFailed = Number.POSITIVE_INFINITY } = + options; + const sorted = [...results].sort( + (a, b) => + ORDER[a.status] - ORDER[b.status] || + a.harness.localeCompare(b.harness) || + a.provider.localeCompare(b.provider) || + a.tag.localeCompare(b.tag) || + a.model.localeCompare(b.model), + ); + const count = (status: Result["status"]) => + results.filter((row) => row.status === status).length; + const failed = sorted.filter((row) => row.status === "fail"); + const skipped = sorted.filter((row) => row.status === "skipped"); + const passed = sorted.filter((row) => row.status === "pass"); + const missing = [ + ...new Set( + skipped.map( + (row) => + PROVIDERS.find((p) => p.label === row.provider)?.secret ?? + row.provider, + ), + ), + ]; + const parts = [ + "## Model compatibility", + "", + `${count("fail")} failed, ${count("pass")} passed, ${count("skipped")} skipped.`, + "", + ]; + if (missing.length) + parts.push( + `Skipped for want of a key, not failed: add the repository secret${missing.length > 1 ? "s" : ""} ${missing.map((name) => `\`${name}\``).join(", ")} to check ${missing.length > 1 ? "those providers" : "that provider"}.`, + "", + ); + if (failed.length) { + parts.push("### Failed", "", renderTable(failed.slice(0, maxFailed)), ""); + if (failed.length > maxFailed) + parts.push( + `And ${failed.length - maxFailed} more failures, in the run's summary.`, + "", + ); + } + if (skipped.length) parts.push("### Skipped", "", renderTable(skipped), ""); + if (passed.length && showPassed) + parts.push( + "
Passed", + "", + renderTable(passed), + "", + "
", + "", + ); + return parts.join("\n"); +} + +// --------------------------------------------------------------------------------------------- +// Running it. +// --------------------------------------------------------------------------------------------- + +type Options = { + envFile?: string; + harnesses?: string[]; + providers?: string[]; + model?: RegExp; + tags: string[]; + concurrency: number; + out?: string; + exitZero: boolean; +}; + +function parseOptions(argv: string[]): Options { + const options: Options = { + tags: ["latest", SOURCE], + concurrency: 6, + exitZero: false, + }; + const list = (value: string | undefined) => + (value ?? "") + .split(/[\s,]+/) + .map((item) => item.trim()) + .filter(Boolean); + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined) throw new Error(`${flag} needs a value`); + return next; + }; + if (flag === "--env-file") options.envFile = value(); + else if (flag === "--harness") options.harnesses = list(value()); + else if (flag === "--provider") options.providers = list(value()); + else if (flag === "--model") options.model = new RegExp(value()); + else if (flag === "--tags") options.tags = list(value()); + else if (flag === "--concurrency") + options.concurrency = Math.max(1, Number(value()) || 1); + else if (flag === "--out") options.out = value(); + else if (flag === "--exit-zero") options.exitZero = true; + else throw new Error(`Unknown option ${flag}`); + } + return options; +} + +/** The provider keys, from the environment, or from an env file for the names it lacks. */ +function readKeys(envFile: string | undefined): Map { + const fromFile = new Map(); + if (envFile) + for (const line of readFileSync(envFile, "utf8").split("\n")) { + const match = /^\s*(?:export\s+)?([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line); + if (!match) continue; + const [, name = "", raw = ""] = match; + fromFile.set(name, raw.trim().replace(/^(['"])(.*)\1$/, "$2")); + } + const keys = new Map(); + for (const row of PROVIDERS) { + const value = ( + process.env[row.secret] || + fromFile.get(row.secret) || + "" + ).trim(); + if (value) keys.set(row.id, value); + } + return keys; +} + +function scrubber(keys: Map) { + const secrets = [...keys.values()].filter((value) => value.length >= 8); + return (text: string) => + secrets.reduce( + (scrubbed, secret) => scrubbed.split(secret).join("[key]"), + text, + ); +} + +async function fetchJson( + url: string, + headers: Record = {}, +): Promise<{ ok: true; body: unknown } | { ok: false; error: string }> { + try { + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(30_000), + }); + const text = await response.text(); + if (!response.ok) + return { ok: false, error: `HTTP ${response.status}: ${oneLine(text)}` }; + return { ok: true, body: JSON.parse(text) }; + } catch (error) { + return { ok: false, error: String(error) }; + } +} + +async function exec( + args: string[], + env: Record = {}, +): Promise<{ code: number; stdout: string; stderr: string }> { + const child = Bun.spawn(args, { + env: { ...process.env, ...env }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + return { code, stdout, stderr }; +} + +/** + * `source` is the image built from this checkout, which is what the next release ships, so a pin + * bumped here is checked before it is released. Any other tag is the published image: `latest` is + * what people run today. + */ +function imageOf(harness: Harness, tag: string): string { + return tag === SOURCE + ? `model-compat/${harness.image}:source` + : `${REGISTRY}${harness.image}:${tag}`; +} + +async function prepare(harness: Harness, tag: string) { + return tag === SOURCE + ? exec([ + "docker", + "build", + "--quiet", + "-f", + join(ROOT, harness.image ?? "", "Dockerfile"), + "-t", + imageOf(harness, tag), + ROOT, + ]) + : exec(["docker", "pull", "--quiet", imageOf(harness, tag)]); +} + +/** + * The model-client packages whose versions decide what a provider accepts, by the names each + * ecosystem installs them under. Whichever an image has are recorded in the lock beside the + * models they were proven with, so a lock entry says what it was true for. + */ +export const PYTHON_CLIENTS = [ + "claude-agent-sdk", + "ag-ui-claude-sdk", + "openai", + "anthropic", + "litellm", + "google-genai", + "google-adk", + "ag2", + "agno", + "crewai", + "langgraph", + "ag-ui-langgraph", + "langchain-core", + "langchain-openai", + "langchain-anthropic", + "langchain-google-genai", + "langroid", + "llama-index-core", + "llama-index-llms-openai", + "llama-index-llms-anthropic", + "llama-index-llms-google-genai", + "pydantic-ai-slim", + "strands-agents", + "agent-framework", + "agent-framework-core", +]; +export const NODE_CLIENTS = [ + "@mastra/core", + "ai", + "@ai-sdk/openai", + "@ai-sdk/anthropic", + "@ai-sdk/google", + // The LangGraph (TypeScript) engine: its graph, the LangChain chat models that call each + // provider, and the AG-UI packages it speaks through. + "@langchain/langgraph", + "@langchain/core", + "@langchain/anthropic", + "@langchain/openai", + "@langchain/google", + "@langchain/google-genai", + "@ag-ui/core", + "@ag-ui/client", + "@ag-ui/encoder", + "@ag-ui/langgraph", +]; + +const PYTHON_VERSIONS = ` +import json, importlib.metadata as m +found = {} +for name in ${JSON.stringify(PYTHON_CLIENTS)}: + try: + found[name] = m.version(name) + except Exception: + pass +try: + import claude_agent_sdk, pathlib, subprocess + for cli in pathlib.Path(claude_agent_sdk.__file__).parent.glob("_bundled/claude*"): + said = subprocess.run([str(cli), "--version"], capture_output=True, text=True, timeout=30).stdout.split() + if said: + found["claude-code"] = said[0] + break +except Exception: + pass +print(json.dumps(found)) +`; +/** + * Read under `bun -e` or plain `node -e` (a Node image may have no Bun), from the image's working + * directory up, the way Node resolves a package. The manifest is read as a file rather than + * required, since a package whose `exports` map leaves out `./package.json` refuses the require. + */ +export const NODE_VERSIONS = ` +const { readFileSync } = require("node:fs"); +const { dirname, join } = require("node:path"); +const found = {}; +for (const name of ${JSON.stringify(NODE_CLIENTS)}) { + for (let dir = process.cwd(); ; dir = dirname(dir)) { + try { + found[name] = JSON.parse(readFileSync(join(dir, "node_modules", name, "package.json"), "utf8")).version; + break; + } catch {} + if (dirname(dir) === dir) break; + } +} +console.log(JSON.stringify(found)); +`; + +/** The model clients an image carries, or an empty record when it says nothing. */ +async function clientsIn(image: string): Promise> { + for (const args of [ + ["python", "-c", PYTHON_VERSIONS], + ["bun", "-e", NODE_VERSIONS], + ["node", "-e", NODE_VERSIONS], + ] as const) { + const ran = await exec([ + "docker", + "run", + "--rm", + "--entrypoint", + args[0], + image, + args[1], + args[2], + ]); + if (ran.code !== 0) continue; + try { + const found = JSON.parse(ran.stdout.trim().split("\n").at(-1) ?? ""); + if (found && typeof found === "object" && Object.keys(found).length) + return found as Record; + } catch { + // Not this runtime's answer: try the next. + } + } + return {}; +} + +let serial = 0; + +/** Starts the harness on one provider and model, sends one run, and stops it. */ +async function runOne( + row: Extract, + key: string, + scrub: (text: string) => string, +): Promise { + const harness = row.harness; + const name = `model-compat-${process.pid}-${++serial}`; + const env = { + ...provider(row.provider).harnessEnv(key, harness), + BOT_MODEL: row.model, + }; + const token = randomBytes(12).toString("hex"); + // Each value goes to Docker through the environment and is named here, so no key is in argv. + const started = await exec( + [ + "docker", + "run", + "-d", + "--rm", + "--name", + name, + "-p", + `127.0.0.1::${harness.port}`, + "-e", + "MANAGED_AGENT_TOKEN", + ...Object.keys(env).flatMap((variable) => ["-e", variable]), + imageOf(harness, row.tag), + ], + { ...env, MANAGED_AGENT_TOKEN: token }, + ); + if (started.code !== 0) + return { + status: "fail", + detail: scrub(`the harness did not start: ${oneLine(started.stderr)}`), + }; + try { + const port = await exec(["docker", "port", name, `${harness.port}/tcp`]); + const address = port.stdout.trim().split("\n")[0]?.trim(); + if (!address) + return { status: "fail", detail: "the harness published no port" }; + const base = `http://${address}`; + const deadline = Date.now() + START_TIMEOUT_MS; + let healthy = false; + while (Date.now() < deadline) { + try { + const health = await fetch(`${base}/health`, { + signal: AbortSignal.timeout(5_000), + }); + if (health.ok) { + healthy = true; + break; + } + } catch { + // Not listening yet. + } + const state = await exec([ + "docker", + "inspect", + "-f", + "{{.State.Running}}", + name, + ]); + if (state.stdout.trim() !== "true") break; + await Bun.sleep(1_000); + } + if (!healthy) { + const logs = await exec(["docker", "logs", "--tail", "5", name]); + return { + status: "fail", + detail: scrub( + `the harness never became healthy: ${oneLine(logs.stdout + logs.stderr)}`, + ), + }; + } + const id = randomBytes(6).toString("hex"); + const mastra = harness.image === "agent-mastra"; + let response: { status: number; body: string }; + try { + const reply = await fetch( + mastra + ? `${base}/api/agents/openbot/generate` + : `${base}${harness.runPath || "/"}`, + { + method: "POST", + headers: { + "content-type": "application/json", + accept: mastra ? "application/json" : "text/event-stream", + "x-openbot-agent-token": token, + }, + body: JSON.stringify( + mastra + ? { messages: [{ role: "user", content: PROMPT }] } + : { + threadId: `model-compat-${id}`, + runId: `run-${id}`, + state: {}, + messages: [ + { id: `message-${id}`, role: "user", content: PROMPT }, + ], + // A Bot's run always carries tools, and a client's tool path is where models + // part ways (Responses-only models, parallel tool calls), so this one does too. + tools: [PROBE_TOOL], + context: [], + // What the server tells the LangGraph and Claude Agent SDK harnesses; the others + // read BOT_MODEL, set above. + forwardedProps: { openbotModel: { model: row.model } }, + }, + ), + signal: AbortSignal.timeout(RUN_TIMEOUT_MS), + }, + ); + response = { status: reply.status, body: await reply.text() }; + } catch (error) { + return { + status: "fail", + detail: scrub(`no answer: ${oneLine(String(error))}`), + }; + } + const outcome = mastra ? classifyMastra(response) : classifyRun(response); + if (outcome.status === "fail" && !response.body.includes("RUN_ERROR")) { + // A harness that swallowed the error still logged it. + const logs = await exec(["docker", "logs", "--tail", "40", name]); + const said = (logs.stdout + logs.stderr) + .split("\n") + .filter((line) => /error|exception/i.test(line)) + .at(-1); + if (said) + outcome.detail = `${outcome.detail}; logged: ${oneLine(said, 300)}`; + } + return { status: outcome.status, detail: scrub(outcome.detail) }; + } finally { + await exec(["docker", "rm", "-f", name]); + } +} + +async function pool( + items: readonly T[], + width: number, + work: (item: T) => Promise, +) { + let next = 0; + await Promise.all( + Array.from({ length: Math.min(width, items.length) }, async () => { + for (let item = items[next++]; item !== undefined; item = items[next++]) + await work(item); + }), + ); +} + +async function check(argv: string[]): Promise { + const options = parseOptions(argv); + const keys = readKeys(options.envFile); + const scrub = scrubber(keys); + + const harnesses = readProbed(options.harnesses); + if (!harnesses.length) + throw new Error( + "No harness matches --harness: it must be published, or named in shared/model-lock.json, and have a row in shared/harnesses.json.", + ); + + const wanted = new Set( + harnesses + .flatMap(providersFor) + .filter((id) => !options.providers || options.providers.includes(id)), + ); + const capabilities = await fetchJson(MODELS_DEV); + const known = capabilities.ok + ? parseCapabilities(capabilities.body) + : new Map(); + const listings = new Map(); + for (const id of wanted) { + const key = keys.get(id); + if (!key) continue; + const row = provider(id); + const request = row.listing(key); + const listed = await fetchJson(request.url, request.headers); + listings.set( + id, + listed.ok + ? { + models: usableModels( + row.source, + listed.body, + known, + Date.now(), + ).filter((model) => !options.model || options.model.test(model)), + } + : { error: scrub(`the model listing failed: ${listed.error}`) }, + ); + } + const matrix = buildMatrix({ + harnesses, + tags: options.tags, + listings, + }).filter( + (row) => !options.providers || options.providers.includes(row.provider), + ); + + const results: Result[] = []; + const label = (id: string) => provider(id).label; + for (const row of matrix) { + if (row.kind === "no-key") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + providerId: row.provider, + model: "(all)", + status: "skipped", + detail: `skipped: no key (${provider(row.provider).secret})`, + }); + if (row.kind === "listing-failed") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + providerId: row.provider, + model: "(all)", + status: "fail", + detail: row.detail, + }); + } + + // Every image once, before any run, so a pull or build is never counted against a run's time. + const images = harnesses.flatMap((harness) => + options.tags.map((tag) => ({ harness, tag })), + ); + const unpulled = new Map(); + await pool(images, 3, async ({ harness, tag }) => { + const began = Date.now(); + const ready = await prepare(harness, tag); + console.error( + `${tag === SOURCE ? "built" : "pulled"} ${imageOf(harness, tag)} in ${Math.round((Date.now() - began) / 1000)}s${ready.code ? " (failed)" : ""}`, + ); + if (ready.code !== 0) + unpulled.set( + imageOf(harness, tag), + oneLine(ready.stderr.split("\n").slice(-5).join(" ")), + ); + }); + + // The model clients each image carries, which the lock records beside what passed. + const clients: NonNullable = {}; + for (const { harness, tag } of images) { + const image = imageOf(harness, tag); + if (unpulled.has(image) || !harness.image) continue; + const found = await clientsIn(image); + clients[harness.image] = { ...clients[harness.image], [tag]: found }; + } + + // The ChatGPT plan's Codex client version, in whichever images carry the plan. + const codex = await fetchJson(CODEX_CATALOGUE); + if (!options.providers || options.providers.includes("openai")) + for (const harness of harnesses) + for (const tag of options.tags) { + const image = imageOf(harness, tag); + if (unpulled.has(image)) continue; + const version = await exec([ + "docker", + "run", + "--rm", + "--entrypoint", + "python", + image, + "-c", + "from src.chatgpt_models import client_version; print(client_version())", + ]); + if (version.code !== 0) continue; + const sends = version.stdout.trim(); + // What the plan's backend is told, recorded with the image's other clients. + if (harness.image) + clients[harness.image] = { + ...clients[harness.image], + [tag]: { ...clients[harness.image]?.[tag], "codex-client": sends }, + }; + if (!codex.ok) { + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + providerId: "chatgpt", + model: "(all)", + status: "fail", + detail: `could not read Codex's model catalogue: ${codex.error}`, + }); + continue; + } + for (const gate of codexGate(sends, codex.body)) + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + providerId: "chatgpt", + ...gate, + }); + } + + const runs = matrix.filter( + (row): row is Extract => row.kind === "run", + ); + let done = 0; + await pool(runs, options.concurrency, async (row) => { + const image = imageOf(row.harness, row.tag); + const ask = () => runOne(row, keys.get(row.provider) ?? "", scrub); + /* + * A failure is asked once more before it counts. A provider's refusal says the same thing + * twice; a run that timed out once does not (gemini-4-carbon through Antigravity's source + * image on 2026-11-18, which answered through the published image in the same run and through + * every other engine), and must not fail a pull request or leave the lock. + */ + let outcome: Outcome = unpulled.has(image) + ? { + status: "fail" as const, + detail: `could not get ${image}: ${unpulled.get(image)}`, + } + : await ask(); + if (outcome.status === "fail" && !unpulled.has(image)) + outcome = await ask(); + results.push({ + harness: row.harness.image ?? "", + tag: row.tag, + provider: + row.provider === "anthropic" && row.harness.credential === "anthropic" + ? "Anthropic (key or Claude plan)" + : label(row.provider), + providerId: row.provider, + model: row.model, + ...outcome, + }); + done++; + console.error( + `[${done}/${runs.length}] ${row.harness.image}:${row.tag} ${row.provider} ${row.model}: ${outcome.status}`, + ); + }); + + if (options.out) + writeFileSync( + options.out, + `${JSON.stringify({ results, clients } satisfies Results, null, 2)}\n`, + ); + console.log(renderSummary(results)); + return results.some((row) => row.status === "fail") && !options.exitZero + ? 1 + : 0; +} + +// --------------------------------------------------------------------------------------------- +// The model lock. +// --------------------------------------------------------------------------------------------- + +const PROVIDER_IDS_BY_LABEL: Record = { + OpenAI: "openai", + Anthropic: "anthropic", + "Anthropic (key or Claude plan)": "anthropic", + Google: "google", + xAI: "xai", + "ChatGPT plan": "chatgpt", +}; + +/** The lock's name for a row's provider. Results written before rows carried it are read by label. */ +export function providerIdOf(row: { + provider: string; + providerId?: string; +}): LockProvider | undefined { + if (row.providerId) return row.providerId as LockProvider; + return PROVIDER_IDS_BY_LABEL[row.provider]; +} + +/** + * A failure that says nothing about the model: the harness never started, the image could not be + * had, or the run timed out with no word from the provider. The lock keeps what it knew before + * for such a model rather than take it away on a bad day for the runner. + */ +const INCONCLUSIVE = + /^(no answer:|the harness did not start|the harness never became healthy|the harness published no port|could not get |the stream ended without finishing the run)/; + +/** A provider limiting how fast the check's key may call it: about the key's day, not the model. */ +const RATE_LIMITED = + /\b429\b|rate.?limit|RESOURCE_EXHAUSTED|too many requests/i; + +/** + * A model this key's account can't see, which another account may: a bare 404, or one that names + * no reason. About the key, not the client. A 404 that names the client's endpoint ("only supported + * in v1/responses") is the client's, and stays a refusal. + */ +const UNAVAILABLE_TO_KEY = + /is not found for API version|does not exist or you do not have access|do not have access to (?:the )?model|^(?:NotFoundError\(')?Error code: 404'?\)?$|^Not Found$/i; + +/** + * A provider retiring a model: Google's "no longer available to new users, use …". The check + * working as designed: the lock refuses it, so a person set to it is told and moved to the default + * instead of meeting the provider's 404. Not a client's fault, so it never fails a pull request. + */ +const RETIRED = + /no longer available to new users|has been (?:deprecated|retired|shut down)|(?:was|is) (?:deprecated|retired) and (?:is )?no longer/i; + +export function retiredByProvider(detail: string): boolean { + return RETIRED.test(detail); +} + +/** A failure about the check's key, not the harness's client: a rate limit, or a model it can't see. */ +export function aboutTheKey(detail: string): boolean { + return RATE_LIMITED.test(detail) || UNAVAILABLE_TO_KEY.test(detail); +} + +export function inconclusive(detail: string): boolean { + return INCONCLUSIVE.test(detail) || aboutTheKey(detail); +} + +function sortedRecord(record: Record): Record { + return Object.fromEntries( + Object.entries(record).sort(([a], [b]) => a.localeCompare(b)), + ); +} + +/** + * The tag a harness's lock is read from: `source`, the image built from this commit, which is what + * the next release ships. A run of the published images alone says nothing about this commit. + */ +function lockTag(rows: readonly Result[]): string | undefined { + return rows.some((row) => row.tag === SOURCE) ? SOURCE : undefined; +} + +/** + * The model lock from a run's results, over the lock before it. + * + * For each harness, the `source` image's rows: what passed is verified, what the provider refused + * is refused, in its words. What the run could not decide (an inconclusive failure), a provider + * it had no key for or could not list, and a harness it did not run, keep what the lock said + * before. A model the provider no longer lists drops out. The date moves only when the lock does, + * so a day with nothing new changes nothing. + */ +export function buildLock( + previous: ModelLock | undefined, + runs: readonly Results[], + date: string, +): ModelLock { + const results = runs.flatMap((run) => run.results); + const clients = Object.assign({}, ...runs.map((run) => run.clients ?? {})); + const engines: Record = { + ...(previous?.engines ?? {}), + }; + const harnesses = [...new Set(results.map((row) => row.harness))]; + for (const harness of harnesses) { + const rows = results.filter((row) => row.harness === harness); + const tag = lockTag(rows); + const before = previous?.engines[harness]; + const providers: LockedEngine["providers"] = { + ...(before?.providers ?? {}), + }; + const ran = rows.filter((row) => row.tag === tag && row.model !== "(all)"); + for (const id of new Set(ran.map(providerIdOf))) { + if (!id) continue; + const earlier = before?.providers[id]; + const verified = new Set(); + const refused: Record = {}; + for (const row of ran.filter((row) => providerIdOf(row) === id)) { + if (row.status === "pass") verified.add(row.model); + else if (row.status !== "fail") continue; + else if (!inconclusive(row.detail)) refused[row.model] = row.detail; + else if (earlier?.verified.includes(row.model)) verified.add(row.model); + else if (earlier?.refused[row.model] !== undefined) + refused[row.model] = earlier.refused[row.model] ?? ""; + } + providers[id] = { + verified: [...verified].sort(), + refused: sortedRecord(refused), + }; + } + if (!Object.keys(providers).length) continue; + const found = tag ? clients[harness]?.[tag] : undefined; + engines[harness] = { + clients: sortedRecord( + found && Object.keys(found).length ? found : (before?.clients ?? {}), + ), + providers: sortedRecord(providers), + }; + } + const next = sortedRecord(engines); + const same = + previous !== undefined && + JSON.stringify(previous.engines) === JSON.stringify(next); + return { checkedAt: same ? previous.checkedAt : date, engines: next }; +} + +/** + * What changed between two locks, as a pull request says it: models that newly answer, models + * that stopped, models that dropped out of a provider's listing, and clients that moved. + */ +export function describeLockChange( + before: ModelLock | undefined, + after: ModelLock, +): string { + const lines: string[] = []; + for (const [engine, entry] of Object.entries(after.engines)) { + const earlier = before?.engines[engine]; + const changes: string[] = []; + for (const [provider, now] of Object.entries(entry.providers)) { + if (!now) continue; + const was = earlier?.providers[provider as LockProvider]; + const known = new Set([ + ...(was?.verified ?? []), + ...Object.keys(was?.refused ?? {}), + ]); + const passes = now.verified.filter( + (model) => !was?.verified.includes(model), + ); + const stopped = Object.keys(now.refused).filter((model) => + was?.verified.includes(model), + ); + const newlyRefused = Object.keys(now.refused).filter( + (model) => !known.has(model), + ); + const gone = [...known].filter( + (model) => !now.verified.includes(model) && !(model in now.refused), + ); + if (passes.length) + changes.push( + `${provider}: now answers ${passes.map((m) => `\`${m}\``).join(", ")}`, + ); + if (stopped.length) + changes.push( + `${provider}: **stopped answering** ${stopped.map((m) => `\`${m}\` (${cell(now.refused[m] ?? "")})`).join(", ")}`, + ); + if (newlyRefused.length) + changes.push( + `${provider}: refused ${newlyRefused.map((m) => `\`${m}\` (${cell(now.refused[m] ?? "")})`).join(", ")}`, + ); + if (gone.length) + changes.push( + `${provider}: no longer listed ${gone.map((m) => `\`${m}\``).join(", ")}`, + ); + } + const moved = Object.entries(entry.clients) + .filter(([name, version]) => earlier?.clients[name] !== version) + .map(([name, version]) => + earlier?.clients[name] + ? `${name} ${earlier.clients[name]} to ${version}` + : `${name} ${version}`, + ); + if (moved.length) changes.push(`clients: ${moved.join(", ")}`); + if (changes.length) + lines.push( + `- **${engine}**`, + ...changes.map((change) => ` - ${change}`), + ); + } + return lines.length ? lines.join("\n") : "No model changed."; +} + +/** + * The model-client versions a harness pins in this repository, by the names the lock records + * them under: exact `name==version` lines in its `requirements.txt`, the versions its `bun.lock` + * resolves for the Node clients, and the Codex client version the LangGraph harness sends a + * ChatGPT plan. Unpinned packages are left out: only what the repository fixes can disagree. + */ +export function pinnedClients( + root: string, + harness: string, +): Record { + const pinned: Record = {}; + const read = (file: string) => { + const path = join(root, harness, file); + return existsSync(path) ? readFileSync(path, "utf8") : ""; + }; + for (const line of read("requirements.txt").split("\n")) { + const match = /^\s*([A-Za-z0-9_.-]+)(?:\[[^\]]*\])?\s*==\s*([^\s;#]+)/.exec( + line, + ); + if (match?.[1] && match[2]) + pinned[match[1].toLowerCase().replace(/_/g, "-")] = match[2]; + } + const lockfile = read("bun.lock"); + for (const name of NODE_CLIENTS) { + const escaped = name.replace(/[/.]/g, "\\$&"); + const found = new RegExp(`"${escaped}": \\["${escaped}@([^"]+)"`).exec( + lockfile, + )?.[1]; + if (found) pinned[name] = found; + } + // npm's lockfile, for a harness installed with `npm ci` (the LangGraph (TypeScript) engine). + const npmLock = read("package-lock.json"); + if (npmLock) { + const packages = + ( + JSON.parse(npmLock) as { + packages?: Record; + } + ).packages ?? {}; + for (const name of NODE_CLIENTS) { + const version = packages[`node_modules/${name}`]?.version; + if (version) pinned[name] = version; + } + } + const codex = /^DEFAULT_CLIENT_VERSION\s*=\s*"([^"]+)"/m.exec( + read("src/chatgpt_models.py"), + )?.[1]; + if (codex) pinned["codex-client"] = codex; + return pinned; +} + +/** + * Where the lock and the repository disagree about a harness's clients: each pinned client the + * lock recorded at another version, or didn't record. A lock that disagrees was proven on other + * clients than the ones this commit ships, so it may hide models that now work, or offer ones that + * no longer do. Regenerate it (`lock`) from a run of this commit. + */ +export function staleClients( + lock: ModelLock, + root: string, +): { harness: string; client: string; pinned: string; locked?: string }[] { + const stale: { + harness: string; + client: string; + pinned: string; + locked?: string; + }[] = []; + for (const [harness, entry] of Object.entries(lock.engines)) { + for (const [client, pinned] of Object.entries( + pinnedClients(root, harness), + )) { + const locked = entry.clients[client]; + if (locked !== pinned) + stale.push({ harness, client, pinned, ...(locked ? { locked } : {}) }); + } + } + return stale; +} + +/** Every model the lock verifies that now fails: what a client bump must not break. */ +export function brokenVerified( + lock: ModelLock, + results: readonly Result[], +): Result[] { + /* + * What the published image did with the same key in the same run, when the run checked it too: a + * model that fails there as well is the provider's or the key's today, not this change's. + */ + const failsOnPublished = (row: Result) => + results.some( + (other) => + other.tag !== SOURCE && + other.status === "fail" && + other.harness === row.harness && + other.model === row.model && + providerIdOf(other) === providerIdOf(row), + ); + return results.filter((row) => { + if (row.status !== "fail" || row.model === "(all)") return false; + if (row.tag !== SOURCE) return false; + // The key's account, not the change's client: says nothing about the change. + if ( + aboutTheKey(row.detail) || + retiredByProvider(row.detail) || + failsOnPublished(row) + ) + return false; + const id = providerIdOf(row); + return ( + !!id && + (lock.engines[row.harness]?.providers[id]?.verified.includes(row.model) ?? + false) + ); + }); +} + +function readResults(files: readonly string[]): Results[] { + return files.map((file) => JSON.parse(readFileSync(file, "utf8")) as Results); +} + +function lockCommand(argv: string[]): number { + let path = join(ROOT, "shared/model-lock.json"); + let date = new Date().toISOString().slice(0, 10); + let changes: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--lock") path = argv[++i] ?? path; + else if (arg === "--date") date = argv[++i] ?? date; + else if (arg === "--changes") changes = argv[++i]; + else if (arg) files.push(arg); + } + const previous = existsSync(path) + ? (JSON.parse(readFileSync(path, "utf8")) as ModelLock) + : undefined; + const next = buildLock(previous, readResults(files), date); + const text = `${JSON.stringify(next, null, 2)}\n`; + const changed = !existsSync(path) || readFileSync(path, "utf8") !== text; + if (changed) writeFileSync(path, text); + if (changes) + writeFileSync(changes, `${describeLockChange(previous, next)}\n`); + console.log( + changed + ? `The model lock changed: ${path}` + : "The model lock is unchanged.", + ); + return 0; +} + +function guardCommand(argv: string[]): number { + let path = join(ROOT, "shared/model-lock.json"); + let summary: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--lock") path = argv[++i] ?? path; + else if (arg === "--summary") summary = argv[++i]; + else if (arg) files.push(arg); + } + const lock = JSON.parse(readFileSync(path, "utf8")) as ModelLock; + const results = readResults(files).flatMap((run) => run.results); + const broken = brokenVerified(lock, results); + const markdown = [ + "## Model lock", + "", + broken.length + ? `${broken.length} model${broken.length === 1 ? "" : "s"} the lock verifies no longer answer${broken.length === 1 ? "s" : ""} through this change's client. A person set to one would get the provider's refusal on every message. Fix the client, or, if the model really can't run on this harness any more, say so in the pull request: the next daily check takes it out of the lock.` + : "Every model the lock verifies for the harnesses this change touches still answers.", + "", + ...(broken.length ? [renderTable(broken), ""] : []), + renderSummary(results), + ].join("\n"); + if (summary) writeFileSync(summary, `${markdown}\n`); + else console.log(markdown); + return broken.length ? 1 : 0; +} + +function report(argv: string[]): number { + let summary: string | undefined; + let issue: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--summary") summary = argv[++i]; + else if (arg === "--issue") issue = argv[++i]; + else if (arg) files.push(arg); + } + const results = files.flatMap( + (file) => + (JSON.parse(readFileSync(file, "utf8")) as { results: Result[] }).results, + ); + const markdown = renderSummary(results); + if (summary) writeFileSync(summary, `${markdown}\n`); + else console.log(markdown); + if (issue) + writeFileSync( + issue, + `${renderSummary(results, { passed: false, maxFailed: 150 })}\n`, + ); + if (!results.length) { + console.error( + "No results were found: every harness job failed before it wrote any.", + ); + return 1; + } + return results.some((row) => row.status === "fail") ? 1 : 0; +} + +if (import.meta.main) { + const argv = process.argv.slice(2); + if (argv[0] === "harnesses") { + // The workflows' matrix: one job per harness this check runs, as compact JSON. + console.log( + JSON.stringify(readProbed().map((harness) => harness.image ?? "")), + ); + process.exit(0); + } + const code = + argv[0] === "report" + ? report(argv.slice(1)) + : argv[0] === "lock" + ? lockCommand(argv.slice(1)) + : argv[0] === "guard" + ? guardCommand(argv.slice(1)) + : await check(argv); + process.exit(code); +} diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts new file mode 100644 index 0000000000..dec044c612 --- /dev/null +++ b/scripts/model-lock.test.ts @@ -0,0 +1,415 @@ +import { expect, test } from "bun:test"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + DEFAULT_ENGINES, + emptyPairs, + MODEL_LOCK, + type ModelLock, +} from "../shared/model-lock"; +import { + brokenVerified, + buildLock, + describeLockChange, + inconclusive, + PYTHON_CLIENTS, + pinnedClients, + providerIdOf, + type Result, + type Results, + staleClients, +} from "./model-compat"; + +const ROOT = join(import.meta.dir, ".."); + +/* + * THE LOCK MATCHES WHAT THIS COMMIT SHIPS. Written on a commit whose Claude Agent SDK harness + * pinned claude-agent-sdk 0.2.152 (Claude Code 2.1.259), the lock refused Opus 5.5 and Fable 5.2; + * main then bumped the pin to a Claude Code that runs both, and the lock kept hiding them. A lock + * proven on other clients than the ones pinned here is stale, and fails here until it is written + * again from a run of this commit. + */ +test("the lock was proven on the model clients each harness pins in this commit", () => { + expect(staleClients(MODEL_LOCK, ROOT)).toEqual([]); +}); + +/* + * A DEFAULT ENGINE ALWAYS HAS SOMETHING TO OFFER. The LangGraph (TypeScript) engine was first + * locked with its pairs empty, so nothing would be offered on it before a run proved a model: had + * that reached main before a run, every new Bot on it would have had no model at all. + */ +test("every engine new Bots run on by default verifies a model on each provider it is locked for", () => { + for (const engine of DEFAULT_ENGINES) { + expect(MODEL_LOCK.engines[engine], engine).toBeDefined(); + expect(emptyPairs(MODEL_LOCK, engine), engine).toEqual([]); + } +}); + +test("every exact pin in a harness's requirements is a client the check records", () => { + const harnesses = readdirSync(ROOT).filter((name) => + name.startsWith("agent-"), + ); + for (const harness of harnesses) { + let text = ""; + try { + text = readFileSync(join(ROOT, harness, "requirements.txt"), "utf8"); + } catch { + continue; + } + for (const match of text.matchAll( + /^\s*([A-Za-z0-9_.-]+)(?:\[[^\]]*\])?\s*==/gm, + )) + expect(PYTHON_CLIENTS, `${harness}: ${match[1]}`).toContain( + (match[1] ?? "").toLowerCase().replace(/_/g, "-"), + ); + } +}); + +test("a pin that moved past the lock is caught", () => { + const claude = pinnedClients(ROOT, "agent-claude-sdk"); + expect(Object.keys(claude)).toContain("claude-agent-sdk"); + const old: ModelLock = { + checkedAt: "2026-11-17", + engines: { + "agent-claude-sdk": { + clients: { ...claude, "claude-agent-sdk": "0.2.152" }, + providers: {}, + }, + }, + }; + expect(staleClients(old, ROOT)).toEqual([ + { + harness: "agent-claude-sdk", + client: "claude-agent-sdk", + pinned: claude["claude-agent-sdk"] ?? "", + locked: "0.2.152", + }, + ]); + // The Codex version the LangGraph harness sends a ChatGPT plan, and Mastra's resolved clients. + expect(Object.keys(pinnedClients(ROOT, "agent-langgraph-agui"))).toContain( + "codex-client", + ); + expect(Object.keys(pinnedClients(ROOT, "agent-mastra"))).toContain( + "@mastra/core", + ); + // npm's lockfile, for the LangGraph (TypeScript) engine. + expect(Object.keys(pinnedClients(ROOT, "agent-langgraph-ts"))).toContain( + "@langchain/core", + ); +}); + +/* + * Writing the model lock from the model check's results, and holding a client bump to it. The + * rows are what the check reported on 2026-11-17, in its own words. + */ + +const OPUS_REFUSED = + "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again."; + +const row = ( + model: string, + status: Result["status"], + detail = "OK", + extra: Partial = {}, +): Result => ({ + harness: "agent-claude-sdk", + tag: "source", + provider: "Anthropic (key or Claude plan)", + providerId: "anthropic", + model, + status, + detail, + ...extra, +}); + +const run: Results = { + results: [ + row("claude-sonnet-5-5", "pass"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "fail", OPUS_REFUSED), + // `latest` is what people run today; the lock is what the next release ships. + row("claude-sonnet-5-5", "fail", OPUS_REFUSED, { tag: "latest" }), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.152", "claude-code": "2.1.259" }, + latest: { "claude-agent-sdk": "0.2.150" }, + }, + }, +}; + +test("what answered on the source image is verified, what was refused is kept in the provider's words", () => { + const lock = buildLock(undefined, [run], "2026-11-17"); + expect(lock).toEqual({ + checkedAt: "2026-11-17", + engines: { + "agent-claude-sdk": { + clients: { "claude-agent-sdk": "0.2.152", "claude-code": "2.1.259" }, + providers: { + anthropic: { + verified: ["claude-haiku-5-5", "claude-sonnet-5-5"], + refused: { "claude-opus-5-5": OPUS_REFUSED }, + }, + }, + }, + }, + }); +}); + +test("a bad day for the runner takes nothing away", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const flaky: Results = { + results: [ + row( + "claude-sonnet-5-5", + "fail", + "no answer: TimeoutError: The operation timed out.", + ), + row( + "claude-haiku-5-5", + "fail", + "the harness never became healthy: exit 1", + ), + // What Antigravity said for gemma-4-31b-it on 2026-11-17, while it answered on `latest`. + row( + "claude-opus-5-5", + "fail", + 'AntigravityExecutionError: Agent execution terminated due to error. ("agent executor error: Error 429, Message: Resource has been exhausted")', + ), + ], + }; + const after = buildLock(before, [flaky], "2026-11-18"); + expect(after).toEqual(before); + // Nothing changed, so neither did the date, and the daily check opens no pull request. + expect(after.checkedAt).toBe("2026-11-17"); +}); + +test("a model that starts answering after a client bump is verified, and the clients move with it", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const bumped: Results = { + results: [ + row("claude-sonnet-5-5", "pass"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "pass"), + row("claude-fable-5-2", "pass"), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + }, + }, + }; + const after = buildLock(before, [bumped], "2026-11-18"); + expect(after.checkedAt).toBe("2026-11-18"); + expect(after.engines["agent-claude-sdk"]).toEqual({ + clients: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + providers: { + anthropic: { + verified: [ + "claude-fable-5-2", + "claude-haiku-5-5", + "claude-opus-5-5", + "claude-sonnet-5-5", + ], + refused: {}, + }, + }, + }); +}); + +test("a provider with no key and a harness that did not run keep what was known", () => { + const before: ModelLock = { + checkedAt: "2026-11-10", + engines: { + "agent-agno": { + clients: { agno: "2.0.0" }, + providers: { + xai: { verified: ["grok-4.7"], refused: {} }, + openai: { verified: ["gpt-6.1-terra"], refused: {} }, + }, + }, + "agent-adk": { + clients: { "google-adk": "1.0.0" }, + providers: { openai: { verified: ["gpt-5.5"], refused: {} } }, + }, + }, + }; + const today: Results = { + results: [ + { + harness: "agent-agno", + tag: "latest, source", + provider: "xAI", + providerId: "xai", + model: "(all)", + status: "skipped", + detail: "skipped: no key (XAI_API_KEY)", + }, + { + harness: "agent-agno", + tag: "source", + provider: "OpenAI", + providerId: "openai", + model: "gpt-6.1-terra", + status: "pass", + detail: "OK", + }, + ], + }; + const after = buildLock(before, [today], "2026-11-17"); + expect(after.engines["agent-agno"]?.providers.xai).toEqual({ + verified: ["grok-4.7"], + refused: {}, + }); + expect(after.engines["agent-adk"]).toEqual(before.engines["agent-adk"]!); + // No clients in the results: the ones recorded before stand. + expect(after.engines["agent-agno"]?.clients).toEqual({ agno: "2.0.0" }); +}); + +test("the refresh pull request says which models started and stopped answering", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const after = buildLock( + before, + [ + { + results: [ + row( + "claude-sonnet-5-5", + "fail", + "API Error: 400 invalid_request_error", + ), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "pass"), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + }, + }, + }, + ], + "2026-11-18", + ); + const said = describeLockChange(before, after); + expect(said).toContain("now answers `claude-opus-5-5`"); + expect(said).toContain("**stopped answering** `claude-sonnet-5-5`"); + expect(said).toContain("claude-code 2.1.259 to 2.1.291"); + expect(describeLockChange(before, before)).toBe("No model changed."); +}); + +test("a run of the published images alone does not move the lock", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const latestOnly: Results = { + results: [row("claude-opus-5-5", "pass", "OK", { tag: "latest" })], + }; + expect(buildLock(before, [latestOnly], "2026-11-18")).toEqual(before); +}); + +test("results written before rows named their provider are read by its label", () => { + expect(providerIdOf({ provider: "Anthropic (key or Claude plan)" })).toBe( + "anthropic", + ); + expect(providerIdOf({ provider: "ChatGPT plan" })).toBe("chatgpt"); + expect(providerIdOf({ provider: "Google" })).toBe("google"); +}); + +test("a client bump that breaks a verified model is caught, and one the lock already refuses is not", () => { + const lock = buildLock(undefined, [run], "2026-11-17"); + const afterBump: Result[] = [ + row("claude-sonnet-5-5", "fail", "API Error: 400 invalid_request_error"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "fail", OPUS_REFUSED), + ]; + expect(brokenVerified(lock, afterBump).map((broken) => broken.model)).toEqual( + ["claude-sonnet-5-5"], + ); +}); + +test("a model the provider retires is refused, and fails no pull request", () => { + // What Google answered through Antigravity on 2026-11-18, for a model the lock had verified. + const RETIRED_SAID = + 'AntigravityExecutionError: Agent execution terminated due to error. ("agent executor error: Error 404, Message: This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements.'; + const google = (status: Result["status"], detail = "OK", tag = "source") => + row("gemini-2.5-flash-lite", status, detail, { + harness: "agent-antigravity", + provider: "Google", + providerId: "google", + tag, + }); + const before = buildLock( + undefined, + [{ results: [google("pass")] }], + "2026-11-17", + ); + const after = buildLock( + before, + [{ results: [google("fail", RETIRED_SAID)] }], + "2026-11-18", + ); + const pair = after.engines["agent-antigravity"]?.providers.google; + expect(pair?.verified).toEqual([]); + expect(pair?.refused["gemini-2.5-flash-lite"]).toContain( + "no longer available to new users", + ); + // The provider's doing, on every image: not the change's. + expect(brokenVerified(before, [google("fail", RETIRED_SAID)])).toEqual([]); +}); + +test("a model the check's key can't use is the key's, not the client's", () => { + const google = (model: string, status: Result["status"], detail = "OK") => + row(model, status, detail, { + harness: "agent-langgraph-agui", + provider: "Google", + providerId: "google", + }); + // One Gemini key's run on 2026-11-17; another key, through the same image, disagreed. + const lock = buildLock( + undefined, + [ + { + results: [ + google("gemini-2.5-flash-lite", "pass"), + google("gemini-decision-1", "fail", "Error code: 404"), + google( + "gemini-omni-1.1-flash", + "fail", + "Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.'}}]", + ), + ], + }, + ], + "2026-11-17", + ); + const pair = lock.engines["agent-langgraph-agui"]?.providers.google; + expect(pair?.verified).toEqual(["gemini-2.5-flash-lite"]); + // A bare 404 isn't refused: another key may run it, and a later run with one proves it. + expect(Object.keys(pair?.refused ?? {})).toEqual(["gemini-omni-1.1-flash"]); + expect( + brokenVerified(lock, [ + google("gemini-2.5-flash-lite", "fail", "Error code: 404"), + ]), + ).toEqual([]); + // CrewAI hides the provider's words; the published image failing too, with the same key in the + // same run, says it isn't the change. + const hidden = () => + google( + "gemini-2.5-flash-lite", + "fail", + "CrewAI flow failed; see server logs", + ); + expect( + brokenVerified(lock, [hidden(), { ...hidden(), tag: "latest" }]), + ).toEqual([]); + expect( + brokenVerified(lock, [ + hidden(), + { ...google("gemini-2.5-flash-lite", "pass"), tag: "latest" }, + ]).map((broken) => broken.model), + ).toEqual(["gemini-2.5-flash-lite"]); + // A 404 that names the client's endpoint is the client's. + expect( + inconclusive( + "NotFoundError('Error code: 404 - This model is only supported in v1/responses and not in v1/chat/completions.')", + ), + ).toBe(false); +}); diff --git a/shared/fixtures/model-listings/anthropic-models.json b/shared/fixtures/model-listings/anthropic-models.json new file mode 100644 index 0000000000..478a3cc111 --- /dev/null +++ b/shared/fixtures/model-listings/anthropic-models.json @@ -0,0 +1,1327 @@ +{ + "data": [ + { + "type": "model", + "id": "claude-fable-5-2", + "display_name": "Claude Fable 5.2", + "created_at": "2026-11-09T18:30:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-haiku-5-5", + "display_name": "Claude Haiku 5.5", + "created_at": "2026-10-07T18:00:00Z", + "line": "haiku", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-5-5", + "display_name": "Claude Sonnet 5.5", + "created_at": "2026-09-28T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": true + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-5-5", + "display_name": "Claude Opus 5.5", + "created_at": "2026-09-21T16:24:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-fable-5-1", + "display_name": "Claude Fable 5.1", + "created_at": "2026-08-28T00:00:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-5", + "display_name": "Claude Opus 5", + "created_at": "2026-07-24T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-5", + "display_name": "Claude Sonnet 5", + "created_at": "2026-06-29T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-fable-5", + "display_name": "Claude Fable 5", + "created_at": "2026-06-07T00:00:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-8", + "display_name": "Claude Opus 4.8", + "created_at": "2026-05-28T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-7", + "display_name": "Claude Opus 4.7", + "created_at": "2026-04-14T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-4-6", + "display_name": "Claude Sonnet 4.6", + "created_at": "2026-02-17T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-6", + "display_name": "Claude Opus 4.6", + "created_at": "2026-02-04T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-5-20251101", + "display_name": "Claude Opus 4.5", + "created_at": "2025-11-24T00:00:00Z", + "line": "opus", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-haiku-4-5-20251001", + "display_name": "Claude Haiku 4.5", + "created_at": "2025-10-15T00:00:00Z", + "line": "haiku", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": false + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": false, + "low": { + "supported": false + }, + "medium": { + "supported": false + }, + "high": { + "supported": false + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-4-5-20250929", + "display_name": "Claude Sonnet 4.5", + "created_at": "2025-09-29T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": false, + "low": { + "supported": false + }, + "medium": { + "supported": false + }, + "high": { + "supported": false + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "deprecated", + "deprecated_at": "2026-09-30T18:30:00Z", + "retires_at": "2026-11-30T17:00:00Z" + } + ], + "has_more": false, + "first_id": "claude-fable-5-2", + "last_id": "claude-sonnet-4-5-20250929" +} diff --git a/shared/fixtures/model-listings/openai-models.json b/shared/fixtures/model-listings/openai-models.json new file mode 100644 index 0000000000..f4c33cd814 --- /dev/null +++ b/shared/fixtures/model-listings/openai-models.json @@ -0,0 +1,705 @@ +{ + "object": "list", + "data": [ + { + "id": "gpt-5-nano-2025-08-07", + "object": "model", + "created": 1754426303, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o-mini-tts-2025-03-20", + "object": "model", + "created": 1765610731, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-audio-mini", + "object": "model", + "created": 1759512027, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-transcribe", + "object": "model", + "created": 1742068463, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5.2", + "object": "model", + "created": 1765313051, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-mini-2024-07-18", + "object": "model", + "created": 1721172717, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4.1-2025-04-14", + "object": "model", + "created": 1744315746, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4.1-mini-2025-04-14", + "object": "model", + "created": 1744317547, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-5.4-nano-2026-03-17", + "object": "model", + "created": 1773450837, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-image-2.5-flare", + "object": "model", + "created": 1788563147, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.1-sol", + "object": "model", + "created": 1790552874, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5", + "object": "model", + "created": 1754425777, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o", + "object": "model", + "created": 1715367049, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4o-mini-transcribe", + "object": "model", + "created": 1742068596, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-image-2-2026-04-21", + "object": "model", + "created": 1776399994, + "owned_by": "system", + "shutdown_date": "2027-02-10" + }, + { + "id": "gpt-4o-mini-tts-2025-12-15", + "object": "model", + "created": 1765610837, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-audio", + "object": "model", + "created": 1756339249, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.4-nano", + "object": "model", + "created": 1773450870, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5.3-codex", + "object": "model", + "created": 1770537915, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5-search-api-2025-10-14", + "object": "model", + "created": 1760043960, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-mini-2025-12-15", + "object": "model", + "created": 1765760008, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-transcribe", + "object": "model", + "created": 1785168027, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "tts-1", + "object": "model", + "created": 1681940951, + "owned_by": "openai-internal", + "shutdown_date": "2027-01-06" + }, + { + "id": "omni-moderation-2024-09-26", + "object": "model", + "created": 1732734466, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2.1-mini", + "object": "model", + "created": 1782254706, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2025-08-28", + "object": "model", + "created": 1756271773, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-live-transcribe", + "object": "model", + "created": 1785168034, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-mini", + "object": "model", + "created": 1754425928, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5.4-2026-03-05", + "object": "model", + "created": 1772654062, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-mini-2025-08-07", + "object": "model", + "created": 1754425867, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5-nano", + "object": "model", + "created": 1754426384, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5.6-luna", + "object": "model", + "created": 1782228658, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.5-sunburst-2026-09-08", + "object": "model", + "created": 1788851010, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.6-flare", + "object": "model", + "created": 1793222647, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-2025-08-07", + "object": "model", + "created": 1754075360, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o-mini-transcribe-2025-12-15", + "object": "model", + "created": 1765610407, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "o3-pro-2025-06-10", + "object": "model", + "created": 1749166761, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "chat-latest", + "object": "model", + "created": 1777704602, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-pro", + "object": "model", + "created": 1776894349, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-translate", + "object": "model", + "created": 1777950216, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-2025-08-28", + "object": "model", + "created": 1756256146, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-mini-transcribe-2025-03-20", + "object": "model", + "created": 1765610545, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.4-pro-2026-03-05", + "object": "model", + "created": 1772659657, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-pro-2025-10-06", + "object": "model", + "created": 1759469707, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "tts-1-1106", + "object": "model", + "created": 1699053241, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-pro-2026-04-23", + "object": "model", + "created": 1776894470, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.2-pro-2025-12-11", + "object": "model", + "created": 1765343959, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.6-sol", + "object": "model", + "created": 1782228018, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.2-astra", + "object": "model", + "created": 1793430042, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3-2025-04-16", + "object": "model", + "created": 1744133301, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4.1-mini", + "object": "model", + "created": 1744318173, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-5.2-2025-12-11", + "object": "model", + "created": 1765313028, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "chatgpt-image-latest", + "object": "model", + "created": 1765925279, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-4o-mini", + "object": "model", + "created": 1721172741, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-image-2.5-sunburst", + "object": "model", + "created": 1788563162, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.5-flare-2026-09-08", + "object": "model", + "created": 1788851006, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-luna", + "object": "model", + "created": 1789406102, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "tts-1-hd-1106", + "object": "model", + "created": 1699053533, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-mini", + "object": "model", + "created": 1759517133, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-transcribe-diarize", + "object": "model", + "created": 1750798887, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5.4", + "object": "model", + "created": 1772691852, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-1.5", + "object": "model", + "created": 1771550885, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-1-mini", + "object": "model", + "created": 1758845821, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-image-1.5", + "object": "model", + "created": 1764030620, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-5.1-2025-11-13", + "object": "model", + "created": 1762800353, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "text-embedding-3-small", + "object": "model", + "created": 1705948997, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.6-terra", + "object": "model", + "created": 1782228459, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-astra", + "object": "model", + "created": 1787853604, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.1-terra", + "object": "model", + "created": 1791908780, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.2-sol", + "object": "model", + "created": 1793642820, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "omni-moderation-latest", + "object": "model", + "created": 1731689265, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.4-mini", + "object": "model", + "created": 1773451123, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-2024-11-20", + "object": "model", + "created": 1739331543, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "tts-1-hd", + "object": "model", + "created": 1699046015, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-realtime-1.5", + "object": "model", + "created": 1771461469, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2", + "object": "model", + "created": 1778006032, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime", + "object": "model", + "created": 1756271701, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.1", + "object": "model", + "created": 1762800673, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5-search-api", + "object": "model", + "created": 1759514629, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-2026-04-23", + "object": "model", + "created": 1776839241, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "whisper-1", + "object": "model", + "created": 1677532384, + "owned_by": "openai-internal", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5-pro", + "object": "model", + "created": 1759469822, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-realtime-whisper", + "object": "model", + "created": 1778012060, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.2-pro", + "object": "model", + "created": 1765343983, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-live-1", + "object": "model", + "created": 1788889407, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.6-sunburst", + "object": "model", + "created": 1793222670, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.4-mini-2026-03-17", + "object": "model", + "created": 1773451076, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3-pro", + "object": "model", + "created": 1748475349, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-image-2", + "object": "model", + "created": 1776399795, + "owned_by": "system", + "shutdown_date": "2027-02-10" + }, + { + "id": "gpt-5.4-pro", + "object": "model", + "created": 1772659601, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-mini-tts", + "object": "model", + "created": 1742403959, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-realtime-mini-2025-12-15", + "object": "model", + "created": 1765612007, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "text-embedding-ada-002", + "object": "model", + "created": 1671217299, + "owned_by": "openai-internal", + "shutdown_date": null + }, + { + "id": "gpt-4o-2024-08-06", + "object": "model", + "created": 1722814719, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-realtime-2.1", + "object": "model", + "created": 1782254687, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3", + "object": "model", + "created": 1744225308, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4.1", + "object": "model", + "created": 1744316542, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "text-embedding-3-large", + "object": "model", + "created": 1705953180, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5", + "object": "model", + "created": 1776824847, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-sol", + "object": "model", + "created": 1789405937, + "owned_by": "system", + "shutdown_date": null + } + ] +} diff --git a/shared/fixtures/model-lock-before-claude-code-bump.json b/shared/fixtures/model-lock-before-claude-code-bump.json new file mode 100644 index 0000000000..4bbabc631d --- /dev/null +++ b/shared/fixtures/model-lock-before-claude-code-bump.json @@ -0,0 +1,36 @@ +{ + "checkedAt": "2026-11-17", + "engines": { + "agent-claude-sdk": { + "clients": { + "ag-ui-claude-sdk": "0.1.5", + "anthropic": "2.10.0", + "claude-agent-sdk": "0.2.152", + "claude-code": "2.1.259" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": { + "claude-fable-5-2": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.", + "claude-opus-5-5": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again." + } + } + } + } + } +} diff --git a/shared/model-lock.json b/shared/model-lock.json new file mode 100644 index 0000000000..3437cab84b --- /dev/null +++ b/shared/model-lock.json @@ -0,0 +1,1203 @@ +{ + "checkedAt": "2026-11-18", + "engines": { + "agent-adk": { + "clients": { + "google-adk": "2.14.1", + "google-genai": "2.34.0", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_...", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow the instruction exactly.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-ag2": { + "clients": { + "ag2": "1.3.2", + "anthropic": "2.10.0", + "openai": "3.55.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-omni-1.1-flash": "BadRequestError('Error code: 400')", + "gemini-omni-1.2-flash": "BadRequestError('Error code: 400')", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with the instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user is asking for a specific single-word response: \"OK\". No tool calls are needed. The response should be exactly \"OK\".OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-agno": { + "clients": { + "agno": "3.2.1", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [], + "refused": { + "gemini-2.5-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-2.5-flash-lite": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-2.5-pro": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3-flash-preview": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-flash-lite": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-flash-lite-preview": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-pro-preview": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-pro-preview-customtools": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.5-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.5-flash-lite": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.6-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.7-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.8-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-4-argon": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-4-carbon": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-4-hydrogen": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-decision-1": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-flash-latest": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-flash-lite-latest": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-pro-latest": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemma-4-26b-a4b-it": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemma-4-31b-it": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei..." + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-antigravity": { + "clients": { + "google-genai": "3.5.0" + }, + "providers": { + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 404, Message: This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started)., Status: NOT_FOU...", + "gemini-omni-1.1-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)", + "gemini-omni-1.2-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)" + } + } + } + }, + "agent-claude-sdk": { + "clients": { + "ag-ui-claude-sdk": "0.1.5", + "anthropic": "2.10.0", + "claude-agent-sdk": "0.2.182" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + } + } + }, + "agent-crewai": { + "clients": { + "crewai": "1.15.41", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "thread=model-compat-811af2816788 run=run-811af2816788: CrewAI flow failed; see server logs", + "gemini-decision-1": "thread=model-compat-53e7cc2c2ba7 run=run-53e7cc2c2ba7: CrewAI flow failed; see server logs", + "gemini-omni-1.1-flash": "thread=model-compat-bb5958474292 run=run-bb5958474292: CrewAI flow failed; see server logs", + "gemini-omni-1.2-flash": "thread=model-compat-d99e32be6322 run=run-d99e32be6322: CrewAI flow failed; see server logs", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". This is a direct instruction. I should comply.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". This is a simple constraint following task. No tools are needed.OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-langgraph-agui": { + "clients": { + "ag-ui-langgraph": "0.0.45", + "anthropic": "2.10.0", + "codex-client": "0.167.0", + "google-genai": "2.34.0", + "langchain-anthropic": "1.7.10", + "langchain-core": "1.8.2", + "langchain-google-genai": "4.4.3", + "langchain-openai": "1.8.1", + "langgraph": "1.2.22", + "openai": "3.55.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "chatgpt": { + "verified": [ + "gpt-5.5", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-omni-1.1-flash": "Error code: 400", + "gemini-omni-1.2-flash": "Error code: 400", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single-word response: \"OK\". This is a direct instruction to follow a specific format. Plan: 1. Acknowledge the instruction. 2. Output ...", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a specific single-word response: \"OK\". I should comply with this request exactly.OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-langgraph-ts": { + "clients": { + "@ag-ui/client": "1.0.5", + "@ag-ui/core": "1.0.5", + "@ag-ui/encoder": "1.0.5", + "@langchain/anthropic": "1.6.5", + "@langchain/core": "1.4.1", + "@langchain/google": "0.2.15", + "@langchain/langgraph": "1.7.5", + "@langchain/openai": "1.8.4" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "Error code: 404 - [{\"error\":{\"code\":404,\"message\":\"This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).\",\"status\":\"NOT_FOUND\"}}] Troubleshooting URL: https://docs.langchain.com/oss/...", + "gemini-omni-1.1-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", + "gemini-omni-1.2-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". This is a direct instruction for a specific output. I should comply.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a single word response: \"OK\".OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-langroid": { + "clients": { + "google-genai": "3.5.0", + "langroid": "0.69.24", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "Agent error: Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'}}]", + "gemini-omni-1.1-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with this instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-llamaindex": { + "clients": { + "litellm": "1.105.7", + "llama-index-core": "0.14.26", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_...", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with this instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-mastra": { + "clients": { + "@ai-sdk/anthropic": "4.0.97", + "@ai-sdk/openai": "4.0.109", + "@mastra/core": "1.82.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "HTTP 500: Not Found", + "gemini-decision-1": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", + "gemini-omni-1.1-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", + "gemini-omni-1.2-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". * Prompt: \"Reply with the single word OK.\" * Instruction: Answer the question briefly and correctl...", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a response consisting of a single word: \"OK\". Constraint checklist & Confidence score: 1. Single word? Yes. 2. Word is \"OK\"? Yes. Confid..." + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-microsoft": { + "clients": { + "agent-framework-core": "1.26.0", + "anthropic": "1.17.0", + "openai": "3.55.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [], + "refused": { + "gemini-2.5-flash": "An internal error has occurred while streaming events.", + "gemini-2.5-flash-lite": "An internal error has occurred while streaming events.", + "gemini-2.5-pro": "An internal error has occurred while streaming events.", + "gemini-3-flash-preview": "An internal error has occurred while streaming events.", + "gemini-3.1-flash-lite": "An internal error has occurred while streaming events.", + "gemini-3.1-flash-lite-preview": "An internal error has occurred while streaming events.", + "gemini-3.1-pro-preview": "An internal error has occurred while streaming events.", + "gemini-3.1-pro-preview-customtools": "An internal error has occurred while streaming events.", + "gemini-3.5-flash": "An internal error has occurred while streaming events.", + "gemini-3.5-flash-lite": "An internal error has occurred while streaming events.", + "gemini-3.6-flash": "An internal error has occurred while streaming events.", + "gemini-3.7-flash": "An internal error has occurred while streaming events.", + "gemini-3.8-flash": "An internal error has occurred while streaming events.", + "gemini-4-argon": "An internal error has occurred while streaming events.", + "gemini-4-carbon": "An internal error has occurred while streaming events.", + "gemini-4-hydrogen": "An internal error has occurred while streaming events.", + "gemini-decision-1": "An internal error has occurred while streaming events.", + "gemini-flash-latest": "An internal error has occurred while streaming events.", + "gemini-flash-lite-latest": "An internal error has occurred while streaming events.", + "gemini-omni-1.1-flash": "An internal error has occurred while streaming events.", + "gemini-omni-1.2-flash": "An internal error has occurred while streaming events.", + "gemini-pro-latest": "An internal error has occurred while streaming events.", + "gemma-4-26b-a4b-it": "An internal error has occurred while streaming events.", + "gemma-4-31b-it": "An internal error has occurred while streaming events." + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-pydantic-ai": { + "clients": { + "anthropic": "1.17.0", + "google-genai": "2.34.0", + "openai": "3.55.0", + "pydantic-ai-slim": "2.70.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [], + "refused": { + "gemini-2.5-flash": "status_code: 404, model_name: gemini-2.5-flash, body:", + "gemini-2.5-flash-lite": "status_code: 404, model_name: gemini-2.5-flash-lite, body:", + "gemini-2.5-pro": "status_code: 404, model_name: gemini-2.5-pro, body:", + "gemini-3-flash-preview": "status_code: 404, model_name: gemini-3-flash-preview, body:", + "gemini-3.1-flash-lite": "status_code: 404, model_name: gemini-3.1-flash-lite, body:", + "gemini-3.1-flash-lite-preview": "status_code: 404, model_name: gemini-3.1-flash-lite-preview, body:", + "gemini-3.1-pro-preview": "status_code: 404, model_name: gemini-3.1-pro-preview, body:", + "gemini-3.1-pro-preview-customtools": "status_code: 404, model_name: gemini-3.1-pro-preview-customtools, body:", + "gemini-3.5-flash": "status_code: 404, model_name: gemini-3.5-flash, body:", + "gemini-3.5-flash-lite": "status_code: 404, model_name: gemini-3.5-flash-lite, body:", + "gemini-3.6-flash": "status_code: 404, model_name: gemini-3.6-flash, body:", + "gemini-3.7-flash": "status_code: 404, model_name: gemini-3.7-flash, body:", + "gemini-3.8-flash": "status_code: 404, model_name: gemini-3.8-flash, body:", + "gemini-4-argon": "status_code: 404, model_name: gemini-4-argon, body:", + "gemini-4-carbon": "status_code: 404, model_name: gemini-4-carbon, body:", + "gemini-4-hydrogen": "status_code: 404, model_name: gemini-4-hydrogen, body:", + "gemini-decision-1": "status_code: 404, model_name: gemini-decision-1, body:", + "gemini-flash-latest": "status_code: 404, model_name: gemini-flash-latest, body:", + "gemini-flash-lite-latest": "status_code: 404, model_name: gemini-flash-lite-latest, body:", + "gemini-omni-1.1-flash": "status_code: 404, model_name: gemini-omni-1.1-flash, body:", + "gemini-omni-1.2-flash": "status_code: 404, model_name: gemini-omni-1.2-flash, body:", + "gemini-pro-latest": "status_code: 404, model_name: gemini-pro-latest, body:", + "gemma-4-26b-a4b-it": "status_code: 404, model_name: gemma-4-26b-a4b-it, body:", + "gemma-4-31b-it": "status_code: 404, model_name: gemma-4-31b-it, body:" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-strands": { + "clients": { + "litellm": "1.105.7", + "openai": "2.54.0", + "strands-agents": "1.65.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_...", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single-word response: \"OK\". No tools are needed. Compliance with the constraint is required.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a specific response: \"OK\". The instruction is very clear and simple. No tool calls are needed.OK" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + } + } +} diff --git a/shared/model-lock.test.ts b/shared/model-lock.test.ts new file mode 100644 index 0000000000..de8d9bec45 --- /dev/null +++ b/shared/model-lock.test.ts @@ -0,0 +1,144 @@ +import { expect, test } from "bun:test"; +import anthropicListing from "./fixtures/model-listings/anthropic-models.json"; +import openaiListing from "./fixtures/model-listings/openai-models.json"; +import before from "./fixtures/model-lock-before-claude-code-bump.json"; +import { + lockAllows, + lockVerdict, + MODEL_LOCK, + type ModelLock, +} from "./model-lock"; + +/* + * The lock's rule against real data: the model listings OpenAI and Anthropic answered on + * 2026-11-17 (shared/fixtures/model-listings), and two locks written that day. `before` is the + * Claude Agent SDK harness's entry from a run of claude-agent-sdk 0.2.152 (Claude Code 2.1.259), + * which Anthropic refused Opus 5.5 and Fable 5.2: every message a Claude plan sent them failed. + * `MODEL_LOCK` is the lock this commit ships, written from a run of its own pins. + */ + +const BEFORE = before as ModelLock; +const anthropicIds = anthropicListing.data.map((model) => model.id); +const openaiIds = openaiListing.data.map((model) => model.id); + +const offered = ( + lock: ModelLock, + engine: string | undefined, + provider: "anthropic" | "openai", + ids: string[], +) => ids.filter((id) => lockAllows(lock, engine, provider, id)); + +test("a harness is not offered the models its client is refused", () => { + const claude = offered(BEFORE, "agent-claude-sdk", "anthropic", anthropicIds); + expect(anthropicIds).toContain("claude-opus-5-5"); + expect(anthropicIds).toContain("claude-fable-5-2"); + expect(claude).not.toContain("claude-opus-5-5"); + expect(claude).not.toContain("claude-fable-5-2"); + expect(claude).toContain("claude-sonnet-5-5"); + expect(claude).toHaveLength(anthropicIds.length - 2); +}); + +test("a refused model carries the provider's own words", () => { + const verdict = lockVerdict( + BEFORE, + "agent-claude-sdk", + "anthropic", + "claude-opus-5-5", + ); + expect(verdict.status).toBe("refused"); + expect(verdict.status === "refused" && verdict.said).toMatch( + /does not support this model; version [\d.]+ or newer is required/, + ); +}); + +test("once the pinned client runs them, the shipped lock offers them", () => { + // The Claude Code bump this commit pins. + expect( + MODEL_LOCK.engines["agent-claude-sdk"]?.clients["claude-code"], + ).not.toBe("2.1.259"); + const claude = offered( + MODEL_LOCK, + "agent-claude-sdk", + "anthropic", + anthropicIds, + ); + expect(claude).toContain("claude-opus-5-5"); + expect(claude).toContain("claude-fable-5-2"); +}); + +test("the shipped lock still holds back what a provider really refuses", () => { + // Google takes gemini-omni-1.x-flash only through its Interactions API, so no client here runs it. + const verdict = lockVerdict( + MODEL_LOCK, + "agent-langgraph-agui", + "google", + "gemini-omni-1.1-flash", + ); + expect(verdict.status).toBe("refused"); + expect( + lockAllows( + MODEL_LOCK, + "agent-langgraph-agui", + "google", + "gemini-omni-1.1-flash", + ), + ).toBe(false); + expect( + lockAllows( + MODEL_LOCK, + "agent-langgraph-agui", + "google", + "gemini-3.8-flash", + ), + ).toBe(true); +}); + +test("a harness that takes OpenAI's newest models through the Responses API is offered them", () => { + const ag2 = offered(MODEL_LOCK, "agent-ag2", "openai", openaiIds); + for (const model of ["gpt-6.1-terra", "gpt-5-pro"]) + if (openaiIds.includes(model)) expect(ag2).toContain(model); +}); + +test("a model released after the last check waits for it", () => { + expect( + lockVerdict(MODEL_LOCK, "agent-claude-sdk", "anthropic", "claude-opus-6"), + ).toEqual({ status: "unchecked" }); + expect( + lockAllows(MODEL_LOCK, "agent-claude-sdk", "anthropic", "claude-opus-6"), + ).toBe(false); +}); + +test("what the check never ran is open, so nobody loses their models to missing data", () => { + // No engine named, or one the lock has no entry for. + expect(offered(MODEL_LOCK, undefined, "anthropic", anthropicIds)).toEqual( + anthropicIds, + ); + // No key for xAI in CI. + expect(lockVerdict(MODEL_LOCK, "agent-agno", "xai", "grok-4.7")).toEqual({ + status: "open", + }); + expect( + lockAllows(MODEL_LOCK, "agent-unknown", "openai", "gpt-6.1-terra"), + ).toBe(true); +}); + +// Also why the lock can't reach main before an engine it names: agent-langgraph-ts needs its row in +// shared/harnesses.json first. +test("every engine the lock names is a harness OpenBot knows, with the clients it ran", async () => { + const { harnesses } = (await import("./harnesses.json")).default as { + harnesses: { image: string | null }[]; + }; + const images = harnesses.map((harness) => harness.image); + for (const [engine, entry] of Object.entries(MODEL_LOCK.engines)) { + expect(images, engine).toContain(engine); + // An engine locked before its first run (agent-langgraph-ts) has run no client yet, and + // offers nothing until it has. + const ran = Object.values(entry.providers).some( + (pair) => + !!pair && + (pair.verified.length > 0 || Object.keys(pair.refused).length > 0), + ); + if (ran) + expect(Object.keys(entry.clients).length, engine).toBeGreaterThan(0); + } +}); diff --git a/shared/model-lock.ts b/shared/model-lock.ts new file mode 100644 index 0000000000..1346132372 --- /dev/null +++ b/shared/model-lock.ts @@ -0,0 +1,109 @@ +import manifest from "./model-lock.json"; + +/** + * THE MODEL LOCK: which models each harness has been proven to run, with the client versions it + * was proven on. + * + * WHY. The model picker lists each provider's models live, but every harness image carries its + * model client at a fixed version, and a provider can refuse a new model to an older client + * ("Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required"). A + * Bot set to such a model fails on every message. So a live model is offered only when the daily + * model check (scripts/model-compat.ts) has sent it one real request through that harness's + * own client and it answered. + * + * WHERE THE DATA COMES FROM. `model-lock.json` is written by `bun scripts/model-compat.ts lock` + * from the check's results, never by hand. The daily check opens a pull request when a model newly + * passes or starts failing, and a pull request that changes a harness's client checks that + * harness before it merges. See docs/releasing.md, "Model lock". + * + * THE RULE. A harness and provider pair the check has run is locked: only the models that passed + * are offered. A pair it has never run (no key for that provider in CI, or an engine it has no + * entry for) is open: nothing is known, so nothing is taken away. A model the provider lists that the + * check has not reached yet (released since the last run) is not offered until it passes. + */ + +/** Who serves the models, as the lock names them. `chatgpt` is a ChatGPT plan's Codex backend. */ +export type LockProvider = + | "openai" + | "anthropic" + | "google" + | "xai" + | "chatgpt"; + +export type LockedProvider = { + /** The model ids that answered through this harness's client. */ + verified: string[]; + /** The model ids the provider refused through this client, with what it said. */ + refused: Record; +}; + +export type LockedEngine = { + /** The model-client packages in the image the check ran, with their versions. */ + clients: Record; + providers: Partial>; +}; + +export type ModelLock = { + /** The day of the run the lock was written from, `YYYY-MM-DD`. */ + checkedAt: string; + /** By harness image directory (`agent-claude-sdk`), as `shared/harnesses.json` names it. */ + engines: Record; +}; + +export const MODEL_LOCK: ModelLock = manifest as ModelLock; + +/** + * The engines new Bots run on unless somebody picks another. A locked pair on one of these with + * nothing verified would leave every new Bot with no model to offer on that provider, so a test + * holds each to at least one (scripts/model-lock.test.ts). + */ +export const DEFAULT_ENGINES: readonly string[] = ["agent-langgraph-ts"]; + +/** The locked pairs of an engine that verify nothing: no model would be offered on them. */ +export function emptyPairs(lock: ModelLock, engine: string): LockProvider[] { + const providers = lock.engines[engine]?.providers ?? {}; + return (Object.keys(providers) as LockProvider[]).filter( + (provider) => (providers[provider]?.verified.length ?? 0) === 0, + ); +} + +/** + * What the lock says about one model on one harness and provider. + * + * - `verified`: it answered through the harness's own client. + * - `refused`: the provider turned it down through that client; `said` is the provider's words. + * - `unchecked`: the pair is locked, and this model was not in the run (released since). + * - `open`: the pair was never checked, or the lock has no entry for the engine. + */ +export type LockVerdict = + | { status: "verified" } + | { status: "refused"; said: string } + | { status: "unchecked" } + | { status: "open" }; + +export function lockVerdict( + lock: ModelLock, + engine: string | undefined, + provider: LockProvider | undefined, + model: string, +): LockVerdict { + const pair = + engine && provider ? lock.engines[engine]?.providers[provider] : undefined; + if (!pair) return { status: "open" }; + if (pair.verified.includes(model)) return { status: "verified" }; + const said = pair.refused[model]; + return said === undefined + ? { status: "unchecked" } + : { status: "refused", said }; +} + +/** Whether a model may be offered: proven on the pair, or the pair is not locked. */ +export function lockAllows( + lock: ModelLock, + engine: string | undefined, + provider: LockProvider | undefined, + model: string, +): boolean { + const { status } = lockVerdict(lock, engine, provider, model); + return status === "verified" || status === "open"; +}