diff --git a/.github/workflows/model-compat.yml b/.github/workflows/model-compat.yml new file mode 100644 index 0000000000..1db03a8315 --- /dev/null +++ b/.github/workflows/model-compat.yml @@ -0,0 +1,261 @@ +name: Model compatibility + +# Does every model the picker offers still answer through the clients OpenBot ships? The picker +# lists each provider's models live, but every harness image carries its model client at a fixed +# version (the Claude Agent SDK harness bundles a Claude Code CLI, the LangGraph harness sends the +# ChatGPT plan's backend a fixed Codex client version, the others carry the provider SDKs they were +# built with), and a provider can refuse a new model to an older client. scripts/model-compat.ts +# starts each harness image, one job per harness, and sends it one request per listed model on +# every provider that has a key here. See docs/releasing.md, "Model compatibility". +# +# Keys are repository secrets, one per provider. A provider without one is reported as skipped, +# not failed. On any failure the run fails and opens (or updates) one issue titled "Model +# compatibility check failing"; a passing run closes it. +# +# What answered on the `source` image is the model lock (shared/model-lock.json), the models the +# picker may offer for each harness. When a model newly answers or stops answering, the run opens +# or updates one pull request that refreshes the lock. See docs/releasing.md, "Model lock". +# +# Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever +# runs with these keys. +on: + schedule: + - cron: "41 6 * * *" + workflow_dispatch: + inputs: + tags: + description: Harness images to check, comma-separated. `latest` is the published release people run; `source` is built from this commit, which is what the next release ships. + type: string + default: latest,source + required: false + +permissions: {} + +concurrency: + group: model-compat + cancel-in-progress: false + +jobs: + # The published harnesses, and every engine the model lock names whether it is published yet or + # not (an engine new Bots run on is locked, and checked, from before its first release). + harnesses: + name: List the harnesses to check + if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + images: ${{ steps.list.outputs.images }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - id: list + run: echo "images=$(bun scripts/model-compat.ts harnesses)" >> "$GITHUB_OUTPUT" + + check: + name: ${{ matrix.harness }} + needs: harnesses + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + harness: ${{ fromJSON(needs.harnesses.outputs.images) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + # Writes its results and exits 0 whatever they say: the report job below decides. + - name: One request per model through this harness + env: + HARNESS: ${{ matrix.harness }} + TAGS: ${{ inputs.tags || 'latest,source' }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + run: >- + bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero + --out "$RUNNER_TEMP/model-compat-$HARNESS.json" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: model-compat-${{ matrix.harness }} + path: ${{ runner.temp }}/model-compat-${{ matrix.harness }}.json + if-no-files-found: error + retention-days: 14 + + report: + name: Report + needs: [harnesses, check] + if: always() && needs.harnesses.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write # open, update or close the one "Model compatibility check failing" issue + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TITLE: Model compatibility check failing + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: model-compat-* + path: ${{ runner.temp }}/results + merge-multiple: true + - name: Summarize + id: summary + run: | + set +e + shopt -s nullglob + bun scripts/model-compat.ts report --summary "$RUNNER_TEMP/summary.md" \ + --issue "$RUNNER_TEMP/issue-table.md" "$RUNNER_TEMP"/results/*.json + code=$? + echo "code=$code" >> "$GITHUB_OUTPUT" + if [ -f "$RUNNER_TEMP/summary.md" ]; then cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY"; fi + exit 0 + - name: Open or update the issue + if: steps.summary.outputs.code != '0' + run: | + set -euo pipefail + body="$RUNNER_TEMP/issue.md" + { + echo "The model compatibility check failed on $(date -u +%Y-%m-%d): $RUN_URL" + echo + echo "A model refused by a pinned client means that client needs bumping: see docs/releasing.md, \"Model compatibility\". This issue is updated by every failing run and closed by the first passing one." + echo + if [ -f "$RUNNER_TEMP/issue-table.md" ]; then + cat "$RUNNER_TEMP/issue-table.md" + else + echo "No harness job wrote results. See the run." + fi + } > "$body" + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$body" + fi + - name: Close the issue when everything passes + if: steps.summary.outputs.code == '0' + run: | + set -euo pipefail + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Passing again: $RUN_URL" + fi + - name: Fail when a model failed + if: steps.summary.outputs.code != '0' + run: | + echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"." + exit 1 + + lock: + name: Refresh the model lock + needs: [harnesses, check] + if: always() && needs.harnesses.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # push the one refresh branch + pull-requests: write # open or update its pull request + env: + BRANCH: model-lock/refresh + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: model-compat-* + path: ${{ runner.temp }}/results + merge-multiple: true + - name: Write the lock from the source images' results + id: lock + run: | + set -euo pipefail + shopt -s nullglob + files=("$RUNNER_TEMP"/results/*.json) + if [ "${#files[@]}" -eq 0 ]; then + echo "No harness job wrote results; the lock stays as it is." + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + bun scripts/model-compat.ts lock --changes "$RUNNER_TEMP/lock-changes.md" "${files[@]}" + if git diff --quiet -- shared/model-lock.json; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + { echo "## Model lock changes"; echo; cat "$RUNNER_TEMP/lock-changes.md"; } >> "$GITHUB_STEP_SUMMARY" + fi + # GITHUB_TOKEN's pull requests get CI runs that wait for a maintainer to approve them. A + # GitHub App installation token, when one is configured, has no such limit. + - id: app-token + if: steps.lock.outputs.changed == 'true' && vars.OPENBOT_UPGRADE_APP_CLIENT_ID != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ vars.OPENBOT_UPGRADE_APP_CLIENT_ID }} + private-key: ${{ secrets.OPENBOT_UPGRADE_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + - name: Open or update the refresh pull request + if: steps.lock.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }} + TOKEN_KIND: ${{ steps.app-token.outputs.token && 'app' || 'github-token' }} + # Credentials for this one push, through gh, and no hook can run. + GIT_CONFIG_COUNT: "3" + GIT_CONFIG_KEY_0: credential.helper + GIT_CONFIG_VALUE_0: "" + GIT_CONFIG_KEY_1: credential.helper + GIT_CONFIG_VALUE_1: "!gh auth git-credential" + GIT_CONFIG_KEY_2: core.hooksPath + GIT_CONFIG_VALUE_2: /dev/null + run: | + set -euo pipefail + git switch -C "$BRANCH" + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -m "Refresh the model lock from the daily model check" -- shared/model-lock.json + git push --force "https://github.com/$GITHUB_REPOSITORY.git" "HEAD:refs/heads/$BRANCH" + body="$RUNNER_TEMP/lock-pr.md" + { + echo "The daily model check found models that started or stopped answering through the clients the harnesses ship. This refreshes \`shared/model-lock.json\`, the models the picker may offer for each harness." + echo + cat "$RUNNER_TEMP/lock-changes.md" + echo + echo "From $RUN_URL. Every run that finds a change updates this pull request. See docs/releasing.md, \"Model lock\"." + if [ "$TOKEN_KIND" != app ]; then + echo + echo "**CI must be started by hand.** This pull request was opened with \`GITHUB_TOKEN\`, so its CI runs wait for a maintainer to approve them." + fi + } > "$body" + number=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq ".[0].number // empty") + if [ -n "$number" ]; then + gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" + else + gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$BRANCH" \ + --title "Refresh the model lock" --body-file "$body" + fi diff --git a/.github/workflows/model-lock.yml b/.github/workflows/model-lock.yml new file mode 100644 index 0000000000..998cbc458c --- /dev/null +++ b/.github/workflows/model-lock.yml @@ -0,0 +1,111 @@ +name: Model lock + +# A pull request that changes a harness's model client (its requirements, its package.json or +# lockfile, its Dockerfile, or the ChatGPT plan's Codex client version) must not break a model the +# model lock (shared/model-lock.json) says that harness runs: a person set to it would get the +# provider's refusal on every message. This builds the changed harnesses from the pull request, +# sends each listed model one request through them (scripts/model-compat.ts), and fails when a +# model the lock verifies no longer answers. See docs/releasing.md, "Model lock". +# +# Only for pull requests from this repository: a fork's pull request is never given the provider +# keys, so it is skipped and the daily check covers it once it merges. +on: + pull_request: + paths: + - "agent-*/requirements.txt" + - "agent-*/Dockerfile" + - "agent-*/package.json" + - "agent-*/bun.lock" + - "agent-*/package-lock.json" + - "agent-langgraph-agui/src/chatgpt_models.py" + - "shared/model-lock.json" + - "scripts/model-compat.ts" + - ".github/workflows/model-lock.yml" + +permissions: {} + +concurrency: + group: model-lock-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + changed: + name: Harnesses whose client changed + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + harnesses: ${{ steps.list.outputs.harnesses }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - id: list + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA") + # The harnesses the check runs: the published ones, and every engine the lock names, + # published yet or not, that has a row in shared/harnesses.json. + checked=$(bun scripts/model-compat.ts harnesses) + # The check itself, or the lock, changed: every harness the lock names is checked. + if grep -qxE 'shared/model-lock.json|scripts/model-compat.ts|\.github/workflows/model-lock.yml' <<<"$changed"; then + harnesses=$(jq -c --argjson checked "$checked" \ + '[.engines | keys[] | select(. as $h | $checked | index($h))]' shared/model-lock.json) + else + harnesses=$(grep -oE '^agent-[a-z0-9-]+/' <<<"$changed" | tr -d / | sort -u \ + | jq -R . | jq -sc --argjson checked "$checked" \ + 'map(select(. as $h | $checked | index($h)))') + fi + echo "harnesses=$harnesses" >> "$GITHUB_OUTPUT" + echo "Harnesses to check: $harnesses" + + check: + name: ${{ matrix.harness }} + needs: changed + if: needs.changed.outputs.harnesses != '[]' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + harness: ${{ fromJSON(needs.changed.outputs.harnesses) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - name: One request per model through this pull request's harness + env: + HARNESS: ${{ matrix.harness }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + run: >- + bun scripts/model-compat.ts --harness "$HARNESS" --tags source --exit-zero + --out "$RUNNER_TEMP/model-compat.json" + - name: Every model the lock verifies still answers + env: + HARNESS: ${{ matrix.harness }} + run: | + set +e + bun scripts/model-compat.ts guard --summary "$RUNNER_TEMP/guard.md" "$RUNNER_TEMP/model-compat.json" + code=$? + cat "$RUNNER_TEMP/guard.md" >> "$GITHUB_STEP_SUMMARY" + if [ "$code" -ne 0 ]; then + echo "::error::A model the lock verifies for $HARNESS no longer answers through this change's client. See the summary." + fi + exit "$code" diff --git a/api/client-boundary.allow.ts b/api/client-boundary.allow.ts index dec6e96ca6..dc3a6b9a03 100644 --- a/api/client-boundary.allow.ts +++ b/api/client-boundary.allow.ts @@ -392,6 +392,12 @@ export const ALLOWED: Allowed[] = [ rule: "third-party", why: "image registries, never OpenBot's server", }, + { + file: "scripts/model-compat.ts", + what: /^(fetch\(\)|\/health|\/api\/agents\/openbot\/generate)$/, + rule: "third-party", + why: "model providers' listings and the harness images the model check starts on this machine, never OpenBot's server", + }, // Scripts that run and stop a server on this machine. { file: "scripts/update-local.ts", diff --git a/api/openapi.yaml b/api/openapi.yaml index b7f7a8ebb4..97eabbff48 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -23579,6 +23579,8 @@ components: type: - string - 'null' + unverified: + $ref: '#/components/schemas/UnverifiedModel' ModelChoiceInput: type: object additionalProperties: false @@ -24689,6 +24691,19 @@ components: text: type: string description: What was said. + UnverifiedModel: + type: object + description: The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason. + required: + - model + - label + properties: + model: + type: string + description: The chosen model's id. + label: + type: string + description: Its name, as a model picker shows it. AppSource: type: string enum: diff --git a/api/openapi/model-connections.yaml b/api/openapi/model-connections.yaml index e42d9df16a..590bb25e74 100644 --- a/api/openapi/model-connections.yaml +++ b/api/openapi/model-connections.yaml @@ -587,6 +587,23 @@ components: type: [string, 'null'] effort: type: [string, 'null'] + unverified: + $ref: '#/components/schemas/UnverifiedModel' + UnverifiedModel: + type: object + description: >- + The model a person chose for this conversation, set aside because it + hasn't been checked to work with the client this Bot's harness + ships. Runs answer on the default model until it has been. Absent when + the choice is in use, or set aside for another reason. + required: [model, label] + properties: + model: + type: string + description: The chosen model's id. + label: + type: string + description: Its name, as a model picker shows it. ModelChoiceInput: type: object additionalProperties: false diff --git a/app/src/lib/api/generated/model/index.ts b/app/src/lib/api/generated/model/index.ts index dccf1b19d1..7c3de1a363 100644 --- a/app/src/lib/api/generated/model/index.ts +++ b/app/src/lib/api/generated/model/index.ts @@ -1449,6 +1449,7 @@ export * from "./unpinBotHeaders"; export * from "./unpublishComponentHeaders"; export * from "./unpublishTeamBotHeaders"; export * from "./unsupportedFileResponse"; +export * from "./unverifiedModel"; export * from "./updateAppHeaders"; export * from "./updateAppRequest"; export * from "./updateApprovalPreferencesHeaders"; diff --git a/app/src/lib/api/generated/model/modelChoice.ts b/app/src/lib/api/generated/model/modelChoice.ts index 3b7cddda63..6d84c877ce 100644 --- a/app/src/lib/api/generated/model/modelChoice.ts +++ b/app/src/lib/api/generated/model/modelChoice.ts @@ -1,6 +1,7 @@ /** * Generated by Orval from api/openapi.yaml. Do not edit: change the spec and run `bun run api:generate`. */ +import type { UnverifiedModel } from "./unverifiedModel"; /** * The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default. @@ -10,4 +11,5 @@ export interface ModelChoice { model: string | null; /** @nullable */ effort: string | null; + unverified?: UnverifiedModel; } diff --git a/app/src/lib/api/generated/model/unverifiedModel.ts b/app/src/lib/api/generated/model/unverifiedModel.ts new file mode 100644 index 0000000000..c2cedf19df --- /dev/null +++ b/app/src/lib/api/generated/model/unverifiedModel.ts @@ -0,0 +1,13 @@ +/** + * Generated by Orval from api/openapi.yaml. Do not edit: change the spec and run `bun run api:generate`. + */ + +/** + * The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason. + */ +export interface UnverifiedModel { + /** The chosen model's id. */ + model: string; + /** Its name, as a model picker shows it. */ + label: string; +} diff --git a/app/src/lib/api/generated/models/models.zod.ts b/app/src/lib/api/generated/models/models.zod.ts index 296e895d13..ea9d04eb29 100644 --- a/app/src/lib/api/generated/models/models.zod.ts +++ b/app/src/lib/api/generated/models/models.zod.ts @@ -134,6 +134,15 @@ export const GetModelChoiceResponse = zod .object({ model: zod.string().nullable(), effort: zod.string().nullable(), + unverified: zod + .object({ + model: zod.string().describe("The chosen model's id."), + label: zod.string().describe("Its name, as a model picker shows it."), + }) + .optional() + .describe( + "The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason.", + ), }) .describe( "The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default.", @@ -216,6 +225,15 @@ export const ReplaceModelChoiceResponse = zod .object({ model: zod.string().nullable(), effort: zod.string().nullable(), + unverified: zod + .object({ + model: zod.string().describe("The chosen model's id."), + label: zod.string().describe("Its name, as a model picker shows it."), + }) + .optional() + .describe( + "The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason.", + ), }) .describe( "The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default.", diff --git a/app/src/lib/models/queries.ts b/app/src/lib/models/queries.ts index 9b126f7d76..f0a4612e24 100644 --- a/app/src/lib/models/queries.ts +++ b/app/src/lib/models/queries.ts @@ -80,6 +80,12 @@ export type AvailableModels = { export type ConversationModel = { model: string | null; effort: string | null; + /** + * The model the person chose here, set aside because it hasn't been checked to work with this + * Bot's harness (shared/model-lock.ts). Runs use the default until it has been. Only ever on a + * read from the server. + */ + unverified?: { model: string; label: string }; }; export const modelKeys = { diff --git a/app/src/lib/models/run-model.ts b/app/src/lib/models/run-model.ts index c90768375c..7937dda6bf 100644 --- a/app/src/lib/models/run-model.ts +++ b/app/src/lib/models/run-model.ts @@ -28,6 +28,15 @@ export function withdrawnModelNotice( return `${picked} ${gone} any more, so this conversation now uses ${now}. Pick another model to change it.`; } +/** + * What the composer says when the model a person chose here hasn't been proven to work with this + * Bot (shared/model-lock.ts): the provider would turn down every message, so runs use the default + * until a check proves it. + */ +export function unverifiedModelNotice(picked: string, now: string): string { + return `${picked} doesn't work with this Bot yet, so this conversation uses ${now}. Pick another model to change it.`; +} + /** * The model and thinking effort one conversation runs on, as its picker shows them, and the way * to change them. @@ -133,8 +142,19 @@ export function useConversationModel( ? undefined : shownPick(picked, catalogue), saving: save.isPending, - /** What the composer says about the pick, when the picked model stopped being offered. */ - notice: withdrawn?.channelId === channelId ? withdrawn.notice : null, + /** + * What the composer says about the pick: the chosen model doesn't work with this Bot yet, or + * it stopped being offered. + */ + notice: + (!save.isPending && saved.data?.unverified && catalogue + ? unverifiedModelNotice( + saved.data.unverified.label, + catalogue.models.find((option) => option.id === catalogue.default) + ?.label ?? catalogue.default, + ) + : null) ?? + (withdrawn?.channelId === channelId ? withdrawn.notice : null), // Read the ref at submission time, including a choice made before React renders again. beforeSend: async () => { await pendingSave.current; diff --git a/docs/releasing.md b/docs/releasing.md index bd92907919..fd181bcd60 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -993,6 +993,97 @@ it when it is not. Signed desktop builds are the exception: they read the `macos `windows-signing` environments described above and in [Windows signing](windows-signing.md). Downloads sync signs in to AWS with GitHub's OIDC token rather than a stored key. +## Model compatibility + +The model picker lists each provider's models live, but every harness image carries its model +client at a fixed version: the Claude Agent SDK harness bundles a Claude Code CLI (through +`claude-agent-sdk` in `agent-claude-sdk/requirements.txt`), the LangGraph harness sends the ChatGPT +plan's backend a fixed Codex client version (`DEFAULT_CLIENT_VERSION` in +`agent-langgraph-agui/src/chatgpt_models.py`), and the others carry the provider SDKs they were built +with. A provider can refuse a new model to an older client while the picker keeps offering it: +"Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required". + +**Model compatibility** (`.github/workflows/model-compat.yml`) checks for that daily, and by hand +from the Actions tab. One job per harness in `.github/published-images.json` runs +[`scripts/model-compat.ts`](../scripts/model-compat.ts), which: + +- lists each provider's models from the same source, with the same filter, as the picker (see + [The model list](configuration.md#the-model-list)); +- starts the harness image on each provider the way the Desktop configures it, and sends one AG-UI + run per model, so the harness's own client makes the call. It checks `latest`, the published + image people run, and `source`, the image built from this commit; +- compares the LangGraph harness's Codex client version with the minimum each ChatGPT plan model + needs, from openai/codex's published model catalogue. A plan sign-in cannot run in CI. A Claude + plan runs through the same Claude Code CLI as an Anthropic key, so the key's rows cover it. + +It writes a pass, fail or skipped row per harness, image, provider and model, with the provider's +own words, to the run's summary. On any failure the run fails and opens or updates one issue, +**Model compatibility check failing**, and the first passing run closes it. It runs only on a +schedule or by hand on `main`, never on a pull request. + +Each provider needs its key as a repository secret: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, +`GEMINI_API_KEY` (Google, also used for Antigravity) and `XAI_API_KEY`. A provider without one is +reported as skipped, not failed, and the summary names the missing secrets. Unbiased AI is not +checked: it is a gateway serving its own model, not a provider's listing. + +**When it fails**, read the provider's words in the table: + +- A version refusal (a model "not supported" by the client's version, or a ChatGPT plan model that + needs a newer Codex) means bumping the pinned client: `claude-agent-sdk` in + `agent-claude-sdk/requirements.txt` (each release bundles a newer Claude Code CLI), or + `DEFAULT_CLIENT_VERSION` in `agent-langgraph-agui/src/chatgpt_models.py`, or the harness's + provider SDK in its `requirements.txt` or `package.json`. Run the check by hand: the `source` rows + show whether the bump fixed it before a release ships it, and the `latest` rows stay red until + one does. +- A refused parameter or endpoint (a model that takes tools only through `/v1/responses`, or + refuses a `temperature`) means that harness builds its request in a way the model no longer + accepts: fix how the harness sets up its model, or bump the framework if a newer one does. +- Billing, rate limits or a refused key are the key's problem, not a client's: fix the key. + +To run it locally (needs Docker), with keys in the environment or an env file: + +```sh +bun scripts/model-compat.ts --env-file .env --harness agent-claude-sdk --provider anthropic +``` + +`--model ` narrows the models, `--tags latest` skips building from source. + +### Model lock + +The picker offers a model only when this check has proven it on the Bot's harness. +`shared/model-lock.json` holds, per harness, the model clients its `source` image carried and, per +provider, the models that answered and the ones refused, in the provider's words. The server's model +catalogue reads it (`shared/model-lock.ts` has the rule): a live model the lock doesn't verify for +the harness is hidden, and a conversation already set to one runs on the default and says so. + +- **Which harness.** A Claude plan runs every Bot through `agent-claude-sdk`, a ChatGPT plan through + `agent-langgraph-agui`, and with a key, the harness picked in setup when it takes OpenBot's model + choice. OpenBot's own runtime is not a harness image, so its models are not locked. +- **What is locked.** Only a harness and provider pair this check has run. A pair it never ran (a + provider with no key here, such as xAI today) stays open, and a gateway is never locked. +- **A new engine** can be locked before its first release, by giving it empty pairs for the + providers it runs on. Nothing is offered on it until a run proves a model, including on a + provider the check has no key for. The check runs every engine the lock names, published or not + yet, from its `source` image (`bun scripts/model-compat.ts harnesses` lists them). The LangGraph + (TypeScript) engine, `agent-langgraph-ts`, starts this way on Anthropic, OpenAI and Google. +- **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull + request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that + started and stopped answering and the clients that moved. Merging it lets the picker offer them. + A failure that says nothing about the model (the harness never started, a timeout) keeps what the + lock said before. +- **A client bump** is held to the lock: a pull request from this repository that changes a + harness's `requirements.txt`, `package.json`, `bun.lock`, `Dockerfile` or the Codex client version + runs the **Model lock** workflow, which builds that harness, sends every listed model a request, + and fails when a model the lock verifies no longer answers. Models the bump newly fixes are added + by the next daily refresh. + +To write the lock from results by hand: + +```sh +bun scripts/model-compat.ts lock --changes changes.md results/*.json +bun scripts/model-compat.ts guard results/*.json +``` + ## The one thing CI cannot do The smoke journey in `tests/smoke` is the only check that proves the parts are wired to each other: diff --git a/scripts/model-compat-langgraph-ts.test.ts b/scripts/model-compat-langgraph-ts.test.ts new file mode 100644 index 0000000000..1f0f15b5f2 --- /dev/null +++ b/scripts/model-compat-langgraph-ts.test.ts @@ -0,0 +1,351 @@ +import { expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import anthropicListing from "../shared/fixtures/model-listings/anthropic-models.json"; +import openaiListing from "../shared/fixtures/model-listings/openai-models.json"; +import { lockAllows, MODEL_LOCK, type ModelLock } from "../shared/model-lock"; +import { + buildLock, + buildMatrix, + type Harness, + NODE_VERSIONS, + probedHarnesses, + providersFor, + type Result, + type Results, + usableModels, +} from "./model-compat"; + +/* + * The LangGraph (TypeScript) engine, `agent-langgraph-ts`, is locked before its first release: + * it becomes the engine new Bots run on, so a model is offered on it only once the model check has + * sent that model a real request through the engine's own client and it answered. Until then, + * nothing is offered on it, for any provider, even one CI has no key for. + */ + +const ENGINE = "agent-langgraph-ts"; +const ROOT = join(import.meta.dir, ".."); +const NOW = Date.parse("2026-11-17T00:00:00Z"); + +/** Its row in shared/harnesses.json, as the shared contract names it. */ +const LANGGRAPH_TS: Harness & { id: string } = { + id: "langgraph-ts", + image: ENGINE, + port: 4215, + runPath: "", + credential: "any-provider", +}; + +const anthropicIds = usableModels( + "anthropic", + anthropicListing, + new Map(), + NOW, +); +const openaiIds = usableModels("openai", openaiListing, new Map(), NOW); +const geminiIds = ["gemini-3.8-flash", "gemini-4-argon", "gemini-3.5-pro"]; + +const UNRUN: ModelLock = { + checkedAt: "2026-11-16", + engines: { + [ENGINE]: { + clients: {}, + providers: { + anthropic: { verified: [], refused: {} }, + openai: { verified: [], refused: {} }, + google: { verified: [], refused: {} }, + }, + }, + }, +}; + +test("before its first run, nothing is offered on langgraph-ts, on any provider", () => { + expect(anthropicIds.length).toBeGreaterThan(0); + expect(openaiIds.length).toBeGreaterThan(0); + for (const [provider, ids] of [ + ["anthropic", anthropicIds], + ["openai", openaiIds], + ["google", geminiIds], + ] as const) + for (const model of ids) + expect( + lockAllows(UNRUN, ENGINE, provider, model), + `${provider} ${model}`, + ).toBe(false); +}); + +const row = ( + providerId: "anthropic" | "openai" | "google", + model: string, + status: Result["status"], + detail = "OK", + tag = "source", +): Result => ({ + harness: ENGINE, + tag, + provider: { anthropic: "Anthropic", openai: "OpenAI", google: "Google" }[ + providerId + ], + providerId, + model, + status, + detail, +}); + +test("after a run, exactly the models that answered are offered on langgraph-ts", () => { + const run: Results = { + results: [ + row("anthropic", "claude-sonnet-5-5", "pass"), + row("anthropic", "claude-opus-5-5", "pass"), + row( + "anthropic", + "claude-fable-5-2", + "fail", + "400 invalid_request_error: model: claude-fable-5-2", + ), + // The runner's bad day: says nothing about the model, and verifies nothing. + row( + "anthropic", + "claude-sonnet-4-5-20250929", + "fail", + "no answer: TimeoutError: The operation timed out.", + ), + row("openai", "gpt-6.1-terra", "pass"), + row( + "openai", + "gpt-5.5", + "fail", + "the stream ended without finishing the run", + ), + // The published image is what people ran before; the lock follows `source` alone. + row("openai", "gpt-6.2-sol", "pass", "OK", "latest"), + { + harness: ENGINE, + tag: "latest, source", + provider: "Google", + providerId: "google", + model: "(all)", + status: "skipped", + detail: "skipped: no key (GEMINI_API_KEY)", + }, + ], + clients: { + [ENGINE]: { + source: { + "@langchain/langgraph": "1.7.5", + "@langchain/anthropic": "1.3.1", + }, + }, + }, + }; + const lock = buildLock(UNRUN, [run], "2026-11-17"); + const allowed = ( + provider: "anthropic" | "openai" | "google", + ids: string[], + ) => ids.filter((model) => lockAllows(lock, ENGINE, provider, model)); + + expect(allowed("anthropic", anthropicIds).sort()).toEqual([ + "claude-opus-5-5", + "claude-sonnet-5-5", + ]); + expect(openaiIds).toContain("gpt-6.1-terra"); + expect(allowed("openai", openaiIds)).toEqual(["gpt-6.1-terra"]); + // No key for Gemini today: the pair stays locked, and nothing is offered on it. + expect(allowed("google", geminiIds)).toEqual([]); + expect(lock.engines[ENGINE]?.providers.google).toEqual({ + verified: [], + refused: {}, + }); + expect(lock.engines[ENGINE]?.providers.anthropic?.refused).toEqual({ + "claude-fable-5-2": "400 invalid_request_error: model: claude-fable-5-2", + }); + expect(lock.engines[ENGINE]?.clients).toEqual({ + "@langchain/anthropic": "1.3.1", + "@langchain/langgraph": "1.7.5", + }); +}); + +test("an inconclusive failure never verifies a model langgraph-ts has not answered", () => { + const before: ModelLock = { + checkedAt: "2026-11-16", + engines: { + [ENGINE]: { + clients: { "@langchain/langgraph": "1.7.5" }, + providers: { + anthropic: { verified: ["claude-sonnet-5-5"], refused: {} }, + openai: { verified: [], refused: {} }, + google: { verified: [], refused: {} }, + }, + }, + }, + }; + const inconclusive = [ + "no answer: TimeoutError: The operation timed out.", + "the harness did not start: port is already allocated", + "the harness never became healthy: Error: Cannot find module", + `could not get ghcr.io/copilotkit/openbot-${ENGINE}:latest: manifest unknown`, + "the stream ended without finishing the run", + ]; + const run: Results = { + results: inconclusive.flatMap((detail, index) => [ + row("anthropic", `claude-model-${index}`, "fail", detail), + row("openai", `gpt-model-${index}`, "fail", detail), + // One it had answered before keeps its place: a bad day takes nothing away. + row("anthropic", "claude-sonnet-5-5", "fail", detail), + ]), + }; + const lock = buildLock(before, [run], "2026-11-17"); + expect(lock.engines[ENGINE]?.providers.anthropic?.verified).toEqual([ + "claude-sonnet-5-5", + ]); + expect(lock.engines[ENGINE]?.providers.openai?.verified).toEqual([]); + for (let index = 0; index < inconclusive.length; index++) { + expect(lockAllows(lock, ENGINE, "anthropic", `claude-model-${index}`)).toBe( + false, + ); + expect(lockAllows(lock, ENGINE, "openai", `gpt-model-${index}`)).toBe( + false, + ); + } +}); + +test("the lock names langgraph-ts, with every provider a key reaches locked", () => { + const entry = MODEL_LOCK.engines[ENGINE]; + expect(entry).toBeDefined(); + for (const provider of ["anthropic", "openai", "google"] as const) + expect(entry?.providers[provider], provider).toBeDefined(); +}); + +test("an engine the lock names is checked before it is published", () => { + const published = ["agent-claude-sdk", "agent-langgraph-agui"]; + const known: Harness[] = [ + { + image: "agent-claude-sdk", + credential: "anthropic", + port: 4212, + runPath: "", + }, + { + image: "agent-langgraph-agui", + credential: "any-provider", + port: 4201, + runPath: "", + }, + LANGGRAPH_TS, + { + image: "agent-unlisted", + credential: "any-provider", + port: 4299, + runPath: "", + }, + ]; + const lock: ModelLock = { + checkedAt: "2026-11-17", + engines: { + [ENGINE]: { clients: {}, providers: {} }, + // A lock entry with no harness row has nothing to start, so it is not run. + "agent-gone": { clients: {}, providers: {} }, + }, + }; + expect( + probedHarnesses(known, published, lock).map((harness) => harness.image), + ).toEqual(["agent-claude-sdk", "agent-langgraph-agui", ENGINE]); + expect( + probedHarnesses(known, published, lock, [ENGINE]).map( + (harness) => harness.image, + ), + ).toEqual([ENGINE]); +}); + +test("langgraph-ts is run on Anthropic, OpenAI and Gemini, one row per listed model", () => { + expect(providersFor(LANGGRAPH_TS)).toEqual( + expect.arrayContaining(["anthropic", "openai", "google"]), + ); + const rows = buildMatrix({ + harnesses: [LANGGRAPH_TS], + tags: ["source"], + listings: new Map([ + ["anthropic", { models: anthropicIds }], + ["openai", { models: openaiIds }], + ["google", { models: geminiIds }], + ]), + }); + const runs = rows.filter((matrixRow) => matrixRow.kind === "run"); + expect(runs.length).toBe( + anthropicIds.length + openaiIds.length + geminiIds.length, + ); + for (const [provider, ids] of [ + ["anthropic", anthropicIds], + ["openai", openaiIds], + ["google", geminiIds], + ] as const) + for (const model of ids) + expect( + runs.some( + (run) => + run.kind === "run" && + run.provider === provider && + run.model === model, + ), + `${provider} ${model}`, + ).toBe(true); +}); + +test("the probe reads an engine's LangChain and AG-UI clients under plain Node", async () => { + // A Node 22 image's working directory, with its packages installed beside it (and one hoisted + // to the directory above, as a workspace install leaves it). + const image = mkdtempSync(join(tmpdir(), "model-compat-node-")); + const app = join(image, "app"); + const installed: Record = { + "@langchain/langgraph": "1.7.5", + "@langchain/core": "1.2.0", + "@langchain/anthropic": "1.3.1", + "@langchain/openai": "1.4.0", + "@langchain/google-genai": "2.1.0", + // The engine's Gemini client, the one its package.json names. + "@langchain/google": "0.2.15", + "@ag-ui/client": "0.0.46", + }; + for (const [name, version] of Object.entries(installed)) { + const dir = join(app, "node_modules", name); + mkdirSync(dir, { recursive: true }); + // An `exports` map without `./package.json`, as some packages ship: a plain require of the + // package's manifest is refused, so the probe reads the file. + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name, version, exports: { ".": "./index.js" } }), + ); + } + const hoisted = join(image, "node_modules", "@ag-ui", "core"); + mkdirSync(hoisted, { recursive: true }); + writeFileSync( + join(hoisted, "package.json"), + JSON.stringify({ name: "@ag-ui/core", version: "0.0.46" }), + ); + + const child = Bun.spawn(["node", "-e", NODE_VERSIONS], { + cwd: app, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, code] = await Promise.all([ + new Response(child.stdout).text(), + child.exited, + ]); + expect(code).toBe(0); + expect(JSON.parse(stdout.trim())).toEqual({ + ...installed, + "@ag-ui/core": "0.0.46", + }); +}); + +test("both workflows check every engine the lock names, not only the published ones", () => { + for (const workflow of ["model-compat.yml", "model-lock.yml"]) { + const text = readFileSync( + join(ROOT, ".github/workflows", workflow), + "utf8", + ); + expect(text, workflow).toContain("bun scripts/model-compat.ts harnesses"); + } +}); diff --git a/scripts/model-compat.test.ts b/scripts/model-compat.test.ts new file mode 100644 index 0000000000..c4ce089879 --- /dev/null +++ b/scripts/model-compat.test.ts @@ -0,0 +1,392 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + buildMatrix, + classifyMastra, + classifyRun, + codexGate, + type Harness, + type MatrixRow, + NOT_CHECKED, + PROVIDERS, + parseCapabilities, + providersFor, + renderTable, + usableModels, +} from "./model-compat"; + +const ROOT = join(import.meta.dir, ".."); +const NOW = Date.parse("2026-11-17T12:00:00Z"); + +// The listings below are cut from what each provider answered on 2026-11-17. +const openaiListing = { + data: [ + { id: "gpt-6.1-terra", created: 1792000000 }, + { id: "gpt-6.2-sol", created: 1793000000 }, + { id: "gpt-5.5", created: 1776900000 }, + { id: "gpt-5.5-2026-04-23", created: 1776900000 }, + { id: "gpt-5.3-codex", created: 1770000000 }, + { id: "gpt-5-search-api", created: 1760000000 }, + { id: "chat-latest", created: 1760000000 }, + { id: "gpt-4o-mini-tts", created: 1740000000 }, + { id: "gpt-realtime-2", created: 1780000000 }, + { id: "gpt-image-2", created: 1776000000 }, + { id: "text-embedding-3-large", created: 1700000000 }, + { id: "whisper-1", created: 1680000000 }, + { id: "gpt-live-1", created: 1790000000 }, + { id: "gpt-4-turbo", created: 1700000000 }, + { id: "o1", created: 1720000000 }, + { id: "o3", created: 1744000000, shutdown_date: "2026-11-01" }, + { id: "o3-pro", created: 1749000000, shutdown_date: "2026-12-11" }, + ], +}; + +const anthropicListing = { + data: [ + { + id: "claude-fable-5-2", + created_at: "2026-11-09T18:30:00Z", + lifecycle: "active", + }, + { + id: "claude-opus-5-5", + created_at: "2026-09-21T16:24:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-5-5", + created_at: "2026-09-28T00:00:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-4-5-20250929", + created_at: "2025-09-29T00:00:00Z", + retires_at: "2026-11-30T17:00:00Z", + lifecycle: "deprecated", + }, + { + id: "claude-3-opus-20240229", + created_at: "2024-02-29T00:00:00Z", + retires_at: "2026-01-05T00:00:00Z", + lifecycle: "retired", + }, + ], +}; + +const geminiListing = { + object: "list", + data: [ + { id: "models/gemini-3.8-flash", object: "model", owned_by: "google" }, + { id: "models/gemini-4-argon", object: "model", owned_by: "google" }, + { id: "models/gemini-embedding-001", object: "model", owned_by: "google" }, + { + id: "models/imagen-4.0-generate-001", + object: "model", + owned_by: "google", + }, + { id: "models/veo-3.0-generate-001", object: "model", owned_by: "google" }, + { id: "models/gemini-3.8-flash-live", object: "model", owned_by: "google" }, + { + id: "models/gemini-robotics-er-1.5", + object: "model", + owned_by: "google", + }, + { id: "models/aqa", object: "model", owned_by: "google" }, + { + id: "models/gemini-2.5-flash-preview-tts", + object: "model", + owned_by: "google", + }, + ], +}; + +test("an OpenAI key is offered only the chat models a Bot can run, newest first", () => { + expect(usableModels("openai", openaiListing, new Map(), NOW)).toEqual([ + "gpt-6.2-sol", + "gpt-6.1-terra", + "gpt-5.5", + "o3-pro", + ]); +}); + +test("models.dev saying a model takes no tools drops it, as the picker does", () => { + const capabilities = parseCapabilities({ + openai: { models: { "gpt-6.1-terra": { tool_call: false } } }, + }); + expect( + usableModels("openai", openaiListing, capabilities, NOW), + ).not.toContain("gpt-6.1-terra"); +}); + +test("an Anthropic key is offered every current Claude, retired ones dropped", () => { + expect(usableModels("anthropic", anthropicListing, new Map(), NOW)).toEqual([ + "claude-fable-5-2", + "claude-sonnet-5-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + ]); +}); + +test("a Google key keeps the Gemini chat models and drops media, live, robots and aqa", () => { + expect(usableModels("google", geminiListing, new Map(), NOW).sort()).toEqual([ + "gemini-3.8-flash", + "gemini-4-argon", + ]); +}); + +const harnesses: Harness[] = [ + { + image: "agent-claude-sdk", + credential: "anthropic", + port: 4212, + runPath: "", + }, + { image: "agent-antigravity", credential: "google", port: 4214, runPath: "" }, + { + image: "agent-agno", + credential: "any-provider", + port: 4203, + runPath: "/agui", + }, +]; + +test("each harness is paired with the providers it can run on", () => { + expect(providersFor(harnesses[0]!)).toEqual(["anthropic"]); + expect(providersFor(harnesses[1]!)).toEqual(["google"]); + expect(providersFor(harnesses[2]!)).toEqual( + PROVIDERS.map((provider) => provider.id), + ); +}); + +test("the matrix has one row per harness, image tag and model, and skips a provider with no key", () => { + const rows = buildMatrix({ + harnesses, + tags: ["latest", "main"], + listings: new Map([ + ["anthropic", { models: ["claude-opus-5-5", "claude-sonnet-5-5"] }], + ["openai", { error: "401 Incorrect API key provided" }], + ]), + }); + const runs = rows.filter( + (row): row is Extract => row.kind === "run", + ); + // claude-sdk and agno on Anthropic: 2 harnesses x 2 tags x 2 models. + expect(runs).toHaveLength(8); + expect( + runs + .filter((row) => row.harness.image === "agent-claude-sdk") + .map((row) => `${row.tag} ${row.model}`), + ).toEqual([ + "latest claude-opus-5-5", + "latest claude-sonnet-5-5", + "main claude-opus-5-5", + "main claude-sonnet-5-5", + ]); + const listingFailures = rows.filter((row) => row.kind === "listing-failed"); + expect( + listingFailures.map((row) => [row.harness.image, row.provider]), + ).toEqual([["agent-agno", "openai"]]); + const skipped = rows.filter((row) => row.kind === "no-key"); + expect( + skipped.map((row) => [row.harness.image, row.provider]).sort(), + ).toEqual([ + ["agent-agno", "google"], + ["agent-agno", "xai"], + ["agent-antigravity", "google"], + ]); +}); + +// What the Claude Agent SDK harness at claude-agent-sdk==0.2.152 streamed for claude-opus-5-5. +const claudeSdkRefusal = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"CUSTOM","name":"system:status","value":{"status":"requesting"}}', + `data: {"type":"RUN_ERROR","message":"API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.","code":"OPENBOT_MODEL_REFUSED"}`, + "", +].join("\n\n"); + +// What the Agno harness streamed for claude-sonnet-5-5, and for a model Anthropic does not have. +const agnoReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"TEXT_MESSAGE_START","messageId":"m","role":"assistant"}', + 'data: {"type":"TEXT_MESSAGE_CONTENT","messageId":"m","delta":"OK"}', + 'data: {"type":"TEXT_MESSAGE_END","messageId":"m"}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', +].join("\n\n"); +const agnoNotFound = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + `data: {"type":"RUN_ERROR","rawEvent":{"event":"RunError","content":"litellm.NotFoundError: AnthropicException - b'{\\"type\\":\\"error\\",\\"error\\":{\\"type\\":\\"not_found_error\\",\\"message\\":\\"model: claude-nonexistent-9\\"}}'"}}`, +].join("\n\n"); + +test("a run the provider refused fails with the provider's own words", () => { + const result = classifyRun({ status: 200, body: claudeSdkRefusal }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain( + "Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required", + ); +}); + +test("a run that answered passes", () => { + expect(classifyRun({ status: 200, body: agnoReply })).toEqual({ + status: "pass", + detail: "OK", + }); +}); + +test("an error a framework puts in its raw event is still the failure's text", () => { + const result = classifyRun({ status: 200, body: agnoNotFound }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("not_found_error"); +}); + +test("a run that finished without a reply, or answered with an error as text, fails", () => { + expect( + classifyRun({ + status: 200, + body: 'data: {"type":"RUN_STARTED"}\n\ndata: {"type":"RUN_FINISHED"}', + }).status, + ).toBe("fail"); + const asText = classifyRun({ + status: 200, + body: [ + 'data: {"type":"TEXT_MESSAGE_CONTENT","delta":"Error code: 400 - model not supported"}', + 'data: {"type":"RUN_FINISHED"}', + ].join("\n\n"), + }); + expect(asText.status).toBe("fail"); + expect(asText.detail).toContain("model not supported"); +}); + +test("a reply that arrives only as a messages snapshot, as CrewAI's bridge sends it, passes", () => { + const crewaiReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"MESSAGES_SNAPSHOT","messages":[{"id":"u","role":"user","content":"Reply with the single word OK."},{"id":"a","role":"assistant","content":"OK"}]}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', + ].join("\n\n"); + expect(classifyRun({ status: 200, body: crewaiReply })).toEqual({ + status: "pass", + detail: "OK", + }); + // The person's own message in the snapshot is not a reply. + const onlyThePrompt = crewaiReply.replace( + ',{"id":"a","role":"assistant","content":"OK"}', + "", + ); + expect(classifyRun({ status: 200, body: onlyThePrompt }).status).toBe("fail"); +}); + +test("Mastra's own API answer is read as a reply or a refusal", () => { + expect( + classifyMastra({ + status: 200, + body: '{"text":"OK","finishReason":"stop"}', + }), + ).toEqual({ status: "pass", detail: "OK" }); + expect( + classifyMastra({ + status: 500, + body: '{"error":"This model is only supported in v1/responses and not in v1/chat/completions."}', + }), + ).toEqual({ + status: "fail", + detail: + "HTTP 500: This model is only supported in v1/responses and not in v1/chat/completions.", + }); +}); + +test("a refused request fails with its status and body", () => { + expect( + classifyRun({ status: 401, body: '{"error":"unauthorised"}' }), + ).toEqual({ + status: "fail", + detail: 'HTTP 401: {"error":"unauthorised"}', + }); +}); + +// Shaped as openai/codex publishes codex-rs/models-manager/models.json. +const codexCatalogue = { + models: [ + { + slug: "gpt-6.1-terra", + visibility: "list", + minimal_client_version: "0.153.0", + }, + { + slug: "gpt-6.3-sol", + visibility: "list", + minimal_client_version: "0.170.0", + }, + { + slug: "gpt-daybreak-blue-latest", + visibility: "hide", + minimal_client_version: "0.142.2", + }, + ], +}; + +test("a ChatGPT plan model newer than the harness's Codex client version fails the gate", () => { + expect(codexGate("0.167.0", codexCatalogue)).toEqual([ + { + model: "gpt-6.1-terra", + status: "pass", + detail: "needs Codex 0.153.0, harness sends 0.167.0", + }, + { + model: "gpt-6.3-sol", + status: "fail", + detail: "needs Codex 0.170.0, harness sends 0.167.0", + }, + ]); + // Compared as numbers, not text. + expect( + codexGate("0.99.0", { + models: [ + { slug: "m", visibility: "list", minimal_client_version: "0.100.0" }, + ], + })[0]!.status, + ).toBe("fail"); +}); + +test("the table escapes what would break a Markdown row", () => { + const table = renderTable([ + { + harness: "agent-agno", + tag: "latest", + provider: "openai", + model: "gpt-6.1-terra", + status: "fail", + detail: "a | b\nc", + }, + ]); + expect(table).toContain("a \\| b c"); +}); + +test("every provider the Desktop offers by address is checked", () => { + const picker = readFileSync( + join(ROOT, "desktop/src/ProviderPicker.tsx"), + "utf8", + ); + const presets = [...picker.matchAll(/baseUrl: "(https:[^"]+)"/g)].map( + (match) => match[1]!.replace(/\/+$/, ""), + ); + expect(presets.length).toBeGreaterThan(0); + const checked = [ + ...PROVIDERS.map((provider) => provider.baseUrl), + ...NOT_CHECKED.map((left) => left.baseUrl), + ].map((url) => url?.replace(/\/+$/, "")); + for (const preset of presets) expect(checked).toContain(preset); +}); + +test("every published harness is one the matrix can run", () => { + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const { harnesses: known } = JSON.parse( + readFileSync(join(ROOT, "shared/harnesses.json"), "utf8"), + ) as { harnesses: Harness[] }; + for (const image of published) { + const harness = known.find((row) => row.image === image); + expect(harness, image).toBeDefined(); + expect(providersFor(harness!).length, image).toBeGreaterThan(0); + } +}); diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts new file mode 100644 index 0000000000..9e415d5707 --- /dev/null +++ b/scripts/model-compat.ts @@ -0,0 +1,1592 @@ +/** + * Does every model the picker offers still answer through the clients OpenBot ships? + * + * A harness image carries its own model client at a fixed version: the Claude Agent SDK harness + * bundles a Claude Code CLI, the LangGraph harness sends the ChatGPT plan's Codex backend a fixed + * client version, and every other harness carries the provider SDKs (LiteLLM, LangChain, the AI + * SDK, ...) it was built with. A provider can start refusing a model to an older client ("Claude + * Code 2.1.259 does not support this model; version 2.1.290 or newer is required") while the picker, + * which lists models live, keeps offering it. Nothing else calls today's models through the + * published images, so this does, once per model: + * + * 1. Lists each provider's models from the same source, with the same filter, as the picker + * (docs/configuration.md, "The model list"). + * 2. Starts each harness image on that provider, as the Desktop's settings would, and sends it + * one AG-UI run per model. The harness's own client makes the call, at the version that image + * ships. `latest` is the published image people run (ghcr.io/copilotkit/openbot-); + * `source` is the image built from this checkout, which shows whether a bumped pin fixed it. + * 3. Checks the ChatGPT plan's Codex client version in each image against the minimum each + * listed Codex model needs, from openai/codex's published model catalogue. A plan sign-in + * cannot run here, and the plan's version gate is a fact about the client version. + * + * A Claude plan runs through the same bundled CLI as an Anthropic key, so the key's rows cover it. + * + * Keys come from the environment (or --env-file), one per provider: ANTHROPIC_API_KEY, + * OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY. A provider with no key is reported as skipped, + * not failed. Keys are never printed: every detail is scrubbed of them. + * + * What passed on the `source` image becomes the model lock (shared/model-lock.ts): the models + * the picker may offer for each harness. `lock` writes it from the results; `guard` fails when a + * model the lock verifies no longer answers, for a pull request that changes a harness's client. + * + * bun scripts/model-compat.ts [--env-file .env] [--harness agent-claude-sdk,...] + * [--provider anthropic,...] [--model ] [--tags latest,source] [--concurrency 6] + * [--out results.json] [--exit-zero] + * bun scripts/model-compat.ts report --summary [--issue ] results/*.json + * bun scripts/model-compat.ts lock [--lock shared/model-lock.json] [--date YYYY-MM-DD] results/*.json + * bun scripts/model-compat.ts guard [--lock shared/model-lock.json] [--summary ] results/*.json + * bun scripts/model-compat.ts harnesses + * + * The harnesses it runs are the published ones and every engine the lock names, published or not + * yet (`harnesses` prints them, for the workflows' matrix). An engine is put in the lock with its + * providers' pairs empty before its first release, so nothing is offered on it until it has + * passed here: the LangGraph (TypeScript) engine, agent-langgraph-ts, starts that way. + * + * Needs Docker. Exits 1 when anything failed (unless --exit-zero); `report` merges the results of + * several runs, writes the Markdown summary, and exits 1 when any of them failed. `lock` exits 0, + * and says whether the lock changed. `guard` exits 1 when a verified model failed. + */ + +import { randomBytes } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { + LockedEngine, + LockProvider, + ModelLock, +} from "../shared/model-lock"; + +const ROOT = join(import.meta.dir, ".."); +const REGISTRY = "ghcr.io/copilotkit/openbot-"; +const SOURCE = "source"; +const MODELS_DEV = "https://models.dev/api.json"; +const CODEX_CATALOGUE = + "https://raw.githubusercontent.com/openai/codex/main/codex-rs/models-manager/models.json"; +const PROMPT = "Reply with the single word OK."; +/** One tool the prompt has no use for, so the request takes the client's tool path. */ +const PROBE_TOOL = { + name: "look_up_order", + description: "Look up the status of an order by its number.", + parameters: { + type: "object", + properties: { order: { type: "string", description: "The order number." } }, + required: ["order"], + }, +}; +const RUN_TIMEOUT_MS = 300_000; +const START_TIMEOUT_MS = 120_000; + +// --------------------------------------------------------------------------------------------- +// The picker's list. These rules are the ones the server's model catalogue applies, so this offers +// what a person can pick: the provider's own listing, minus what no Bot can chat on. +// --------------------------------------------------------------------------------------------- + +export type Source = "openai" | "anthropic" | "google" | "xai" | "gateway"; + +/** Models no chat can run on, by any provider or gateway's naming. */ +const NEVER_CHAT = + /(embed|tts|whisper|transcribe|dall-e|image|moderation|realtime|audio|sora|translate|video|lyria|imagen|veo-|(^|[-/])live([-.]|$))/i; +/** Chat models in OpenAI's listing a Bot is still not put on. */ +const OPENAI_NOT_FOR_BOTS = + /(search|deep-research|computer-use|chat-latest|codex|instruct|^gpt-live|^gpt-(3\.5|4)(-|$)|^o1(-|$)|^(babbage|davinci))/i; +/** The same, for the other providers a deployment names directly. */ +const VENDOR_NOT_FOR_BOTS = + /(deep-research|computer-use|robotics|antigravity|nano-banana|multi-agent|(^|-)aqa$)/i; +/** OpenAI's dated snapshots, which repeat a model already listed under its own name. */ +const DATED_SNAPSHOT = /-(\d{4}-\d{2}-\d{2}|\d{4})$/; + +const VENDOR_FAMILIES: readonly [RegExp, string][] = [ + [/^(gpt-|o\d|chatgpt-)/i, "openai"], + [/^claude/i, "anthropic"], + [/^(gemini|gemma)/i, "google"], + [/^grok/i, "xai"], +]; + +function modelVendor(id: string): string | undefined { + const slash = id.indexOf("/"); + if (slash > 0) return id.slice(0, slash).toLowerCase().replace(/^~/, ""); + return VENDOR_FAMILIES.find(([family]) => family.test(id))?.[1]; +} + +function chatModel(id: string, officialOpenAI: boolean): boolean { + if (NEVER_CHAT.test(id)) return false; + if (!officialOpenAI) return true; + return ( + /^(gpt-|o\d)/i.test(id) && + !OPENAI_NOT_FOR_BOTS.test(id) && + !DATED_SNAPSHOT.test(id) + ); +} + +function seconds(value: unknown): number { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed / 1000 : 0; + } + return 0; +} + +function pastDate(value: unknown, now: number): boolean { + if (typeof value !== "string" || !value) return false; + const at = Date.parse(value); + return Number.isFinite(at) && at <= now; +} + +/** What models.dev says a model can do, by `vendor/id`. */ +export type Capabilities = Map< + string, + { tools?: boolean; text?: boolean; released?: number } +>; + +export function parseCapabilities(body: unknown): Capabilities { + const found: Capabilities = new Map(); + if (!body || typeof body !== "object") return found; + for (const vendor of ["openai", "anthropic", "google", "xai"]) { + const models = (body as Record)[vendor] + ?.models; + if (!models || typeof models !== "object") continue; + for (const [id, raw] of Object.entries(models as Record)) { + if (!raw || typeof raw !== "object") continue; + const model = raw as Record; + const outputs = (model.modalities as { output?: unknown } | undefined) + ?.output; + found.set(`${vendor}/${id}`, { + ...(typeof model.tool_call === "boolean" + ? { tools: model.tool_call } + : {}), + ...(Array.isArray(outputs) ? { text: outputs.includes("text") } : {}), + ...(typeof model.release_date === "string" + ? { released: seconds(model.release_date) } + : {}), + }); + } + } + return found; +} + +/** The models the picker offers from one listing, newest first. */ +export function usableModels( + source: Source, + body: unknown, + capabilities: Capabilities, + now: number, +): string[] { + const record = + body && typeof body === "object" ? (body as Record) : {}; + const data = Array.isArray(record.data) + ? record.data + : Array.isArray(record.models) + ? record.models + : []; + const vendorOfSource = source === "gateway" ? undefined : source; + const usable: { id: string; created: number }[] = []; + for (const raw of data) { + if (!raw || typeof raw !== "object") continue; + const item = raw as Record; + const named = + typeof item.id === "string" + ? item.id + : typeof item.name === "string" + ? item.name + : ""; + const id = named.replace(/^models\//, ""); + if (!id) continue; + const outputs = item.output_modalities; + const methods = item.supportedGenerationMethods; + const notChat = + (Array.isArray(outputs) && !outputs.includes("text")) || + (Array.isArray(methods) && !methods.includes("generateContent")); + const retired = + pastDate(item.shutdown_date, now) || + pastDate(item.retires_at, now) || + item.lifecycle === "retired"; + const bare = id.split("/").at(-1) ?? id; + const vendor = vendorOfSource ?? modelVendor(id); + const known = vendor ? capabilities.get(`${vendor}/${bare}`) : undefined; + const chat = !notChat && known?.tools !== false && known?.text !== false; + const offered = + !retired && + chat && + (source === "openai" + ? chatModel(id, true) + : source === "gateway" + ? chatModel(id, false) + : !NEVER_CHAT.test(id) && !VENDOR_NOT_FOR_BOTS.test(id)); + if (offered) + usable.push({ + id, + created: + seconds(item.created ?? item.created_at) || known?.released || 0, + }); + } + return usable + .sort((a, b) => b.created - a.created || a.id.localeCompare(b.id)) + .map((entry) => entry.id); +} + +// --------------------------------------------------------------------------------------------- +// Providers and harnesses. +// --------------------------------------------------------------------------------------------- + +/** A row of shared/harnesses.json, the fields this reads. */ +export type Harness = { + image: string | null; + credential: string; + port: number | null; + runPath: string; +}; + +export type ProviderId = "openai" | "anthropic" | "google" | "xai"; + +export type Provider = { + id: ProviderId; + label: string; + /** The repository secret, and the environment variable, that holds this provider's key. */ + secret: string; + source: Source; + /** The OpenAI-compatible address the Desktop writes as OPENAI_BASE_URL for this provider. */ + baseUrl?: string; + listing: (key: string) => { url: string; headers: Record }; + /** The environment a harness is started with on this provider, as the Desktop writes it. */ + harnessEnv: (key: string, harness: Harness) => Record; +}; + +function compatible(baseUrl: string): Pick { + return { + baseUrl, + harnessEnv: (key) => ({ + OPENAI_API_KEY: key, + OPENAI_BASE_URL: baseUrl, + BOT_PROVIDER: "openai", + }), + }; +} + +function listedAt(url: string) { + return (key: string) => ({ + url, + headers: { authorization: `Bearer ${key}` }, + }); +} + +export const PROVIDERS: readonly Provider[] = [ + { + id: "openai", + label: "OpenAI", + secret: "OPENAI_API_KEY", + source: "openai", + listing: listedAt("https://api.openai.com/v1/models"), + harnessEnv: (key) => ({ OPENAI_API_KEY: key, BOT_PROVIDER: "openai" }), + }, + { + id: "anthropic", + label: "Anthropic", + secret: "ANTHROPIC_API_KEY", + source: "anthropic", + listing: (key) => ({ + url: "https://api.anthropic.com/v1/models?limit=1000", + headers: { "x-api-key": key, "anthropic-version": "2023-06-01" }, + }), + harnessEnv: (key) => ({ + ANTHROPIC_API_KEY: key, + BOT_PROVIDER: "anthropic", + }), + }, + { + id: "google", + label: "Google", + secret: "GEMINI_API_KEY", + source: "google", + listing: listedAt( + "https://generativelanguage.googleapis.com/v1beta/openai/models", + ), + ...compatible("https://generativelanguage.googleapis.com/v1beta/openai/"), + // Antigravity is pinned to Gemini's own API and reads the key under Google's name. + harnessEnv: (key, harness) => + harness.credential === "google" + ? { GOOGLE_API_KEY: key } + : compatible( + "https://generativelanguage.googleapis.com/v1beta/openai/", + ).harnessEnv(key, harness), + }, + { + id: "xai", + label: "xAI", + secret: "XAI_API_KEY", + source: "xai", + listing: listedAt("https://api.x.ai/v1/language-models"), + ...compatible("https://api.x.ai/v1"), + }, +]; + +/** + * Providers the Desktop offers by address that this check leaves out on purpose, with why. A + * gateway serves its own aliases, not a provider's models, so the lock has nothing to say about + * it and leaves its models open. + */ +export const NOT_CHECKED: readonly { baseUrl: string; why: string }[] = [ + { + baseUrl: "https://api.unbiased.ai/v1", + why: "Unbiased AI is a gateway serving its own `pareto` model, not a provider's listing", + }, +]; + +function provider(id: string): Provider { + const found = PROVIDERS.find((row) => row.id === id); + if (!found) throw new Error(`No provider ${id}`); + return found; +} + +/** The providers a harness can run on, by the credential shared/harnesses.json gives it. */ +export function providersFor(harness: Harness): ProviderId[] { + if (!harness.image || !harness.port) return []; + switch (harness.credential) { + case "anthropic": + return ["anthropic"]; + case "google": + return ["google"]; + case "any-provider": + return PROVIDERS.map((row) => row.id); + default: + return []; + } +} + +/** + * The harnesses the check runs: every published harness, and every engine the model lock names + * whether it is published yet or not. An engine the lock names offers a model only once it has + * passed here, so it is checked from before its first release (its `source` image, built from this + * checkout; a `latest` that isn't published yet is an inconclusive row). A lock entry with no row + * in shared/harnesses.json has nothing to start, so it is not run. `only` narrows to the named + * image directories. + */ +export function probedHarnesses( + known: readonly Harness[], + published: readonly string[], + lock: ModelLock | undefined, + only?: readonly string[], +): Harness[] { + const locked = Object.keys(lock?.engines ?? {}); + return known.filter( + (row) => + !!row.image && + (published.includes(row.image) || locked.includes(row.image)) && + (!only || only.includes(row.image)), + ); +} + +function readProbed(only?: readonly string[]): Harness[] { + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const { harnesses } = JSON.parse( + readFileSync(join(ROOT, "shared/harnesses.json"), "utf8"), + ) as { harnesses: Harness[] }; + const lockPath = join(ROOT, "shared/model-lock.json"); + const lock = existsSync(lockPath) + ? (JSON.parse(readFileSync(lockPath, "utf8")) as ModelLock) + : undefined; + return probedHarnesses(harnesses, published, lock, only); +} + +export type Listing = { models: string[] } | { error: string }; + +export type MatrixRow = + | { + kind: "run"; + harness: Harness; + tag: string; + provider: ProviderId; + model: string; + } + | { kind: "no-key"; harness: Harness; provider: ProviderId } + | { + kind: "listing-failed"; + harness: Harness; + provider: ProviderId; + detail: string; + }; + +/** + * One run per harness, image tag and listed model. A provider with no key is one skipped row per + * harness, and a listing that failed one failed row, since there are no models to name. + */ +export function buildMatrix(options: { + harnesses: readonly Harness[]; + tags: readonly string[]; + listings: ReadonlyMap; +}): MatrixRow[] { + const rows: MatrixRow[] = []; + for (const harness of options.harnesses) { + for (const id of providersFor(harness)) { + const listing = options.listings.get(id); + if (!listing) rows.push({ kind: "no-key", harness, provider: id }); + else if ("error" in listing) + rows.push({ + kind: "listing-failed", + harness, + provider: id, + detail: listing.error, + }); + else + for (const tag of options.tags) + for (const model of listing.models) + rows.push({ kind: "run", harness, tag, provider: id, model }); + } + } + return rows; +} + +// --------------------------------------------------------------------------------------------- +// Reading a run. +// --------------------------------------------------------------------------------------------- + +export type Outcome = { status: "pass" | "fail"; detail: string }; + +function oneLine(text: string, limit = 400): string { + const flat = text.replace(/\s+/g, " ").trim(); + return flat.length > limit ? `${flat.slice(0, limit)}...` : flat; +} + +function errorText(event: Record): string { + if (typeof event.message === "string" && event.message) return event.message; + const raw = event.rawEvent as Record | undefined; + for (const field of ["content", "error", "message"]) { + const value = raw?.[field]; + if (typeof value === "string" && value) return value; + } + return JSON.stringify(event); +} + +/** Pass when the harness streamed a reply that says OK; otherwise fail, in the provider's words. */ +export function classifyRun(response: { + status: number; + body: string; +}): Outcome { + if (response.status !== 200) + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(response.body)}`, + }; + let streamed = ""; + // Some bridges (CrewAI's) send the reply only as the conversation's new state. + let snapshot = ""; + let finished = false; + let calledTool = false; + for (const line of response.body.split("\n")) { + if (!line.startsWith("data:")) continue; + let event: Record; + try { + event = JSON.parse(line.slice(5)); + } catch { + continue; + } + if (event.type === "RUN_ERROR") + return { status: "fail", detail: oneLine(errorText(event)) }; + if ( + (event.type === "TEXT_MESSAGE_CONTENT" || + event.type === "TEXT_MESSAGE_CHUNK") && + typeof event.delta === "string" + ) + streamed += event.delta; + if (event.type === "MESSAGES_SNAPSHOT" && Array.isArray(event.messages)) { + const last = event.messages.at(-1) as + | { role?: unknown; content?: unknown } + | undefined; + snapshot = + last?.role === "assistant" && typeof last.content === "string" + ? last.content + : ""; + } + if (event.type === "TOOL_CALL_START") calledTool = true; + if (event.type === "RUN_FINISHED") finished = true; + } + return judgeReply( + streamed || snapshot, + calledTool + ? "the model called the probe's tool instead of replying" + : finished + ? "the run finished without a reply" + : "the stream ended without finishing the run", + ); +} + +function judgeReply(text: string, silence: string): Outcome { + if (/\bok(ay)?\b/i.test(text)) + return { status: "pass", detail: oneLine(text, 80) }; + if (text) + return { status: "fail", detail: `answered without OK: ${oneLine(text)}` }; + return { status: "fail", detail: silence }; +} + +/** + * Mastra serves no AG-UI route: OpenBot reaches it through Mastra's own agent API, so its run is a + * `generate` call on that API, answered with the reply's `text` or an `error`. + */ +export function classifyMastra(response: { + status: number; + body: string; +}): Outcome { + let parsed: { text?: unknown; error?: unknown; message?: unknown } = {}; + try { + parsed = JSON.parse(response.body); + } catch { + // Not JSON: reported as it is, below. + } + if (response.status !== 200) { + const said = [parsed.error, parsed.message].find( + (value): value is string => typeof value === "string" && !!value, + ); + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(said ?? response.body)}`, + }; + } + return judgeReply( + typeof parsed.text === "string" ? parsed.text : "", + "the agent answered without text", + ); +} + +// --------------------------------------------------------------------------------------------- +// The ChatGPT plan's version gate. +// --------------------------------------------------------------------------------------------- + +function versionParts(version: string): number[] { + return version.split(/[.-]/).map((part) => Number.parseInt(part, 10) || 0); +} + +export function compareVersions(a: string, b: string): number { + const left = versionParts(a); + const right = versionParts(b); + for (let i = 0; i < Math.max(left.length, right.length); i++) { + const difference = (left[i] ?? 0) - (right[i] ?? 0); + if (difference) return Math.sign(difference); + } + return 0; +} + +/** Each model the Codex picker lists, against the client version the harness sends the plan. */ +export function codexGate( + clientVersion: string, + catalogue: unknown, +): { model: string; status: "pass" | "fail"; detail: string }[] { + const models = (catalogue as { models?: unknown })?.models; + if (!Array.isArray(models)) return []; + return models + .filter( + (row): row is { slug: string; minimal_client_version?: string } => + !!row && row.visibility === "list" && typeof row.slug === "string", + ) + .map((row) => { + const needs = row.minimal_client_version ?? "0.0.0"; + return { + model: row.slug, + status: compareVersions(clientVersion, needs) >= 0 ? "pass" : "fail", + detail: `needs Codex ${needs}, harness sends ${clientVersion}`, + }; + }); +} + +// --------------------------------------------------------------------------------------------- +// Results. +// --------------------------------------------------------------------------------------------- + +export type Result = { + harness: string; + tag: string; + /** What a person reads: "Anthropic (key or Claude plan)", "ChatGPT plan". */ + provider: string; + /** What the lock files it under. */ + providerId: LockProvider; + model: string; + status: "pass" | "fail" | "skipped"; + detail: string; +}; + +/** One results file: the rows, and the model clients each image carried. */ +export type Results = { + results: Result[]; + /** By harness, then image tag: package name to version. */ + clients?: Record>>; +}; + +function cell(text: string): string { + return oneLine(text).replace(/\|/g, "\\|"); +} + +export function renderTable(results: readonly Result[]): string { + const lines = [ + "| Harness | Image | Provider | Model | Result | Provider's answer |", + "| --- | --- | --- | --- | --- | --- |", + ]; + for (const row of results) + lines.push( + `| ${cell(row.harness)} | ${cell(row.tag)} | ${cell(row.provider)} | ${cell(row.model)} | ${row.status} | ${cell(row.detail)} |`, + ); + return lines.join("\n"); +} + +const ORDER = { fail: 0, skipped: 1, pass: 2 } as const; + +/** + * The summary: what failed first, then what was skipped for want of a key, then what passed. The + * issue's copy leaves out what passed and keeps at most `maxFailed` failures, since an issue body + * holds 65,536 characters and the run's own summary has the rest. + */ +export function renderSummary( + results: readonly Result[], + options: { passed?: boolean; maxFailed?: number } = {}, +): string { + const { passed: showPassed = true, maxFailed = Number.POSITIVE_INFINITY } = + options; + const sorted = [...results].sort( + (a, b) => + ORDER[a.status] - ORDER[b.status] || + a.harness.localeCompare(b.harness) || + a.provider.localeCompare(b.provider) || + a.tag.localeCompare(b.tag) || + a.model.localeCompare(b.model), + ); + const count = (status: Result["status"]) => + results.filter((row) => row.status === status).length; + const failed = sorted.filter((row) => row.status === "fail"); + const skipped = sorted.filter((row) => row.status === "skipped"); + const passed = sorted.filter((row) => row.status === "pass"); + const missing = [ + ...new Set( + skipped.map( + (row) => + PROVIDERS.find((p) => p.label === row.provider)?.secret ?? + row.provider, + ), + ), + ]; + const parts = [ + "## Model compatibility", + "", + `${count("fail")} failed, ${count("pass")} passed, ${count("skipped")} skipped.`, + "", + ]; + if (missing.length) + parts.push( + `Skipped for want of a key, not failed: add the repository secret${missing.length > 1 ? "s" : ""} ${missing.map((name) => `\`${name}\``).join(", ")} to check ${missing.length > 1 ? "those providers" : "that provider"}.`, + "", + ); + if (failed.length) { + parts.push("### Failed", "", renderTable(failed.slice(0, maxFailed)), ""); + if (failed.length > maxFailed) + parts.push( + `And ${failed.length - maxFailed} more failures, in the run's summary.`, + "", + ); + } + if (skipped.length) parts.push("### Skipped", "", renderTable(skipped), ""); + if (passed.length && showPassed) + parts.push( + "
Passed", + "", + renderTable(passed), + "", + "
", + "", + ); + return parts.join("\n"); +} + +// --------------------------------------------------------------------------------------------- +// Running it. +// --------------------------------------------------------------------------------------------- + +type Options = { + envFile?: string; + harnesses?: string[]; + providers?: string[]; + model?: RegExp; + tags: string[]; + concurrency: number; + out?: string; + exitZero: boolean; +}; + +function parseOptions(argv: string[]): Options { + const options: Options = { + tags: ["latest", SOURCE], + concurrency: 6, + exitZero: false, + }; + const list = (value: string | undefined) => + (value ?? "") + .split(/[\s,]+/) + .map((item) => item.trim()) + .filter(Boolean); + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined) throw new Error(`${flag} needs a value`); + return next; + }; + if (flag === "--env-file") options.envFile = value(); + else if (flag === "--harness") options.harnesses = list(value()); + else if (flag === "--provider") options.providers = list(value()); + else if (flag === "--model") options.model = new RegExp(value()); + else if (flag === "--tags") options.tags = list(value()); + else if (flag === "--concurrency") + options.concurrency = Math.max(1, Number(value()) || 1); + else if (flag === "--out") options.out = value(); + else if (flag === "--exit-zero") options.exitZero = true; + else throw new Error(`Unknown option ${flag}`); + } + return options; +} + +/** The provider keys, from the environment, or from an env file for the names it lacks. */ +function readKeys(envFile: string | undefined): Map { + const fromFile = new Map(); + if (envFile) + for (const line of readFileSync(envFile, "utf8").split("\n")) { + const match = /^\s*(?:export\s+)?([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line); + if (!match) continue; + const [, name = "", raw = ""] = match; + fromFile.set(name, raw.trim().replace(/^(['"])(.*)\1$/, "$2")); + } + const keys = new Map(); + for (const row of PROVIDERS) { + const value = ( + process.env[row.secret] || + fromFile.get(row.secret) || + "" + ).trim(); + if (value) keys.set(row.id, value); + } + return keys; +} + +function scrubber(keys: Map) { + const secrets = [...keys.values()].filter((value) => value.length >= 8); + return (text: string) => + secrets.reduce( + (scrubbed, secret) => scrubbed.split(secret).join("[key]"), + text, + ); +} + +async function fetchJson( + url: string, + headers: Record = {}, +): Promise<{ ok: true; body: unknown } | { ok: false; error: string }> { + try { + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(30_000), + }); + const text = await response.text(); + if (!response.ok) + return { ok: false, error: `HTTP ${response.status}: ${oneLine(text)}` }; + return { ok: true, body: JSON.parse(text) }; + } catch (error) { + return { ok: false, error: String(error) }; + } +} + +async function exec( + args: string[], + env: Record = {}, +): Promise<{ code: number; stdout: string; stderr: string }> { + const child = Bun.spawn(args, { + env: { ...process.env, ...env }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + return { code, stdout, stderr }; +} + +/** + * `source` is the image built from this checkout, which is what the next release ships, so a pin + * bumped here is checked before it is released. Any other tag is the published image: `latest` is + * what people run today. + */ +function imageOf(harness: Harness, tag: string): string { + return tag === SOURCE + ? `model-compat/${harness.image}:source` + : `${REGISTRY}${harness.image}:${tag}`; +} + +async function prepare(harness: Harness, tag: string) { + return tag === SOURCE + ? exec([ + "docker", + "build", + "--quiet", + "-f", + join(ROOT, harness.image ?? "", "Dockerfile"), + "-t", + imageOf(harness, tag), + ROOT, + ]) + : exec(["docker", "pull", "--quiet", imageOf(harness, tag)]); +} + +/** + * The model-client packages whose versions decide what a provider accepts, by the names each + * ecosystem installs them under. Whichever an image has are recorded in the lock beside the + * models they were proven with, so a lock entry says what it was true for. + */ +const PYTHON_CLIENTS = [ + "claude-agent-sdk", + "ag-ui-claude-sdk", + "openai", + "anthropic", + "litellm", + "google-genai", + "google-adk", + "ag2", + "agno", + "crewai", + "langgraph", + "langchain-core", + "langchain-openai", + "langchain-anthropic", + "langchain-google-genai", + "langroid", + "llama-index-core", + "llama-index-llms-openai", + "llama-index-llms-anthropic", + "llama-index-llms-google-genai", + "pydantic-ai-slim", + "strands-agents", + "agent-framework", + "agent-framework-core", +]; +const NODE_CLIENTS = [ + "@mastra/core", + "ai", + "@ai-sdk/openai", + "@ai-sdk/anthropic", + "@ai-sdk/google", + // The LangGraph (TypeScript) engine: its graph, the LangChain chat models that call each + // provider, and the AG-UI packages it speaks through. + "@langchain/langgraph", + "@langchain/core", + "@langchain/anthropic", + "@langchain/openai", + "@langchain/google", + "@langchain/google-genai", + "@ag-ui/core", + "@ag-ui/client", + "@ag-ui/encoder", + "@ag-ui/langgraph", +]; + +const PYTHON_VERSIONS = ` +import json, importlib.metadata as m +found = {} +for name in ${JSON.stringify(PYTHON_CLIENTS)}: + try: + found[name] = m.version(name) + except Exception: + pass +try: + import claude_agent_sdk, pathlib, subprocess + for cli in pathlib.Path(claude_agent_sdk.__file__).parent.glob("_bundled/claude*"): + said = subprocess.run([str(cli), "--version"], capture_output=True, text=True, timeout=30).stdout.split() + if said: + found["claude-code"] = said[0] + break +except Exception: + pass +print(json.dumps(found)) +`; +/** + * Read under `bun -e` or plain `node -e` (a Node image may have no Bun), from the image's working + * directory up, the way Node resolves a package. The manifest is read as a file rather than + * required, since a package whose `exports` map leaves out `./package.json` refuses the require. + */ +export const NODE_VERSIONS = ` +const { readFileSync } = require("node:fs"); +const { dirname, join } = require("node:path"); +const found = {}; +for (const name of ${JSON.stringify(NODE_CLIENTS)}) { + for (let dir = process.cwd(); ; dir = dirname(dir)) { + try { + found[name] = JSON.parse(readFileSync(join(dir, "node_modules", name, "package.json"), "utf8")).version; + break; + } catch {} + if (dirname(dir) === dir) break; + } +} +console.log(JSON.stringify(found)); +`; + +/** The model clients an image carries, or an empty record when it says nothing. */ +async function clientsIn(image: string): Promise> { + for (const args of [ + ["python", "-c", PYTHON_VERSIONS], + ["bun", "-e", NODE_VERSIONS], + ["node", "-e", NODE_VERSIONS], + ] as const) { + const ran = await exec([ + "docker", + "run", + "--rm", + "--entrypoint", + args[0], + image, + args[1], + args[2], + ]); + if (ran.code !== 0) continue; + try { + const found = JSON.parse(ran.stdout.trim().split("\n").at(-1) ?? ""); + if (found && typeof found === "object" && Object.keys(found).length) + return found as Record; + } catch { + // Not this runtime's answer: try the next. + } + } + return {}; +} + +let serial = 0; + +/** Starts the harness on one provider and model, sends one run, and stops it. */ +async function runOne( + row: Extract, + key: string, + scrub: (text: string) => string, +): Promise { + const harness = row.harness; + const name = `model-compat-${process.pid}-${++serial}`; + const env = { + ...provider(row.provider).harnessEnv(key, harness), + BOT_MODEL: row.model, + }; + const token = randomBytes(12).toString("hex"); + // Each value goes to Docker through the environment and is named here, so no key is in argv. + const started = await exec( + [ + "docker", + "run", + "-d", + "--rm", + "--name", + name, + "-p", + `127.0.0.1::${harness.port}`, + "-e", + "MANAGED_AGENT_TOKEN", + ...Object.keys(env).flatMap((variable) => ["-e", variable]), + imageOf(harness, row.tag), + ], + { ...env, MANAGED_AGENT_TOKEN: token }, + ); + if (started.code !== 0) + return { + status: "fail", + detail: scrub(`the harness did not start: ${oneLine(started.stderr)}`), + }; + try { + const port = await exec(["docker", "port", name, `${harness.port}/tcp`]); + const address = port.stdout.trim().split("\n")[0]?.trim(); + if (!address) + return { status: "fail", detail: "the harness published no port" }; + const base = `http://${address}`; + const deadline = Date.now() + START_TIMEOUT_MS; + let healthy = false; + while (Date.now() < deadline) { + try { + const health = await fetch(`${base}/health`, { + signal: AbortSignal.timeout(5_000), + }); + if (health.ok) { + healthy = true; + break; + } + } catch { + // Not listening yet. + } + const state = await exec([ + "docker", + "inspect", + "-f", + "{{.State.Running}}", + name, + ]); + if (state.stdout.trim() !== "true") break; + await Bun.sleep(1_000); + } + if (!healthy) { + const logs = await exec(["docker", "logs", "--tail", "5", name]); + return { + status: "fail", + detail: scrub( + `the harness never became healthy: ${oneLine(logs.stdout + logs.stderr)}`, + ), + }; + } + const id = randomBytes(6).toString("hex"); + const mastra = harness.image === "agent-mastra"; + let response: { status: number; body: string }; + try { + const reply = await fetch( + mastra + ? `${base}/api/agents/openbot/generate` + : `${base}${harness.runPath || "/"}`, + { + method: "POST", + headers: { + "content-type": "application/json", + accept: mastra ? "application/json" : "text/event-stream", + "x-openbot-agent-token": token, + }, + body: JSON.stringify( + mastra + ? { messages: [{ role: "user", content: PROMPT }] } + : { + threadId: `model-compat-${id}`, + runId: `run-${id}`, + state: {}, + messages: [ + { id: `message-${id}`, role: "user", content: PROMPT }, + ], + // A Bot's run always carries tools, and a client's tool path is where models + // part ways (Responses-only models, parallel tool calls), so this one does too. + tools: [PROBE_TOOL], + context: [], + // What the server tells the LangGraph and Claude Agent SDK harnesses; the others + // read BOT_MODEL, set above. + forwardedProps: { openbotModel: { model: row.model } }, + }, + ), + signal: AbortSignal.timeout(RUN_TIMEOUT_MS), + }, + ); + response = { status: reply.status, body: await reply.text() }; + } catch (error) { + return { + status: "fail", + detail: scrub(`no answer: ${oneLine(String(error))}`), + }; + } + const outcome = mastra ? classifyMastra(response) : classifyRun(response); + if (outcome.status === "fail" && !response.body.includes("RUN_ERROR")) { + // A harness that swallowed the error still logged it. + const logs = await exec(["docker", "logs", "--tail", "40", name]); + const said = (logs.stdout + logs.stderr) + .split("\n") + .filter((line) => /error|exception/i.test(line)) + .at(-1); + if (said) + outcome.detail = `${outcome.detail}; logged: ${oneLine(said, 300)}`; + } + return { status: outcome.status, detail: scrub(outcome.detail) }; + } finally { + await exec(["docker", "rm", "-f", name]); + } +} + +async function pool( + items: readonly T[], + width: number, + work: (item: T) => Promise, +) { + let next = 0; + await Promise.all( + Array.from({ length: Math.min(width, items.length) }, async () => { + for (let item = items[next++]; item !== undefined; item = items[next++]) + await work(item); + }), + ); +} + +async function check(argv: string[]): Promise { + const options = parseOptions(argv); + const keys = readKeys(options.envFile); + const scrub = scrubber(keys); + + const harnesses = readProbed(options.harnesses); + if (!harnesses.length) + throw new Error( + "No harness matches --harness: it must be published, or named in shared/model-lock.json, and have a row in shared/harnesses.json.", + ); + + const wanted = new Set( + harnesses + .flatMap(providersFor) + .filter((id) => !options.providers || options.providers.includes(id)), + ); + const capabilities = await fetchJson(MODELS_DEV); + const known = capabilities.ok + ? parseCapabilities(capabilities.body) + : new Map(); + const listings = new Map(); + for (const id of wanted) { + const key = keys.get(id); + if (!key) continue; + const row = provider(id); + const request = row.listing(key); + const listed = await fetchJson(request.url, request.headers); + listings.set( + id, + listed.ok + ? { + models: usableModels( + row.source, + listed.body, + known, + Date.now(), + ).filter((model) => !options.model || options.model.test(model)), + } + : { error: scrub(`the model listing failed: ${listed.error}`) }, + ); + } + const matrix = buildMatrix({ + harnesses, + tags: options.tags, + listings, + }).filter( + (row) => !options.providers || options.providers.includes(row.provider), + ); + + const results: Result[] = []; + const label = (id: string) => provider(id).label; + for (const row of matrix) { + if (row.kind === "no-key") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + providerId: row.provider, + model: "(all)", + status: "skipped", + detail: `skipped: no key (${provider(row.provider).secret})`, + }); + if (row.kind === "listing-failed") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + providerId: row.provider, + model: "(all)", + status: "fail", + detail: row.detail, + }); + } + + // Every image once, before any run, so a pull or build is never counted against a run's time. + const images = harnesses.flatMap((harness) => + options.tags.map((tag) => ({ harness, tag })), + ); + const unpulled = new Map(); + await pool(images, 3, async ({ harness, tag }) => { + const began = Date.now(); + const ready = await prepare(harness, tag); + console.error( + `${tag === SOURCE ? "built" : "pulled"} ${imageOf(harness, tag)} in ${Math.round((Date.now() - began) / 1000)}s${ready.code ? " (failed)" : ""}`, + ); + if (ready.code !== 0) + unpulled.set( + imageOf(harness, tag), + oneLine(ready.stderr.split("\n").slice(-5).join(" ")), + ); + }); + + // The model clients each image carries, which the lock records beside what passed. + const clients: NonNullable = {}; + for (const { harness, tag } of images) { + const image = imageOf(harness, tag); + if (unpulled.has(image) || !harness.image) continue; + const found = await clientsIn(image); + clients[harness.image] = { ...clients[harness.image], [tag]: found }; + } + + // The ChatGPT plan's Codex client version, in whichever images carry the plan. + const codex = await fetchJson(CODEX_CATALOGUE); + if (!options.providers || options.providers.includes("openai")) + for (const harness of harnesses) + for (const tag of options.tags) { + const image = imageOf(harness, tag); + if (unpulled.has(image)) continue; + const version = await exec([ + "docker", + "run", + "--rm", + "--entrypoint", + "python", + image, + "-c", + "from src.chatgpt_models import client_version; print(client_version())", + ]); + if (version.code !== 0) continue; + const sends = version.stdout.trim(); + if (!codex.ok) { + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + providerId: "chatgpt", + model: "(all)", + status: "fail", + detail: `could not read Codex's model catalogue: ${codex.error}`, + }); + continue; + } + for (const gate of codexGate(sends, codex.body)) + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + providerId: "chatgpt", + ...gate, + }); + } + + const runs = matrix.filter( + (row): row is Extract => row.kind === "run", + ); + let done = 0; + await pool(runs, options.concurrency, async (row) => { + const image = imageOf(row.harness, row.tag); + const outcome = unpulled.has(image) + ? { + status: "fail" as const, + detail: `could not get ${image}: ${unpulled.get(image)}`, + } + : await runOne(row, keys.get(row.provider) ?? "", scrub); + results.push({ + harness: row.harness.image ?? "", + tag: row.tag, + provider: + row.provider === "anthropic" && row.harness.credential === "anthropic" + ? "Anthropic (key or Claude plan)" + : label(row.provider), + providerId: row.provider, + model: row.model, + ...outcome, + }); + done++; + console.error( + `[${done}/${runs.length}] ${row.harness.image}:${row.tag} ${row.provider} ${row.model}: ${outcome.status}`, + ); + }); + + if (options.out) + writeFileSync( + options.out, + `${JSON.stringify({ results, clients } satisfies Results, null, 2)}\n`, + ); + console.log(renderSummary(results)); + return results.some((row) => row.status === "fail") && !options.exitZero + ? 1 + : 0; +} + +// --------------------------------------------------------------------------------------------- +// The model lock. +// --------------------------------------------------------------------------------------------- + +const PROVIDER_IDS_BY_LABEL: Record = { + OpenAI: "openai", + Anthropic: "anthropic", + "Anthropic (key or Claude plan)": "anthropic", + Google: "google", + xAI: "xai", + "ChatGPT plan": "chatgpt", +}; + +/** The lock's name for a row's provider. Results written before rows carried it are read by label. */ +export function providerIdOf(row: { + provider: string; + providerId?: string; +}): LockProvider | undefined { + if (row.providerId) return row.providerId as LockProvider; + return PROVIDER_IDS_BY_LABEL[row.provider]; +} + +/** + * A failure that says nothing about the model: the harness never started, the image could not be + * had, or the run timed out with no word from the provider. The lock keeps what it knew before + * for such a model rather than take it away on a bad day for the runner. + */ +const INCONCLUSIVE = + /^(no answer:|the harness did not start|the harness never became healthy|the harness published no port|could not get |the stream ended without finishing the run)/; + +export function inconclusive(detail: string): boolean { + return INCONCLUSIVE.test(detail); +} + +function sortedRecord(record: Record): Record { + return Object.fromEntries( + Object.entries(record).sort(([a], [b]) => a.localeCompare(b)), + ); +} + +/** + * The tag a harness's lock is read from: `source`, the image built from this commit, which is what + * the next release ships. A run of the published images alone says nothing about this commit. + */ +function lockTag(rows: readonly Result[]): string | undefined { + return rows.some((row) => row.tag === SOURCE) ? SOURCE : undefined; +} + +/** + * The model lock from a run's results, over the lock before it. + * + * For each harness, the `source` image's rows: what passed is verified, what the provider refused + * is refused, in its words. What the run could not decide (an inconclusive failure), a provider + * it had no key for or could not list, and a harness it did not run, keep what the lock said + * before. A model the provider no longer lists drops out. The date moves only when the lock does, + * so a day with nothing new changes nothing. + */ +export function buildLock( + previous: ModelLock | undefined, + runs: readonly Results[], + date: string, +): ModelLock { + const results = runs.flatMap((run) => run.results); + const clients = Object.assign({}, ...runs.map((run) => run.clients ?? {})); + const engines: Record = { + ...(previous?.engines ?? {}), + }; + const harnesses = [...new Set(results.map((row) => row.harness))]; + for (const harness of harnesses) { + const rows = results.filter((row) => row.harness === harness); + const tag = lockTag(rows); + const before = previous?.engines[harness]; + const providers: LockedEngine["providers"] = { + ...(before?.providers ?? {}), + }; + const ran = rows.filter((row) => row.tag === tag && row.model !== "(all)"); + for (const id of new Set(ran.map(providerIdOf))) { + if (!id) continue; + const earlier = before?.providers[id]; + const verified = new Set(); + const refused: Record = {}; + for (const row of ran.filter((row) => providerIdOf(row) === id)) { + if (row.status === "pass") verified.add(row.model); + else if (row.status !== "fail") continue; + else if (!inconclusive(row.detail)) refused[row.model] = row.detail; + else if (earlier?.verified.includes(row.model)) verified.add(row.model); + else if (earlier?.refused[row.model] !== undefined) + refused[row.model] = earlier.refused[row.model] ?? ""; + } + providers[id] = { + verified: [...verified].sort(), + refused: sortedRecord(refused), + }; + } + if (!Object.keys(providers).length) continue; + const found = tag ? clients[harness]?.[tag] : undefined; + engines[harness] = { + clients: sortedRecord( + found && Object.keys(found).length ? found : (before?.clients ?? {}), + ), + providers: sortedRecord(providers), + }; + } + const next = sortedRecord(engines); + const same = + previous !== undefined && + JSON.stringify(previous.engines) === JSON.stringify(next); + return { checkedAt: same ? previous.checkedAt : date, engines: next }; +} + +/** + * What changed between two locks, as a pull request says it: models that newly answer, models + * that stopped, models that dropped out of a provider's listing, and clients that moved. + */ +export function describeLockChange( + before: ModelLock | undefined, + after: ModelLock, +): string { + const lines: string[] = []; + for (const [engine, entry] of Object.entries(after.engines)) { + const earlier = before?.engines[engine]; + const changes: string[] = []; + for (const [provider, now] of Object.entries(entry.providers)) { + if (!now) continue; + const was = earlier?.providers[provider as LockProvider]; + const known = new Set([ + ...(was?.verified ?? []), + ...Object.keys(was?.refused ?? {}), + ]); + const passes = now.verified.filter( + (model) => !was?.verified.includes(model), + ); + const stopped = Object.keys(now.refused).filter((model) => + was?.verified.includes(model), + ); + const newlyRefused = Object.keys(now.refused).filter( + (model) => !known.has(model), + ); + const gone = [...known].filter( + (model) => !now.verified.includes(model) && !(model in now.refused), + ); + if (passes.length) + changes.push( + `${provider}: now answers ${passes.map((m) => `\`${m}\``).join(", ")}`, + ); + if (stopped.length) + changes.push( + `${provider}: **stopped answering** ${stopped.map((m) => `\`${m}\` (${cell(now.refused[m] ?? "")})`).join(", ")}`, + ); + if (newlyRefused.length) + changes.push( + `${provider}: refused ${newlyRefused.map((m) => `\`${m}\` (${cell(now.refused[m] ?? "")})`).join(", ")}`, + ); + if (gone.length) + changes.push( + `${provider}: no longer listed ${gone.map((m) => `\`${m}\``).join(", ")}`, + ); + } + const moved = Object.entries(entry.clients) + .filter(([name, version]) => earlier?.clients[name] !== version) + .map(([name, version]) => + earlier?.clients[name] + ? `${name} ${earlier.clients[name]} to ${version}` + : `${name} ${version}`, + ); + if (moved.length) changes.push(`clients: ${moved.join(", ")}`); + if (changes.length) + lines.push( + `- **${engine}**`, + ...changes.map((change) => ` - ${change}`), + ); + } + return lines.length ? lines.join("\n") : "No model changed."; +} + +/** Every model the lock verifies that now fails: what a client bump must not break. */ +export function brokenVerified( + lock: ModelLock, + results: readonly Result[], +): Result[] { + return results.filter((row) => { + if (row.status !== "fail" || row.model === "(all)") return false; + const id = providerIdOf(row); + return ( + !!id && + (lock.engines[row.harness]?.providers[id]?.verified.includes(row.model) ?? + false) + ); + }); +} + +function readResults(files: readonly string[]): Results[] { + return files.map((file) => JSON.parse(readFileSync(file, "utf8")) as Results); +} + +function lockCommand(argv: string[]): number { + let path = join(ROOT, "shared/model-lock.json"); + let date = new Date().toISOString().slice(0, 10); + let changes: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--lock") path = argv[++i] ?? path; + else if (arg === "--date") date = argv[++i] ?? date; + else if (arg === "--changes") changes = argv[++i]; + else if (arg) files.push(arg); + } + const previous = existsSync(path) + ? (JSON.parse(readFileSync(path, "utf8")) as ModelLock) + : undefined; + const next = buildLock(previous, readResults(files), date); + const text = `${JSON.stringify(next, null, 2)}\n`; + const changed = !existsSync(path) || readFileSync(path, "utf8") !== text; + if (changed) writeFileSync(path, text); + if (changes) + writeFileSync(changes, `${describeLockChange(previous, next)}\n`); + console.log( + changed + ? `The model lock changed: ${path}` + : "The model lock is unchanged.", + ); + return 0; +} + +function guardCommand(argv: string[]): number { + let path = join(ROOT, "shared/model-lock.json"); + let summary: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--lock") path = argv[++i] ?? path; + else if (arg === "--summary") summary = argv[++i]; + else if (arg) files.push(arg); + } + const lock = JSON.parse(readFileSync(path, "utf8")) as ModelLock; + const results = readResults(files).flatMap((run) => run.results); + const broken = brokenVerified(lock, results); + const markdown = [ + "## Model lock", + "", + broken.length + ? `${broken.length} model${broken.length === 1 ? "" : "s"} the lock verifies no longer answer${broken.length === 1 ? "s" : ""} through this change's client. A person set to one would get the provider's refusal on every message. Fix the client, or, if the model really can't run on this harness any more, say so in the pull request: the next daily check takes it out of the lock.` + : "Every model the lock verifies for the harnesses this change touches still answers.", + "", + ...(broken.length ? [renderTable(broken), ""] : []), + renderSummary(results), + ].join("\n"); + if (summary) writeFileSync(summary, `${markdown}\n`); + else console.log(markdown); + return broken.length ? 1 : 0; +} + +function report(argv: string[]): number { + let summary: string | undefined; + let issue: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--summary") summary = argv[++i]; + else if (arg === "--issue") issue = argv[++i]; + else if (arg) files.push(arg); + } + const results = files.flatMap( + (file) => + (JSON.parse(readFileSync(file, "utf8")) as { results: Result[] }).results, + ); + const markdown = renderSummary(results); + if (summary) writeFileSync(summary, `${markdown}\n`); + else console.log(markdown); + if (issue) + writeFileSync( + issue, + `${renderSummary(results, { passed: false, maxFailed: 150 })}\n`, + ); + if (!results.length) { + console.error( + "No results were found: every harness job failed before it wrote any.", + ); + return 1; + } + return results.some((row) => row.status === "fail") ? 1 : 0; +} + +if (import.meta.main) { + const argv = process.argv.slice(2); + if (argv[0] === "harnesses") { + // The workflows' matrix: one job per harness this check runs, as compact JSON. + console.log( + JSON.stringify(readProbed().map((harness) => harness.image ?? "")), + ); + process.exit(0); + } + const code = + argv[0] === "report" + ? report(argv.slice(1)) + : argv[0] === "lock" + ? lockCommand(argv.slice(1)) + : argv[0] === "guard" + ? guardCommand(argv.slice(1)) + : await check(argv); + process.exit(code); +} diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts new file mode 100644 index 0000000000..705048f544 --- /dev/null +++ b/scripts/model-lock.test.ts @@ -0,0 +1,232 @@ +import { expect, test } from "bun:test"; +import type { ModelLock } from "../shared/model-lock"; +import { + brokenVerified, + buildLock, + describeLockChange, + providerIdOf, + type Result, + type Results, +} from "./model-compat"; + +/* + * Writing the model lock from the model check's results, and holding a client bump to it. The + * rows are what the check reported on 2026-11-17, in its own words. + */ + +const OPUS_REFUSED = + "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again."; + +const row = ( + model: string, + status: Result["status"], + detail = "OK", + extra: Partial = {}, +): Result => ({ + harness: "agent-claude-sdk", + tag: "source", + provider: "Anthropic (key or Claude plan)", + providerId: "anthropic", + model, + status, + detail, + ...extra, +}); + +const run: Results = { + results: [ + row("claude-sonnet-5-5", "pass"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "fail", OPUS_REFUSED), + // `latest` is what people run today; the lock is what the next release ships. + row("claude-sonnet-5-5", "fail", OPUS_REFUSED, { tag: "latest" }), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.152", "claude-code": "2.1.259" }, + latest: { "claude-agent-sdk": "0.2.150" }, + }, + }, +}; + +test("what answered on the source image is verified, what was refused is kept in the provider's words", () => { + const lock = buildLock(undefined, [run], "2026-11-17"); + expect(lock).toEqual({ + checkedAt: "2026-11-17", + engines: { + "agent-claude-sdk": { + clients: { "claude-agent-sdk": "0.2.152", "claude-code": "2.1.259" }, + providers: { + anthropic: { + verified: ["claude-haiku-5-5", "claude-sonnet-5-5"], + refused: { "claude-opus-5-5": OPUS_REFUSED }, + }, + }, + }, + }, + }); +}); + +test("a bad day for the runner takes nothing away", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const flaky: Results = { + results: [ + row( + "claude-sonnet-5-5", + "fail", + "no answer: TimeoutError: The operation timed out.", + ), + row( + "claude-haiku-5-5", + "fail", + "the harness never became healthy: exit 1", + ), + row("claude-opus-5-5", "fail", "no answer: TimeoutError"), + ], + }; + const after = buildLock(before, [flaky], "2026-11-18"); + expect(after).toEqual(before); + // Nothing changed, so neither did the date, and the daily check opens no pull request. + expect(after.checkedAt).toBe("2026-11-17"); +}); + +test("a model that starts answering after a client bump is verified, and the clients move with it", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const bumped: Results = { + results: [ + row("claude-sonnet-5-5", "pass"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "pass"), + row("claude-fable-5-2", "pass"), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + }, + }, + }; + const after = buildLock(before, [bumped], "2026-11-18"); + expect(after.checkedAt).toBe("2026-11-18"); + expect(after.engines["agent-claude-sdk"]).toEqual({ + clients: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + providers: { + anthropic: { + verified: [ + "claude-fable-5-2", + "claude-haiku-5-5", + "claude-opus-5-5", + "claude-sonnet-5-5", + ], + refused: {}, + }, + }, + }); +}); + +test("a provider with no key and a harness that did not run keep what was known", () => { + const before: ModelLock = { + checkedAt: "2026-11-10", + engines: { + "agent-agno": { + clients: { agno: "2.0.0" }, + providers: { + xai: { verified: ["grok-4.7"], refused: {} }, + openai: { verified: ["gpt-6.1-terra"], refused: {} }, + }, + }, + "agent-adk": { + clients: { "google-adk": "1.0.0" }, + providers: { openai: { verified: ["gpt-5.5"], refused: {} } }, + }, + }, + }; + const today: Results = { + results: [ + { + harness: "agent-agno", + tag: "latest, source", + provider: "xAI", + providerId: "xai", + model: "(all)", + status: "skipped", + detail: "skipped: no key (XAI_API_KEY)", + }, + { + harness: "agent-agno", + tag: "source", + provider: "OpenAI", + providerId: "openai", + model: "gpt-6.1-terra", + status: "pass", + detail: "OK", + }, + ], + }; + const after = buildLock(before, [today], "2026-11-17"); + expect(after.engines["agent-agno"]?.providers.xai).toEqual({ + verified: ["grok-4.7"], + refused: {}, + }); + expect(after.engines["agent-adk"]).toEqual(before.engines["agent-adk"]!); + // No clients in the results: the ones recorded before stand. + expect(after.engines["agent-agno"]?.clients).toEqual({ agno: "2.0.0" }); +}); + +test("the refresh pull request says which models started and stopped answering", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const after = buildLock( + before, + [ + { + results: [ + row( + "claude-sonnet-5-5", + "fail", + "API Error: 400 invalid_request_error", + ), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "pass"), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + }, + }, + }, + ], + "2026-11-18", + ); + const said = describeLockChange(before, after); + expect(said).toContain("now answers `claude-opus-5-5`"); + expect(said).toContain("**stopped answering** `claude-sonnet-5-5`"); + expect(said).toContain("claude-code 2.1.259 to 2.1.291"); + expect(describeLockChange(before, before)).toBe("No model changed."); +}); + +test("a run of the published images alone does not move the lock", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const latestOnly: Results = { + results: [row("claude-opus-5-5", "pass", "OK", { tag: "latest" })], + }; + expect(buildLock(before, [latestOnly], "2026-11-18")).toEqual(before); +}); + +test("results written before rows named their provider are read by its label", () => { + expect(providerIdOf({ provider: "Anthropic (key or Claude plan)" })).toBe( + "anthropic", + ); + expect(providerIdOf({ provider: "ChatGPT plan" })).toBe("chatgpt"); + expect(providerIdOf({ provider: "Google" })).toBe("google"); +}); + +test("a client bump that breaks a verified model is caught, and one the lock already refuses is not", () => { + const lock = buildLock(undefined, [run], "2026-11-17"); + const afterBump: Result[] = [ + row("claude-sonnet-5-5", "fail", "API Error: 400 invalid_request_error"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "fail", OPUS_REFUSED), + ]; + expect(brokenVerified(lock, afterBump).map((broken) => broken.model)).toEqual( + ["claude-sonnet-5-5"], + ); +}); diff --git a/shared/fixtures/model-listings/anthropic-models.json b/shared/fixtures/model-listings/anthropic-models.json new file mode 100644 index 0000000000..478a3cc111 --- /dev/null +++ b/shared/fixtures/model-listings/anthropic-models.json @@ -0,0 +1,1327 @@ +{ + "data": [ + { + "type": "model", + "id": "claude-fable-5-2", + "display_name": "Claude Fable 5.2", + "created_at": "2026-11-09T18:30:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-haiku-5-5", + "display_name": "Claude Haiku 5.5", + "created_at": "2026-10-07T18:00:00Z", + "line": "haiku", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-5-5", + "display_name": "Claude Sonnet 5.5", + "created_at": "2026-09-28T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": true + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-5-5", + "display_name": "Claude Opus 5.5", + "created_at": "2026-09-21T16:24:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-fable-5-1", + "display_name": "Claude Fable 5.1", + "created_at": "2026-08-28T00:00:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-5", + "display_name": "Claude Opus 5", + "created_at": "2026-07-24T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-5", + "display_name": "Claude Sonnet 5", + "created_at": "2026-06-29T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-fable-5", + "display_name": "Claude Fable 5", + "created_at": "2026-06-07T00:00:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-8", + "display_name": "Claude Opus 4.8", + "created_at": "2026-05-28T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-7", + "display_name": "Claude Opus 4.7", + "created_at": "2026-04-14T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-4-6", + "display_name": "Claude Sonnet 4.6", + "created_at": "2026-02-17T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-6", + "display_name": "Claude Opus 4.6", + "created_at": "2026-02-04T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-5-20251101", + "display_name": "Claude Opus 4.5", + "created_at": "2025-11-24T00:00:00Z", + "line": "opus", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-haiku-4-5-20251001", + "display_name": "Claude Haiku 4.5", + "created_at": "2025-10-15T00:00:00Z", + "line": "haiku", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": false + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": false, + "low": { + "supported": false + }, + "medium": { + "supported": false + }, + "high": { + "supported": false + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-4-5-20250929", + "display_name": "Claude Sonnet 4.5", + "created_at": "2025-09-29T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": false, + "low": { + "supported": false + }, + "medium": { + "supported": false + }, + "high": { + "supported": false + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "deprecated", + "deprecated_at": "2026-09-30T18:30:00Z", + "retires_at": "2026-11-30T17:00:00Z" + } + ], + "has_more": false, + "first_id": "claude-fable-5-2", + "last_id": "claude-sonnet-4-5-20250929" +} diff --git a/shared/fixtures/model-listings/openai-models.json b/shared/fixtures/model-listings/openai-models.json new file mode 100644 index 0000000000..f4c33cd814 --- /dev/null +++ b/shared/fixtures/model-listings/openai-models.json @@ -0,0 +1,705 @@ +{ + "object": "list", + "data": [ + { + "id": "gpt-5-nano-2025-08-07", + "object": "model", + "created": 1754426303, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o-mini-tts-2025-03-20", + "object": "model", + "created": 1765610731, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-audio-mini", + "object": "model", + "created": 1759512027, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-transcribe", + "object": "model", + "created": 1742068463, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5.2", + "object": "model", + "created": 1765313051, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-mini-2024-07-18", + "object": "model", + "created": 1721172717, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4.1-2025-04-14", + "object": "model", + "created": 1744315746, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4.1-mini-2025-04-14", + "object": "model", + "created": 1744317547, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-5.4-nano-2026-03-17", + "object": "model", + "created": 1773450837, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-image-2.5-flare", + "object": "model", + "created": 1788563147, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.1-sol", + "object": "model", + "created": 1790552874, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5", + "object": "model", + "created": 1754425777, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o", + "object": "model", + "created": 1715367049, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4o-mini-transcribe", + "object": "model", + "created": 1742068596, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-image-2-2026-04-21", + "object": "model", + "created": 1776399994, + "owned_by": "system", + "shutdown_date": "2027-02-10" + }, + { + "id": "gpt-4o-mini-tts-2025-12-15", + "object": "model", + "created": 1765610837, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-audio", + "object": "model", + "created": 1756339249, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.4-nano", + "object": "model", + "created": 1773450870, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5.3-codex", + "object": "model", + "created": 1770537915, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5-search-api-2025-10-14", + "object": "model", + "created": 1760043960, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-mini-2025-12-15", + "object": "model", + "created": 1765760008, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-transcribe", + "object": "model", + "created": 1785168027, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "tts-1", + "object": "model", + "created": 1681940951, + "owned_by": "openai-internal", + "shutdown_date": "2027-01-06" + }, + { + "id": "omni-moderation-2024-09-26", + "object": "model", + "created": 1732734466, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2.1-mini", + "object": "model", + "created": 1782254706, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2025-08-28", + "object": "model", + "created": 1756271773, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-live-transcribe", + "object": "model", + "created": 1785168034, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-mini", + "object": "model", + "created": 1754425928, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5.4-2026-03-05", + "object": "model", + "created": 1772654062, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-mini-2025-08-07", + "object": "model", + "created": 1754425867, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5-nano", + "object": "model", + "created": 1754426384, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5.6-luna", + "object": "model", + "created": 1782228658, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.5-sunburst-2026-09-08", + "object": "model", + "created": 1788851010, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.6-flare", + "object": "model", + "created": 1793222647, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-2025-08-07", + "object": "model", + "created": 1754075360, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o-mini-transcribe-2025-12-15", + "object": "model", + "created": 1765610407, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "o3-pro-2025-06-10", + "object": "model", + "created": 1749166761, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "chat-latest", + "object": "model", + "created": 1777704602, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-pro", + "object": "model", + "created": 1776894349, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-translate", + "object": "model", + "created": 1777950216, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-2025-08-28", + "object": "model", + "created": 1756256146, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-mini-transcribe-2025-03-20", + "object": "model", + "created": 1765610545, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.4-pro-2026-03-05", + "object": "model", + "created": 1772659657, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-pro-2025-10-06", + "object": "model", + "created": 1759469707, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "tts-1-1106", + "object": "model", + "created": 1699053241, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-pro-2026-04-23", + "object": "model", + "created": 1776894470, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.2-pro-2025-12-11", + "object": "model", + "created": 1765343959, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.6-sol", + "object": "model", + "created": 1782228018, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.2-astra", + "object": "model", + "created": 1793430042, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3-2025-04-16", + "object": "model", + "created": 1744133301, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4.1-mini", + "object": "model", + "created": 1744318173, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-5.2-2025-12-11", + "object": "model", + "created": 1765313028, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "chatgpt-image-latest", + "object": "model", + "created": 1765925279, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-4o-mini", + "object": "model", + "created": 1721172741, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-image-2.5-sunburst", + "object": "model", + "created": 1788563162, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.5-flare-2026-09-08", + "object": "model", + "created": 1788851006, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-luna", + "object": "model", + "created": 1789406102, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "tts-1-hd-1106", + "object": "model", + "created": 1699053533, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-mini", + "object": "model", + "created": 1759517133, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-transcribe-diarize", + "object": "model", + "created": 1750798887, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5.4", + "object": "model", + "created": 1772691852, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-1.5", + "object": "model", + "created": 1771550885, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-1-mini", + "object": "model", + "created": 1758845821, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-image-1.5", + "object": "model", + "created": 1764030620, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-5.1-2025-11-13", + "object": "model", + "created": 1762800353, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "text-embedding-3-small", + "object": "model", + "created": 1705948997, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.6-terra", + "object": "model", + "created": 1782228459, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-astra", + "object": "model", + "created": 1787853604, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.1-terra", + "object": "model", + "created": 1791908780, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.2-sol", + "object": "model", + "created": 1793642820, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "omni-moderation-latest", + "object": "model", + "created": 1731689265, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.4-mini", + "object": "model", + "created": 1773451123, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-2024-11-20", + "object": "model", + "created": 1739331543, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "tts-1-hd", + "object": "model", + "created": 1699046015, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-realtime-1.5", + "object": "model", + "created": 1771461469, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2", + "object": "model", + "created": 1778006032, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime", + "object": "model", + "created": 1756271701, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.1", + "object": "model", + "created": 1762800673, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5-search-api", + "object": "model", + "created": 1759514629, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-2026-04-23", + "object": "model", + "created": 1776839241, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "whisper-1", + "object": "model", + "created": 1677532384, + "owned_by": "openai-internal", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5-pro", + "object": "model", + "created": 1759469822, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-realtime-whisper", + "object": "model", + "created": 1778012060, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.2-pro", + "object": "model", + "created": 1765343983, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-live-1", + "object": "model", + "created": 1788889407, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.6-sunburst", + "object": "model", + "created": 1793222670, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.4-mini-2026-03-17", + "object": "model", + "created": 1773451076, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3-pro", + "object": "model", + "created": 1748475349, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-image-2", + "object": "model", + "created": 1776399795, + "owned_by": "system", + "shutdown_date": "2027-02-10" + }, + { + "id": "gpt-5.4-pro", + "object": "model", + "created": 1772659601, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-mini-tts", + "object": "model", + "created": 1742403959, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-realtime-mini-2025-12-15", + "object": "model", + "created": 1765612007, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "text-embedding-ada-002", + "object": "model", + "created": 1671217299, + "owned_by": "openai-internal", + "shutdown_date": null + }, + { + "id": "gpt-4o-2024-08-06", + "object": "model", + "created": 1722814719, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-realtime-2.1", + "object": "model", + "created": 1782254687, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3", + "object": "model", + "created": 1744225308, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4.1", + "object": "model", + "created": 1744316542, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "text-embedding-3-large", + "object": "model", + "created": 1705953180, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5", + "object": "model", + "created": 1776824847, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-sol", + "object": "model", + "created": 1789405937, + "owned_by": "system", + "shutdown_date": null + } + ] +} diff --git a/shared/model-lock.json b/shared/model-lock.json new file mode 100644 index 0000000000..6370421824 --- /dev/null +++ b/shared/model-lock.json @@ -0,0 +1,22 @@ +{ + "checkedAt": "2026-11-17", + "engines": { + "agent-langgraph-ts": { + "clients": {}, + "providers": { + "anthropic": { + "verified": [], + "refused": {} + }, + "google": { + "verified": [], + "refused": {} + }, + "openai": { + "verified": [], + "refused": {} + } + } + } + } +} diff --git a/shared/model-lock.test.ts b/shared/model-lock.test.ts new file mode 100644 index 0000000000..c695ac3993 --- /dev/null +++ b/shared/model-lock.test.ts @@ -0,0 +1,98 @@ +import { expect, test } from "bun:test"; +import anthropicListing from "./fixtures/model-listings/anthropic-models.json"; +import openaiListing from "./fixtures/model-listings/openai-models.json"; +import { lockAllows, lockVerdict, MODEL_LOCK } from "./model-lock"; + +/* + * The lock's rule against real data: the model listings OpenAI and Anthropic answered on + * 2026-11-17 (shared/fixtures/model-listings), and the lock the model check wrote that day. The + * Claude plan's failure that day was the Claude Agent SDK harness offering Opus 5.5 and Fable + * 5.2, which its pinned Claude Code was refused on every message. + */ + +const anthropicIds = anthropicListing.data.map((model) => model.id); +const openaiIds = openaiListing.data.map((model) => model.id); + +const offered = ( + engine: string | undefined, + provider: "anthropic" | "openai", + ids: string[], +) => ids.filter((id) => lockAllows(MODEL_LOCK, engine, provider, id)); + +test("the Claude Agent SDK harness is not offered the models its Claude Code is refused", () => { + const claude = offered("agent-claude-sdk", "anthropic", anthropicIds); + expect(anthropicIds).toContain("claude-opus-5-5"); + expect(anthropicIds).toContain("claude-fable-5-2"); + expect(claude).not.toContain("claude-opus-5-5"); + expect(claude).not.toContain("claude-fable-5-2"); + expect(claude).toContain("claude-sonnet-5-5"); + expect(claude).toContain("claude-haiku-5-5"); + // Everything else Anthropic listed that day answered. + expect(claude).toHaveLength(anthropicIds.length - 2); +}); + +test("a refused model carries the provider's own words", () => { + const verdict = lockVerdict( + MODEL_LOCK, + "agent-claude-sdk", + "anthropic", + "claude-opus-5-5", + ); + expect(verdict.status).toBe("refused"); + expect(verdict.status === "refused" && verdict.said).toMatch( + /does not support this model; version [\d.]+ or newer is required/, + ); +}); + +test("a harness whose client takes every model is offered every model", () => { + expect(offered("agent-langgraph-agui", "anthropic", anthropicIds)).toEqual( + anthropicIds, + ); +}); + +test("the Responses-only OpenAI models are not offered on a Chat Completions harness", () => { + const ag2 = offered("agent-ag2", "openai", openaiIds); + for (const model of ["gpt-5-pro", "o3-pro"]) + if (openaiIds.includes(model)) expect(ag2).not.toContain(model); + expect(ag2.length).toBeGreaterThan(0); +}); + +test("a model released after the last check waits for it", () => { + expect( + lockVerdict(MODEL_LOCK, "agent-claude-sdk", "anthropic", "claude-opus-6"), + ).toEqual({ status: "unchecked" }); + expect( + lockAllows(MODEL_LOCK, "agent-claude-sdk", "anthropic", "claude-opus-6"), + ).toBe(false); +}); + +test("what the check never ran is open, so nobody loses their models to missing data", () => { + // OpenBot's own runtime is no harness image. + expect(offered(undefined, "anthropic", anthropicIds)).toEqual(anthropicIds); + // No key for xAI in CI. + expect(lockVerdict(MODEL_LOCK, "agent-agno", "xai", "grok-4.7")).toEqual({ + status: "open", + }); + expect( + lockAllows(MODEL_LOCK, "agent-unknown", "openai", "gpt-6.1-terra"), + ).toBe(true); +}); + +test("every engine the lock names is a harness OpenBot publishes, with the clients it ran", async () => { + const { harnesses } = (await import("./harnesses.json")).default as { + harnesses: { image: string | null }[]; + }; + const images = harnesses.map((harness) => harness.image); + for (const [engine, entry] of Object.entries(MODEL_LOCK.engines)) { + expect(images, engine).toContain(engine); + // An engine locked before its first run (agent-langgraph-ts) has run no client yet, and + // offers nothing until it has. + const ran = Object.values(entry.providers).some( + (pair) => + !!pair && + (pair.verified.length > 0 || Object.keys(pair.refused).length > 0), + ); + if (ran) + expect(Object.keys(entry.clients).length, engine).toBeGreaterThan(0); + } +}); diff --git a/shared/model-lock.ts b/shared/model-lock.ts new file mode 100644 index 0000000000..b4f1d8e8ce --- /dev/null +++ b/shared/model-lock.ts @@ -0,0 +1,94 @@ +import manifest from "./model-lock.json"; + +/** + * THE MODEL LOCK: which models each harness has been proven to run, with the client versions it + * was proven on. + * + * WHY. The model picker lists each provider's models live, but every harness image carries its + * model client at a fixed version, and a provider can refuse a new model to an older client + * ("Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required"). A + * Bot set to such a model fails on every message. So a live model is offered only when the daily + * model check (scripts/model-compat.ts) has sent it one real request through that harness's + * own client and it answered. + * + * WHERE THE DATA COMES FROM. `model-lock.json` is written by `bun scripts/model-compat.ts lock` + * from the check's results, never by hand. The daily check opens a pull request when a model newly + * passes or starts failing, and a pull request that changes a harness's client checks that + * harness before it merges. See docs/releasing.md, "Model lock". + * + * THE RULE. A harness and provider pair the check has run is locked: only the models that passed + * are offered. A pair it has never run (no key for that provider in CI, or a harness it does not + * start) is open: nothing is known, so nothing is taken away. A model the provider lists that the + * check has not reached yet (released since the last run) is not offered until it passes. + */ + +/** Who serves the models, as the lock names them. `chatgpt` is a ChatGPT plan's Codex backend. */ +export type LockProvider = + | "openai" + | "anthropic" + | "google" + | "xai" + | "chatgpt"; + +export type LockedProvider = { + /** The model ids that answered through this harness's client. */ + verified: string[]; + /** The model ids the provider refused through this client, with what it said. */ + refused: Record; +}; + +export type LockedEngine = { + /** The model-client packages in the image the check ran, with their versions. */ + clients: Record; + providers: Partial>; +}; + +export type ModelLock = { + /** The day of the run the lock was written from, `YYYY-MM-DD`. */ + checkedAt: string; + /** By harness image directory (`agent-claude-sdk`), as `shared/harnesses.json` names it. */ + engines: Record; +}; + +export const MODEL_LOCK: ModelLock = manifest as ModelLock; + +/** + * What the lock says about one model on one harness and provider. + * + * - `verified`: it answered through the harness's own client. + * - `refused`: the provider turned it down through that client; `said` is the provider's words. + * - `unchecked`: the pair is locked, and this model was not in the run (released since). + * - `open`: the pair was never checked, or there is no harness to check (OpenBot's own runtime). + */ +export type LockVerdict = + | { status: "verified" } + | { status: "refused"; said: string } + | { status: "unchecked" } + | { status: "open" }; + +export function lockVerdict( + lock: ModelLock, + engine: string | undefined, + provider: LockProvider | undefined, + model: string, +): LockVerdict { + const pair = + engine && provider ? lock.engines[engine]?.providers[provider] : undefined; + if (!pair) return { status: "open" }; + if (pair.verified.includes(model)) return { status: "verified" }; + const said = pair.refused[model]; + return said === undefined + ? { status: "unchecked" } + : { status: "refused", said }; +} + +/** Whether a model may be offered: proven on the pair, or the pair is not locked. */ +export function lockAllows( + lock: ModelLock, + engine: string | undefined, + provider: LockProvider | undefined, + model: string, +): boolean { + const { status } = lockVerdict(lock, engine, provider, model); + return status === "verified" || status === "open"; +}