diff --git a/.github/workflows/model-compat.yml b/.github/workflows/model-compat.yml new file mode 100644 index 0000000000..bc650d18f2 --- /dev/null +++ b/.github/workflows/model-compat.yml @@ -0,0 +1,160 @@ +name: Model compatibility + +# Does every model the picker offers still answer through the clients OpenBot ships? The picker +# lists each provider's models live, but every harness image carries its model client at a fixed +# version (the Claude Agent SDK harness bundles a Claude Code CLI, the LangGraph harness sends the +# ChatGPT plan's backend a fixed Codex client version, the others carry the provider SDKs they were +# built with), and a provider can refuse a new model to an older client. scripts/model-compat.ts +# starts each harness image, one job per harness, and sends it one request per listed model on +# every provider that has a key here. See docs/releasing.md, "Model compatibility". +# +# Keys are repository secrets, one per provider. A provider without one is reported as skipped, +# not failed. On any failure the run fails and opens (or updates) one issue titled "Model +# compatibility check failing"; a passing run closes it. +# +# Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever +# runs with these keys. +on: + schedule: + - cron: "41 6 * * *" + workflow_dispatch: + inputs: + tags: + description: Harness images to check, comma-separated. `latest` is the published release people run; `source` is built from this commit, which is what the next release ships. + type: string + default: latest,source + required: false + +permissions: {} + +concurrency: + group: model-compat + cancel-in-progress: false + +jobs: + harnesses: + name: List the published harnesses + if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + images: ${{ steps.list.outputs.images }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - id: list + run: echo "images=$(jq -c . .github/published-images.json)" >> "$GITHUB_OUTPUT" + + check: + name: ${{ matrix.harness }} + needs: harnesses + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + harness: ${{ fromJSON(needs.harnesses.outputs.images) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + # Writes its results and exits 0 whatever they say: the report job below decides. + - name: One request per model through this harness + env: + HARNESS: ${{ matrix.harness }} + TAGS: ${{ inputs.tags || 'latest,source' }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + run: >- + bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero + --out "$RUNNER_TEMP/model-compat-$HARNESS.json" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: model-compat-${{ matrix.harness }} + path: ${{ runner.temp }}/model-compat-${{ matrix.harness }}.json + if-no-files-found: error + retention-days: 14 + + report: + name: Report + needs: [harnesses, check] + if: always() && needs.harnesses.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write # open, update or close the one "Model compatibility check failing" issue + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TITLE: Model compatibility check failing + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: model-compat-* + path: ${{ runner.temp }}/results + merge-multiple: true + - name: Summarize + id: summary + run: | + set +e + shopt -s nullglob + bun scripts/model-compat.ts report --summary "$RUNNER_TEMP/summary.md" \ + --issue "$RUNNER_TEMP/issue-table.md" "$RUNNER_TEMP"/results/*.json + code=$? + echo "code=$code" >> "$GITHUB_OUTPUT" + if [ -f "$RUNNER_TEMP/summary.md" ]; then cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY"; fi + exit 0 + - name: Open or update the issue + if: steps.summary.outputs.code != '0' + run: | + set -euo pipefail + body="$RUNNER_TEMP/issue.md" + { + echo "The model compatibility check failed on $(date -u +%Y-%m-%d): $RUN_URL" + echo + echo "A model refused by a pinned client means that client needs bumping: see docs/releasing.md, \"Model compatibility\". This issue is updated by every failing run and closed by the first passing one." + echo + if [ -f "$RUNNER_TEMP/issue-table.md" ]; then + cat "$RUNNER_TEMP/issue-table.md" + else + echo "No harness job wrote results. See the run." + fi + } > "$body" + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$body" + fi + - name: Close the issue when everything passes + if: steps.summary.outputs.code == '0' + run: | + set -euo pipefail + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Passing again: $RUN_URL" + fi + - name: Fail when a model failed + if: steps.summary.outputs.code != '0' + run: | + echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"." + exit 1 diff --git a/api/client-boundary.allow.ts b/api/client-boundary.allow.ts index dec6e96ca6..dc3a6b9a03 100644 --- a/api/client-boundary.allow.ts +++ b/api/client-boundary.allow.ts @@ -392,6 +392,12 @@ export const ALLOWED: Allowed[] = [ rule: "third-party", why: "image registries, never OpenBot's server", }, + { + file: "scripts/model-compat.ts", + what: /^(fetch\(\)|\/health|\/api\/agents\/openbot\/generate)$/, + rule: "third-party", + why: "model providers' listings and the harness images the model check starts on this machine, never OpenBot's server", + }, // Scripts that run and stop a server on this machine. { file: "scripts/update-local.ts", diff --git a/docs/releasing.md b/docs/releasing.md index bd92907919..3adf687224 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -993,6 +993,61 @@ it when it is not. Signed desktop builds are the exception: they read the `macos `windows-signing` environments described above and in [Windows signing](windows-signing.md). Downloads sync signs in to AWS with GitHub's OIDC token rather than a stored key. +## Model compatibility + +The model picker lists each provider's models live, but every harness image carries its model +client at a fixed version: the Claude Agent SDK harness bundles a Claude Code CLI (through +`claude-agent-sdk` in `agent-claude-sdk/requirements.txt`), the LangGraph harness sends the ChatGPT +plan's backend a fixed Codex client version (`DEFAULT_CLIENT_VERSION` in +`agent-langgraph-agui/src/chatgpt_models.py`), and the others carry the provider SDKs they were built +with. A provider can refuse a new model to an older client while the picker keeps offering it: +"Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required". + +**Model compatibility** (`.github/workflows/model-compat.yml`) checks for that daily, and by hand +from the Actions tab. One job per harness in `.github/published-images.json` runs +[`scripts/model-compat.ts`](../scripts/model-compat.ts), which: + +- lists each provider's models from the same source, with the same filter, as the picker (see + [The model list](configuration.md#the-model-list)); +- starts the harness image on each provider the way the Desktop configures it, and sends one AG-UI + run per model, so the harness's own client makes the call. It checks `latest`, the published + image people run, and `source`, the image built from this commit; +- compares the LangGraph harness's Codex client version with the minimum each ChatGPT plan model + needs, from openai/codex's published model catalogue. A plan sign-in cannot run in CI. A Claude + plan runs through the same Claude Code CLI as an Anthropic key, so the key's rows cover it. + +It writes a pass, fail or skipped row per harness, image, provider and model, with the provider's +own words, to the run's summary. On any failure the run fails and opens or updates one issue, +**Model compatibility check failing**, and the first passing run closes it. It runs only on a +schedule or by hand on `main`, never on a pull request. + +Each provider needs its key as a repository secret: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, +`GEMINI_API_KEY` (Google, also used for Antigravity) and `XAI_API_KEY`. A provider without one is +reported as skipped, not failed, and the summary names the missing secrets. Unbiased AI is not +checked: it is a gateway serving its own model, not a provider's listing. + +**When it fails**, read the provider's words in the table: + +- A version refusal (a model "not supported" by the client's version, or a ChatGPT plan model that + needs a newer Codex) means bumping the pinned client: `claude-agent-sdk` in + `agent-claude-sdk/requirements.txt` (each release bundles a newer Claude Code CLI), or + `DEFAULT_CLIENT_VERSION` in `agent-langgraph-agui/src/chatgpt_models.py`, or the harness's + provider SDK in its `requirements.txt` or `package.json`. Run the check by hand: the `source` rows + show whether the bump fixed it before a release ships it, and the `latest` rows stay red until + one does. +- A refused parameter or endpoint (a model that takes tools only through `/v1/responses`, or + refuses a `temperature`) means that harness builds its request in a way the model no longer + accepts: fix how the harness sets up its model, or bump the framework if a newer one does. +- Billing, rate limits or a refused key are the key's problem, not a client's: fix the key. + +To run it locally (needs Docker), with keys in the environment or an env file: + +```sh +bun scripts/model-compat.ts --env-file .env --harness agent-claude-sdk --provider anthropic +``` + +`--model ` narrows the models, `--tags latest` skips building from source. + ## The one thing CI cannot do The smoke journey in `tests/smoke` is the only check that proves the parts are wired to each other: diff --git a/scripts/model-compat.test.ts b/scripts/model-compat.test.ts new file mode 100644 index 0000000000..c4ce089879 --- /dev/null +++ b/scripts/model-compat.test.ts @@ -0,0 +1,392 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + buildMatrix, + classifyMastra, + classifyRun, + codexGate, + type Harness, + type MatrixRow, + NOT_CHECKED, + PROVIDERS, + parseCapabilities, + providersFor, + renderTable, + usableModels, +} from "./model-compat"; + +const ROOT = join(import.meta.dir, ".."); +const NOW = Date.parse("2026-11-17T12:00:00Z"); + +// The listings below are cut from what each provider answered on 2026-11-17. +const openaiListing = { + data: [ + { id: "gpt-6.1-terra", created: 1792000000 }, + { id: "gpt-6.2-sol", created: 1793000000 }, + { id: "gpt-5.5", created: 1776900000 }, + { id: "gpt-5.5-2026-04-23", created: 1776900000 }, + { id: "gpt-5.3-codex", created: 1770000000 }, + { id: "gpt-5-search-api", created: 1760000000 }, + { id: "chat-latest", created: 1760000000 }, + { id: "gpt-4o-mini-tts", created: 1740000000 }, + { id: "gpt-realtime-2", created: 1780000000 }, + { id: "gpt-image-2", created: 1776000000 }, + { id: "text-embedding-3-large", created: 1700000000 }, + { id: "whisper-1", created: 1680000000 }, + { id: "gpt-live-1", created: 1790000000 }, + { id: "gpt-4-turbo", created: 1700000000 }, + { id: "o1", created: 1720000000 }, + { id: "o3", created: 1744000000, shutdown_date: "2026-11-01" }, + { id: "o3-pro", created: 1749000000, shutdown_date: "2026-12-11" }, + ], +}; + +const anthropicListing = { + data: [ + { + id: "claude-fable-5-2", + created_at: "2026-11-09T18:30:00Z", + lifecycle: "active", + }, + { + id: "claude-opus-5-5", + created_at: "2026-09-21T16:24:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-5-5", + created_at: "2026-09-28T00:00:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-4-5-20250929", + created_at: "2025-09-29T00:00:00Z", + retires_at: "2026-11-30T17:00:00Z", + lifecycle: "deprecated", + }, + { + id: "claude-3-opus-20240229", + created_at: "2024-02-29T00:00:00Z", + retires_at: "2026-01-05T00:00:00Z", + lifecycle: "retired", + }, + ], +}; + +const geminiListing = { + object: "list", + data: [ + { id: "models/gemini-3.8-flash", object: "model", owned_by: "google" }, + { id: "models/gemini-4-argon", object: "model", owned_by: "google" }, + { id: "models/gemini-embedding-001", object: "model", owned_by: "google" }, + { + id: "models/imagen-4.0-generate-001", + object: "model", + owned_by: "google", + }, + { id: "models/veo-3.0-generate-001", object: "model", owned_by: "google" }, + { id: "models/gemini-3.8-flash-live", object: "model", owned_by: "google" }, + { + id: "models/gemini-robotics-er-1.5", + object: "model", + owned_by: "google", + }, + { id: "models/aqa", object: "model", owned_by: "google" }, + { + id: "models/gemini-2.5-flash-preview-tts", + object: "model", + owned_by: "google", + }, + ], +}; + +test("an OpenAI key is offered only the chat models a Bot can run, newest first", () => { + expect(usableModels("openai", openaiListing, new Map(), NOW)).toEqual([ + "gpt-6.2-sol", + "gpt-6.1-terra", + "gpt-5.5", + "o3-pro", + ]); +}); + +test("models.dev saying a model takes no tools drops it, as the picker does", () => { + const capabilities = parseCapabilities({ + openai: { models: { "gpt-6.1-terra": { tool_call: false } } }, + }); + expect( + usableModels("openai", openaiListing, capabilities, NOW), + ).not.toContain("gpt-6.1-terra"); +}); + +test("an Anthropic key is offered every current Claude, retired ones dropped", () => { + expect(usableModels("anthropic", anthropicListing, new Map(), NOW)).toEqual([ + "claude-fable-5-2", + "claude-sonnet-5-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + ]); +}); + +test("a Google key keeps the Gemini chat models and drops media, live, robots and aqa", () => { + expect(usableModels("google", geminiListing, new Map(), NOW).sort()).toEqual([ + "gemini-3.8-flash", + "gemini-4-argon", + ]); +}); + +const harnesses: Harness[] = [ + { + image: "agent-claude-sdk", + credential: "anthropic", + port: 4212, + runPath: "", + }, + { image: "agent-antigravity", credential: "google", port: 4214, runPath: "" }, + { + image: "agent-agno", + credential: "any-provider", + port: 4203, + runPath: "/agui", + }, +]; + +test("each harness is paired with the providers it can run on", () => { + expect(providersFor(harnesses[0]!)).toEqual(["anthropic"]); + expect(providersFor(harnesses[1]!)).toEqual(["google"]); + expect(providersFor(harnesses[2]!)).toEqual( + PROVIDERS.map((provider) => provider.id), + ); +}); + +test("the matrix has one row per harness, image tag and model, and skips a provider with no key", () => { + const rows = buildMatrix({ + harnesses, + tags: ["latest", "main"], + listings: new Map([ + ["anthropic", { models: ["claude-opus-5-5", "claude-sonnet-5-5"] }], + ["openai", { error: "401 Incorrect API key provided" }], + ]), + }); + const runs = rows.filter( + (row): row is Extract => row.kind === "run", + ); + // claude-sdk and agno on Anthropic: 2 harnesses x 2 tags x 2 models. + expect(runs).toHaveLength(8); + expect( + runs + .filter((row) => row.harness.image === "agent-claude-sdk") + .map((row) => `${row.tag} ${row.model}`), + ).toEqual([ + "latest claude-opus-5-5", + "latest claude-sonnet-5-5", + "main claude-opus-5-5", + "main claude-sonnet-5-5", + ]); + const listingFailures = rows.filter((row) => row.kind === "listing-failed"); + expect( + listingFailures.map((row) => [row.harness.image, row.provider]), + ).toEqual([["agent-agno", "openai"]]); + const skipped = rows.filter((row) => row.kind === "no-key"); + expect( + skipped.map((row) => [row.harness.image, row.provider]).sort(), + ).toEqual([ + ["agent-agno", "google"], + ["agent-agno", "xai"], + ["agent-antigravity", "google"], + ]); +}); + +// What the Claude Agent SDK harness at claude-agent-sdk==0.2.152 streamed for claude-opus-5-5. +const claudeSdkRefusal = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"CUSTOM","name":"system:status","value":{"status":"requesting"}}', + `data: {"type":"RUN_ERROR","message":"API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.","code":"OPENBOT_MODEL_REFUSED"}`, + "", +].join("\n\n"); + +// What the Agno harness streamed for claude-sonnet-5-5, and for a model Anthropic does not have. +const agnoReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"TEXT_MESSAGE_START","messageId":"m","role":"assistant"}', + 'data: {"type":"TEXT_MESSAGE_CONTENT","messageId":"m","delta":"OK"}', + 'data: {"type":"TEXT_MESSAGE_END","messageId":"m"}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', +].join("\n\n"); +const agnoNotFound = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + `data: {"type":"RUN_ERROR","rawEvent":{"event":"RunError","content":"litellm.NotFoundError: AnthropicException - b'{\\"type\\":\\"error\\",\\"error\\":{\\"type\\":\\"not_found_error\\",\\"message\\":\\"model: claude-nonexistent-9\\"}}'"}}`, +].join("\n\n"); + +test("a run the provider refused fails with the provider's own words", () => { + const result = classifyRun({ status: 200, body: claudeSdkRefusal }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain( + "Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required", + ); +}); + +test("a run that answered passes", () => { + expect(classifyRun({ status: 200, body: agnoReply })).toEqual({ + status: "pass", + detail: "OK", + }); +}); + +test("an error a framework puts in its raw event is still the failure's text", () => { + const result = classifyRun({ status: 200, body: agnoNotFound }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("not_found_error"); +}); + +test("a run that finished without a reply, or answered with an error as text, fails", () => { + expect( + classifyRun({ + status: 200, + body: 'data: {"type":"RUN_STARTED"}\n\ndata: {"type":"RUN_FINISHED"}', + }).status, + ).toBe("fail"); + const asText = classifyRun({ + status: 200, + body: [ + 'data: {"type":"TEXT_MESSAGE_CONTENT","delta":"Error code: 400 - model not supported"}', + 'data: {"type":"RUN_FINISHED"}', + ].join("\n\n"), + }); + expect(asText.status).toBe("fail"); + expect(asText.detail).toContain("model not supported"); +}); + +test("a reply that arrives only as a messages snapshot, as CrewAI's bridge sends it, passes", () => { + const crewaiReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"MESSAGES_SNAPSHOT","messages":[{"id":"u","role":"user","content":"Reply with the single word OK."},{"id":"a","role":"assistant","content":"OK"}]}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', + ].join("\n\n"); + expect(classifyRun({ status: 200, body: crewaiReply })).toEqual({ + status: "pass", + detail: "OK", + }); + // The person's own message in the snapshot is not a reply. + const onlyThePrompt = crewaiReply.replace( + ',{"id":"a","role":"assistant","content":"OK"}', + "", + ); + expect(classifyRun({ status: 200, body: onlyThePrompt }).status).toBe("fail"); +}); + +test("Mastra's own API answer is read as a reply or a refusal", () => { + expect( + classifyMastra({ + status: 200, + body: '{"text":"OK","finishReason":"stop"}', + }), + ).toEqual({ status: "pass", detail: "OK" }); + expect( + classifyMastra({ + status: 500, + body: '{"error":"This model is only supported in v1/responses and not in v1/chat/completions."}', + }), + ).toEqual({ + status: "fail", + detail: + "HTTP 500: This model is only supported in v1/responses and not in v1/chat/completions.", + }); +}); + +test("a refused request fails with its status and body", () => { + expect( + classifyRun({ status: 401, body: '{"error":"unauthorised"}' }), + ).toEqual({ + status: "fail", + detail: 'HTTP 401: {"error":"unauthorised"}', + }); +}); + +// Shaped as openai/codex publishes codex-rs/models-manager/models.json. +const codexCatalogue = { + models: [ + { + slug: "gpt-6.1-terra", + visibility: "list", + minimal_client_version: "0.153.0", + }, + { + slug: "gpt-6.3-sol", + visibility: "list", + minimal_client_version: "0.170.0", + }, + { + slug: "gpt-daybreak-blue-latest", + visibility: "hide", + minimal_client_version: "0.142.2", + }, + ], +}; + +test("a ChatGPT plan model newer than the harness's Codex client version fails the gate", () => { + expect(codexGate("0.167.0", codexCatalogue)).toEqual([ + { + model: "gpt-6.1-terra", + status: "pass", + detail: "needs Codex 0.153.0, harness sends 0.167.0", + }, + { + model: "gpt-6.3-sol", + status: "fail", + detail: "needs Codex 0.170.0, harness sends 0.167.0", + }, + ]); + // Compared as numbers, not text. + expect( + codexGate("0.99.0", { + models: [ + { slug: "m", visibility: "list", minimal_client_version: "0.100.0" }, + ], + })[0]!.status, + ).toBe("fail"); +}); + +test("the table escapes what would break a Markdown row", () => { + const table = renderTable([ + { + harness: "agent-agno", + tag: "latest", + provider: "openai", + model: "gpt-6.1-terra", + status: "fail", + detail: "a | b\nc", + }, + ]); + expect(table).toContain("a \\| b c"); +}); + +test("every provider the Desktop offers by address is checked", () => { + const picker = readFileSync( + join(ROOT, "desktop/src/ProviderPicker.tsx"), + "utf8", + ); + const presets = [...picker.matchAll(/baseUrl: "(https:[^"]+)"/g)].map( + (match) => match[1]!.replace(/\/+$/, ""), + ); + expect(presets.length).toBeGreaterThan(0); + const checked = [ + ...PROVIDERS.map((provider) => provider.baseUrl), + ...NOT_CHECKED.map((left) => left.baseUrl), + ].map((url) => url?.replace(/\/+$/, "")); + for (const preset of presets) expect(checked).toContain(preset); +}); + +test("every published harness is one the matrix can run", () => { + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const { harnesses: known } = JSON.parse( + readFileSync(join(ROOT, "shared/harnesses.json"), "utf8"), + ) as { harnesses: Harness[] }; + for (const image of published) { + const harness = known.find((row) => row.image === image); + expect(harness, image).toBeDefined(); + expect(providersFor(harness!).length, image).toBeGreaterThan(0); + } +}); diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts new file mode 100644 index 0000000000..fa4bd8ad47 --- /dev/null +++ b/scripts/model-compat.ts @@ -0,0 +1,1133 @@ +/** + * Does every model the picker offers still answer through the clients OpenBot ships? + * + * A harness image carries its own model client at a fixed version: the Claude Agent SDK harness + * bundles a Claude Code CLI, the LangGraph harness sends the ChatGPT plan's Codex backend a fixed + * client version, and every other harness carries the provider SDKs (LiteLLM, LangChain, the AI + * SDK, ...) it was built with. A provider can start refusing a model to an older client ("Claude + * Code 2.1.259 does not support this model; version 2.1.290 or newer is required") while the picker, + * which lists models live, keeps offering it. Nothing else calls today's models through the + * published images, so this does, once per model: + * + * 1. Lists each provider's models from the same source, with the same filter, as the picker + * (docs/configuration.md, "The model list"). + * 2. Starts each harness image on that provider, as the Desktop's settings would, and sends it + * one AG-UI run per model. The harness's own client makes the call, at the version that image + * ships. `latest` is the published image people run (ghcr.io/copilotkit/openbot-); + * `source` is the image built from this checkout, which shows whether a bumped pin fixed it. + * 3. Checks the ChatGPT plan's Codex client version in each image against the minimum each + * listed Codex model needs, from openai/codex's published model catalogue. A plan sign-in + * cannot run here, and the plan's version gate is a fact about the client version. + * + * A Claude plan runs through the same bundled CLI as an Anthropic key, so the key's rows cover it. + * + * Keys come from the environment (or --env-file), one per provider: ANTHROPIC_API_KEY, + * OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY. A provider with no key is reported as skipped, + * not failed. Keys are never printed: every detail is scrubbed of them. + * + * bun scripts/model-compat.ts [--env-file .env] [--harness agent-claude-sdk,...] + * [--provider anthropic,...] [--model ] [--tags latest,source] [--concurrency 6] + * [--out results.json] [--exit-zero] + * bun scripts/model-compat.ts report --summary [--issue ] results/*.json + * + * Needs Docker. Exits 1 when anything failed (unless --exit-zero); `report` merges the results of + * several runs, writes the Markdown summary, and exits 1 when any of them failed. + */ + +import { randomBytes } from "node:crypto"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const ROOT = join(import.meta.dir, ".."); +const REGISTRY = "ghcr.io/copilotkit/openbot-"; +const SOURCE = "source"; +const MODELS_DEV = "https://models.dev/api.json"; +const CODEX_CATALOGUE = + "https://raw.githubusercontent.com/openai/codex/main/codex-rs/models-manager/models.json"; +const PROMPT = "Reply with the single word OK."; +/** One tool the prompt has no use for, so the request takes the client's tool path. */ +const PROBE_TOOL = { + name: "look_up_order", + description: "Look up the status of an order by its number.", + parameters: { + type: "object", + properties: { order: { type: "string", description: "The order number." } }, + required: ["order"], + }, +}; +const RUN_TIMEOUT_MS = 300_000; +const START_TIMEOUT_MS = 120_000; + +// --------------------------------------------------------------------------------------------- +// The picker's list. These rules are the ones the server's model catalogue applies, so this offers +// what a person can pick: the provider's own listing, minus what no Bot can chat on. +// --------------------------------------------------------------------------------------------- + +export type Source = "openai" | "anthropic" | "google" | "xai" | "gateway"; + +/** Models no chat can run on, by any provider or gateway's naming. */ +const NEVER_CHAT = + /(embed|tts|whisper|transcribe|dall-e|image|moderation|realtime|audio|sora|translate|video|lyria|imagen|veo-|(^|[-/])live([-.]|$))/i; +/** Chat models in OpenAI's listing a Bot is still not put on. */ +const OPENAI_NOT_FOR_BOTS = + /(search|deep-research|computer-use|chat-latest|codex|instruct|^gpt-live|^gpt-(3\.5|4)(-|$)|^o1(-|$)|^(babbage|davinci))/i; +/** The same, for the other providers a deployment names directly. */ +const VENDOR_NOT_FOR_BOTS = + /(deep-research|computer-use|robotics|antigravity|nano-banana|multi-agent|(^|-)aqa$)/i; +/** OpenAI's dated snapshots, which repeat a model already listed under its own name. */ +const DATED_SNAPSHOT = /-(\d{4}-\d{2}-\d{2}|\d{4})$/; + +const VENDOR_FAMILIES: readonly [RegExp, string][] = [ + [/^(gpt-|o\d|chatgpt-)/i, "openai"], + [/^claude/i, "anthropic"], + [/^(gemini|gemma)/i, "google"], + [/^grok/i, "xai"], +]; + +function modelVendor(id: string): string | undefined { + const slash = id.indexOf("/"); + if (slash > 0) return id.slice(0, slash).toLowerCase().replace(/^~/, ""); + return VENDOR_FAMILIES.find(([family]) => family.test(id))?.[1]; +} + +function chatModel(id: string, officialOpenAI: boolean): boolean { + if (NEVER_CHAT.test(id)) return false; + if (!officialOpenAI) return true; + return ( + /^(gpt-|o\d)/i.test(id) && + !OPENAI_NOT_FOR_BOTS.test(id) && + !DATED_SNAPSHOT.test(id) + ); +} + +function seconds(value: unknown): number { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed / 1000 : 0; + } + return 0; +} + +function pastDate(value: unknown, now: number): boolean { + if (typeof value !== "string" || !value) return false; + const at = Date.parse(value); + return Number.isFinite(at) && at <= now; +} + +/** What models.dev says a model can do, by `vendor/id`. */ +export type Capabilities = Map< + string, + { tools?: boolean; text?: boolean; released?: number } +>; + +export function parseCapabilities(body: unknown): Capabilities { + const found: Capabilities = new Map(); + if (!body || typeof body !== "object") return found; + for (const vendor of ["openai", "anthropic", "google", "xai"]) { + const models = (body as Record)[vendor] + ?.models; + if (!models || typeof models !== "object") continue; + for (const [id, raw] of Object.entries(models as Record)) { + if (!raw || typeof raw !== "object") continue; + const model = raw as Record; + const outputs = (model.modalities as { output?: unknown } | undefined) + ?.output; + found.set(`${vendor}/${id}`, { + ...(typeof model.tool_call === "boolean" + ? { tools: model.tool_call } + : {}), + ...(Array.isArray(outputs) ? { text: outputs.includes("text") } : {}), + ...(typeof model.release_date === "string" + ? { released: seconds(model.release_date) } + : {}), + }); + } + } + return found; +} + +/** The models the picker offers from one listing, newest first. */ +export function usableModels( + source: Source, + body: unknown, + capabilities: Capabilities, + now: number, +): string[] { + const record = + body && typeof body === "object" ? (body as Record) : {}; + const data = Array.isArray(record.data) + ? record.data + : Array.isArray(record.models) + ? record.models + : []; + const vendorOfSource = source === "gateway" ? undefined : source; + const usable: { id: string; created: number }[] = []; + for (const raw of data) { + if (!raw || typeof raw !== "object") continue; + const item = raw as Record; + const named = + typeof item.id === "string" + ? item.id + : typeof item.name === "string" + ? item.name + : ""; + const id = named.replace(/^models\//, ""); + if (!id) continue; + const outputs = item.output_modalities; + const methods = item.supportedGenerationMethods; + const notChat = + (Array.isArray(outputs) && !outputs.includes("text")) || + (Array.isArray(methods) && !methods.includes("generateContent")); + const retired = + pastDate(item.shutdown_date, now) || + pastDate(item.retires_at, now) || + item.lifecycle === "retired"; + const bare = id.split("/").at(-1) ?? id; + const vendor = vendorOfSource ?? modelVendor(id); + const known = vendor ? capabilities.get(`${vendor}/${bare}`) : undefined; + const chat = !notChat && known?.tools !== false && known?.text !== false; + const offered = + !retired && + chat && + (source === "openai" + ? chatModel(id, true) + : source === "gateway" + ? chatModel(id, false) + : !NEVER_CHAT.test(id) && !VENDOR_NOT_FOR_BOTS.test(id)); + if (offered) + usable.push({ + id, + created: + seconds(item.created ?? item.created_at) || known?.released || 0, + }); + } + return usable + .sort((a, b) => b.created - a.created || a.id.localeCompare(b.id)) + .map((entry) => entry.id); +} + +// --------------------------------------------------------------------------------------------- +// Providers and harnesses. +// --------------------------------------------------------------------------------------------- + +/** A row of shared/harnesses.json, the fields this reads. */ +export type Harness = { + image: string | null; + credential: string; + port: number | null; + runPath: string; +}; + +export type ProviderId = "openai" | "anthropic" | "google" | "xai"; + +export type Provider = { + id: ProviderId; + label: string; + /** The repository secret, and the environment variable, that holds this provider's key. */ + secret: string; + source: Source; + /** The OpenAI-compatible address the Desktop writes as OPENAI_BASE_URL for this provider. */ + baseUrl?: string; + listing: (key: string) => { url: string; headers: Record }; + /** The environment a harness is started with on this provider, as the Desktop writes it. */ + harnessEnv: (key: string, harness: Harness) => Record; +}; + +function compatible(baseUrl: string): Pick { + return { + baseUrl, + harnessEnv: (key) => ({ + OPENAI_API_KEY: key, + OPENAI_BASE_URL: baseUrl, + BOT_PROVIDER: "openai", + }), + }; +} + +function listedAt(url: string) { + return (key: string) => ({ + url, + headers: { authorization: `Bearer ${key}` }, + }); +} + +export const PROVIDERS: readonly Provider[] = [ + { + id: "openai", + label: "OpenAI", + secret: "OPENAI_API_KEY", + source: "openai", + listing: listedAt("https://api.openai.com/v1/models"), + harnessEnv: (key) => ({ OPENAI_API_KEY: key, BOT_PROVIDER: "openai" }), + }, + { + id: "anthropic", + label: "Anthropic", + secret: "ANTHROPIC_API_KEY", + source: "anthropic", + listing: (key) => ({ + url: "https://api.anthropic.com/v1/models?limit=1000", + headers: { "x-api-key": key, "anthropic-version": "2023-06-01" }, + }), + harnessEnv: (key) => ({ + ANTHROPIC_API_KEY: key, + BOT_PROVIDER: "anthropic", + }), + }, + { + id: "google", + label: "Google", + secret: "GEMINI_API_KEY", + source: "google", + listing: listedAt( + "https://generativelanguage.googleapis.com/v1beta/openai/models", + ), + ...compatible("https://generativelanguage.googleapis.com/v1beta/openai/"), + // Antigravity is pinned to Gemini's own API and reads the key under Google's name. + harnessEnv: (key, harness) => + harness.credential === "google" + ? { GOOGLE_API_KEY: key } + : compatible( + "https://generativelanguage.googleapis.com/v1beta/openai/", + ).harnessEnv(key, harness), + }, + { + id: "xai", + label: "xAI", + secret: "XAI_API_KEY", + source: "xai", + listing: listedAt("https://api.x.ai/v1/language-models"), + ...compatible("https://api.x.ai/v1"), + }, +]; + +/** + * Providers the Desktop offers by address that this check leaves out on purpose, with why. A + * gateway serves its own aliases, not a provider's models. + */ +export const NOT_CHECKED: readonly { baseUrl: string; why: string }[] = [ + { + baseUrl: "https://api.unbiased.ai/v1", + why: "Unbiased AI is a gateway serving its own `pareto` model, not a provider's listing", + }, +]; + +function provider(id: string): Provider { + const found = PROVIDERS.find((row) => row.id === id); + if (!found) throw new Error(`No provider ${id}`); + return found; +} + +/** The providers a harness can run on, by the credential shared/harnesses.json gives it. */ +export function providersFor(harness: Harness): ProviderId[] { + if (!harness.image || !harness.port) return []; + switch (harness.credential) { + case "anthropic": + return ["anthropic"]; + case "google": + return ["google"]; + case "any-provider": + return PROVIDERS.map((row) => row.id); + default: + return []; + } +} + +export type Listing = { models: string[] } | { error: string }; + +export type MatrixRow = + | { + kind: "run"; + harness: Harness; + tag: string; + provider: ProviderId; + model: string; + } + | { kind: "no-key"; harness: Harness; provider: ProviderId } + | { + kind: "listing-failed"; + harness: Harness; + provider: ProviderId; + detail: string; + }; + +/** + * One run per harness, image tag and listed model. A provider with no key is one skipped row per + * harness, and a listing that failed one failed row, since there are no models to name. + */ +export function buildMatrix(options: { + harnesses: readonly Harness[]; + tags: readonly string[]; + listings: ReadonlyMap; +}): MatrixRow[] { + const rows: MatrixRow[] = []; + for (const harness of options.harnesses) { + for (const id of providersFor(harness)) { + const listing = options.listings.get(id); + if (!listing) rows.push({ kind: "no-key", harness, provider: id }); + else if ("error" in listing) + rows.push({ + kind: "listing-failed", + harness, + provider: id, + detail: listing.error, + }); + else + for (const tag of options.tags) + for (const model of listing.models) + rows.push({ kind: "run", harness, tag, provider: id, model }); + } + } + return rows; +} + +// --------------------------------------------------------------------------------------------- +// Reading a run. +// --------------------------------------------------------------------------------------------- + +export type Outcome = { status: "pass" | "fail"; detail: string }; + +function oneLine(text: string, limit = 400): string { + const flat = text.replace(/\s+/g, " ").trim(); + return flat.length > limit ? `${flat.slice(0, limit)}...` : flat; +} + +function errorText(event: Record): string { + if (typeof event.message === "string" && event.message) return event.message; + const raw = event.rawEvent as Record | undefined; + for (const field of ["content", "error", "message"]) { + const value = raw?.[field]; + if (typeof value === "string" && value) return value; + } + return JSON.stringify(event); +} + +/** Pass when the harness streamed a reply that says OK; otherwise fail, in the provider's words. */ +export function classifyRun(response: { + status: number; + body: string; +}): Outcome { + if (response.status !== 200) + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(response.body)}`, + }; + let streamed = ""; + // Some bridges (CrewAI's) send the reply only as the conversation's new state. + let snapshot = ""; + let finished = false; + let calledTool = false; + for (const line of response.body.split("\n")) { + if (!line.startsWith("data:")) continue; + let event: Record; + try { + event = JSON.parse(line.slice(5)); + } catch { + continue; + } + if (event.type === "RUN_ERROR") + return { status: "fail", detail: oneLine(errorText(event)) }; + if ( + (event.type === "TEXT_MESSAGE_CONTENT" || + event.type === "TEXT_MESSAGE_CHUNK") && + typeof event.delta === "string" + ) + streamed += event.delta; + if (event.type === "MESSAGES_SNAPSHOT" && Array.isArray(event.messages)) { + const last = event.messages.at(-1) as + | { role?: unknown; content?: unknown } + | undefined; + snapshot = + last?.role === "assistant" && typeof last.content === "string" + ? last.content + : ""; + } + if (event.type === "TOOL_CALL_START") calledTool = true; + if (event.type === "RUN_FINISHED") finished = true; + } + return judgeReply( + streamed || snapshot, + calledTool + ? "the model called the probe's tool instead of replying" + : finished + ? "the run finished without a reply" + : "the stream ended without finishing the run", + ); +} + +function judgeReply(text: string, silence: string): Outcome { + if (/\bok(ay)?\b/i.test(text)) + return { status: "pass", detail: oneLine(text, 80) }; + if (text) + return { status: "fail", detail: `answered without OK: ${oneLine(text)}` }; + return { status: "fail", detail: silence }; +} + +/** + * Mastra serves no AG-UI route: OpenBot reaches it through Mastra's own agent API, so its run is a + * `generate` call on that API, answered with the reply's `text` or an `error`. + */ +export function classifyMastra(response: { + status: number; + body: string; +}): Outcome { + let parsed: { text?: unknown; error?: unknown; message?: unknown } = {}; + try { + parsed = JSON.parse(response.body); + } catch { + // Not JSON: reported as it is, below. + } + if (response.status !== 200) { + const said = [parsed.error, parsed.message].find( + (value): value is string => typeof value === "string" && !!value, + ); + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(said ?? response.body)}`, + }; + } + return judgeReply( + typeof parsed.text === "string" ? parsed.text : "", + "the agent answered without text", + ); +} + +// --------------------------------------------------------------------------------------------- +// The ChatGPT plan's version gate. +// --------------------------------------------------------------------------------------------- + +function versionParts(version: string): number[] { + return version.split(/[.-]/).map((part) => Number.parseInt(part, 10) || 0); +} + +export function compareVersions(a: string, b: string): number { + const left = versionParts(a); + const right = versionParts(b); + for (let i = 0; i < Math.max(left.length, right.length); i++) { + const difference = (left[i] ?? 0) - (right[i] ?? 0); + if (difference) return Math.sign(difference); + } + return 0; +} + +/** Each model the Codex picker lists, against the client version the harness sends the plan. */ +export function codexGate( + clientVersion: string, + catalogue: unknown, +): { model: string; status: "pass" | "fail"; detail: string }[] { + const models = (catalogue as { models?: unknown })?.models; + if (!Array.isArray(models)) return []; + return models + .filter( + (row): row is { slug: string; minimal_client_version?: string } => + !!row && row.visibility === "list" && typeof row.slug === "string", + ) + .map((row) => { + const needs = row.minimal_client_version ?? "0.0.0"; + return { + model: row.slug, + status: compareVersions(clientVersion, needs) >= 0 ? "pass" : "fail", + detail: `needs Codex ${needs}, harness sends ${clientVersion}`, + }; + }); +} + +// --------------------------------------------------------------------------------------------- +// Results. +// --------------------------------------------------------------------------------------------- + +export type Result = { + harness: string; + tag: string; + provider: string; + model: string; + status: "pass" | "fail" | "skipped"; + detail: string; +}; + +function cell(text: string): string { + return oneLine(text).replace(/\|/g, "\\|"); +} + +export function renderTable(results: readonly Result[]): string { + const lines = [ + "| Harness | Image | Provider | Model | Result | Provider's answer |", + "| --- | --- | --- | --- | --- | --- |", + ]; + for (const row of results) + lines.push( + `| ${cell(row.harness)} | ${cell(row.tag)} | ${cell(row.provider)} | ${cell(row.model)} | ${row.status} | ${cell(row.detail)} |`, + ); + return lines.join("\n"); +} + +const ORDER = { fail: 0, skipped: 1, pass: 2 } as const; + +/** + * The summary: what failed first, then what was skipped for want of a key, then what passed. The + * issue's copy leaves out what passed and keeps at most `maxFailed` failures, since an issue body + * holds 65,536 characters and the run's own summary has the rest. + */ +export function renderSummary( + results: readonly Result[], + options: { passed?: boolean; maxFailed?: number } = {}, +): string { + const { passed: showPassed = true, maxFailed = Number.POSITIVE_INFINITY } = + options; + const sorted = [...results].sort( + (a, b) => + ORDER[a.status] - ORDER[b.status] || + a.harness.localeCompare(b.harness) || + a.provider.localeCompare(b.provider) || + a.tag.localeCompare(b.tag) || + a.model.localeCompare(b.model), + ); + const count = (status: Result["status"]) => + results.filter((row) => row.status === status).length; + const failed = sorted.filter((row) => row.status === "fail"); + const skipped = sorted.filter((row) => row.status === "skipped"); + const passed = sorted.filter((row) => row.status === "pass"); + const missing = [ + ...new Set( + skipped.map( + (row) => + PROVIDERS.find((p) => p.label === row.provider)?.secret ?? + row.provider, + ), + ), + ]; + const parts = [ + "## Model compatibility", + "", + `${count("fail")} failed, ${count("pass")} passed, ${count("skipped")} skipped.`, + "", + ]; + if (missing.length) + parts.push( + `Skipped for want of a key, not failed: add the repository secret${missing.length > 1 ? "s" : ""} ${missing.map((name) => `\`${name}\``).join(", ")} to check ${missing.length > 1 ? "those providers" : "that provider"}.`, + "", + ); + if (failed.length) { + parts.push("### Failed", "", renderTable(failed.slice(0, maxFailed)), ""); + if (failed.length > maxFailed) + parts.push( + `And ${failed.length - maxFailed} more failures, in the run's summary.`, + "", + ); + } + if (skipped.length) parts.push("### Skipped", "", renderTable(skipped), ""); + if (passed.length && showPassed) + parts.push( + "
Passed", + "", + renderTable(passed), + "", + "
", + "", + ); + return parts.join("\n"); +} + +// --------------------------------------------------------------------------------------------- +// Running it. +// --------------------------------------------------------------------------------------------- + +type Options = { + envFile?: string; + harnesses?: string[]; + providers?: string[]; + model?: RegExp; + tags: string[]; + concurrency: number; + out?: string; + exitZero: boolean; +}; + +function parseOptions(argv: string[]): Options { + const options: Options = { + tags: ["latest", SOURCE], + concurrency: 6, + exitZero: false, + }; + const list = (value: string | undefined) => + (value ?? "") + .split(/[\s,]+/) + .map((item) => item.trim()) + .filter(Boolean); + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined) throw new Error(`${flag} needs a value`); + return next; + }; + if (flag === "--env-file") options.envFile = value(); + else if (flag === "--harness") options.harnesses = list(value()); + else if (flag === "--provider") options.providers = list(value()); + else if (flag === "--model") options.model = new RegExp(value()); + else if (flag === "--tags") options.tags = list(value()); + else if (flag === "--concurrency") + options.concurrency = Math.max(1, Number(value()) || 1); + else if (flag === "--out") options.out = value(); + else if (flag === "--exit-zero") options.exitZero = true; + else throw new Error(`Unknown option ${flag}`); + } + return options; +} + +/** The provider keys, from the environment, or from an env file for the names it lacks. */ +function readKeys(envFile: string | undefined): Map { + const fromFile = new Map(); + if (envFile) + for (const line of readFileSync(envFile, "utf8").split("\n")) { + const match = /^\s*(?:export\s+)?([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line); + if (!match) continue; + const [, name = "", raw = ""] = match; + fromFile.set(name, raw.trim().replace(/^(['"])(.*)\1$/, "$2")); + } + const keys = new Map(); + for (const row of PROVIDERS) { + const value = ( + process.env[row.secret] || + fromFile.get(row.secret) || + "" + ).trim(); + if (value) keys.set(row.id, value); + } + return keys; +} + +function scrubber(keys: Map) { + const secrets = [...keys.values()].filter((value) => value.length >= 8); + return (text: string) => + secrets.reduce( + (scrubbed, secret) => scrubbed.split(secret).join("[key]"), + text, + ); +} + +async function fetchJson( + url: string, + headers: Record = {}, +): Promise<{ ok: true; body: unknown } | { ok: false; error: string }> { + try { + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(30_000), + }); + const text = await response.text(); + if (!response.ok) + return { ok: false, error: `HTTP ${response.status}: ${oneLine(text)}` }; + return { ok: true, body: JSON.parse(text) }; + } catch (error) { + return { ok: false, error: String(error) }; + } +} + +async function exec( + args: string[], + env: Record = {}, +): Promise<{ code: number; stdout: string; stderr: string }> { + const child = Bun.spawn(args, { + env: { ...process.env, ...env }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + return { code, stdout, stderr }; +} + +/** + * `source` is the image built from this checkout, which is what the next release ships, so a pin + * bumped here is checked before it is released. Any other tag is the published image: `latest` is + * what people run today. + */ +function imageOf(harness: Harness, tag: string): string { + return tag === SOURCE + ? `model-compat/${harness.image}:source` + : `${REGISTRY}${harness.image}:${tag}`; +} + +async function prepare(harness: Harness, tag: string) { + return tag === SOURCE + ? exec([ + "docker", + "build", + "--quiet", + "-f", + join(ROOT, harness.image ?? "", "Dockerfile"), + "-t", + imageOf(harness, tag), + ROOT, + ]) + : exec(["docker", "pull", "--quiet", imageOf(harness, tag)]); +} + +let serial = 0; + +/** Starts the harness on one provider and model, sends one run, and stops it. */ +async function runOne( + row: Extract, + key: string, + scrub: (text: string) => string, +): Promise { + const harness = row.harness; + const name = `model-compat-${process.pid}-${++serial}`; + const env = { + ...provider(row.provider).harnessEnv(key, harness), + BOT_MODEL: row.model, + }; + const token = randomBytes(12).toString("hex"); + // Each value goes to Docker through the environment and is named here, so no key is in argv. + const started = await exec( + [ + "docker", + "run", + "-d", + "--rm", + "--name", + name, + "-p", + `127.0.0.1::${harness.port}`, + "-e", + "MANAGED_AGENT_TOKEN", + ...Object.keys(env).flatMap((variable) => ["-e", variable]), + imageOf(harness, row.tag), + ], + { ...env, MANAGED_AGENT_TOKEN: token }, + ); + if (started.code !== 0) + return { + status: "fail", + detail: scrub(`the harness did not start: ${oneLine(started.stderr)}`), + }; + try { + const port = await exec(["docker", "port", name, `${harness.port}/tcp`]); + const address = port.stdout.trim().split("\n")[0]?.trim(); + if (!address) + return { status: "fail", detail: "the harness published no port" }; + const base = `http://${address}`; + const deadline = Date.now() + START_TIMEOUT_MS; + let healthy = false; + while (Date.now() < deadline) { + try { + const health = await fetch(`${base}/health`, { + signal: AbortSignal.timeout(5_000), + }); + if (health.ok) { + healthy = true; + break; + } + } catch { + // Not listening yet. + } + const state = await exec([ + "docker", + "inspect", + "-f", + "{{.State.Running}}", + name, + ]); + if (state.stdout.trim() !== "true") break; + await Bun.sleep(1_000); + } + if (!healthy) { + const logs = await exec(["docker", "logs", "--tail", "5", name]); + return { + status: "fail", + detail: scrub( + `the harness never became healthy: ${oneLine(logs.stdout + logs.stderr)}`, + ), + }; + } + const id = randomBytes(6).toString("hex"); + const mastra = harness.image === "agent-mastra"; + let response: { status: number; body: string }; + try { + const reply = await fetch( + mastra + ? `${base}/api/agents/openbot/generate` + : `${base}${harness.runPath || "/"}`, + { + method: "POST", + headers: { + "content-type": "application/json", + accept: mastra ? "application/json" : "text/event-stream", + "x-openbot-agent-token": token, + }, + body: JSON.stringify( + mastra + ? { messages: [{ role: "user", content: PROMPT }] } + : { + threadId: `model-compat-${id}`, + runId: `run-${id}`, + state: {}, + messages: [ + { id: `message-${id}`, role: "user", content: PROMPT }, + ], + // A Bot's run always carries tools, and a client's tool path is where models + // part ways (Responses-only models, parallel tool calls), so this one does too. + tools: [PROBE_TOOL], + context: [], + // What the server tells the LangGraph and Claude Agent SDK harnesses; the others + // read BOT_MODEL, set above. + forwardedProps: { openbotModel: { model: row.model } }, + }, + ), + signal: AbortSignal.timeout(RUN_TIMEOUT_MS), + }, + ); + response = { status: reply.status, body: await reply.text() }; + } catch (error) { + return { + status: "fail", + detail: scrub(`no answer: ${oneLine(String(error))}`), + }; + } + const outcome = mastra ? classifyMastra(response) : classifyRun(response); + if (outcome.status === "fail" && !response.body.includes("RUN_ERROR")) { + // A harness that swallowed the error still logged it. + const logs = await exec(["docker", "logs", "--tail", "40", name]); + const said = (logs.stdout + logs.stderr) + .split("\n") + .filter((line) => /error|exception/i.test(line)) + .at(-1); + if (said) + outcome.detail = `${outcome.detail}; logged: ${oneLine(said, 300)}`; + } + return { status: outcome.status, detail: scrub(outcome.detail) }; + } finally { + await exec(["docker", "rm", "-f", name]); + } +} + +async function pool( + items: readonly T[], + width: number, + work: (item: T) => Promise, +) { + let next = 0; + await Promise.all( + Array.from({ length: Math.min(width, items.length) }, async () => { + for (let item = items[next++]; item !== undefined; item = items[next++]) + await work(item); + }), + ); +} + +async function check(argv: string[]): Promise { + const options = parseOptions(argv); + const keys = readKeys(options.envFile); + const scrub = scrubber(keys); + + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const harnesses = ( + JSON.parse(readFileSync(join(ROOT, "shared/harnesses.json"), "utf8")) as { + harnesses: Harness[]; + } + ).harnesses.filter( + (row) => + row.image && + published.includes(row.image) && + (!options.harnesses || options.harnesses.includes(row.image)), + ); + if (!harnesses.length) + throw new Error("No published harness matches --harness."); + + const wanted = new Set( + harnesses + .flatMap(providersFor) + .filter((id) => !options.providers || options.providers.includes(id)), + ); + const capabilities = await fetchJson(MODELS_DEV); + const known = capabilities.ok + ? parseCapabilities(capabilities.body) + : new Map(); + const listings = new Map(); + for (const id of wanted) { + const key = keys.get(id); + if (!key) continue; + const row = provider(id); + const request = row.listing(key); + const listed = await fetchJson(request.url, request.headers); + listings.set( + id, + listed.ok + ? { + models: usableModels( + row.source, + listed.body, + known, + Date.now(), + ).filter((model) => !options.model || options.model.test(model)), + } + : { error: scrub(`the model listing failed: ${listed.error}`) }, + ); + } + const matrix = buildMatrix({ + harnesses, + tags: options.tags, + listings, + }).filter( + (row) => !options.providers || options.providers.includes(row.provider), + ); + + const results: Result[] = []; + const label = (id: string) => provider(id).label; + for (const row of matrix) { + if (row.kind === "no-key") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + model: "(all)", + status: "skipped", + detail: `skipped: no key (${provider(row.provider).secret})`, + }); + if (row.kind === "listing-failed") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + model: "(all)", + status: "fail", + detail: row.detail, + }); + } + + // Every image once, before any run, so a pull or build is never counted against a run's time. + const images = harnesses.flatMap((harness) => + options.tags.map((tag) => ({ harness, tag })), + ); + const unpulled = new Map(); + await pool(images, 3, async ({ harness, tag }) => { + const began = Date.now(); + const ready = await prepare(harness, tag); + console.error( + `${tag === SOURCE ? "built" : "pulled"} ${imageOf(harness, tag)} in ${Math.round((Date.now() - began) / 1000)}s${ready.code ? " (failed)" : ""}`, + ); + if (ready.code !== 0) + unpulled.set( + imageOf(harness, tag), + oneLine(ready.stderr.split("\n").slice(-5).join(" ")), + ); + }); + + // The ChatGPT plan's Codex client version, in whichever images carry the plan. + const codex = await fetchJson(CODEX_CATALOGUE); + if (!options.providers || options.providers.includes("openai")) + for (const harness of harnesses) + for (const tag of options.tags) { + const image = imageOf(harness, tag); + if (unpulled.has(image)) continue; + const version = await exec([ + "docker", + "run", + "--rm", + "--entrypoint", + "python", + image, + "-c", + "from src.chatgpt_models import client_version; print(client_version())", + ]); + if (version.code !== 0) continue; + const sends = version.stdout.trim(); + if (!codex.ok) { + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + model: "(all)", + status: "fail", + detail: `could not read Codex's model catalogue: ${codex.error}`, + }); + continue; + } + for (const gate of codexGate(sends, codex.body)) + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + ...gate, + }); + } + + const runs = matrix.filter( + (row): row is Extract => row.kind === "run", + ); + let done = 0; + await pool(runs, options.concurrency, async (row) => { + const image = imageOf(row.harness, row.tag); + const outcome = unpulled.has(image) + ? { + status: "fail" as const, + detail: `could not get ${image}: ${unpulled.get(image)}`, + } + : await runOne(row, keys.get(row.provider) ?? "", scrub); + results.push({ + harness: row.harness.image ?? "", + tag: row.tag, + provider: + row.provider === "anthropic" && row.harness.credential === "anthropic" + ? "Anthropic (key or Claude plan)" + : label(row.provider), + model: row.model, + ...outcome, + }); + done++; + console.error( + `[${done}/${runs.length}] ${row.harness.image}:${row.tag} ${row.provider} ${row.model}: ${outcome.status}`, + ); + }); + + if (options.out) + writeFileSync(options.out, `${JSON.stringify({ results }, null, 2)}\n`); + console.log(renderSummary(results)); + return results.some((row) => row.status === "fail") && !options.exitZero + ? 1 + : 0; +} + +function report(argv: string[]): number { + let summary: string | undefined; + let issue: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--summary") summary = argv[++i]; + else if (arg === "--issue") issue = argv[++i]; + else if (arg) files.push(arg); + } + const results = files.flatMap( + (file) => + (JSON.parse(readFileSync(file, "utf8")) as { results: Result[] }).results, + ); + const markdown = renderSummary(results); + if (summary) writeFileSync(summary, `${markdown}\n`); + else console.log(markdown); + if (issue) + writeFileSync( + issue, + `${renderSummary(results, { passed: false, maxFailed: 150 })}\n`, + ); + if (!results.length) { + console.error( + "No results were found: every harness job failed before it wrote any.", + ); + return 1; + } + return results.some((row) => row.status === "fail") ? 1 : 0; +} + +if (import.meta.main) { + const argv = process.argv.slice(2); + const code = argv[0] === "report" ? report(argv.slice(1)) : await check(argv); + process.exit(code); +}