Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
261 changes: 261 additions & 0 deletions .github/workflows/model-compat.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,261 @@
name: Model compatibility

# Does every model the picker offers still answer through the clients OpenBot ships? The picker
# lists each provider's models live, but every harness image carries its model client at a fixed
# version (the Claude Agent SDK harness bundles a Claude Code CLI, the LangGraph harness sends the
# ChatGPT plan's backend a fixed Codex client version, the others carry the provider SDKs they were
# built with), and a provider can refuse a new model to an older client. scripts/model-compat.ts
# starts each harness image, one job per harness, and sends it one request per listed model on
# every provider that has a key here. See docs/releasing.md, "Model compatibility".
#
# Keys are repository secrets, one per provider. A provider without one is reported as skipped,
# not failed. On any failure the run fails and opens (or updates) one issue titled "Model
# compatibility check failing"; a passing run closes it.
#
# What answered on the `source` image is the model lock (shared/model-lock.json), the models the
# picker may offer for each harness. When a model newly answers or stops answering, the run opens
# or updates one pull request that refreshes the lock. See docs/releasing.md, "Model lock".
#
# Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever
# runs with these keys.
on:
schedule:
- cron: "41 6 * * *"
workflow_dispatch:
inputs:
tags:
description: Harness images to check, comma-separated. `latest` is the published release people run; `source` is built from this commit, which is what the next release ships.
type: string
default: latest,source
required: false

permissions: {}

concurrency:
group: model-compat
cancel-in-progress: false

jobs:
# The published harnesses, and every engine the model lock names whether it is published yet or
# not (an engine new Bots run on is locked, and checked, from before its first release).
harnesses:
name: List the harnesses to check
if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
images: ${{ steps.list.outputs.images }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.4.3
- id: list
run: echo "images=$(bun scripts/model-compat.ts harnesses)" >> "$GITHUB_OUTPUT"

check:
name: ${{ matrix.harness }}
needs: harnesses
runs-on: ubuntu-latest
timeout-minutes: 90
permissions:
contents: read
strategy:
fail-fast: false
matrix:
harness: ${{ fromJSON(needs.harnesses.outputs.images) }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.4.3
# Writes its results and exits 0 whatever they say: the report job below decides.
- name: One request per model through this harness
env:
HARNESS: ${{ matrix.harness }}
TAGS: ${{ inputs.tags || 'latest,source' }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
run: >-
bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero
--out "$RUNNER_TEMP/model-compat-$HARNESS.json"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: model-compat-${{ matrix.harness }}
path: ${{ runner.temp }}/model-compat-${{ matrix.harness }}.json
if-no-files-found: error
retention-days: 14

report:
name: Report
needs: [harnesses, check]
if: always() && needs.harnesses.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write # open, update or close the one "Model compatibility check failing" issue
env:
GH_TOKEN: ${{ github.token }}
ISSUE_TITLE: Model compatibility check failing
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.4.3
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: model-compat-*
path: ${{ runner.temp }}/results
merge-multiple: true
- name: Summarize
id: summary
run: |
set +e
shopt -s nullglob
bun scripts/model-compat.ts report --summary "$RUNNER_TEMP/summary.md" \
--issue "$RUNNER_TEMP/issue-table.md" "$RUNNER_TEMP"/results/*.json
code=$?
echo "code=$code" >> "$GITHUB_OUTPUT"
if [ -f "$RUNNER_TEMP/summary.md" ]; then cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY"; fi
exit 0
- name: Open or update the issue
if: steps.summary.outputs.code != '0'
run: |
set -euo pipefail
body="$RUNNER_TEMP/issue.md"
{
echo "The model compatibility check failed on $(date -u +%Y-%m-%d): $RUN_URL"
echo
echo "A model refused by a pinned client means that client needs bumping: see docs/releasing.md, \"Model compatibility\". This issue is updated by every failing run and closed by the first passing one."
echo
if [ -f "$RUNNER_TEMP/issue-table.md" ]; then
cat "$RUNNER_TEMP/issue-table.md"
else
echo "No harness job wrote results. See the run."
fi
} > "$body"
number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \
--json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty")
if [ -n "$number" ]; then
gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body"
else
gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$body"
fi
- name: Close the issue when everything passes
if: steps.summary.outputs.code == '0'
run: |
set -euo pipefail
number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \
--json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty")
if [ -n "$number" ]; then
gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Passing again: $RUN_URL"
fi
- name: Fail when a model failed
if: steps.summary.outputs.code != '0'
run: |
echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"."
exit 1

lock:
name: Refresh the model lock
needs: [harnesses, check]
if: always() && needs.harnesses.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: write # push the one refresh branch
pull-requests: write # open or update its pull request
env:
BRANCH: model-lock/refresh
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.4.3
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: model-compat-*
path: ${{ runner.temp }}/results
merge-multiple: true
- name: Write the lock from the source images' results
id: lock
run: |
set -euo pipefail
shopt -s nullglob
files=("$RUNNER_TEMP"/results/*.json)
if [ "${#files[@]}" -eq 0 ]; then
echo "No harness job wrote results; the lock stays as it is."
echo "changed=false" >> "$GITHUB_OUTPUT"
exit 0
fi
bun scripts/model-compat.ts lock --changes "$RUNNER_TEMP/lock-changes.md" "${files[@]}"
if git diff --quiet -- shared/model-lock.json; then
echo "changed=false" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
{ echo "## Model lock changes"; echo; cat "$RUNNER_TEMP/lock-changes.md"; } >> "$GITHUB_STEP_SUMMARY"
fi
# GITHUB_TOKEN's pull requests get CI runs that wait for a maintainer to approve them. A
# GitHub App installation token, when one is configured, has no such limit.
- id: app-token
if: steps.lock.outputs.changed == 'true' && vars.OPENBOT_UPGRADE_APP_CLIENT_ID != ''
uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
with:
client-id: ${{ vars.OPENBOT_UPGRADE_APP_CLIENT_ID }}
private-key: ${{ secrets.OPENBOT_UPGRADE_APP_PRIVATE_KEY }}
permission-contents: write
permission-pull-requests: write
- name: Open or update the refresh pull request
if: steps.lock.outputs.changed == 'true'
env:
GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }}
TOKEN_KIND: ${{ steps.app-token.outputs.token && 'app' || 'github-token' }}
# Credentials for this one push, through gh, and no hook can run.
GIT_CONFIG_COUNT: "3"
GIT_CONFIG_KEY_0: credential.helper
GIT_CONFIG_VALUE_0: ""
GIT_CONFIG_KEY_1: credential.helper
GIT_CONFIG_VALUE_1: "!gh auth git-credential"
GIT_CONFIG_KEY_2: core.hooksPath
GIT_CONFIG_VALUE_2: /dev/null
run: |
set -euo pipefail
git switch -C "$BRANCH"
git -c user.name="github-actions[bot]" \
-c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
commit -m "Refresh the model lock from the daily model check" -- shared/model-lock.json
git push --force "https://github.com/$GITHUB_REPOSITORY.git" "HEAD:refs/heads/$BRANCH"
body="$RUNNER_TEMP/lock-pr.md"
{
echo "The daily model check found models that started or stopped answering through the clients the harnesses ship. This refreshes \`shared/model-lock.json\`, the models the picker may offer for each harness."
echo
cat "$RUNNER_TEMP/lock-changes.md"
echo
echo "From $RUN_URL. Every run that finds a change updates this pull request. See docs/releasing.md, \"Model lock\"."
if [ "$TOKEN_KIND" != app ]; then
echo
echo "**CI must be started by hand.** This pull request was opened with \`GITHUB_TOKEN\`, so its CI runs wait for a maintainer to approve them."
fi
} > "$body"
number=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq ".[0].number // empty")
if [ -n "$number" ]; then
gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body"
else
gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$BRANCH" \
--title "Refresh the model lock" --body-file "$body"
fi
111 changes: 111 additions & 0 deletions .github/workflows/model-lock.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
name: Model lock

# A pull request that changes a harness's model client (its requirements, its package.json or
# lockfile, its Dockerfile, or the ChatGPT plan's Codex client version) must not break a model the
# model lock (shared/model-lock.json) says that harness runs: a person set to it would get the
# provider's refusal on every message. This builds the changed harnesses from the pull request,
# sends each listed model one request through them (scripts/model-compat.ts), and fails when a
# model the lock verifies no longer answers. See docs/releasing.md, "Model lock".
#
# Only for pull requests from this repository: a fork's pull request is never given the provider
# keys, so it is skipped and the daily check covers it once it merges.
on:
pull_request:
paths:
- "agent-*/requirements.txt"
- "agent-*/Dockerfile"
- "agent-*/package.json"
- "agent-*/bun.lock"
- "agent-*/package-lock.json"
- "agent-langgraph-agui/src/chatgpt_models.py"
- "shared/model-lock.json"
- "scripts/model-compat.ts"
- ".github/workflows/model-lock.yml"

permissions: {}

concurrency:
group: model-lock-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
changed:
name: Harnesses whose client changed
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
harnesses: ${{ steps.list.outputs.harnesses }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
fetch-depth: 0
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.4.3
- id: list
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
set -euo pipefail
changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA")
# The harnesses the check runs: the published ones, and every engine the lock names,
# published yet or not, that has a row in shared/harnesses.json.
checked=$(bun scripts/model-compat.ts harnesses)
# The check itself, or the lock, changed: every harness the lock names is checked.
if grep -qxE 'shared/model-lock.json|scripts/model-compat.ts|\.github/workflows/model-lock.yml' <<<"$changed"; then
harnesses=$(jq -c --argjson checked "$checked" \
'[.engines | keys[] | select(. as $h | $checked | index($h))]' shared/model-lock.json)
else
harnesses=$(grep -oE '^agent-[a-z0-9-]+/' <<<"$changed" | tr -d / | sort -u \
| jq -R . | jq -sc --argjson checked "$checked" \
'map(select(. as $h | $checked | index($h)))')
fi
echo "harnesses=$harnesses" >> "$GITHUB_OUTPUT"
echo "Harnesses to check: $harnesses"

check:
name: ${{ matrix.harness }}
needs: changed
if: needs.changed.outputs.harnesses != '[]'
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
contents: read
strategy:
fail-fast: false
matrix:
harness: ${{ fromJSON(needs.changed.outputs.harnesses) }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.4.3
- name: One request per model through this pull request's harness
env:
HARNESS: ${{ matrix.harness }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
run: >-
bun scripts/model-compat.ts --harness "$HARNESS" --tags source --exit-zero
--out "$RUNNER_TEMP/model-compat.json"
- name: Every model the lock verifies still answers
env:
HARNESS: ${{ matrix.harness }}
run: |
set +e
bun scripts/model-compat.ts guard --summary "$RUNNER_TEMP/guard.md" "$RUNNER_TEMP/model-compat.json"
code=$?
cat "$RUNNER_TEMP/guard.md" >> "$GITHUB_STEP_SUMMARY"
if [ "$code" -ne 0 ]; then
echo "::error::A model the lock verifies for $HARNESS no longer answers through this change's client. See the summary."
fi
exit "$code"
6 changes: 6 additions & 0 deletions api/client-boundary.allow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,12 @@ export const ALLOWED: Allowed[] = [
rule: "third-party",
why: "image registries, never OpenBot's server",
},
{
file: "scripts/model-compat.ts",
what: /^(fetch\(\)|\/health|\/api\/agents\/openbot\/generate)$/,
rule: "third-party",
why: "model providers' listings and the harness images the model check starts on this machine, never OpenBot's server",
},
// Scripts that run and stop a server on this machine.
{
file: "scripts/update-local.ts",
Expand Down
Loading