From 4b93d6e54dbd57ab50105e8ad8c3a02137f3add6 Mon Sep 17 00:00:00 2001 From: soyuka Date: Tue, 12 May 2026 16:31:16 +0200 Subject: [PATCH] [CI] Add weekly conformance run against latest upstream release The PR pipeline only catches conformance changes when someone opens a PR. Upstream `@modelcontextprotocol/conformance` releases between PRs go untested. This adds a weekly cron + `workflow_dispatch` trigger pinned to `@latest`, and opens a tracking issue on scheduled failures. --- .github/workflows/conformance-weekly.yaml | 163 ++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 .github/workflows/conformance-weekly.yaml diff --git a/.github/workflows/conformance-weekly.yaml b/.github/workflows/conformance-weekly.yaml new file mode 100644 index 00000000..d2a31377 --- /dev/null +++ b/.github/workflows/conformance-weekly.yaml @@ -0,0 +1,163 @@ +name: conformance-weekly + +# Runs the MCP conformance suite on a weekly schedule against the +# latest @modelcontextprotocol/conformance release. The on:pull_request +# pipeline already pins to the version of the conformance package +# available at PR time; this schedule catches upstream releases that +# add new scenarios between PRs and would otherwise stay invisible. +on: + schedule: + # Mondays 06:00 UTC + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + conformance-server: + name: conformance / server (latest) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: '22' + + - name: Install Composer + uses: "ramsey/composer-install@v4" + + - name: Show conformance version + run: npx --yes @modelcontextprotocol/conformance@latest --version + + - name: Start conformance server + run: | + mkdir -p tests/Conformance/sessions tests/Conformance/logs + chmod -R 777 tests/Conformance/sessions tests/Conformance/logs + docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d + sleep 5 + + - name: Run conformance tests + working-directory: ./tests/Conformance + run: npx --yes @modelcontextprotocol/conformance@latest server --url http://localhost:8000/ --expected-failures conformance-baseline.yml + + - name: Show logs on failure + if: failure() + run: | + echo "=== Docker Compose Logs ===" + docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs + echo "" + echo "=== Conformance Log ===" + cat tests/Conformance/logs/conformance.log 2>/dev/null || echo "No conformance log found" + echo "" + echo "=== Test Results (first failed test) ===" + find tests/Conformance/results -name "checks.json" 2>/dev/null | head -3 | while read f; do + echo "--- $f ---" + cat "$f" + echo "" + done || echo "No results found" + + - name: Cleanup + if: always() + run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml down + + conformance-client: + name: conformance / client (latest) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + coverage: "none" + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: '22' + + - name: Install Composer + uses: "ramsey/composer-install@v4" + + - name: Show conformance version + run: npx --yes @modelcontextprotocol/conformance@latest --version + + - name: Create log directory + run: mkdir -p tests/Conformance/logs + + - name: Run client conformance tests + working-directory: ./tests/Conformance + run: npx --yes @modelcontextprotocol/conformance@latest client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml + + - name: Show logs on failure + if: failure() + run: | + echo "=== Client Conformance Log ===" + cat tests/Conformance/logs/client-conformance.log 2>/dev/null || echo "No client conformance log found" + echo "" + echo "=== Test Results ===" + find tests/Conformance/results -name "checks.json" 2>/dev/null | head -3 | while read f; do + echo "--- $f ---" + cat "$f" + echo "" + done || echo "No results found" + + notify: + name: Open issue on failure + runs-on: ubuntu-latest + needs: [conformance-server, conformance-client] + if: failure() && github.event_name == 'schedule' + steps: + - name: File or update tracking issue + uses: actions/github-script@v7 + with: + script: | + const marker = ''; + const title = '[conformance] Weekly conformance run failed'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = [ + marker, + `The weekly conformance workflow against the latest \`@modelcontextprotocol/conformance\` release failed.`, + ``, + `- Workflow run: ${runUrl}`, + `- Triggered: ${context.eventName} on ${new Date().toISOString()}`, + ``, + `This usually means upstream published a new conformance release whose scenarios the SDK does not satisfy. Inspect the run logs and either:`, + ``, + `1. Fix the SDK to satisfy the new scenarios,`, + `2. Update \`tests/Conformance/server.php\` / \`tests/Conformance/client.php\` fixtures, or`, + `3. Add the new expected failure to \`tests/Conformance/conformance-baseline.yml\` with justification.`, + ].join('\n'); + + const existing = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'conformance-weekly', + per_page: 50, + }); + const match = existing.data.find(i => (i.body || '').includes(marker)); + + if (match) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: match.number, + body: `New failure on ${new Date().toISOString()}: ${runUrl}`, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + labels: ['conformance-weekly'], + }); + }