diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml new file mode 100644 index 0000000..ca27f86 --- /dev/null +++ b/.github/workflows/mcp-evals.yml @@ -0,0 +1,62 @@ +name: MCP evals + +# The LLM-driven eval dataset through OpenRouter, weekly and on demand. +on: + workflow_dispatch: + inputs: + model: + description: "OpenRouter model ID for every case (empty keeps the dataset's model)" + required: false + default: "" + schedule: + - cron: "0 7 * * 1" + +permissions: + contents: read + +jobs: + mcp-evals: + runs-on: ubuntu-latest + # The spec itself times out after 90 minutes. + timeout-minutes: 100 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + EVAL_MODEL: ${{ inputs.model }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22.x + + - name: Install dependencies + run: yarn + + # The deployer case runs tflocal; the wrapper would garble its output. + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Install tflocal and the Snowflake CLI + run: pipx install terraform-local && pipx install snowflake-cli + + # Pulled up front so a start inside a case doesn't time out on the download. + - name: Pull the LocalStack images + run: | + docker pull localstack/localstack-pro:latest + docker pull localstack/snowflake:latest + + - name: Run the eval dataset + run: yarn test:mcp:evals + + - name: Keep the eval results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mcp-evals-results + path: .mcp-test-results + if-no-files-found: ignore diff --git a/README.md b/README.md index 17a2f8c..b98d463 100644 --- a/README.md +++ b/README.md @@ -264,12 +264,13 @@ This repository includes [MCP Server Tester](https://github.com/gleanwork/mcp-se ```bash yarn test:mcp:direct ``` -- Run Gemini-based MCP host evals: +- Run MCP host evals through [OpenRouter](https://openrouter.ai) (default model `google/gemini-2.5-flash`; set `EVAL_MODEL` to any OpenRouter model ID to override it): ```bash - export GOOGLE_GENERATIVE_AI_API_KEY="" + export OPENROUTER_API_KEY="" export LOCALSTACK_AUTH_TOKEN="" yarn test:mcp:evals ``` + The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. - Open the latest MCP Server Tester HTML report: ```bash npx mcp-server-tester open diff --git a/data/evals/gemini-comprehensive.json b/data/evals/comprehensive.json similarity index 87% rename from data/evals/gemini-comprehensive.json rename to data/evals/comprehensive.json index a03bc9c..5225343 100644 --- a/data/evals/gemini-comprehensive.json +++ b/data/evals/comprehensive.json @@ -1,13 +1,17 @@ { - "name": "localstack-mcp-gemini-comprehensive", - "description": "Scenario-based Gemini evals covering LocalStack MCP tools.", + "name": "localstack-mcp-comprehensive", + "description": "Scenario-based OpenRouter evals covering LocalStack MCP tools.", "cases": [ { "id": "scenario-management-status", "description": "localstack-management: status check of the AWS emulator.", "mode": "mcp_host", "scenario": "Check the current status of the LocalStack AWS emulator using the LocalStack management tool. If it is not running, start it. Report whether it is ready to accept requests.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -28,7 +32,11 @@ "description": "localstack-deployer: deploy the sample Terraform project.", "mode": "mcp_host", "scenario": "Deploy the Terraform project located in the data/sample-terraform directory of this repository to LocalStack using the LocalStack deployer tool, then report the Terraform outputs.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -45,7 +53,11 @@ "description": "localstack-aws-client: run a pinned AWS CLI command.", "mode": "mcp_host", "scenario": "Using the LocalStack AWS client tool, run the AWS CLI command \"s3 ls\" against LocalStack. Pass the command string exactly as s3 ls, with no aws or awslocal prefix and no extra flags. Then tell me which S3 buckets exist.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -66,7 +78,11 @@ "description": "localstack-logs-analysis: errors analysis of container logs.", "mode": "mcp_host", "scenario": "Use the LocalStack logs analysis tool to run an 'errors' analysis of the recent LocalStack container logs and report which AWS API calls failed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -87,7 +103,11 @@ "description": "localstack-app-inspector: status check plus trace listing.", "mode": "mcp_host", "scenario": "Using the LocalStack App Inspector tool, first check whether App Inspector is currently enabled, enable it only if it is disabled, and then list the most recent App Inspector traces.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -113,7 +133,11 @@ "description": "localstack-iam-policy-analyzer: read mode, then set SOFT_MODE.", "mode": "mcp_host", "scenario": "Check the current IAM enforcement mode in LocalStack using the IAM policy analyzer, then set the enforcement mode to SOFT_MODE with the set-mode action. Perform the set-mode call even if the current mode is already SOFT_MODE, so the setting is explicitly confirmed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -139,7 +163,11 @@ "description": "localstack-state-management: read-only inspect of runtime state.", "mode": "mcp_host", "scenario": "Using the LocalStack state management tool, inspect the in-memory state of the running LocalStack instance and tell me which AWS services currently hold state. Do not export, import, or reset anything.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -161,8 +189,8 @@ "mode": "mcp_host", "scenario": "Using the LocalStack Cloud Pods tool, save the current LocalStack state to a remote Cloud Pod named mcptester-e2e-pod. If saving fails because a pod with that name already exists, delete the existing pod and save again. If a save, load, or delete attempt times out, retry that same operation once before giving up. After saving, load the Cloud Pod mcptester-e2e-pod back into LocalStack. Finally, delete the Cloud Pod mcptester-e2e-pod so nothing is left behind.", "mcpHostConfig": { - "provider": "google", - "model": "gemini-2.5-flash", + "provider": "openrouter", + "model": "google/gemini-2.5-flash", "temperature": 0, "maxToolCalls": 15 }, @@ -196,7 +224,11 @@ "description": "localstack-chaos-injector: inject, verify, and clear latency.", "mode": "mcp_host", "scenario": "Use the LocalStack chaos injector to add 100 milliseconds of network latency to LocalStack, then read back the currently configured latency to verify it was applied, and finally clear the latency so LocalStack returns to normal.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -227,7 +259,11 @@ "description": "localstack-extensions: browse the marketplace catalog.", "mode": "mcp_host", "scenario": "Browse the LocalStack Extensions marketplace and show me which extensions are available to install. I want the marketplace catalog of available extensions, not the list of extensions that are already installed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -248,7 +284,11 @@ "description": "localstack-aws-replicator: read-only supported-resources and job listing.", "mode": "mcp_host", "scenario": "Using the LocalStack AWS replicator tool, first show me which resource types the AWS replicator supports for replication into LocalStack, and then list any replication jobs that currently exist. Do not start any replication job.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -274,7 +314,11 @@ "description": "localstack-ephemeral-instances: read-only listing.", "mode": "mcp_host", "scenario": "List all LocalStack Ephemeral Instances that currently exist in my LocalStack Cloud account using the ephemeral instances tool, and summarize their names and statuses. Do not create, modify, or delete any instance.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -295,7 +339,11 @@ "description": "localstack-docs: documentation search.", "mode": "mcp_host", "scenario": "Search the official LocalStack documentation for how IAM policy enforcement modes work in LocalStack and summarize what the docs say. Use the LocalStack documentation search tool, not the IAM tools that act on the running instance.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -311,8 +359,8 @@ "mode": "mcp_host", "scenario": "Work with the LocalStack Snowflake emulator: 1) Using the LocalStack management tool with service set to snowflake, start the Snowflake emulator. If it reports that it is already running, continue to step 2. If starting fails because a LocalStack container is already running, that container is the AWS emulator: stop it with the management tool, then start the Snowflake emulator with service snowflake again. If a start attempt times out, keep checking the Snowflake status with the management tool (service snowflake) until it reports ready — do not give up on a timeout. 2) Use the LocalStack Snowflake client to check the connection to the emulator. 3) Use the LocalStack Snowflake client to execute the SQL file data/sample-sql/snowflake_test.sql. 4) Run the query SHOW TABLES IN demo_db.public and tell me whether the customers table exists.", "mcpHostConfig": { - "provider": "google", - "model": "gemini-2.5-flash", + "provider": "openrouter", + "model": "google/gemini-2.5-flash", "temperature": 0, "maxToolCalls": 15 }, diff --git a/package.json b/package.json index 1dec23d..1554eaf 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "format": "prettier --write .", "test": "jest", "test:mcp:direct": "yarn build && playwright test -c playwright.config.mjs tests/mcp/direct.spec.mjs tests/mcp/azure-offline.spec.mjs", - "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals-gemini.spec.mjs", + "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals.spec.mjs", "test:mcp": "yarn test:mcp:direct && yarn test:mcp:evals" }, "dependencies": { @@ -26,10 +26,12 @@ "devDependencies": { "@clack/prompts": "^1.8.1", "@gleanwork/mcp-server-tester": "1.1.1", + "@openrouter/ai-sdk-provider": "^2.0.0", "@playwright/test": "^1.64.1", "@types/dockerode": "^4.0.2", "@types/jest": "^30.0.0", "@xmcp-dev/compiler": "^1.9.0", + "ai": "^6.0.0", "audit-ci": "^7.1.0", "esbuild": "^0.28.2", "eslint-config-prettier": "^10.1.8", diff --git a/tests/mcp/evals-gemini.spec.mjs b/tests/mcp/evals.spec.mjs similarity index 71% rename from tests/mcp/evals-gemini.spec.mjs rename to tests/mcp/evals.spec.mjs index 528c337..6a368dc 100644 --- a/tests/mcp/evals-gemini.spec.mjs +++ b/tests/mcp/evals.spec.mjs @@ -9,14 +9,21 @@ function requireEnv(name) { return value; } -test.describe("Gemini comprehensive eval", () => { +test.describe("OpenRouter comprehensive eval", () => { test.describe.configure({ timeout: 5400000 }); test("single comprehensive eval dataset passes", async ({ mcp }, testInfo) => { - requireEnv("GOOGLE_GENERATIVE_AI_API_KEY"); + requireEnv("OPENROUTER_API_KEY"); requireEnv("LOCALSTACK_AUTH_TOKEN"); - const dataset = await loadEvalDataset("./data/evals/gemini-comprehensive.json"); + const dataset = await loadEvalDataset("./data/evals/comprehensive.json"); + // EVAL_MODEL swaps the OpenRouter model for every case, e.g. anthropic/claude-sonnet-4.5. + const model = process.env.EVAL_MODEL?.trim(); + if (model) { + for (const evalCase of dataset.cases) { + if (evalCase.mcpHostConfig) evalCase.mcpHostConfig.model = model; + } + } const result = await runEvalDataset({ dataset }, { mcp, testInfo }); const caseResults = result.caseResults || []; const passed = caseResults.filter((entry) => entry?.pass === true).length; diff --git a/yarn.lock b/yarn.lock index 7546b5e..56a2764 100644 --- a/yarn.lock +++ b/yarn.lock @@ -37,6 +37,15 @@ "@ai-sdk/provider-utils" "4.0.40" "@vercel/oidc" "3.2.0" +"@ai-sdk/gateway@3.0.225": + version "3.0.225" + resolved "https://registry.yarnpkg.com/@ai-sdk/gateway/-/gateway-3.0.225.tgz#5e79b46d73c6bbba7dc84bbfb24fc77f14b708a0" + integrity sha512-WSW7Hk/YER3AqF/ONyh/UounSRDfJ/J4VtVYG75tMcO4/s8DFACqHLkfeMSgfWcmujU1+L3liAlcixwn24rnCw== + dependencies: + "@ai-sdk/provider" "3.0.19" + "@ai-sdk/provider-utils" "4.0.63" + "@vercel/oidc" "3.2.0" + "@ai-sdk/google-vertex@^4.0.68": version "4.0.169" resolved "https://registry.yarnpkg.com/@ai-sdk/google-vertex/-/google-vertex-4.0.169.tgz#9f73aca62e0ac808e80d40c553f8bbf3b8ec649b" @@ -90,6 +99,16 @@ "@standard-schema/spec" "^1.1.0" eventsource-parser "^3.0.8" +"@ai-sdk/provider-utils@4.0.63": + version "4.0.63" + resolved "https://registry.yarnpkg.com/@ai-sdk/provider-utils/-/provider-utils-4.0.63.tgz#bd7b6443981ac37860b5c1e0156bf2ff0c0b4fbe" + integrity sha512-QSrfUn7pnb+2EZLLtpkcY6208ZZVi15QEJFyp/1jAYEH2UecthgAsyYWOJbpMSLtraviCByd6ii2meISIhkTZA== + dependencies: + "@ai-sdk/provider" "3.0.19" + "@standard-schema/spec" "^1.1.0" + eventsource-parser "^3.0.8" + undici "^6.28.0" + "@ai-sdk/provider@3.0.14": version "3.0.14" resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-3.0.14.tgz#5893a90e0b5355ec6a5c148f06bd7f08355c1a6f" @@ -97,6 +116,13 @@ dependencies: json-schema "^0.4.0" +"@ai-sdk/provider@3.0.19": + version "3.0.19" + resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-3.0.19.tgz#9adebbcb183f41d6f726f00efd528485b58d56d3" + integrity sha512-5lWq7zSfRaU5ylDESn3sDQjgGl2AI1vjZKO/6JY3aFW70ct4zcP1kN+ZUYOuTXr28fVsYbyu2YFhddLdIgQFZQ== + dependencies: + json-schema "^0.4.0" + "@ai-sdk/xai@^3.0.0": version "3.0.111" resolved "https://registry.yarnpkg.com/@ai-sdk/xai/-/xai-3.0.111.tgz#d83f0c6f76de3eb718487a72d181edfb6258a030" @@ -1942,6 +1968,16 @@ agent-base@^7.1.2: resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8" integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== +ai@^6.0.0: + version "6.0.319" + resolved "https://registry.yarnpkg.com/ai/-/ai-6.0.319.tgz#d46fff7d0167e96eee1c0f286ac3380502886c97" + integrity sha512-1sMXGSM50gL5R7bJjQrYwqyhdS8YNZgW4LuPZeZ3G+sJ0vHyVyGIgzb59XA3dbgwKxLjvY1hjT+SFAAypfsDsg== + dependencies: + "@ai-sdk/gateway" "3.0.225" + "@ai-sdk/provider" "3.0.19" + "@ai-sdk/provider-utils" "4.0.63" + "@opentelemetry/api" "^1.9.0" + ai@^6.0.97: version "6.0.235" resolved "https://registry.yarnpkg.com/ai/-/ai-6.0.235.tgz#78d0836636fa06ac995cef501f6d0feb0d11ecb8" @@ -5292,6 +5328,11 @@ undici-types@~8.3.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-8.3.0.tgz#44e9fc9f3244648cdea35e4f9bb2d681e9410809" integrity sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ== +undici@^6.28.0: + version "6.29.1" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.29.1.tgz#a3317c5df0236e0633727d9a7bd4f7ed195d8575" + integrity sha512-EvRlUt/QqAEvXdQeRMYUfwOQsfzpDztykrG2owGEE/rBOBoYv9uF39ZzsNDh4lTFHhTr2XOZjKkYgueD12ZmGA== + undici@^7.24.0: version "7.30.1" resolved "https://registry.yarnpkg.com/undici/-/undici-7.30.1.tgz#360b3de104b94abc52729aa1a4a07e027aa71f73"