From d78c2c83227c68d421423bd9b254543eff8b4424 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 14:28:29 +0200 Subject: [PATCH 01/10] feat(ci): coverage reporting with a committed baseline ratchet Adds a `test:coverage` script to every package and a `Coverage` job to the PR workflow that fails when coverage drops. Coverage was previously unrunnable: `@vitest/coverage-v8` was pinned to 4.0.14 in all 50 packages while `vitest` resolved to 4.1.10, so vitest could not resolve the provider at all (`ERR_MODULE_NOT_FOUND`). The two existing `test:coverage` scripts (ai, ai-client) could not have worked. Aligning the provider on 4.1.10 fixes it. - every package gains `test:coverage`, measured over `src/**` with json-summary + text-summary reporters (flags live in the script so no per-package vitest config changes were needed) - `scripts/coverage-check.mjs` diffs the fresh summaries against `coverage-baseline.json` and exits non-zero on a >0.5pp drop in statements/branches/functions/lines; unmeasured packages are skipped rather than counted as 0% - root gains `test:coverage:all`, `test:coverage:check`, `test:coverage:update` - `coverage-baseline.json` seeded from a full run of the 48 packages whose suites pass on this machine - CONTRIBUTING.md documents the workflow and the two known gaps --- .github/workflows/pr.yml | 17 + CLAUDE.md | 10 +- CONTRIBUTING.md | 49 ++- coverage-baseline.json | 290 ++++++++++++++++++ package.json | 3 + packages/ai-acp/package.json | 1 + packages/ai-angular/package.json | 1 + packages/ai-anthropic/package.json | 1 + packages/ai-bedrock/package.json | 1 + packages/ai-claude-code/package.json | 1 + packages/ai-client/package.json | 4 +- packages/ai-code-mode-snippets/package.json | 1 + packages/ai-code-mode/package.json | 1 + packages/ai-codex/package.json | 1 + packages/ai-devtools/package.json | 1 + packages/ai-durable-stream/package.json | 1 + packages/ai-elevenlabs/package.json | 1 + packages/ai-event-client/package.json | 1 + packages/ai-fal/package.json | 1 + packages/ai-gemini/package.json | 1 + packages/ai-grok-build/package.json | 1 + packages/ai-grok/package.json | 1 + packages/ai-groq/package.json | 1 + packages/ai-isolate-cloudflare/package.json | 1 + packages/ai-isolate-node/package.json | 1 + packages/ai-isolate-quickjs/package.json | 1 + packages/ai-mcp/package.json | 2 +- packages/ai-memory/package.json | 1 + packages/ai-mistral/package.json | 1 + packages/ai-ollama/package.json | 1 + packages/ai-openai/package.json | 1 + packages/ai-opencode/package.json | 1 + packages/ai-openrouter/package.json | 1 + packages/ai-persistence/package.json | 1 + packages/ai-preact/package.json | 1 + packages/ai-react-ui/package.json | 1 + packages/ai-react/package.json | 1 + packages/ai-sandbox-cloudflare/package.json | 1 + packages/ai-sandbox-daytona/package.json | 1 + packages/ai-sandbox-docker/package.json | 1 + .../ai-sandbox-local-process/package.json | 1 + packages/ai-sandbox-sprites/package.json | 1 + packages/ai-sandbox-vercel/package.json | 1 + packages/ai-sandbox/package.json | 1 + packages/ai-solid-ui/package.json | 1 + packages/ai-solid/package.json | 1 + packages/ai-svelte/package.json | 1 + packages/ai-utils/package.json | 1 + packages/ai-vue-ui/package.json | 1 + packages/ai-vue/package.json | 1 + packages/ai/package.json | 4 +- packages/openai-base/package.json | 1 + packages/preact-ai-devtools/package.json | 1 + packages/react-ai-devtools/package.json | 1 + packages/solid-ai-devtools/package.json | 1 + scripts/coverage-check.mjs | 151 +++++++++ 56 files changed, 556 insertions(+), 21 deletions(-) create mode 100644 coverage-baseline.json create mode 100644 scripts/coverage-check.mjs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a83eb33a81..a0fcde8152 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -31,6 +31,23 @@ jobs: main-branch-name: main - name: Run Checks run: pnpm run test:pr + coverage: + name: Coverage + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Tools + uses: TanStack/config/.github/setup@190f659075ff0845850e330883eb26d7ffd0671f # main + - name: Get base and head commits for `nx affected` + uses: nrwl/nx-set-shas@3e9ad7370203c1e93d109be57f3b72eb0eb511b1 # v4.4.0 + with: + main-branch-name: main + - name: Check Coverage + run: pnpm run test:coverage:check preview: name: Preview runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index deb7b04a42..6c6f2b03d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,10 @@ pnpm test:lib:dev # Watch mode for unit tests pnpm test:oxlint # Lint affected packages (oxlint, incl. type-aware) pnpm test:types # Type check affected packages pnpm test:build # Verify build artifacts with publint -pnpm test:coverage # Generate coverage reports +pnpm test:coverage # Generate coverage reports (affected) +pnpm test:coverage:all # Generate coverage reports (every package) +pnpm test:coverage:check # Coverage + fail if it dropped vs coverage-baseline.json +pnpm test:coverage:update # Re-baseline coverage-baseline.json pnpm test:knip # Check for unused dependencies pnpm test:sherif # Check pnpm workspace consistency pnpm test:docs # Verify documentation links @@ -318,7 +321,10 @@ Each package uses `exports` field in package.json for subpath exports (e.g., `@t - Unit tests in `*.test.ts` files alongside source - Uses Vitest with happy-dom for DOM testing -- Test coverage via `pnpm test:coverage` +- Test coverage via `pnpm test:coverage`; the `Coverage` CI job runs + `pnpm test:coverage:check`, which fails when any package drops more than + 0.5pp below the percentages committed in `coverage-baseline.json`. Re-baseline + intentional drops with `pnpm test:coverage:update`. See CONTRIBUTING.md. - **E2E tests are mandatory** — see E2E Testing section below ### E2E Testing (REQUIRED) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02200f4bf1..57662287c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,23 +71,44 @@ The workflow pushes with `GITHUB_TOKEN`, so GitHub does not start Test / E2E on All commands are run from the repo root. Nx handles affected detection and caching. -| Goal | Command | -| ----------------------------- | ------------------- | -| Run unit tests (affected) | `pnpm test:lib` | -| Watch unit tests | `pnpm test:lib:dev` | -| Type-check (affected) | `pnpm test:types` | -| Lint (affected) | `pnpm test:eslint` | -| Verify build artifacts | `pnpm test:build` | -| Format the repo | `pnpm format` | -| Build (affected) | `pnpm build` | -| Build everything | `pnpm build:all` | -| Run the full CI suite locally | `pnpm test` | -| Run the affected-PR check | `pnpm test:pr` | -| E2E suite | `pnpm test:e2e` | -| E2E with Playwright UI | `pnpm test:e2e:ui` | +| Goal | Command | +| ----------------------------- | --------------------------- | +| Run unit tests (affected) | `pnpm test:lib` | +| Watch unit tests | `pnpm test:lib:dev` | +| Coverage (affected) | `pnpm test:coverage` | +| Coverage for every package | `pnpm test:coverage:all` | +| Coverage + regression check | `pnpm test:coverage:check` | +| Re-baseline coverage | `pnpm test:coverage:update` | +| Type-check (affected) | `pnpm test:types` | +| Lint (affected) | `pnpm test:eslint` | +| Verify build artifacts | `pnpm test:build` | +| Format the repo | `pnpm format` | +| Build (affected) | `pnpm build` | +| Build everything | `pnpm build:all` | +| Run the full CI suite locally | `pnpm test` | +| Run the affected-PR check | `pnpm test:pr` | +| E2E suite | `pnpm test:e2e` | +| E2E with Playwright UI | `pnpm test:e2e:ui` | Working on a single package? `cd packages/` and use its scripts directly (`pnpm test:lib`, `pnpm test:types`, etc.). +## Coverage + +Every package has a `test:coverage` script (`vitest run --coverage`, measured over `src/**` with the v8 provider). Per-package percentages are committed to `coverage-baseline.json` at the repo root. + +The `Coverage` job on every PR runs `pnpm test:coverage:check`, which measures the affected packages and fails if any metric (statements, branches, functions, lines) drops more than 0.5 percentage points below the baseline. Packages that weren't affected are skipped, not treated as 0%. + +When a drop is intentional — deleting well-tested code, for instance — re-baseline and commit the result: + +```bash +pnpm test:coverage:update +``` + +Two known limitations: + +- Uncovered `.tsx` files can't be remapped by the coverage provider and are dropped from the report with a `Failed to parse ... Excluding it from coverage` warning. `.tsx` files that tests _do_ load are measured normally, so the UI packages read higher than their real coverage. +- `preact-ai-devtools`, `react-ai-devtools`, and `solid-ai-devtools` have no tests and sit at 0%. + ## TypeScript configuration There is a single `tsconfig.base.json` at the repo root with the shared `compilerOptions`. Every package extends it and overrides only what's unique to that package (e.g. `outDir`, JSX runtime, framework lib). diff --git a/coverage-baseline.json b/coverage-baseline.json new file mode 100644 index 0000000000..f4c280aeb4 --- /dev/null +++ b/coverage-baseline.json @@ -0,0 +1,290 @@ +{ + "ai": { + "statements": 88.66, + "branches": 77.81, + "functions": 90.11, + "lines": 89.92 + }, + "ai-angular": { + "statements": 76.57, + "branches": 60.81, + "functions": 63.33, + "lines": 80.17 + }, + "ai-anthropic": { + "statements": 83.87, + "branches": 74.88, + "functions": 85.24, + "lines": 84.4 + }, + "ai-bedrock": { + "statements": 83.7, + "branches": 74.31, + "functions": 82.35, + "lines": 84.16 + }, + "ai-claude-code": { + "statements": 83.2, + "branches": 67.06, + "functions": 86, + "lines": 85.59 + }, + "ai-client": { + "statements": 84.97, + "branches": 76.47, + "functions": 83.01, + "lines": 86.47 + }, + "ai-code-mode": { + "statements": 91.58, + "branches": 85.07, + "functions": 86, + "lines": 92.53 + }, + "ai-code-mode-skills": { + "statements": 80.68, + "branches": 72.75, + "functions": 86.36, + "lines": 81.59 + }, + "ai-codex": { + "statements": 91.72, + "branches": 80.49, + "functions": 89.13, + "lines": 92.61 + }, + "ai-devtools": { + "statements": 80.25, + "branches": 69.87, + "functions": 82.85, + "lines": 81.77 + }, + "ai-durable-stream": { + "statements": 87.42, + "branches": 80.97, + "functions": 93.18, + "lines": 89.7 + }, + "ai-elevenlabs": { + "statements": 65.26, + "branches": 57.6, + "functions": 50.68, + "lines": 66.42 + }, + "ai-event-client": { + "statements": 28.8, + "branches": 19.69, + "functions": 44, + "lines": 29.16 + }, + "ai-fal": { + "statements": 90.12, + "branches": 83.05, + "functions": 90.47, + "lines": 90.19 + }, + "ai-gemini": { + "statements": 59.35, + "branches": 60.25, + "functions": 51.24, + "lines": 60 + }, + "ai-grok": { + "statements": 41.78, + "branches": 43.95, + "functions": 51.63, + "lines": 42.56 + }, + "ai-grok-build": { + "statements": 69.63, + "branches": 51.47, + "functions": 71.02, + "lines": 71.01 + }, + "ai-groq": { + "statements": 39.08, + "branches": 42.6, + "functions": 59.45, + "lines": 40.74 + }, + "ai-isolate-cloudflare": { + "statements": 94.11, + "branches": 70.66, + "functions": 100, + "lines": 93.96 + }, + "ai-isolate-node": { + "statements": 84, + "branches": 43.39, + "functions": 100, + "lines": 83.78 + }, + "ai-isolate-quickjs": { + "statements": 87.23, + "branches": 65.07, + "functions": 88.23, + "lines": 89.05 + }, + "ai-mcp": { + "statements": 95.6, + "branches": 84.23, + "functions": 98.63, + "lines": 98.8 + }, + "ai-memory": { + "statements": 71.42, + "branches": 52.53, + "functions": 77.69, + "lines": 73.21 + }, + "ai-mistral": { + "statements": 68, + "branches": 55.64, + "functions": 76.47, + "lines": 69.16 + }, + "ai-ollama": { + "statements": 28.31, + "branches": 68.12, + "functions": 86.95, + "lines": 28.15 + }, + "ai-openai": { + "statements": 48, + "branches": 50.58, + "functions": 50, + "lines": 48.31 + }, + "ai-opencode": { + "statements": 61.79, + "branches": 52.9, + "functions": 56.62, + "lines": 63.67 + }, + "ai-openrouter": { + "statements": 47.35, + "branches": 49.65, + "functions": 81.7, + "lines": 47.38 + }, + "ai-persistence": { + "statements": 92.15, + "branches": 79.17, + "functions": 99.4, + "lines": 95.44 + }, + "ai-preact": { + "statements": 88.99, + "branches": 78.37, + "functions": 92.85, + "lines": 92.55 + }, + "ai-react": { + "statements": 77.82, + "branches": 66.03, + "functions": 75.86, + "lines": 77.75 + }, + "ai-react-ui": { + "statements": 12.14, + "branches": 5.47, + "functions": 14.28, + "lines": 11.42 + }, + "ai-sandbox-cloudflare": { + "statements": 37.92, + "branches": 32.13, + "functions": 41.98, + "lines": 38.4 + }, + "ai-sandbox-daytona": { + "statements": 12.42, + "branches": 3.07, + "functions": 6.25, + "lines": 13.54 + }, + "ai-sandbox-docker": { + "statements": 62.01, + "branches": 47.36, + "functions": 56.6, + "lines": 64.32 + }, + "ai-sandbox-local-process": { + "statements": 87.71, + "branches": 66.66, + "functions": 89.47, + "lines": 91.33 + }, + "ai-sandbox-sprites": { + "statements": 70.75, + "branches": 55.46, + "functions": 70.87, + "lines": 73.88 + }, + "ai-sandbox-vercel": { + "statements": 5.44, + "branches": 15.47, + "functions": 2.27, + "lines": 6.01 + }, + "ai-solid": { + "statements": 95.27, + "branches": 70.13, + "functions": 93.87, + "lines": 95.81 + }, + "ai-solid-ui": { + "statements": 2.87, + "branches": 3.63, + "functions": 0.94, + "lines": 4.02 + }, + "ai-svelte": { + "statements": 73.51, + "branches": 59.25, + "functions": 72.99, + "lines": 74.72 + }, + "ai-utils": { + "statements": 77.33, + "branches": 75, + "functions": 90.9, + "lines": 77.61 + }, + "ai-vue": { + "statements": 78.18, + "branches": 56.42, + "functions": 81.91, + "lines": 83.95 + }, + "ai-vue-ui": { + "statements": 4.43, + "branches": 4.41, + "functions": 2.56, + "lines": 4.76 + }, + "openai-base": { + "statements": 73.66, + "branches": 57.91, + "functions": 63.82, + "lines": 74.15 + }, + "preact-ai-devtools": { + "statements": 0, + "branches": 0, + "functions": 100, + "lines": 0 + }, + "react-ai-devtools": { + "statements": 0, + "branches": 0, + "functions": 100, + "lines": 0 + }, + "solid-ai-devtools": { + "statements": 0, + "branches": 0, + "functions": 100, + "lines": 0 + } +} diff --git a/package.json b/package.json index bf7b24b987..be2ab17c90 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,9 @@ "test:lib": "nx affected --targets=test:lib --exclude=examples/**,testing/**", "test:lib:dev": "pnpm test:lib && nx watch --all -- pnpm test:lib", "test:coverage": "nx affected --targets=test:coverage --exclude=examples/**,testing/**", + "test:coverage:all": "nx run-many --targets=test:coverage --exclude=examples/**,testing/**", + "test:coverage:check": "pnpm run test:coverage && node scripts/coverage-check.mjs", + "test:coverage:update": "pnpm run test:coverage:all && node scripts/coverage-check.mjs --update", "test:build": "nx affected --target=test:build --exclude=examples/**,testing/**", "test:types": "nx affected --targets=test:types --exclude=examples/**,testing/**", "test:knip": "knip", diff --git a/packages/ai-acp/package.json b/packages/ai-acp/package.json index 614118680e..bd47299a07 100644 --- a/packages/ai-acp/package.json +++ b/packages/ai-acp/package.json @@ -38,6 +38,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run --testTimeout=30000", "test:lib:dev": "vitest --watch", "test:types": "tsc" diff --git a/packages/ai-angular/package.json b/packages/ai-angular/package.json index e18f550973..5fe4fa1bca 100644 --- a/packages/ai-angular/package.json +++ b/packages/ai-angular/package.json @@ -46,6 +46,7 @@ "scripts": { "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "vitest", "test:types": "tsc", diff --git a/packages/ai-anthropic/package.json b/packages/ai-anthropic/package.json index 913893972c..f729726ae6 100644 --- a/packages/ai-anthropic/package.json +++ b/packages/ai-anthropic/package.json @@ -61,6 +61,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-bedrock/package.json b/packages/ai-bedrock/package.json index e8e6d603ff..d870fbebef 100644 --- a/packages/ai-bedrock/package.json +++ b/packages/ai-bedrock/package.json @@ -32,6 +32,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-claude-code/package.json b/packages/ai-claude-code/package.json index 8356574c94..c707ffd0e1 100644 --- a/packages/ai-claude-code/package.json +++ b/packages/ai-claude-code/package.json @@ -46,6 +46,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-client/package.json b/packages/ai-client/package.json index 476bed9921..7781a8493e 100644 --- a/packages/ai-client/package.json +++ b/packages/ai-client/package.json @@ -57,9 +57,9 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", - "test:coverage": "vitest run --coverage", - "test:coverage:watch": "vitest --coverage --watch", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage:watch": "vitest --coverage --watch", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-code-mode-snippets/package.json b/packages/ai-code-mode-snippets/package.json index e0480b2555..61701e622e 100644 --- a/packages/ai-code-mode-snippets/package.json +++ b/packages/ai-code-mode-snippets/package.json @@ -44,6 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-code-mode/package.json b/packages/ai-code-mode/package.json index 4c2d61cd0d..d29d24fd57 100644 --- a/packages/ai-code-mode/package.json +++ b/packages/ai-code-mode/package.json @@ -41,6 +41,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-codex/package.json b/packages/ai-codex/package.json index 41b7de195d..f7278067ba 100644 --- a/packages/ai-codex/package.json +++ b/packages/ai-codex/package.json @@ -45,6 +45,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-devtools/package.json b/packages/ai-devtools/package.json index f2a29d58fa..65b8e060b6 100644 --- a/packages/ai-devtools/package.json +++ b/packages/ai-devtools/package.json @@ -73,6 +73,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-durable-stream/package.json b/packages/ai-durable-stream/package.json index 34f67dc99d..f1e453e788 100644 --- a/packages/ai-durable-stream/package.json +++ b/packages/ai-durable-stream/package.json @@ -36,6 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-elevenlabs/package.json b/packages/ai-elevenlabs/package.json index 38619f88d1..ff667491e6 100644 --- a/packages/ai-elevenlabs/package.json +++ b/packages/ai-elevenlabs/package.json @@ -57,6 +57,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-event-client/package.json b/packages/ai-event-client/package.json index 2ac7f20368..24fbe5a54d 100644 --- a/packages/ai-event-client/package.json +++ b/packages/ai-event-client/package.json @@ -37,6 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-fal/package.json b/packages/ai-fal/package.json index 4912e76acd..f885d8fbab 100644 --- a/packages/ai-fal/package.json +++ b/packages/ai-fal/package.json @@ -40,6 +40,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-gemini/package.json b/packages/ai-gemini/package.json index 96c5f126b7..4b953b2b43 100644 --- a/packages/ai-gemini/package.json +++ b/packages/ai-gemini/package.json @@ -48,6 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-grok-build/package.json b/packages/ai-grok-build/package.json index e64b3d7026..a2492da7d8 100644 --- a/packages/ai-grok-build/package.json +++ b/packages/ai-grok-build/package.json @@ -46,6 +46,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-grok/package.json b/packages/ai-grok/package.json index b0a1c0c92b..4768059fb2 100644 --- a/packages/ai-grok/package.json +++ b/packages/ai-grok/package.json @@ -48,6 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-groq/package.json b/packages/ai-groq/package.json index 9628b55d48..0c878c17ef 100644 --- a/packages/ai-groq/package.json +++ b/packages/ai-groq/package.json @@ -44,6 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-isolate-cloudflare/package.json b/packages/ai-isolate-cloudflare/package.json index dbc798ac08..f03bd01e7b 100644 --- a/packages/ai-isolate-cloudflare/package.json +++ b/packages/ai-isolate-cloudflare/package.json @@ -48,6 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-isolate-node/package.json b/packages/ai-isolate-node/package.json index 5a3d6103ff..cf5d12368f 100644 --- a/packages/ai-isolate-node/package.json +++ b/packages/ai-isolate-node/package.json @@ -40,6 +40,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-isolate-quickjs/package.json b/packages/ai-isolate-quickjs/package.json index 6c1acdb344..e888ae6763 100644 --- a/packages/ai-isolate-quickjs/package.json +++ b/packages/ai-isolate-quickjs/package.json @@ -37,6 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-mcp/package.json b/packages/ai-mcp/package.json index efd68aa30a..d2f48ab202 100644 --- a/packages/ai-mcp/package.json +++ b/packages/ai-mcp/package.json @@ -56,8 +56,8 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", - "test:coverage": "vitest run --coverage", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-memory/package.json b/packages/ai-memory/package.json index cf0cf91b33..855454d941 100644 --- a/packages/ai-memory/package.json +++ b/packages/ai-memory/package.json @@ -50,6 +50,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-mistral/package.json b/packages/ai-mistral/package.json index 31e3e0df90..c92a8db53c 100644 --- a/packages/ai-mistral/package.json +++ b/packages/ai-mistral/package.json @@ -44,6 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-ollama/package.json b/packages/ai-ollama/package.json index 9fa30ae1ce..d5b4791083 100644 --- a/packages/ai-ollama/package.json +++ b/packages/ai-ollama/package.json @@ -40,6 +40,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-openai/package.json b/packages/ai-openai/package.json index 305a7cddc9..4bb2a4e59f 100644 --- a/packages/ai-openai/package.json +++ b/packages/ai-openai/package.json @@ -48,6 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-opencode/package.json b/packages/ai-opencode/package.json index 0432b3e899..3453ac76a1 100644 --- a/packages/ai-opencode/package.json +++ b/packages/ai-opencode/package.json @@ -44,6 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-openrouter/package.json b/packages/ai-openrouter/package.json index 5c23a8e6de..c3314a0379 100644 --- a/packages/ai-openrouter/package.json +++ b/packages/ai-openrouter/package.json @@ -52,6 +52,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-persistence/package.json b/packages/ai-persistence/package.json index 9d16df1b33..afa9958538 100644 --- a/packages/ai-persistence/package.json +++ b/packages/ai-persistence/package.json @@ -43,6 +43,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-preact/package.json b/packages/ai-preact/package.json index 2a203618b0..ea4e872eeb 100644 --- a/packages/ai-preact/package.json +++ b/packages/ai-preact/package.json @@ -38,6 +38,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-react-ui/package.json b/packages/ai-react-ui/package.json index a49b3d1525..85092c76ac 100644 --- a/packages/ai-react-ui/package.json +++ b/packages/ai-react-ui/package.json @@ -34,6 +34,7 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-react/package.json b/packages/ai-react/package.json index c1f528cc8b..4f2abda011 100644 --- a/packages/ai-react/package.json +++ b/packages/ai-react/package.json @@ -38,6 +38,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-sandbox-cloudflare/package.json b/packages/ai-sandbox-cloudflare/package.json index 1c233515e2..7aa4edb1da 100644 --- a/packages/ai-sandbox-cloudflare/package.json +++ b/packages/ai-sandbox-cloudflare/package.json @@ -46,6 +46,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-daytona/package.json b/packages/ai-sandbox-daytona/package.json index da599519f1..cbab67b268 100644 --- a/packages/ai-sandbox-daytona/package.json +++ b/packages/ai-sandbox-daytona/package.json @@ -37,6 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-docker/package.json b/packages/ai-sandbox-docker/package.json index 29f2575ea7..4368b4f273 100644 --- a/packages/ai-sandbox-docker/package.json +++ b/packages/ai-sandbox-docker/package.json @@ -38,6 +38,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-local-process/package.json b/packages/ai-sandbox-local-process/package.json index a0336621c6..7596d7939c 100644 --- a/packages/ai-sandbox-local-process/package.json +++ b/packages/ai-sandbox-local-process/package.json @@ -36,6 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-sprites/package.json b/packages/ai-sandbox-sprites/package.json index a33ffcc967..5dbebd0580 100644 --- a/packages/ai-sandbox-sprites/package.json +++ b/packages/ai-sandbox-sprites/package.json @@ -38,6 +38,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-vercel/package.json b/packages/ai-sandbox-vercel/package.json index 0751d96072..529b063cc3 100644 --- a/packages/ai-sandbox-vercel/package.json +++ b/packages/ai-sandbox-vercel/package.json @@ -37,6 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox/package.json b/packages/ai-sandbox/package.json index a371986999..a1169bcedf 100644 --- a/packages/ai-sandbox/package.json +++ b/packages/ai-sandbox/package.json @@ -50,6 +50,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-solid-ui/package.json b/packages/ai-solid-ui/package.json index 81d69c7582..491bb22994 100644 --- a/packages/ai-solid-ui/package.json +++ b/packages/ai-solid-ui/package.json @@ -36,6 +36,7 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-solid/package.json b/packages/ai-solid/package.json index a58ca6dadf..9fbe78d4f8 100644 --- a/packages/ai-solid/package.json +++ b/packages/ai-solid/package.json @@ -33,6 +33,7 @@ "scripts": { "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-svelte/package.json b/packages/ai-svelte/package.json index bbb918123b..4318d2314d 100644 --- a/packages/ai-svelte/package.json +++ b/packages/ai-svelte/package.json @@ -36,6 +36,7 @@ "clean": "rm -rf ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "svelte-check", diff --git a/packages/ai-utils/package.json b/packages/ai-utils/package.json index f3c738318d..8f052e4d72 100644 --- a/packages/ai-utils/package.json +++ b/packages/ai-utils/package.json @@ -36,6 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-vue-ui/package.json b/packages/ai-vue-ui/package.json index 64ea5e45d7..ab17e2c7f3 100644 --- a/packages/ai-vue-ui/package.json +++ b/packages/ai-vue-ui/package.json @@ -35,6 +35,7 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "vue-tsc --noEmit" diff --git a/packages/ai-vue/package.json b/packages/ai-vue/package.json index 0b9d92c66e..a688a96882 100644 --- a/packages/ai-vue/package.json +++ b/packages/ai-vue/package.json @@ -33,6 +33,7 @@ "scripts": { "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai/package.json b/packages/ai/package.json index 7750bbf783..0f91911e69 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -72,9 +72,9 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", - "test:coverage": "vitest run --coverage", - "test:coverage:watch": "vitest --coverage --watch", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage:watch": "vitest --coverage --watch", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/openai-base/package.json b/packages/openai-base/package.json index bdb1e0b9e9..98bb558301 100644 --- a/packages/openai-base/package.json +++ b/packages/openai-base/package.json @@ -36,6 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/preact-ai-devtools/package.json b/packages/preact-ai-devtools/package.json index 5d0bc3ee25..11fda05472 100644 --- a/packages/preact-ai-devtools/package.json +++ b/packages/preact-ai-devtools/package.json @@ -21,6 +21,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/react-ai-devtools/package.json b/packages/react-ai-devtools/package.json index a2951c2ad0..1ab4b4d461 100644 --- a/packages/react-ai-devtools/package.json +++ b/packages/react-ai-devtools/package.json @@ -21,6 +21,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/solid-ai-devtools/package.json b/packages/solid-ai-devtools/package.json index 70ce24ca58..8b9ffdbe03 100644 --- a/packages/solid-ai-devtools/package.json +++ b/packages/solid-ai-devtools/package.json @@ -42,6 +42,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/scripts/coverage-check.mjs b/scripts/coverage-check.mjs new file mode 100644 index 0000000000..947c10972f --- /dev/null +++ b/scripts/coverage-check.mjs @@ -0,0 +1,151 @@ +// Compares the coverage produced by `test:coverage` against the committed +// baseline in coverage-baseline.json and fails when a package regressed. +// +// Usage: +// node scripts/coverage-check.mjs # compare, exit 1 on a drop +// node scripts/coverage-check.mjs --update # rewrite the baseline +// +// ponytail: a committed baseline file is the whole ratchet. No coverage +// service, no PR comments, no historical database. If we ever want per-PR +// annotations, upload the lcov files a CI step already produces. +import { readFileSync, readdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +const BASELINE = 'coverage-baseline.json' +const METRICS = ['statements', 'branches', 'functions', 'lines'] + +// Coverage percentages wobble slightly between runs (v8 attributes some +// bytes differently depending on JIT timing), so require a real drop. +const TOLERANCE = 0.5 + +const update = process.argv.includes('--update') + +/** @returns {Record>} */ +function readBaseline() { + try { + return JSON.parse(readFileSync(BASELINE, 'utf8')) + } catch (error) { + if (error.code === 'ENOENT') return {} + throw error + } +} + +/** Collect the fresh coverage summaries written by this run. */ +function readCurrent() { + const current = {} + for (const entry of readdirSync('packages', { withFileTypes: true })) { + if (!entry.isDirectory()) continue + const summaryPath = join( + 'packages', + entry.name, + 'coverage', + 'coverage-summary.json', + ) + let summary + try { + summary = JSON.parse(readFileSync(summaryPath, 'utf8')) + } catch (error) { + // Not every package is affected on every run — no summary means the + // package was not measured, which is different from measuring 0%. + if (error.code === 'ENOENT') continue + throw error + } + // A package with no source loaded reports totals of 0/0 as 100%; that is + // not a number worth ratcheting against. + if (summary.total.statements.total === 0) continue + current[entry.name] = Object.fromEntries( + METRICS.map((metric) => [metric, summary.total[metric].pct]), + ) + } + return current +} + +const baseline = readBaseline() +const current = readCurrent() +const names = Object.keys(current).sort() + +if (names.length === 0) { + console.error( + `No coverage summaries found under packages/*/coverage/. Run \`pnpm test:coverage\` first.`, + ) + process.exit(1) +} + +if (update) { + const merged = { ...baseline, ...current } + const sorted = Object.fromEntries( + Object.keys(merged) + .sort() + .map((key) => [key, merged[key]]), + ) + writeFileSync(BASELINE, `${JSON.stringify(sorted, null, 2)}\n`) + console.log(`Updated ${BASELINE} with ${names.length} package(s).`) + process.exit(0) +} + +const regressions = [] +const additions = [] +const rows = [] + +for (const name of names) { + const before = baseline[name] + const after = current[name] + if (!before) { + additions.push(name) + rows.push([name, ...METRICS.map((m) => `${after[m].toFixed(2)}%`), 'new']) + continue + } + const deltas = METRICS.map((metric) => after[metric] - before[metric]) + const dropped = METRICS.filter((metric, i) => deltas[i] < -TOLERANCE) + if (dropped.length > 0) { + regressions.push({ name, before, after, dropped }) + } + rows.push([ + name, + ...METRICS.map((metric, i) => { + const delta = deltas[i] + const sign = delta > 0 ? '+' : '' + return `${after[metric].toFixed(2)}% (${sign}${delta.toFixed(2)})` + }), + dropped.length > 0 ? 'DROP' : 'ok', + ]) +} + +const header = ['package', ...METRICS, ''] +const widths = header.map((_, column) => + Math.max(header[column].length, ...rows.map((row) => row[column].length)), +) +const line = (cells) => + cells + .map((cell, i) => cell.padEnd(widths[i])) + .join(' ') + .trimEnd() + +console.log(line(header)) +console.log(widths.map((width) => '-'.repeat(width)).join(' ')) +for (const row of rows) console.log(line(row)) + +if (additions.length > 0) { + console.log( + `\n${additions.length} package(s) missing from ${BASELINE}: ${additions.join(', ')}` + + `\nRun \`pnpm test:coverage:update\` and commit the baseline.`, + ) +} + +if (regressions.length > 0) { + console.error(`\nCoverage dropped in ${regressions.length} package(s):`) + for (const { name, before, after, dropped } of regressions) { + for (const metric of dropped) { + console.error( + ` ${name} ${metric}: ${before[metric].toFixed(2)}% -> ${after[metric].toFixed(2)}%`, + ) + } + } + console.error( + `\nAdd tests to restore coverage, or run \`pnpm test:coverage:update\` and commit` + + ` the new baseline if the drop is intentional.`, + ) + process.exit(1) +} + +console.log(`\nCoverage held for ${names.length} measured package(s).`) From fc71c7c099a35440b89fd26de39c87292d1deff8 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 14:43:27 +0200 Subject: [PATCH 02/10] feat(ci): surface the coverage table on the PR run summary The check previously only printed to the CI log, so the numbers were invisible unless the job went red. Writing the same per-package table with deltas to $GITHUB_STEP_SUMMARY puts it on the PR's Checks tab on every run, pass or fail, with no token and no `pull-requests: write` permission. --- CONTRIBUTING.md | 2 ++ scripts/coverage-check.mjs | 43 ++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57662287c4..5660e997ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,6 +98,8 @@ Every package has a `test:coverage` script (`vitest run --coverage`, measured ov The `Coverage` job on every PR runs `pnpm test:coverage:check`, which measures the affected packages and fails if any metric (statements, branches, functions, lines) drops more than 0.5 percentage points below the baseline. Packages that weren't affected are skipped, not treated as 0%. +The job writes a per-package table with deltas to the workflow run summary, so you can read the numbers from the PR's Checks tab whether the job passed or failed — click the `Coverage` job and the table is at the top. It is not posted as a PR comment. + When a drop is intentional — deleting well-tested code, for instance — re-baseline and commit the result: ```bash diff --git a/scripts/coverage-check.mjs b/scripts/coverage-check.mjs index 947c10972f..603c4d221f 100644 --- a/scripts/coverage-check.mjs +++ b/scripts/coverage-check.mjs @@ -5,10 +5,16 @@ // node scripts/coverage-check.mjs # compare, exit 1 on a drop // node scripts/coverage-check.mjs --update # rewrite the baseline // -// ponytail: a committed baseline file is the whole ratchet. No coverage -// service, no PR comments, no historical database. If we ever want per-PR -// annotations, upload the lcov files a CI step already produces. -import { readFileSync, readdirSync, writeFileSync } from 'node:fs' +// ponytail: a committed baseline file is the whole ratchet — no coverage +// service and no historical database. Results surface on the PR via +// $GITHUB_STEP_SUMMARY; for per-line annotations, upload the lcov files that +// `test:coverage` already writes. +import { + appendFileSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs' import { join } from 'node:path' const BASELINE = 'coverage-baseline.json' @@ -132,6 +138,35 @@ if (additions.length > 0) { ) } +// ponytail: $GITHUB_STEP_SUMMARY renders on the run page with no token, no +// permissions and no API call. A sticky PR comment needs `pull-requests: +// write` — add that only if the summary tab turns out to be too easy to miss. +if (process.env.GITHUB_STEP_SUMMARY) { + const verdict = + regressions.length > 0 + ? `❌ Coverage dropped in ${regressions.length} package(s).` + : `✅ Coverage held for ${names.length} measured package(s).` + const md = [ + `## Coverage`, + ``, + verdict, + ``, + `Baseline: \`${BASELINE}\`. Fails on a drop of more than ${TOLERANCE}pp. Packages not affected by this PR are not measured and not listed.`, + ``, + `| package | ${METRICS.join(' | ')} | |`, + `| --- | ${METRICS.map(() => '---:').join(' | ')} | --- |`, + ...rows.map((row) => `| ${row.join(' | ')} |`), + ] + if (additions.length > 0) { + md.push( + ``, + `> ${additions.length} package(s) are not in the baseline yet: ${additions.join(', ')}.`, + `> Run \`pnpm test:coverage:update\` and commit the baseline.`, + ) + } + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${md.join('\n')}\n`) +} + if (regressions.length > 0) { console.error(`\nCoverage dropped in ${regressions.length} package(s):`) for (const { name, before, after, dropped } of regressions) { From 744693560af89335f9bc0873c7d5581b9a758619 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 14:55:42 +0200 Subject: [PATCH 03/10] fix(ci): baseline the three platform-divergent packages from Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first CI run confirmed the gate works — 47 of 50 packages reproduced the locally generated numbers to the hundredth — and surfaced the three that could not be measured correctly on Windows: - ai-acp and ai-sandbox have tests that fail on Windows for reasons unrelated to coverage, so they were missing from the baseline entirely - ai-sandbox-local-process reads ~20pp higher on Windows because its process-spawn code branches on platform, which tripped the gate All three now carry the numbers from the Linux CI run. CONTRIBUTING.md spells out that the baseline must come from Linux, since CI is what enforces it. --- CONTRIBUTING.md | 5 ++++- coverage-baseline.json | 20 ++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5660e997ef..e9e20e9dd8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,10 +106,13 @@ When a drop is intentional — deleting well-tested code, for instance — re-ba pnpm test:coverage:update ``` -Two known limitations: +**The baseline is Linux numbers, because CI is the thing enforcing it.** A few packages cover different amounts on different platforms — `ai-sandbox-local-process` reads ~20pp higher on Windows, since its process-spawn code branches on platform. If you re-baseline on macOS or Windows you will commit numbers CI can't reproduce and the gate will fail for everyone. Take the values from the `Coverage` job's table on a green run instead, or re-baseline in a Linux container. + +Three known limitations: - Uncovered `.tsx` files can't be remapped by the coverage provider and are dropped from the report with a `Failed to parse ... Excluding it from coverage` warning. `.tsx` files that tests _do_ load are measured normally, so the UI packages read higher than their real coverage. - `preact-ai-devtools`, `react-ai-devtools`, and `solid-ai-devtools` have no tests and sit at 0%. +- `ai-acp` and `ai-sandbox` have tests that fail on Windows for unrelated reasons (`EBUSY` on temp-dir unlink; a POSIX-vs-Windows path assertion), so they can't be measured there at all. ## TypeScript configuration diff --git a/coverage-baseline.json b/coverage-baseline.json index f4c280aeb4..3dc14f80bb 100644 --- a/coverage-baseline.json +++ b/coverage-baseline.json @@ -5,6 +5,12 @@ "functions": 90.11, "lines": 89.92 }, + "ai-acp": { + "statements": 75.93, + "branches": 62.82, + "functions": 68.75, + "lines": 77.73 + }, "ai-angular": { "statements": 76.57, "branches": 60.81, @@ -191,6 +197,12 @@ "functions": 14.28, "lines": 11.42 }, + "ai-sandbox": { + "statements": 84.69, + "branches": 75.7, + "functions": 83.18, + "lines": 86.53 + }, "ai-sandbox-cloudflare": { "statements": 37.92, "branches": 32.13, @@ -210,10 +222,10 @@ "lines": 64.32 }, "ai-sandbox-local-process": { - "statements": 87.71, - "branches": 66.66, - "functions": 89.47, - "lines": 91.33 + "statements": 68.42, + "branches": 50.57, + "functions": 82.45, + "lines": 71.33 }, "ai-sandbox-sprites": { "statements": 70.75, From 4747ca380bdfef91d22d634a61cbd3991dc528d0 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 15:10:26 +0200 Subject: [PATCH 04/10] docs(ci): make coverage explicitly CI-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage was already absent from `test`, `test:pr`, `test:ci` and the git hooks, but the docs still presented it as part of the local workflow — four coverage rows in CONTRIBUTING's day-to-day command table, and an instruction to re-baseline with `pnpm test:coverage:update` on your own machine. That last one was an active trap: the committed numbers are Linux numbers and a few packages measure differently per platform, so following the doc on a Mac or Windows box reintroduces exactly the ai-sandbox-local-process failure the previous commit fixed. The check now prints a paste-ready JSON block for every package the baseline needs to absorb, in both the log and the run summary, so re-baselining is a copy out of CI rather than a local run. `test:coverage:update` stays as the maintainer escape hatch but is no longer the documented path. --- CLAUDE.md | 19 ++++++++------- CONTRIBUTING.md | 48 +++++++++++++++++--------------------- scripts/coverage-check.mjs | 32 +++++++++++++++++++------ 3 files changed, 58 insertions(+), 41 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6c6f2b03d3..669dadeeff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,10 +54,7 @@ pnpm test:lib:dev # Watch mode for unit tests pnpm test:oxlint # Lint affected packages (oxlint, incl. type-aware) pnpm test:types # Type check affected packages pnpm test:build # Verify build artifacts with publint -pnpm test:coverage # Generate coverage reports (affected) -pnpm test:coverage:all # Generate coverage reports (every package) -pnpm test:coverage:check # Coverage + fail if it dropped vs coverage-baseline.json -pnpm test:coverage:update # Re-baseline coverage-baseline.json +# Coverage is CI-only — see the Coverage section below. Don't run it locally. pnpm test:knip # Check for unused dependencies pnpm test:sherif # Check pnpm workspace consistency pnpm test:docs # Verify documentation links @@ -321,10 +318,16 @@ Each package uses `exports` field in package.json for subpath exports (e.g., `@t - Unit tests in `*.test.ts` files alongside source - Uses Vitest with happy-dom for DOM testing -- Test coverage via `pnpm test:coverage`; the `Coverage` CI job runs - `pnpm test:coverage:check`, which fails when any package drops more than - 0.5pp below the percentages committed in `coverage-baseline.json`. Re-baseline - intentional drops with `pnpm test:coverage:update`. See CONTRIBUTING.md. +- **Coverage is CI-only.** Don't run it locally and don't add it to local + gates — it is deliberately absent from `test`, `test:pr`, `test:ci` and the + git hooks. The `Coverage` job on each PR runs `test:coverage:check`, which + fails when a package drops more than 0.5pp below the numbers committed in + `coverage-baseline.json`. There are no target percentages; each package's + current number is its own floor. When a drop is intentional, copy the + paste-ready JSON block the job prints into `coverage-baseline.json` — never + regenerate the baseline on a dev machine, the committed numbers are Linux + numbers and some packages measure differently per platform. See + CONTRIBUTING.md. - **E2E tests are mandatory** — see E2E Testing section below ### E2E Testing (REQUIRED) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e9e20e9dd8..5467c77782 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,42 +71,38 @@ The workflow pushes with `GITHUB_TOKEN`, so GitHub does not start Test / E2E on All commands are run from the repo root. Nx handles affected detection and caching. -| Goal | Command | -| ----------------------------- | --------------------------- | -| Run unit tests (affected) | `pnpm test:lib` | -| Watch unit tests | `pnpm test:lib:dev` | -| Coverage (affected) | `pnpm test:coverage` | -| Coverage for every package | `pnpm test:coverage:all` | -| Coverage + regression check | `pnpm test:coverage:check` | -| Re-baseline coverage | `pnpm test:coverage:update` | -| Type-check (affected) | `pnpm test:types` | -| Lint (affected) | `pnpm test:eslint` | -| Verify build artifacts | `pnpm test:build` | -| Format the repo | `pnpm format` | -| Build (affected) | `pnpm build` | -| Build everything | `pnpm build:all` | -| Run the full CI suite locally | `pnpm test` | -| Run the affected-PR check | `pnpm test:pr` | -| E2E suite | `pnpm test:e2e` | -| E2E with Playwright UI | `pnpm test:e2e:ui` | +| Goal | Command | +| ----------------------------- | ------------------- | +| Run unit tests (affected) | `pnpm test:lib` | +| Watch unit tests | `pnpm test:lib:dev` | +| Type-check (affected) | `pnpm test:types` | +| Lint (affected) | `pnpm test:eslint` | +| Verify build artifacts | `pnpm test:build` | +| Format the repo | `pnpm format` | +| Build (affected) | `pnpm build` | +| Build everything | `pnpm build:all` | +| Run the full CI suite locally | `pnpm test` | +| Run the affected-PR check | `pnpm test:pr` | +| E2E suite | `pnpm test:e2e` | +| E2E with Playwright UI | `pnpm test:e2e:ui` | Working on a single package? `cd packages/` and use its scripts directly (`pnpm test:lib`, `pnpm test:types`, etc.). ## Coverage -Every package has a `test:coverage` script (`vitest run --coverage`, measured over `src/**` with the v8 provider). Per-package percentages are committed to `coverage-baseline.json` at the repo root. +**Coverage runs in CI only. It is not part of `pnpm test`, `pnpm test:pr`, or any git hook, and you are not expected to run it locally.** -The `Coverage` job on every PR runs `pnpm test:coverage:check`, which measures the affected packages and fails if any metric (statements, branches, functions, lines) drops more than 0.5 percentage points below the baseline. Packages that weren't affected are skipped, not treated as 0%. +The `Coverage` job on every PR runs `test:coverage:check`: it measures the affected packages with the v8 provider over `src/**` and fails if any metric (statements, branches, functions, lines) drops more than 0.5 percentage points below `coverage-baseline.json`. Packages your PR didn't affect are skipped, not treated as 0%. -The job writes a per-package table with deltas to the workflow run summary, so you can read the numbers from the PR's Checks tab whether the job passed or failed — click the `Coverage` job and the table is at the top. It is not posted as a PR comment. +There are no target percentages to hit. Each package's committed number is its own floor, so the gate only stops coverage getting _worse_ — it never blocks a PR for being below some repo-wide bar. -When a drop is intentional — deleting well-tested code, for instance — re-baseline and commit the result: +Read the numbers from the PR's Checks tab: open the `Coverage` job and its summary has a per-package table with deltas, on every run whether it passed or failed. It is not posted as a PR comment. -```bash -pnpm test:coverage:update -``` +### If the job says coverage dropped + +Add tests to cover what you changed. If the drop is genuinely intended — you deleted well-tested code, say — the job prints a paste-ready JSON block; copy those entries into `coverage-baseline.json` and push. -**The baseline is Linux numbers, because CI is the thing enforcing it.** A few packages cover different amounts on different platforms — `ai-sandbox-local-process` reads ~20pp higher on Windows, since its process-spawn code branches on platform. If you re-baseline on macOS or Windows you will commit numbers CI can't reproduce and the gate will fail for everyone. Take the values from the `Coverage` job's table on a green run instead, or re-baseline in a Linux container. +**Don't regenerate the baseline on your own machine.** The committed numbers are Linux numbers because Linux is what enforces them, and a few packages genuinely measure differently per platform — `ai-sandbox-local-process` reads ~20pp higher on Windows, since its process-spawn code branches on platform. Re-baselining locally commits numbers CI can't reproduce, which breaks the gate for everyone. (`test:coverage:update` exists for maintainers running in a Linux container; it is not the normal path.) Three known limitations: diff --git a/scripts/coverage-check.mjs b/scripts/coverage-check.mjs index 603c4d221f..34d24550ab 100644 --- a/scripts/coverage-check.mjs +++ b/scripts/coverage-check.mjs @@ -131,10 +131,20 @@ console.log(line(header)) console.log(widths.map((width) => '-'.repeat(width)).join(' ')) for (const row of rows) console.log(line(row)) +// Anything the baseline needs to absorb, rendered paste-ready. Coverage is a +// CI-only job and a few packages measure differently per platform, so a +// contributor must never regenerate the baseline on their own machine — they +// copy these entries out of the run instead. +const staleNames = [ + ...new Set([...additions, ...regressions.map((r) => r.name)]), +].sort() +const pasteBlock = staleNames + .map((name) => ` ${JSON.stringify(name)}: ${JSON.stringify(current[name])},`) + .join('\n') + if (additions.length > 0) { console.log( - `\n${additions.length} package(s) missing from ${BASELINE}: ${additions.join(', ')}` + - `\nRun \`pnpm test:coverage:update\` and commit the baseline.`, + `\n${additions.length} package(s) missing from ${BASELINE}: ${additions.join(', ')}`, ) } @@ -157,11 +167,18 @@ if (process.env.GITHUB_STEP_SUMMARY) { `| --- | ${METRICS.map(() => '---:').join(' | ')} | --- |`, ...rows.map((row) => `| ${row.join(' | ')} |`), ] - if (additions.length > 0) { + if (pasteBlock) { md.push( ``, - `> ${additions.length} package(s) are not in the baseline yet: ${additions.join(', ')}.`, - `> Run \`pnpm test:coverage:update\` and commit the baseline.`, + `### Updating the baseline`, + ``, + `Coverage runs in CI only, and some packages measure differently per platform —`, + `do not regenerate the baseline on your own machine. If these numbers are the`, + `intended ones, paste the entries below into \`${BASELINE}\` and push:`, + ``, + '```json', + pasteBlock, + '```', ) } appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${md.join('\n')}\n`) @@ -177,8 +194,9 @@ if (regressions.length > 0) { } } console.error( - `\nAdd tests to restore coverage, or run \`pnpm test:coverage:update\` and commit` + - ` the new baseline if the drop is intentional.`, + `\nAdd tests to restore coverage. If the drop is intentional, paste these` + + ` entries into ${BASELINE} and push — do not regenerate the baseline` + + ` locally, some packages measure differently per platform:\n\n${pasteBlock}`, ) process.exit(1) } From f87ed2c63f6d2fec04e9887f66a88b140521f8b4 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 15:21:27 +0200 Subject: [PATCH 05/10] fix(ci): quote the coverage include glob, correct stale lint references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: - quote `--coverage.include='src/**'` in all 50 test:coverage scripts so a POSIX shell can't expand the pattern before vitest parses it. Verified the quoted form behaves identically on Windows (pnpm normalises it) and yields byte-identical percentages, so it is safe cross-platform. - CONTRIBUTING referenced `pnpm test:eslint`, which does not exist — the repo moved to oxlint. Fixed that and the two other stale ESLint mentions, and added the Coverage gate to the list of what CI runs. --- CONTRIBUTING.md | 8 +++++--- packages/ai-acp/package.json | 2 +- packages/ai-angular/package.json | 2 +- packages/ai-anthropic/package.json | 2 +- packages/ai-bedrock/package.json | 2 +- packages/ai-claude-code/package.json | 2 +- packages/ai-client/package.json | 2 +- packages/ai-code-mode-snippets/package.json | 2 +- packages/ai-code-mode/package.json | 2 +- packages/ai-codex/package.json | 2 +- packages/ai-devtools/package.json | 2 +- packages/ai-durable-stream/package.json | 2 +- packages/ai-elevenlabs/package.json | 2 +- packages/ai-event-client/package.json | 2 +- packages/ai-fal/package.json | 2 +- packages/ai-gemini/package.json | 2 +- packages/ai-grok-build/package.json | 2 +- packages/ai-grok/package.json | 2 +- packages/ai-groq/package.json | 2 +- packages/ai-isolate-cloudflare/package.json | 2 +- packages/ai-isolate-node/package.json | 2 +- packages/ai-isolate-quickjs/package.json | 2 +- packages/ai-mcp/package.json | 2 +- packages/ai-memory/package.json | 2 +- packages/ai-mistral/package.json | 2 +- packages/ai-ollama/package.json | 2 +- packages/ai-openai/package.json | 2 +- packages/ai-opencode/package.json | 2 +- packages/ai-openrouter/package.json | 2 +- packages/ai-persistence/package.json | 2 +- packages/ai-preact/package.json | 2 +- packages/ai-react-ui/package.json | 2 +- packages/ai-react/package.json | 2 +- packages/ai-sandbox-cloudflare/package.json | 2 +- packages/ai-sandbox-daytona/package.json | 2 +- packages/ai-sandbox-docker/package.json | 2 +- packages/ai-sandbox-local-process/package.json | 2 +- packages/ai-sandbox-sprites/package.json | 2 +- packages/ai-sandbox-vercel/package.json | 2 +- packages/ai-sandbox/package.json | 2 +- packages/ai-solid-ui/package.json | 2 +- packages/ai-solid/package.json | 2 +- packages/ai-svelte/package.json | 2 +- packages/ai-utils/package.json | 2 +- packages/ai-vue-ui/package.json | 2 +- packages/ai-vue/package.json | 2 +- packages/ai/package.json | 2 +- packages/openai-base/package.json | 2 +- packages/preact-ai-devtools/package.json | 2 +- packages/react-ai-devtools/package.json | 2 +- packages/solid-ai-devtools/package.json | 2 +- 51 files changed, 55 insertions(+), 53 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5467c77782..4b42bbf2b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,7 +76,7 @@ All commands are run from the repo root. Nx handles affected detection and cachi | Run unit tests (affected) | `pnpm test:lib` | | Watch unit tests | `pnpm test:lib:dev` | | Type-check (affected) | `pnpm test:types` | -| Lint (affected) | `pnpm test:eslint` | +| Lint (affected) | `pnpm test:oxlint` | | Verify build artifacts | `pnpm test:build` | | Format the repo | `pnpm format` | | Build (affected) | `pnpm build` | @@ -134,7 +134,7 @@ Tests are included in typecheck. `vite.config.ts` / `vitest.config.ts` are not - Place tests under `packages//tests/` with the suffix `.test.ts` (or `.test.tsx` for JSX). - Vitest's defaults discover anything matching `**/*.{test,spec}.?(c|m)[jt]s?(x)` — no per-package config is needed. -- Tests are typechecked by `tsc` and linted by ESLint. +- Tests are typechecked by `tsc` and linted by oxlint. ## Adding E2E test coverage (required) @@ -194,10 +194,12 @@ The defensive `ignore` list in `.changeset/config.json` blocks accidental public 1. Push your branch and open a PR against `main`. 2. Fill the PR template. Tick **docs** and **changeset** honestly, or say why you skipped them. -3. CI runs: `pnpm test:pr` (sherif workspace check, knip dead-code, docs link verification, ESLint, unit tests, typecheck, build artifacts, build) + the full E2E suite. +3. CI runs: `pnpm test:pr` (sherif workspace check, knip dead-code, docs link verification, oxlint, unit tests, typecheck, build artifacts, build), the `Coverage` regression gate, and the full E2E suite. 4. Address review comments. 5. A maintainer merges. Releases are cut via Changesets. Your changeset entry lands in the next release. +The PR template lists the steps. The `Test plan` section is required — describe how a reviewer can verify your change. + ## Adding a new provider adapter The pattern lives in `packages/ai-openai/`, `packages/ai-anthropic/`, `packages/ai-gemini/`, etc. New core adapters typically: diff --git a/packages/ai-acp/package.json b/packages/ai-acp/package.json index bd47299a07..72f28ec904 100644 --- a/packages/ai-acp/package.json +++ b/packages/ai-acp/package.json @@ -38,7 +38,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run --testTimeout=30000", "test:lib:dev": "vitest --watch", "test:types": "tsc" diff --git a/packages/ai-angular/package.json b/packages/ai-angular/package.json index 5fe4fa1bca..fc792e1965 100644 --- a/packages/ai-angular/package.json +++ b/packages/ai-angular/package.json @@ -46,7 +46,7 @@ "scripts": { "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "vitest", "test:types": "tsc", diff --git a/packages/ai-anthropic/package.json b/packages/ai-anthropic/package.json index f729726ae6..690df1cebf 100644 --- a/packages/ai-anthropic/package.json +++ b/packages/ai-anthropic/package.json @@ -61,7 +61,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-bedrock/package.json b/packages/ai-bedrock/package.json index d870fbebef..76f1a109f3 100644 --- a/packages/ai-bedrock/package.json +++ b/packages/ai-bedrock/package.json @@ -32,7 +32,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-claude-code/package.json b/packages/ai-claude-code/package.json index c707ffd0e1..e25131885e 100644 --- a/packages/ai-claude-code/package.json +++ b/packages/ai-claude-code/package.json @@ -46,7 +46,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-client/package.json b/packages/ai-client/package.json index 7781a8493e..9912b359e3 100644 --- a/packages/ai-client/package.json +++ b/packages/ai-client/package.json @@ -58,7 +58,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:coverage:watch": "vitest --coverage --watch", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", diff --git a/packages/ai-code-mode-snippets/package.json b/packages/ai-code-mode-snippets/package.json index 61701e622e..bf4029c24c 100644 --- a/packages/ai-code-mode-snippets/package.json +++ b/packages/ai-code-mode-snippets/package.json @@ -44,7 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-code-mode/package.json b/packages/ai-code-mode/package.json index d29d24fd57..24764afac8 100644 --- a/packages/ai-code-mode/package.json +++ b/packages/ai-code-mode/package.json @@ -41,7 +41,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-codex/package.json b/packages/ai-codex/package.json index f7278067ba..1d006f876b 100644 --- a/packages/ai-codex/package.json +++ b/packages/ai-codex/package.json @@ -45,7 +45,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-devtools/package.json b/packages/ai-devtools/package.json index 65b8e060b6..82df67a7ca 100644 --- a/packages/ai-devtools/package.json +++ b/packages/ai-devtools/package.json @@ -73,7 +73,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-durable-stream/package.json b/packages/ai-durable-stream/package.json index f1e453e788..e813442cfb 100644 --- a/packages/ai-durable-stream/package.json +++ b/packages/ai-durable-stream/package.json @@ -36,7 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-elevenlabs/package.json b/packages/ai-elevenlabs/package.json index ff667491e6..96f407592e 100644 --- a/packages/ai-elevenlabs/package.json +++ b/packages/ai-elevenlabs/package.json @@ -57,7 +57,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-event-client/package.json b/packages/ai-event-client/package.json index 24fbe5a54d..97ac7651e6 100644 --- a/packages/ai-event-client/package.json +++ b/packages/ai-event-client/package.json @@ -37,7 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-fal/package.json b/packages/ai-fal/package.json index f885d8fbab..0a3895250f 100644 --- a/packages/ai-fal/package.json +++ b/packages/ai-fal/package.json @@ -40,7 +40,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-gemini/package.json b/packages/ai-gemini/package.json index 4b953b2b43..eedd72fb3d 100644 --- a/packages/ai-gemini/package.json +++ b/packages/ai-gemini/package.json @@ -48,7 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-grok-build/package.json b/packages/ai-grok-build/package.json index a2492da7d8..5620c4c569 100644 --- a/packages/ai-grok-build/package.json +++ b/packages/ai-grok-build/package.json @@ -46,7 +46,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-grok/package.json b/packages/ai-grok/package.json index 4768059fb2..3e013d75db 100644 --- a/packages/ai-grok/package.json +++ b/packages/ai-grok/package.json @@ -48,7 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-groq/package.json b/packages/ai-groq/package.json index 0c878c17ef..966d2e26b7 100644 --- a/packages/ai-groq/package.json +++ b/packages/ai-groq/package.json @@ -44,7 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-isolate-cloudflare/package.json b/packages/ai-isolate-cloudflare/package.json index f03bd01e7b..fc05f4ec02 100644 --- a/packages/ai-isolate-cloudflare/package.json +++ b/packages/ai-isolate-cloudflare/package.json @@ -48,7 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-isolate-node/package.json b/packages/ai-isolate-node/package.json index cf5d12368f..0ba6661848 100644 --- a/packages/ai-isolate-node/package.json +++ b/packages/ai-isolate-node/package.json @@ -40,7 +40,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-isolate-quickjs/package.json b/packages/ai-isolate-quickjs/package.json index e888ae6763..55b1e2e8bd 100644 --- a/packages/ai-isolate-quickjs/package.json +++ b/packages/ai-isolate-quickjs/package.json @@ -37,7 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-mcp/package.json b/packages/ai-mcp/package.json index d2f48ab202..4d23ece432 100644 --- a/packages/ai-mcp/package.json +++ b/packages/ai-mcp/package.json @@ -57,7 +57,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-memory/package.json b/packages/ai-memory/package.json index 855454d941..5381884391 100644 --- a/packages/ai-memory/package.json +++ b/packages/ai-memory/package.json @@ -50,7 +50,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-mistral/package.json b/packages/ai-mistral/package.json index c92a8db53c..d4c8841c91 100644 --- a/packages/ai-mistral/package.json +++ b/packages/ai-mistral/package.json @@ -44,7 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-ollama/package.json b/packages/ai-ollama/package.json index d5b4791083..3d42257a2f 100644 --- a/packages/ai-ollama/package.json +++ b/packages/ai-ollama/package.json @@ -40,7 +40,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-openai/package.json b/packages/ai-openai/package.json index 4bb2a4e59f..f9f9071707 100644 --- a/packages/ai-openai/package.json +++ b/packages/ai-openai/package.json @@ -48,7 +48,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-opencode/package.json b/packages/ai-opencode/package.json index 3453ac76a1..5eefc81a0b 100644 --- a/packages/ai-opencode/package.json +++ b/packages/ai-opencode/package.json @@ -44,7 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-openrouter/package.json b/packages/ai-openrouter/package.json index c3314a0379..7bc57cde2b 100644 --- a/packages/ai-openrouter/package.json +++ b/packages/ai-openrouter/package.json @@ -52,7 +52,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-persistence/package.json b/packages/ai-persistence/package.json index afa9958538..0538861e96 100644 --- a/packages/ai-persistence/package.json +++ b/packages/ai-persistence/package.json @@ -43,7 +43,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-preact/package.json b/packages/ai-preact/package.json index ea4e872eeb..3288ee5f32 100644 --- a/packages/ai-preact/package.json +++ b/packages/ai-preact/package.json @@ -38,7 +38,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-react-ui/package.json b/packages/ai-react-ui/package.json index 85092c76ac..451821a44d 100644 --- a/packages/ai-react-ui/package.json +++ b/packages/ai-react-ui/package.json @@ -34,7 +34,7 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-react/package.json b/packages/ai-react/package.json index 4f2abda011..ed0f0b195b 100644 --- a/packages/ai-react/package.json +++ b/packages/ai-react/package.json @@ -38,7 +38,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-sandbox-cloudflare/package.json b/packages/ai-sandbox-cloudflare/package.json index 7aa4edb1da..d29afe5117 100644 --- a/packages/ai-sandbox-cloudflare/package.json +++ b/packages/ai-sandbox-cloudflare/package.json @@ -46,7 +46,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-daytona/package.json b/packages/ai-sandbox-daytona/package.json index cbab67b268..51f0454654 100644 --- a/packages/ai-sandbox-daytona/package.json +++ b/packages/ai-sandbox-daytona/package.json @@ -37,7 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-docker/package.json b/packages/ai-sandbox-docker/package.json index 4368b4f273..f93578aa61 100644 --- a/packages/ai-sandbox-docker/package.json +++ b/packages/ai-sandbox-docker/package.json @@ -38,7 +38,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-local-process/package.json b/packages/ai-sandbox-local-process/package.json index 7596d7939c..78f100bdec 100644 --- a/packages/ai-sandbox-local-process/package.json +++ b/packages/ai-sandbox-local-process/package.json @@ -36,7 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-sprites/package.json b/packages/ai-sandbox-sprites/package.json index 5dbebd0580..918df8db48 100644 --- a/packages/ai-sandbox-sprites/package.json +++ b/packages/ai-sandbox-sprites/package.json @@ -38,7 +38,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox-vercel/package.json b/packages/ai-sandbox-vercel/package.json index 529b063cc3..20ce9a226b 100644 --- a/packages/ai-sandbox-vercel/package.json +++ b/packages/ai-sandbox-vercel/package.json @@ -37,7 +37,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-sandbox/package.json b/packages/ai-sandbox/package.json index a1169bcedf..0ece47c227 100644 --- a/packages/ai-sandbox/package.json +++ b/packages/ai-sandbox/package.json @@ -50,7 +50,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-solid-ui/package.json b/packages/ai-solid-ui/package.json index 491bb22994..50aae37c59 100644 --- a/packages/ai-solid-ui/package.json +++ b/packages/ai-solid-ui/package.json @@ -36,7 +36,7 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-solid/package.json b/packages/ai-solid/package.json index 9fbe78d4f8..b3309301fd 100644 --- a/packages/ai-solid/package.json +++ b/packages/ai-solid/package.json @@ -33,7 +33,7 @@ "scripts": { "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai-svelte/package.json b/packages/ai-svelte/package.json index 4318d2314d..752fcc1b73 100644 --- a/packages/ai-svelte/package.json +++ b/packages/ai-svelte/package.json @@ -36,7 +36,7 @@ "clean": "rm -rf ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "svelte-check", diff --git a/packages/ai-utils/package.json b/packages/ai-utils/package.json index 8f052e4d72..46c4384e96 100644 --- a/packages/ai-utils/package.json +++ b/packages/ai-utils/package.json @@ -36,7 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-vue-ui/package.json b/packages/ai-vue-ui/package.json index ab17e2c7f3..ec29b072f2 100644 --- a/packages/ai-vue-ui/package.json +++ b/packages/ai-vue-ui/package.json @@ -35,7 +35,7 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "vue-tsc --noEmit" diff --git a/packages/ai-vue/package.json b/packages/ai-vue/package.json index a688a96882..8b8d6da61b 100644 --- a/packages/ai-vue/package.json +++ b/packages/ai-vue/package.json @@ -33,7 +33,7 @@ "scripts": { "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/ai/package.json b/packages/ai/package.json index 0f91911e69..e240a5ff4c 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -73,7 +73,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:coverage:watch": "vitest --coverage --watch", "test:lib": "vitest", "test:lib:dev": "pnpm test:lib --watch", diff --git a/packages/openai-base/package.json b/packages/openai-base/package.json index 98bb558301..646d74bde5 100644 --- a/packages/openai-base/package.json +++ b/packages/openai-base/package.json @@ -36,7 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/preact-ai-devtools/package.json b/packages/preact-ai-devtools/package.json index 11fda05472..39e567fc18 100644 --- a/packages/preact-ai-devtools/package.json +++ b/packages/preact-ai-devtools/package.json @@ -21,7 +21,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/react-ai-devtools/package.json b/packages/react-ai-devtools/package.json index 1ab4b4d461..56a70f48f8 100644 --- a/packages/react-ai-devtools/package.json +++ b/packages/react-ai-devtools/package.json @@ -21,7 +21,7 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", diff --git a/packages/solid-ai-devtools/package.json b/packages/solid-ai-devtools/package.json index 8b9ffdbe03..35f2205e6f 100644 --- a/packages/solid-ai-devtools/package.json +++ b/packages/solid-ai-devtools/package.json @@ -42,7 +42,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include=src/** --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" From 296aafb3c8e6bc06c8a1d208d3b3eddf2d4c1968 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 15:58:16 +0200 Subject: [PATCH 06/10] refactor(ci)!: compute the coverage baseline in CI instead of committing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The committed baseline had the maintenance problems it looked like it would: it needed manual syncing, it was platform-sensitive (ai-sandbox-local-process reads ~20pp higher on Windows, which broke the gate on this very PR), and adding or removing a package meant hand-editing a generated file. The Coverage job now measures each affected package twice in one job — on the PR head and on its merge-base with main — and compares those. Nothing to keep in sync, no platform skew (both sides run on the same runner), and package add/remove needs no action. - delete coverage-baseline.json, `test:coverage:check`, `test:coverage:update` and the paste-a-JSON-block flow; all of it existed only to tend the file - coverage-check.mjs gains `--collect ` (snapshot summaries so they survive the checkout between runs) and `--base --head ` - resolve the package list once, with `--with-target=test:coverage`, so both sides measure the same set and a docs-only PR resolves to an empty list rather than erroring in run-many; the base side intersects that list with the projects that exist there, so PR-added packages report as `new` - new coverage.yml runs test:coverage:all on pushes to main purely to populate the Nx Cloud cache: the base-side task hashes then already exist remotely and, because test:coverage declares its coverage dir as an output, restore without re-running tests Verified locally: collect/compare round-trip, a drop failing with only the out-of-tolerance metrics listed, PR-added packages reporting `new` without failing, removed packages ignored, missing base dir, empty head, bad args. --- .github/workflows/coverage.yml | 33 ++++ .github/workflows/pr.yml | 43 ++++- CLAUDE.md | 14 +- CONTRIBUTING.md | 15 +- coverage-baseline.json | 302 --------------------------------- package.json | 2 - scripts/coverage-check.mjs | 212 ++++++++++++----------- 7 files changed, 201 insertions(+), 420 deletions(-) create mode 100644 .github/workflows/coverage.yml delete mode 100644 coverage-baseline.json diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000..40c5739e6f --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,33 @@ +name: Coverage + +# Measures coverage for every package on main. The numbers themselves are not +# asserted here — the point is to populate the Nx Cloud cache, so that when a +# PR re-measures its merge-base the `test:coverage` task hashes already exist +# remotely and the summaries are restored instead of recomputed. +on: + push: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + +permissions: + contents: read + +jobs: + coverage: + name: Warm Coverage Cache + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: TanStack/config/.github/setup@190f659075ff0845850e330883eb26d7ffd0671f # main + - name: Run Coverage + run: pnpm run test:coverage:all diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a0fcde8152..6b7aa6b3a6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -46,8 +46,47 @@ jobs: uses: nrwl/nx-set-shas@3e9ad7370203c1e93d109be57f3b72eb0eb511b1 # v4.4.0 with: main-branch-name: main - - name: Check Coverage - run: pnpm run test:coverage:check + # The set of packages to measure is decided once, here, so that the base + # run measures exactly what the head run does. `--with-target` keeps out + # projects that have no test:coverage script, so a docs-only PR resolves + # to an empty list rather than erroring in run-many. + - name: Resolve affected packages + id: affected + run: | + list=$(pnpm exec nx show projects --affected --with-target=test:coverage --exclude='examples/**,testing/**' --json | jq -r 'sort | join(",")') + echo "list=$list" >> "$GITHUB_OUTPUT" + echo "Measuring: ${list:-}" + - name: Coverage for this PR + if: steps.affected.outputs.list != '' + run: | + pnpm exec nx run-many --targets=test:coverage --projects="${{ steps.affected.outputs.list }}" + node scripts/coverage-check.mjs --collect "$RUNNER_TEMP/cov-head" + # Re-measuring the merge-base is normally a set of Nx Cloud cache + # restores seeded by the Coverage workflow on main: the task hashes are + # identical, and `test:coverage` declares its coverage dir as an output, + # so the summaries come back without the tests re-running. + - name: Coverage for the merge base + if: steps.affected.outputs.list != '' + run: | + git checkout --force --detach "$NX_BASE" + pnpm install --frozen-lockfile + # Packages added by this PR do not exist on the base commit. + printf '%s\n' "${{ steps.affected.outputs.list }}" | tr ',' '\n' | sort > /tmp/wanted + pnpm exec nx show projects --with-target=test:coverage --json | jq -r '.[]' | sort > /tmp/available + base_list=$(comm -12 /tmp/wanted /tmp/available | paste -sd, -) + if [ -z "$base_list" ]; then + echo "No affected package exists on the base commit; nothing to compare against." + exit 0 + fi + pnpm exec nx run-many --targets=test:coverage --projects="$base_list" + node scripts/coverage-check.mjs --collect "$RUNNER_TEMP/cov-base" + - name: Compare + if: steps.affected.outputs.list != '' + run: | + git checkout --force "$GITHUB_SHA" -- scripts/coverage-check.mjs + node scripts/coverage-check.mjs \ + --base "$RUNNER_TEMP/cov-base" \ + --head "$RUNNER_TEMP/cov-head" preview: name: Preview runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index 669dadeeff..54b9c48188 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -320,13 +320,13 @@ Each package uses `exports` field in package.json for subpath exports (e.g., `@t - Uses Vitest with happy-dom for DOM testing - **Coverage is CI-only.** Don't run it locally and don't add it to local gates — it is deliberately absent from `test`, `test:pr`, `test:ci` and the - git hooks. The `Coverage` job on each PR runs `test:coverage:check`, which - fails when a package drops more than 0.5pp below the numbers committed in - `coverage-baseline.json`. There are no target percentages; each package's - current number is its own floor. When a drop is intentional, copy the - paste-ready JSON block the job prints into `coverage-baseline.json` — never - regenerate the baseline on a dev machine, the committed numbers are Linux - numbers and some packages measure differently per platform. See + git hooks. The `Coverage` job on each PR measures every affected package + twice, on the PR head and on its merge-base with `main`, and fails when a + metric drops more than 0.5pp between them. There is **no baseline file** — + don't reintroduce one, it was removed precisely because it needed manual + syncing and was platform-sensitive. The only remedy for a drop is tests. + `.github/workflows/coverage.yml` runs coverage on pushes to `main` purely to + warm the Nx Cloud cache so the base-side run is mostly cache restores. See CONTRIBUTING.md. - **E2E tests are mandatory** — see E2E Testing section below diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4b42bbf2b4..12d38c8329 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,23 +92,24 @@ Working on a single package? `cd packages/` and use its scripts directly (` **Coverage runs in CI only. It is not part of `pnpm test`, `pnpm test:pr`, or any git hook, and you are not expected to run it locally.** -The `Coverage` job on every PR runs `test:coverage:check`: it measures the affected packages with the v8 provider over `src/**` and fails if any metric (statements, branches, functions, lines) drops more than 0.5 percentage points below `coverage-baseline.json`. Packages your PR didn't affect are skipped, not treated as 0%. +The `Coverage` job on every PR measures each affected package **twice** — once on your branch and once on its merge-base with `main` — and compares the two. A drop of more than 0.5 percentage points in any metric (statements, branches, functions, lines) fails the job. Packages your PR didn't affect are never measured. -There are no target percentages to hit. Each package's committed number is its own floor, so the gate only stops coverage getting _worse_ — it never blocks a PR for being below some repo-wide bar. +There is no baseline file to keep in sync, and nothing to update when a package is added or removed: both numbers come from the same job on the same runner. There are also no target percentages to hit — the gate only catches coverage getting _worse_ in what you touched, and never blocks a PR for being below some repo-wide bar. Read the numbers from the PR's Checks tab: open the `Coverage` job and its summary has a per-package table with deltas, on every run whether it passed or failed. It is not posted as a PR comment. -### If the job says coverage dropped +Re-measuring the merge-base is usually close to free. A separate `Coverage` workflow runs `test:coverage:all` on every push to `main`, which populates the Nx Cloud cache; because `test:coverage` declares its `coverage/` directory as a task output, the base-side run generally restores cached summaries rather than re-running any tests. -Add tests to cover what you changed. If the drop is genuinely intended — you deleted well-tested code, say — the job prints a paste-ready JSON block; copy those entries into `coverage-baseline.json` and push. +### If the job says coverage dropped -**Don't regenerate the baseline on your own machine.** The committed numbers are Linux numbers because Linux is what enforces them, and a few packages genuinely measure differently per platform — `ai-sandbox-local-process` reads ~20pp higher on Windows, since its process-spawn code branches on platform. Re-baselining locally commits numbers CI can't reproduce, which breaks the gate for everyone. (`test:coverage:update` exists for maintainers running in a Linux container; it is not the normal path.) +Add tests covering the code you changed. That's the whole remedy — there is no number to override. If you genuinely deleted well-tested code and the drop is expected, say so in the PR and a maintainer can merge past the failing check. -Three known limitations: +Two known limitations: - Uncovered `.tsx` files can't be remapped by the coverage provider and are dropped from the report with a `Failed to parse ... Excluding it from coverage` warning. `.tsx` files that tests _do_ load are measured normally, so the UI packages read higher than their real coverage. - `preact-ai-devtools`, `react-ai-devtools`, and `solid-ai-devtools` have no tests and sit at 0%. -- `ai-acp` and `ai-sandbox` have tests that fail on Windows for unrelated reasons (`EBUSY` on temp-dir unlink; a POSIX-vs-Windows path assertion), so they can't be measured there at all. + +A package your PR adds shows as `new` and can't fail the job — there is no base-commit coverage to compare it against. The same is true for a package whose suite can't run on the base commit. ## TypeScript configuration diff --git a/coverage-baseline.json b/coverage-baseline.json deleted file mode 100644 index 3dc14f80bb..0000000000 --- a/coverage-baseline.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "ai": { - "statements": 88.66, - "branches": 77.81, - "functions": 90.11, - "lines": 89.92 - }, - "ai-acp": { - "statements": 75.93, - "branches": 62.82, - "functions": 68.75, - "lines": 77.73 - }, - "ai-angular": { - "statements": 76.57, - "branches": 60.81, - "functions": 63.33, - "lines": 80.17 - }, - "ai-anthropic": { - "statements": 83.87, - "branches": 74.88, - "functions": 85.24, - "lines": 84.4 - }, - "ai-bedrock": { - "statements": 83.7, - "branches": 74.31, - "functions": 82.35, - "lines": 84.16 - }, - "ai-claude-code": { - "statements": 83.2, - "branches": 67.06, - "functions": 86, - "lines": 85.59 - }, - "ai-client": { - "statements": 84.97, - "branches": 76.47, - "functions": 83.01, - "lines": 86.47 - }, - "ai-code-mode": { - "statements": 91.58, - "branches": 85.07, - "functions": 86, - "lines": 92.53 - }, - "ai-code-mode-skills": { - "statements": 80.68, - "branches": 72.75, - "functions": 86.36, - "lines": 81.59 - }, - "ai-codex": { - "statements": 91.72, - "branches": 80.49, - "functions": 89.13, - "lines": 92.61 - }, - "ai-devtools": { - "statements": 80.25, - "branches": 69.87, - "functions": 82.85, - "lines": 81.77 - }, - "ai-durable-stream": { - "statements": 87.42, - "branches": 80.97, - "functions": 93.18, - "lines": 89.7 - }, - "ai-elevenlabs": { - "statements": 65.26, - "branches": 57.6, - "functions": 50.68, - "lines": 66.42 - }, - "ai-event-client": { - "statements": 28.8, - "branches": 19.69, - "functions": 44, - "lines": 29.16 - }, - "ai-fal": { - "statements": 90.12, - "branches": 83.05, - "functions": 90.47, - "lines": 90.19 - }, - "ai-gemini": { - "statements": 59.35, - "branches": 60.25, - "functions": 51.24, - "lines": 60 - }, - "ai-grok": { - "statements": 41.78, - "branches": 43.95, - "functions": 51.63, - "lines": 42.56 - }, - "ai-grok-build": { - "statements": 69.63, - "branches": 51.47, - "functions": 71.02, - "lines": 71.01 - }, - "ai-groq": { - "statements": 39.08, - "branches": 42.6, - "functions": 59.45, - "lines": 40.74 - }, - "ai-isolate-cloudflare": { - "statements": 94.11, - "branches": 70.66, - "functions": 100, - "lines": 93.96 - }, - "ai-isolate-node": { - "statements": 84, - "branches": 43.39, - "functions": 100, - "lines": 83.78 - }, - "ai-isolate-quickjs": { - "statements": 87.23, - "branches": 65.07, - "functions": 88.23, - "lines": 89.05 - }, - "ai-mcp": { - "statements": 95.6, - "branches": 84.23, - "functions": 98.63, - "lines": 98.8 - }, - "ai-memory": { - "statements": 71.42, - "branches": 52.53, - "functions": 77.69, - "lines": 73.21 - }, - "ai-mistral": { - "statements": 68, - "branches": 55.64, - "functions": 76.47, - "lines": 69.16 - }, - "ai-ollama": { - "statements": 28.31, - "branches": 68.12, - "functions": 86.95, - "lines": 28.15 - }, - "ai-openai": { - "statements": 48, - "branches": 50.58, - "functions": 50, - "lines": 48.31 - }, - "ai-opencode": { - "statements": 61.79, - "branches": 52.9, - "functions": 56.62, - "lines": 63.67 - }, - "ai-openrouter": { - "statements": 47.35, - "branches": 49.65, - "functions": 81.7, - "lines": 47.38 - }, - "ai-persistence": { - "statements": 92.15, - "branches": 79.17, - "functions": 99.4, - "lines": 95.44 - }, - "ai-preact": { - "statements": 88.99, - "branches": 78.37, - "functions": 92.85, - "lines": 92.55 - }, - "ai-react": { - "statements": 77.82, - "branches": 66.03, - "functions": 75.86, - "lines": 77.75 - }, - "ai-react-ui": { - "statements": 12.14, - "branches": 5.47, - "functions": 14.28, - "lines": 11.42 - }, - "ai-sandbox": { - "statements": 84.69, - "branches": 75.7, - "functions": 83.18, - "lines": 86.53 - }, - "ai-sandbox-cloudflare": { - "statements": 37.92, - "branches": 32.13, - "functions": 41.98, - "lines": 38.4 - }, - "ai-sandbox-daytona": { - "statements": 12.42, - "branches": 3.07, - "functions": 6.25, - "lines": 13.54 - }, - "ai-sandbox-docker": { - "statements": 62.01, - "branches": 47.36, - "functions": 56.6, - "lines": 64.32 - }, - "ai-sandbox-local-process": { - "statements": 68.42, - "branches": 50.57, - "functions": 82.45, - "lines": 71.33 - }, - "ai-sandbox-sprites": { - "statements": 70.75, - "branches": 55.46, - "functions": 70.87, - "lines": 73.88 - }, - "ai-sandbox-vercel": { - "statements": 5.44, - "branches": 15.47, - "functions": 2.27, - "lines": 6.01 - }, - "ai-solid": { - "statements": 95.27, - "branches": 70.13, - "functions": 93.87, - "lines": 95.81 - }, - "ai-solid-ui": { - "statements": 2.87, - "branches": 3.63, - "functions": 0.94, - "lines": 4.02 - }, - "ai-svelte": { - "statements": 73.51, - "branches": 59.25, - "functions": 72.99, - "lines": 74.72 - }, - "ai-utils": { - "statements": 77.33, - "branches": 75, - "functions": 90.9, - "lines": 77.61 - }, - "ai-vue": { - "statements": 78.18, - "branches": 56.42, - "functions": 81.91, - "lines": 83.95 - }, - "ai-vue-ui": { - "statements": 4.43, - "branches": 4.41, - "functions": 2.56, - "lines": 4.76 - }, - "openai-base": { - "statements": 73.66, - "branches": 57.91, - "functions": 63.82, - "lines": 74.15 - }, - "preact-ai-devtools": { - "statements": 0, - "branches": 0, - "functions": 100, - "lines": 0 - }, - "react-ai-devtools": { - "statements": 0, - "branches": 0, - "functions": 100, - "lines": 0 - }, - "solid-ai-devtools": { - "statements": 0, - "branches": 0, - "functions": 100, - "lines": 0 - } -} diff --git a/package.json b/package.json index be2ab17c90..5874174c88 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,6 @@ "test:lib:dev": "pnpm test:lib && nx watch --all -- pnpm test:lib", "test:coverage": "nx affected --targets=test:coverage --exclude=examples/**,testing/**", "test:coverage:all": "nx run-many --targets=test:coverage --exclude=examples/**,testing/**", - "test:coverage:check": "pnpm run test:coverage && node scripts/coverage-check.mjs", - "test:coverage:update": "pnpm run test:coverage:all && node scripts/coverage-check.mjs --update", "test:build": "nx affected --target=test:build --exclude=examples/**,testing/**", "test:types": "nx affected --targets=test:types --exclude=examples/**,testing/**", "test:knip": "knip", diff --git a/scripts/coverage-check.mjs b/scripts/coverage-check.mjs index 34d24550ab..6f8ac6dbf0 100644 --- a/scripts/coverage-check.mjs +++ b/scripts/coverage-check.mjs @@ -1,91 +1,117 @@ -// Compares the coverage produced by `test:coverage` against the committed -// baseline in coverage-baseline.json and fails when a package regressed. +// Compares coverage between two runs and fails when a package regressed. +// +// Both sides are measured in the same CI job — the PR head and its merge-base +// with main — so there is no baseline file to keep in sync, no per-platform +// skew, and packages can be added or removed without anyone updating a +// checked-in number. // // Usage: -// node scripts/coverage-check.mjs # compare, exit 1 on a drop -// node scripts/coverage-check.mjs --update # rewrite the baseline +// node scripts/coverage-check.mjs --collect # after a coverage run +// node scripts/coverage-check.mjs --base --head // -// ponytail: a committed baseline file is the whole ratchet — no coverage -// service and no historical database. Results surface on the PR via -// $GITHUB_STEP_SUMMARY; for per-line annotations, upload the lcov files that -// `test:coverage` already writes. +// ponytail: two directories of json-summary files and a subtraction. No +// coverage service, no history, no baseline to maintain. import { appendFileSync, + mkdirSync, readFileSync, readdirSync, writeFileSync, } from 'node:fs' import { join } from 'node:path' -const BASELINE = 'coverage-baseline.json' const METRICS = ['statements', 'branches', 'functions', 'lines'] -// Coverage percentages wobble slightly between runs (v8 attributes some -// bytes differently depending on JIT timing), so require a real drop. +// Coverage percentages wobble slightly between runs (v8 attributes some bytes +// differently depending on JIT timing), so require a real drop. const TOLERANCE = 0.5 -const update = process.argv.includes('--update') +function arg(name) { + const i = process.argv.indexOf(name) + return i === -1 ? undefined : process.argv[i + 1] +} -/** @returns {Record>} */ -function readBaseline() { - try { - return JSON.parse(readFileSync(BASELINE, 'utf8')) - } catch (error) { - if (error.code === 'ENOENT') return {} - throw error - } +/** Reduce a vitest json-summary to the four totals we compare. */ +function totals(summary) { + // A package with no source loaded reports totals of 0/0 as 100%; that is not + // a number worth comparing. + if (summary.total.statements.total === 0) return undefined + return Object.fromEntries( + METRICS.map((metric) => [metric, summary.total[metric].pct]), + ) } -/** Collect the fresh coverage summaries written by this run. */ -function readCurrent() { - const current = {} +/** + * Copy this run's summaries into `dir`, one small file per package, so they + * survive the `git checkout` that happens between the two coverage runs. + */ +function collect(dir) { + mkdirSync(dir, { recursive: true }) + let n = 0 for (const entry of readdirSync('packages', { withFileTypes: true })) { if (!entry.isDirectory()) continue - const summaryPath = join( - 'packages', - entry.name, - 'coverage', - 'coverage-summary.json', - ) let summary try { - summary = JSON.parse(readFileSync(summaryPath, 'utf8')) + summary = JSON.parse( + readFileSync( + join('packages', entry.name, 'coverage', 'coverage-summary.json'), + 'utf8', + ), + ) } catch (error) { - // Not every package is affected on every run — no summary means the - // package was not measured, which is different from measuring 0%. + // No summary means the package was not measured in this run, which is + // different from it measuring 0%. if (error.code === 'ENOENT') continue throw error } - // A package with no source loaded reports totals of 0/0 as 100%; that is - // not a number worth ratcheting against. - if (summary.total.statements.total === 0) continue - current[entry.name] = Object.fromEntries( - METRICS.map((metric) => [metric, summary.total[metric].pct]), - ) + const metrics = totals(summary) + if (!metrics) continue + writeFileSync(join(dir, `${entry.name}.json`), JSON.stringify(metrics)) + n++ } - return current + console.log(`Collected ${n} coverage summary/summaries into ${dir}`) } -const baseline = readBaseline() -const current = readCurrent() -const names = Object.keys(current).sort() +function read(dir) { + let files + try { + files = readdirSync(dir) + } catch (error) { + // A missing directory means that side measured nothing at all. + if (error.code === 'ENOENT') return {} + throw error + } + return Object.fromEntries( + files + .filter((file) => file.endsWith('.json')) + .map((file) => [ + file.replace(/\.json$/, ''), + JSON.parse(readFileSync(join(dir, file), 'utf8')), + ]), + ) +} -if (names.length === 0) { +const collectDir = arg('--collect') +if (collectDir) { + collect(collectDir) + process.exit(0) +} + +const baseDir = arg('--base') +const headDir = arg('--head') +if (!baseDir || !headDir) { console.error( - `No coverage summaries found under packages/*/coverage/. Run \`pnpm test:coverage\` first.`, + 'Usage: coverage-check.mjs --collect | --base --head ', ) - process.exit(1) + process.exit(2) } -if (update) { - const merged = { ...baseline, ...current } - const sorted = Object.fromEntries( - Object.keys(merged) - .sort() - .map((key) => [key, merged[key]]), - ) - writeFileSync(BASELINE, `${JSON.stringify(sorted, null, 2)}\n`) - console.log(`Updated ${BASELINE} with ${names.length} package(s).`) +const base = read(baseDir) +const head = read(headDir) +const names = Object.keys(head).sort() + +if (names.length === 0) { + console.log('No packages were measured — nothing to compare.') process.exit(0) } @@ -94,29 +120,36 @@ const additions = [] const rows = [] for (const name of names) { - const before = baseline[name] - const after = current[name] + const before = base[name] + const after = head[name] if (!before) { + // New package, or one whose suite could not be measured on the base + // commit. There is nothing to regress against. additions.push(name) - rows.push([name, ...METRICS.map((m) => `${after[m].toFixed(2)}%`), 'new']) + rows.push([ + name, + ...METRICS.map((metric) => `${after[metric].toFixed(2)}%`), + 'new', + ]) continue } const deltas = METRICS.map((metric) => after[metric] - before[metric]) const dropped = METRICS.filter((metric, i) => deltas[i] < -TOLERANCE) - if (dropped.length > 0) { - regressions.push({ name, before, after, dropped }) - } + if (dropped.length > 0) regressions.push({ name, before, after, dropped }) rows.push([ name, ...METRICS.map((metric, i) => { - const delta = deltas[i] - const sign = delta > 0 ? '+' : '' - return `${after[metric].toFixed(2)}% (${sign}${delta.toFixed(2)})` + const sign = deltas[i] > 0 ? '+' : '' + return `${after[metric].toFixed(2)}% (${sign}${deltas[i].toFixed(2)})` }), dropped.length > 0 ? 'DROP' : 'ok', ]) } +const removed = Object.keys(base) + .filter((name) => !head[name]) + .sort() + const header = ['package', ...METRICS, ''] const widths = header.map((_, column) => Math.max(header[column].length, ...rows.map((row) => row[column].length)), @@ -131,54 +164,37 @@ console.log(line(header)) console.log(widths.map((width) => '-'.repeat(width)).join(' ')) for (const row of rows) console.log(line(row)) -// Anything the baseline needs to absorb, rendered paste-ready. Coverage is a -// CI-only job and a few packages measure differently per platform, so a -// contributor must never regenerate the baseline on their own machine — they -// copy these entries out of the run instead. -const staleNames = [ - ...new Set([...additions, ...regressions.map((r) => r.name)]), -].sort() -const pasteBlock = staleNames - .map((name) => ` ${JSON.stringify(name)}: ${JSON.stringify(current[name])},`) - .join('\n') - if (additions.length > 0) { console.log( - `\n${additions.length} package(s) missing from ${BASELINE}: ${additions.join(', ')}`, + `\n${additions.length} package(s) had no coverage on the base commit` + + ` (new, or unmeasurable there): ${additions.join(', ')}`, ) } +if (removed.length > 0) { + console.log(`\nNot measured on this PR: ${removed.join(', ')}`) +} -// ponytail: $GITHUB_STEP_SUMMARY renders on the run page with no token, no -// permissions and no API call. A sticky PR comment needs `pull-requests: -// write` — add that only if the summary tab turns out to be too easy to miss. if (process.env.GITHUB_STEP_SUMMARY) { - const verdict = - regressions.length > 0 - ? `❌ Coverage dropped in ${regressions.length} package(s).` - : `✅ Coverage held for ${names.length} measured package(s).` const md = [ `## Coverage`, ``, - verdict, + regressions.length > 0 + ? `❌ Coverage dropped in ${regressions.length} package(s).` + : `✅ Coverage held across ${names.length} measured package(s).`, ``, - `Baseline: \`${BASELINE}\`. Fails on a drop of more than ${TOLERANCE}pp. Packages not affected by this PR are not measured and not listed.`, + `Each package is measured twice in this job — on this PR and on its merge-base` + + ` with \`main\` — and compared. A drop of more than ${TOLERANCE}pp in any metric fails.` + + ` Packages your PR doesn't affect are not measured and not listed.`, ``, `| package | ${METRICS.join(' | ')} | |`, `| --- | ${METRICS.map(() => '---:').join(' | ')} | --- |`, ...rows.map((row) => `| ${row.join(' | ')} |`), ] - if (pasteBlock) { + if (additions.length > 0) { md.push( ``, - `### Updating the baseline`, - ``, - `Coverage runs in CI only, and some packages measure differently per platform —`, - `do not regenerate the baseline on your own machine. If these numbers are the`, - `intended ones, paste the entries below into \`${BASELINE}\` and push:`, - ``, - '```json', - pasteBlock, - '```', + `> No coverage on the base commit, so nothing to compare against:` + + ` ${additions.join(', ')}.`, ) } appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${md.join('\n')}\n`) @@ -193,12 +209,8 @@ if (regressions.length > 0) { ) } } - console.error( - `\nAdd tests to restore coverage. If the drop is intentional, paste these` + - ` entries into ${BASELINE} and push — do not regenerate the baseline` + - ` locally, some packages measure differently per platform:\n\n${pasteBlock}`, - ) + console.error(`\nAdd tests covering the code this PR changed.`) process.exit(1) } -console.log(`\nCoverage held for ${names.length} measured package(s).`) +console.log(`\nCoverage held across ${names.length} measured package(s).`) From d54f5fc14bc5add64018e2da23715bdadbd4a880 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 16:15:26 +0200 Subject: [PATCH 07/10] fix(ci): stage the coverage script outside the working tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge-base step ran `node scripts/coverage-check.mjs --collect` after checking out the base commit, where that file does not exist — this PR is what adds it: Error: Cannot find module '/home/runner/work/ai/ai/scripts/coverage-check.mjs' Not specific to this PR either: any PR that adds or moves the script would hit the same thing. Copying it to $RUNNER_TEMP before the checkout fixes it and also guarantees both runs and the comparison use the same version, which the previous `git checkout $GITHUB_SHA -- scripts/...` in the compare step only approximated. The base coverage run itself was fine, at 3/6 Nx cache hits before coverage.yml has ever run on main. --- .github/workflows/pr.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6b7aa6b3a6..01e2b19e14 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -46,6 +46,12 @@ jobs: uses: nrwl/nx-set-shas@3e9ad7370203c1e93d109be57f3b72eb0eb511b1 # v4.4.0 with: main-branch-name: main + # Stashed outside the working tree because checking out the merge base + # would otherwise remove it — the script may be added or changed by the + # very PR under test. Both runs and the comparison use this copy, so all + # three agree on the version. + - name: Stage the coverage script + run: cp scripts/coverage-check.mjs "$RUNNER_TEMP/coverage-check.mjs" # The set of packages to measure is decided once, here, so that the base # run measures exactly what the head run does. `--with-target` keeps out # projects that have no test:coverage script, so a docs-only PR resolves @@ -60,7 +66,7 @@ jobs: if: steps.affected.outputs.list != '' run: | pnpm exec nx run-many --targets=test:coverage --projects="${{ steps.affected.outputs.list }}" - node scripts/coverage-check.mjs --collect "$RUNNER_TEMP/cov-head" + node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-head" # Re-measuring the merge-base is normally a set of Nx Cloud cache # restores seeded by the Coverage workflow on main: the task hashes are # identical, and `test:coverage` declares its coverage dir as an output, @@ -79,12 +85,11 @@ jobs: exit 0 fi pnpm exec nx run-many --targets=test:coverage --projects="$base_list" - node scripts/coverage-check.mjs --collect "$RUNNER_TEMP/cov-base" + node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-base" - name: Compare if: steps.affected.outputs.list != '' run: | - git checkout --force "$GITHUB_SHA" -- scripts/coverage-check.mjs - node scripts/coverage-check.mjs \ + node "$RUNNER_TEMP/coverage-check.mjs" \ --base "$RUNNER_TEMP/cov-base" \ --head "$RUNNER_TEMP/cov-head" preview: From 43e602bdc80ea0ac62e36be7867904970113b734 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 16:20:59 +0200 Subject: [PATCH 08/10] fix(ci): clear coverage dirs before each run so the base side is real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous run passed for the wrong reason. `coverage/` is gitignored, so checking out the merge base left this PR's coverage files on disk, and the base-side `--collect` read them back: 47 packages were compared against themselves, which is why every delta was exactly 0.00. Both runs now clear `packages/*/coverage` first. Nx restores the directory for cached tasks, so genuine cache hits still work. On this PR the base side will now legitimately collect almost nothing — the merge base predates the json-summary reporter, so there is no base data to compare against and packages report as `new`. That resolves itself once this lands and main has the machinery. --- .github/workflows/pr.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 01e2b19e14..0ddcc2ce9f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -65,6 +65,7 @@ jobs: - name: Coverage for this PR if: steps.affected.outputs.list != '' run: | + rm -rf packages/*/coverage pnpm exec nx run-many --targets=test:coverage --projects="${{ steps.affected.outputs.list }}" node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-head" # Re-measuring the merge-base is normally a set of Nx Cloud cache @@ -75,6 +76,10 @@ jobs: if: steps.affected.outputs.list != '' run: | git checkout --force --detach "$NX_BASE" + # coverage/ is gitignored, so the checkout leaves this PR's results in + # place — without this the base collection silently re-reads head's + # numbers and every delta comes out as 0.00. + rm -rf packages/*/coverage pnpm install --frozen-lockfile # Packages added by this PR do not exist on the base commit. printf '%s\n' "${{ steps.affected.outputs.list }}" | tr ',' '\n' | sort > /tmp/wanted From 6d7b19e157a41123b6929de8c0295c1d6a4f8a17 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Fri, 31 Jul 2026 16:44:07 +0200 Subject: [PATCH 09/10] fix(ci): pass the project list via env to satisfy zizmor zizmor flagged both `${{ steps.affected.outputs.list }}` uses inside run blocks as code injection via template expansion (pr.yml:69, pr.yml:85), and it's right: the list is built from package names in the PR's own files, so on a fork PR a package named with shell metacharacters would be interpolated straight into the script. Passing it through env keeps it out of the shell source entirely. Confirmed clean by running zizmor v1.28.0 locally over both workflows. --- .github/workflows/pr.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0ddcc2ce9f..265e2136be 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -64,9 +64,13 @@ jobs: echo "Measuring: ${list:-}" - name: Coverage for this PR if: steps.affected.outputs.list != '' + # Via env, not `${{ }}` inside run: project names come from the PR's own + # files, so interpolating them into the shell would be injectable. + env: + PROJECTS: ${{ steps.affected.outputs.list }} run: | rm -rf packages/*/coverage - pnpm exec nx run-many --targets=test:coverage --projects="${{ steps.affected.outputs.list }}" + pnpm exec nx run-many --targets=test:coverage --projects="$PROJECTS" node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-head" # Re-measuring the merge-base is normally a set of Nx Cloud cache # restores seeded by the Coverage workflow on main: the task hashes are @@ -74,6 +78,8 @@ jobs: # so the summaries come back without the tests re-running. - name: Coverage for the merge base if: steps.affected.outputs.list != '' + env: + PROJECTS: ${{ steps.affected.outputs.list }} run: | git checkout --force --detach "$NX_BASE" # coverage/ is gitignored, so the checkout leaves this PR's results in @@ -82,7 +88,7 @@ jobs: rm -rf packages/*/coverage pnpm install --frozen-lockfile # Packages added by this PR do not exist on the base commit. - printf '%s\n' "${{ steps.affected.outputs.list }}" | tr ',' '\n' | sort > /tmp/wanted + printf '%s\n' "$PROJECTS" | tr ',' '\n' | sort > /tmp/wanted pnpm exec nx show projects --with-target=test:coverage --json | jq -r '.[]' | sort > /tmp/available base_list=$(comm -12 /tmp/wanted /tmp/available | paste -sd, -) if [ -z "$base_list" ]; then From 380a88456a028388dcc1e0a13f068f7269e12478 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:39:18 +1000 Subject: [PATCH 10/10] fix(ci): fail the coverage gate when a side cannot be compared Missing summaries were treated as out of scope, so this PR's Coverage job compared nothing against main. Collect now requires --expect, packages that vanish from head fail, empty head fails, and both CI runs pass json-summary so the merge-base writes coverage-summary.json. --- .github/workflows/pr.yml | 16 +- AGENTS.md | 7 + CLAUDE.md | 2 +- CONTRIBUTING.md | 10 +- package.json | 2 +- packages/ai-byteplus/package.json | 1 + packages/ai-cohere/package.json | 1 + packages/ai-isolate-daytona/package.json | 1 + packages/ai-isolate-quickjs-bun/package.json | 1 + packages/ai-llmgateway/package.json | 1 + packages/ai-octane/package.json | 1 + packages/ai-perplexity/package.json | 1 + packages/ai-vercel-gateway/package.json | 1 + packages/ai-vertex/package.json | 1 + scripts/coverage-check.mjs | 154 ++++++++---- scripts/coverage-check.test.ts | 236 +++++++++++++++++++ 16 files changed, 382 insertions(+), 54 deletions(-) create mode 100644 scripts/coverage-check.test.ts diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 265e2136be..2eb32ac37b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -70,8 +70,8 @@ jobs: PROJECTS: ${{ steps.affected.outputs.list }} run: | rm -rf packages/*/coverage - pnpm exec nx run-many --targets=test:coverage --projects="$PROJECTS" - node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-head" + pnpm exec nx run-many --targets=test:coverage --projects="$PROJECTS" -- --coverage.reporter=json-summary --coverage.include='src/**' + node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-head" --expect "$PROJECTS" # Re-measuring the merge-base is normally a set of Nx Cloud cache # restores seeded by the Coverage workflow on main: the task hashes are # identical, and `test:coverage` declares its coverage dir as an output, @@ -87,16 +87,20 @@ jobs: # numbers and every delta comes out as 0.00. rm -rf packages/*/coverage pnpm install --frozen-lockfile - # Packages added by this PR do not exist on the base commit. + # Intersect with projects that have test:coverage on the base commit + # (added packages, or packages that gained the script in this PR). printf '%s\n' "$PROJECTS" | tr ',' '\n' | sort > /tmp/wanted pnpm exec nx show projects --with-target=test:coverage --json | jq -r '.[]' | sort > /tmp/available base_list=$(comm -12 /tmp/wanted /tmp/available | paste -sd, -) if [ -z "$base_list" ]; then - echo "No affected package exists on the base commit; nothing to compare against." + echo "No affected package has test:coverage on the base commit; treating all as new." + mkdir -p "$RUNNER_TEMP/cov-base" exit 0 fi - pnpm exec nx run-many --targets=test:coverage --projects="$base_list" - node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-base" + # Extra reporters/include so a merge-base whose scripts are still + # `vitest run --coverage` still writes coverage-summary.json. + pnpm exec nx run-many --targets=test:coverage --projects="$base_list" -- --coverage.reporter=json-summary --coverage.include='src/**' + node "$RUNNER_TEMP/coverage-check.mjs" --collect "$RUNNER_TEMP/cov-base" --expect "$base_list" - name: Compare if: steps.affected.outputs.list != '' run: | diff --git a/AGENTS.md b/AGENTS.md index 6fb6ac40f8..c631bb6ebe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -148,6 +148,13 @@ each of these and confirm they're green before pushing: Do **not** rely on CI as your first signal. Run locally, fix, then push. +## Coverage + +Coverage is CI-only. Do not run it locally and do not add it to `test`, +`test:pr`, `test:ci`, or git hooks. There is no baseline file. The PR +`Coverage` job compares affected packages on the branch against their +merge-base with `main`. See **Coverage** in `CONTRIBUTING.md`. + ## Documentation Load the `docs` skill first (see **Docs skill** above). Then also obey diff --git a/CLAUDE.md b/CLAUDE.md index 54b9c48188..93d1dc02ba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ pnpm test:lib:dev # Watch mode for unit tests pnpm test:oxlint # Lint affected packages (oxlint, incl. type-aware) pnpm test:types # Type check affected packages pnpm test:build # Verify build artifacts with publint -# Coverage is CI-only — see the Coverage section below. Don't run it locally. +# Coverage is CI-only. See CONTRIBUTING.md. Don't run it locally. pnpm test:knip # Check for unused dependencies pnpm test:sherif # Check pnpm workspace consistency pnpm test:docs # Verify documentation links diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12d38c8329..f86d20847c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,11 +92,11 @@ Working on a single package? `cd packages/` and use its scripts directly (` **Coverage runs in CI only. It is not part of `pnpm test`, `pnpm test:pr`, or any git hook, and you are not expected to run it locally.** -The `Coverage` job on every PR measures each affected package **twice** — once on your branch and once on its merge-base with `main` — and compares the two. A drop of more than 0.5 percentage points in any metric (statements, branches, functions, lines) fails the job. Packages your PR didn't affect are never measured. +The `Coverage` job on every PR measures each affected package that has a `test:coverage` target twice: once on your branch and once on its merge-base with `main`. A drop of more than 0.5 percentage points in any metric (statements, branches, functions, lines) fails the job. Packages your PR didn't affect, and packages without that target, are never measured. There is no baseline file to keep in sync, and nothing to update when a package is added or removed: both numbers come from the same job on the same runner. There are also no target percentages to hit — the gate only catches coverage getting _worse_ in what you touched, and never blocks a PR for being below some repo-wide bar. -Read the numbers from the PR's Checks tab: open the `Coverage` job and its summary has a per-package table with deltas, on every run whether it passed or failed. It is not posted as a PR comment. +Read the numbers from the PR's Checks tab: open the `Coverage` job. When at least one package with `test:coverage` is affected, the job summary has a per-package table with deltas, pass or fail. It is not posted as a PR comment. Re-measuring the merge-base is usually close to free. A separate `Coverage` workflow runs `test:coverage:all` on every push to `main`, which populates the Nx Cloud cache; because `test:coverage` declares its `coverage/` directory as a task output, the base-side run generally restores cached summaries rather than re-running any tests. @@ -107,9 +107,11 @@ Add tests covering the code you changed. That's the whole remedy — there is no Two known limitations: - Uncovered `.tsx` files can't be remapped by the coverage provider and are dropped from the report with a `Failed to parse ... Excluding it from coverage` warning. `.tsx` files that tests _do_ load are measured normally, so the UI packages read higher than their real coverage. -- `preact-ai-devtools`, `react-ai-devtools`, and `solid-ai-devtools` have no tests and sit at 0%. +- `preact-ai-devtools`, `react-ai-devtools`, and `solid-ai-devtools` have no tests. They may be omitted (0 statements after `.tsx` remap failure) or show ~0% of remaining `.ts`; they do not gate the job. -A package your PR adds shows as `new` and can't fail the job — there is no base-commit coverage to compare it against. The same is true for a package whose suite can't run on the base commit. +A **new package that defines `test:coverage`** shows as `new` and cannot fail the comparison. Packages without that script are not measured. + +If a package was measured on the base commit and has no summary on this PR, the job fails. A crashing `test:coverage` suite on either side also fails the job. It is not treated as `new`. ## TypeScript configuration diff --git a/package.json b/package.json index 5874174c88..0e8c6ff61e 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "test:knip": "knip", "test:docs": "tsx scripts/verify-links.ts", "test:dts": "node scripts/scan-dangling-dts.mjs", - "test:maintainer": "vitest run --root scripts/maintainer && vitest run scripts/model-sync scripts/publish-staged-docs.test.ts", + "test:maintainer": "vitest run --root scripts/maintainer && vitest run scripts/model-sync scripts/publish-staged-docs.test.ts && vitest run scripts/coverage-check.test.ts", "maintainer:sweep": "tsx scripts/maintainer/sweep.ts", "maintainer:scorecard": "tsx scripts/maintainer/scorecard.ts", "test:kiira": "kiira check", diff --git a/packages/ai-byteplus/package.json b/packages/ai-byteplus/package.json index e887f6d767..806e0db824 100644 --- a/packages/ai-byteplus/package.json +++ b/packages/ai-byteplus/package.json @@ -40,6 +40,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-cohere/package.json b/packages/ai-cohere/package.json index 3af1f45221..fb2a61cf1e 100644 --- a/packages/ai-cohere/package.json +++ b/packages/ai-cohere/package.json @@ -44,6 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-isolate-daytona/package.json b/packages/ai-isolate-daytona/package.json index 9463108e9e..38fe215f75 100644 --- a/packages/ai-isolate-daytona/package.json +++ b/packages/ai-isolate-daytona/package.json @@ -32,6 +32,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:live": "vitest --run tests/live.test.ts", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", diff --git a/packages/ai-isolate-quickjs-bun/package.json b/packages/ai-isolate-quickjs-bun/package.json index 21a4b8abeb..28ca0779b4 100644 --- a/packages/ai-isolate-quickjs-bun/package.json +++ b/packages/ai-isolate-quickjs-bun/package.json @@ -41,6 +41,7 @@ "test:build": "publint --strict", "test:bun": "bun test ./tests", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest --passWithNoTests", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-llmgateway/package.json b/packages/ai-llmgateway/package.json index ee8ceed091..f0fc91dbef 100644 --- a/packages/ai-llmgateway/package.json +++ b/packages/ai-llmgateway/package.json @@ -36,6 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-octane/package.json b/packages/ai-octane/package.json index ce4a83eefb..1a0c61348d 100644 --- a/packages/ai-octane/package.json +++ b/packages/ai-octane/package.json @@ -48,6 +48,7 @@ "scripts": { "lint:fix": "oxlint src --type-aware --ignore-pattern '**/*.tsrx.d.ts' --fix", "test:oxlint": "oxlint src --type-aware --ignore-pattern '**/*.tsrx.d.ts'", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc && tsc -p typetests/tsconfig.json" diff --git a/packages/ai-perplexity/package.json b/packages/ai-perplexity/package.json index a6b549a8ba..9a2c34b5e9 100644 --- a/packages/ai-perplexity/package.json +++ b/packages/ai-perplexity/package.json @@ -44,6 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-vercel-gateway/package.json b/packages/ai-vercel-gateway/package.json index 358a67df6e..5f4f480aeb 100644 --- a/packages/ai-vercel-gateway/package.json +++ b/packages/ai-vercel-gateway/package.json @@ -44,6 +44,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/packages/ai-vertex/package.json b/packages/ai-vertex/package.json index 01bd1ac2ff..cf927ea126 100644 --- a/packages/ai-vertex/package.json +++ b/packages/ai-vertex/package.json @@ -36,6 +36,7 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", "test:lib": "vitest run", "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" diff --git a/scripts/coverage-check.mjs b/scripts/coverage-check.mjs index 6f8ac6dbf0..b0e01278be 100644 --- a/scripts/coverage-check.mjs +++ b/scripts/coverage-check.mjs @@ -6,11 +6,8 @@ // checked-in number. // // Usage: -// node scripts/coverage-check.mjs --collect # after a coverage run +// node scripts/coverage-check.mjs --collect [--expect pkg,pkg] // node scripts/coverage-check.mjs --base --head -// -// ponytail: two directories of json-summary files and a subtraction. No -// coverage service, no history, no baseline to maintain. import { appendFileSync, mkdirSync, @@ -31,7 +28,6 @@ function arg(name) { return i === -1 ? undefined : process.argv[i + 1] } -/** Reduce a vitest json-summary to the four totals we compare. */ function totals(summary) { // A package with no source loaded reports totals of 0/0 as 100%; that is not // a number worth comparing. @@ -41,34 +37,89 @@ function totals(summary) { ) } -/** - * Copy this run's summaries into `dir`, one small file per package, so they - * survive the `git checkout` that happens between the two coverage runs. - */ -function collect(dir) { - mkdirSync(dir, { recursive: true }) - let n = 0 +function packageDirs() { + const dirs = {} for (const entry of readdirSync('packages', { withFileTypes: true })) { if (!entry.isDirectory()) continue - let summary + dirs[entry.name] = entry.name try { - summary = JSON.parse( - readFileSync( - join('packages', entry.name, 'coverage', 'coverage-summary.json'), - 'utf8', - ), + const pkg = JSON.parse( + readFileSync(join('packages', entry.name, 'package.json'), 'utf8'), ) - } catch (error) { - // No summary means the package was not measured in this run, which is - // different from it measuring 0%. - if (error.code === 'ENOENT') continue - throw error + if (pkg.name) dirs[pkg.name] = entry.name + } catch { + // Directory with no package.json is still collectable by folder name. } - const metrics = totals(summary) - if (!metrics) continue - writeFileSync(join(dir, `${entry.name}.json`), JSON.stringify(metrics)) + } + return dirs +} + +function expectedDirs() { + const raw = arg('--expect') + if (raw === undefined) return undefined + const names = raw + .split(',') + .map((name) => name.trim()) + .filter(Boolean) + const dirs = packageDirs() + const resolved = [] + const unknown = [] + for (const name of names) { + const dir = dirs[name] + if (!dir) unknown.push(name) + else resolved.push(dir) + } + if (unknown.length > 0) { + console.error(`Unknown package(s): ${unknown.join(', ')}`) + process.exit(1) + } + return resolved +} + +function readSummary(dirName) { + try { + return JSON.parse( + readFileSync( + join('packages', dirName, 'coverage', 'coverage-summary.json'), + 'utf8', + ), + ) + } catch (error) { + if (error && error.code === 'ENOENT') return null + throw error + } +} + +function collect(dir) { + mkdirSync(dir, { recursive: true }) + const expected = expectedDirs() + const names = + expected ?? + readdirSync('packages', { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + const missing = [] + let n = 0 + for (const name of names) { + const summary = readSummary(name) + if (!summary) { + if (expected) missing.push(`${name} (no coverage-summary.json)`) + continue + } + const values = totals(summary) + if (!values) { + if (expected) missing.push(`${name} (0 statements)`) + continue + } + writeFileSync(join(dir, `${name}.json`), JSON.stringify(values)) n++ } + if (missing.length > 0) { + console.error( + `Coverage collect failed for ${missing.length} package(s):\n ${missing.join('\n ')}`, + ) + process.exit(1) + } console.log(`Collected ${n} coverage summary/summaries into ${dir}`) } @@ -77,8 +128,7 @@ function read(dir) { try { files = readdirSync(dir) } catch (error) { - // A missing directory means that side measured nothing at all. - if (error.code === 'ENOENT') return {} + if (error && error.code === 'ENOENT') return {} throw error } return Object.fromEntries( @@ -101,7 +151,7 @@ const baseDir = arg('--base') const headDir = arg('--head') if (!baseDir || !headDir) { console.error( - 'Usage: coverage-check.mjs --collect | --base --head ', + 'Usage: coverage-check.mjs --collect [--expect pkg,pkg] | --base --head ', ) process.exit(2) } @@ -111,8 +161,8 @@ const head = read(headDir) const names = Object.keys(head).sort() if (names.length === 0) { - console.log('No packages were measured — nothing to compare.') - process.exit(0) + console.error('No packages were measured — nothing to compare.') + process.exit(1) } const regressions = [] @@ -123,8 +173,6 @@ for (const name of names) { const before = base[name] const after = head[name] if (!before) { - // New package, or one whose suite could not be measured on the base - // commit. There is nothing to regress against. additions.push(name) rows.push([ name, @@ -146,9 +194,19 @@ for (const name of names) { ]) } -const removed = Object.keys(base) +for (const name of Object.keys(base) .filter((name) => !head[name]) - .sort() + .sort()) { + const before = base[name] + regressions.push({ name, before, after: null, dropped: ['missing'] }) + rows.push([ + name, + ...METRICS.map((metric) => `missing (${before[metric].toFixed(2)}%)`), + 'DROP', + ]) +} + +const compared = names.filter((name) => base[name]).length const header = ['package', ...METRICS, ''] const widths = header.map((_, column) => @@ -170,17 +228,19 @@ if (additions.length > 0) { ` (new, or unmeasurable there): ${additions.join(', ')}`, ) } -if (removed.length > 0) { - console.log(`\nNot measured on this PR: ${removed.join(', ')}`) -} + +const headline = + regressions.length > 0 + ? `❌ Coverage dropped in ${regressions.length} package(s).` + : compared > 0 + ? `✅ Coverage held across ${compared} compared package(s).` + : `${additions.length} package(s) are new; nothing to compare against.` if (process.env.GITHUB_STEP_SUMMARY) { const md = [ `## Coverage`, ``, - regressions.length > 0 - ? `❌ Coverage dropped in ${regressions.length} package(s).` - : `✅ Coverage held across ${names.length} measured package(s).`, + headline, ``, `Each package is measured twice in this job — on this PR and on its merge-base` + ` with \`main\` — and compared. A drop of more than ${TOLERANCE}pp in any metric fails.` + @@ -203,6 +263,10 @@ if (process.env.GITHUB_STEP_SUMMARY) { if (regressions.length > 0) { console.error(`\nCoverage dropped in ${regressions.length} package(s):`) for (const { name, before, after, dropped } of regressions) { + if (!after) { + console.error(` ${name}: measured on base, missing on this PR`) + continue + } for (const metric of dropped) { console.error( ` ${name} ${metric}: ${before[metric].toFixed(2)}% -> ${after[metric].toFixed(2)}%`, @@ -213,4 +277,10 @@ if (regressions.length > 0) { process.exit(1) } -console.log(`\nCoverage held across ${names.length} measured package(s).`) +if (compared === 0) { + console.log( + `\n${additions.length} package(s) are new; nothing to compare against.`, + ) +} else { + console.log(`\nCoverage held across ${compared} compared package(s).`) +} diff --git a/scripts/coverage-check.test.ts b/scripts/coverage-check.test.ts new file mode 100644 index 0000000000..beeaa6662d --- /dev/null +++ b/scripts/coverage-check.test.ts @@ -0,0 +1,236 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { afterEach, describe, expect, it } from 'vitest' + +const SCRIPT = fileURLToPath(new URL('./coverage-check.mjs', import.meta.url)) + +const temps: Array = [] + +afterEach(() => { + for (const dir of temps.splice(0)) + rmSync(dir, { recursive: true, force: true }) +}) + +function tmp() { + const dir = mkdtempSync(join(tmpdir(), 'cov-check-')) + temps.push(dir) + return dir +} + +function run( + args: Array, + opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, +) { + const env: NodeJS.ProcessEnv = { ...process.env, ...opts.env } + if (!opts.env || !('GITHUB_STEP_SUMMARY' in opts.env)) { + delete env.GITHUB_STEP_SUMMARY + } + return spawnSync(process.execPath, [SCRIPT, ...args], { + encoding: 'utf8', + cwd: opts.cwd, + env, + }) +} + +function metrics(pct: number) { + return { statements: pct, branches: pct, functions: pct, lines: pct } +} + +function writeSnapshot(dir: string, name: string, pct: number) { + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, `${name}.json`), JSON.stringify(metrics(pct))) +} + +function vitestSummary(pct: number, total = 10) { + const cell = { total, covered: 1, pct } + return JSON.stringify({ + total: { + statements: cell, + branches: cell, + functions: cell, + lines: cell, + }, + }) +} + +function writePkg( + root: string, + dirName: string, + summary: string, + pkgName = `@tanstack/${dirName}`, +) { + mkdirSync(join(root, 'packages', dirName, 'coverage'), { recursive: true }) + writeFileSync( + join(root, 'packages', dirName, 'package.json'), + JSON.stringify({ name: pkgName }), + ) + writeFileSync( + join(root, 'packages', dirName, 'coverage', 'coverage-summary.json'), + summary, + ) +} + +describe('usage', () => { + it('exits 2 when --base or --head is missing', () => { + const r = run([]) + expect(r.status).toBe(2) + expect(r.stderr).toMatch(/Usage:/) + }) +}) + +describe('compare', () => { + it('passes a 0.4pp drop and a 0.5pp drop, fails 0.51pp', () => { + const base = tmp() + const head = tmp() + writeSnapshot(base, 'ok', 90) + writeSnapshot(head, 'ok', 89.5) + expect(run(['--base', base, '--head', head]).status).toBe(0) + + writeSnapshot(head, 'ok', 89.6) + expect(run(['--base', base, '--head', head]).status).toBe(0) + + writeSnapshot(head, 'ok', 89.49) + const r = run(['--base', base, '--head', head]) + expect(r.status).toBe(1) + expect(r.stderr).toMatch(/ok statements: 90\.00% -> 89\.49%/) + expect(r.stdout).toMatch(/DROP/) + }) + + it('fails when only branches drop more than 0.5pp', () => { + const base = tmp() + const head = tmp() + mkdirSync(base, { recursive: true }) + mkdirSync(head, { recursive: true }) + writeFileSync( + join(base, 'pkg.json'), + JSON.stringify({ + statements: 90, + branches: 80, + functions: 90, + lines: 90, + }), + ) + writeFileSync( + join(head, 'pkg.json'), + JSON.stringify({ + statements: 90, + branches: 79, + functions: 90, + lines: 90, + }), + ) + const r = run(['--base', base, '--head', head]) + expect(r.status).toBe(1) + expect(r.stderr).toMatch(/pkg branches: 80\.00% -> 79\.00%/) + expect(r.stderr).not.toMatch(/statements/) + }) + + it('treats a head-only package as new and exits 0', () => { + const base = tmp() + const head = tmp() + mkdirSync(base, { recursive: true }) + writeSnapshot(head, 'ai-new', 12) + const r = run(['--base', base, '--head', head]) + expect(r.status).toBe(0) + expect(r.stdout).toMatch(/\bnew\b/) + expect(r.stdout).toMatch(/nothing to compare against/) + expect(r.stdout).not.toMatch(/Coverage held/) + }) + + it('fails when a package was measured on base but not on head', () => { + const base = tmp() + const head = tmp() + writeSnapshot(base, 'ai-old', 90) + writeSnapshot(head, 'other', 90) + const r = run(['--base', base, '--head', head]) + expect(r.status).toBe(1) + expect(r.stderr).toMatch(/ai-old: measured on base, missing on this PR/) + }) + + it('fails when head measured nothing', () => { + const base = tmp() + const head = tmp() + mkdirSync(base, { recursive: true }) + mkdirSync(head, { recursive: true }) + const r = run(['--base', base, '--head', head]) + expect(r.status).toBe(1) + expect(r.stderr).toMatch(/nothing to compare/) + }) + + it('writes the step summary on a failing run', () => { + const base = tmp() + const head = tmp() + const summary = join(tmp(), 'summary.md') + writeSnapshot(base, 'pkg', 90) + writeSnapshot(head, 'pkg', 80) + const r = run(['--base', base, '--head', head], { + env: { ...process.env, GITHUB_STEP_SUMMARY: summary }, + }) + expect(r.status).toBe(1) + const md = require('node:fs').readFileSync(summary, 'utf8') as string + expect(md).toMatch(/Coverage dropped/) + expect(md).toMatch(/pkg/) + }) +}) + +describe('collect', () => { + it('writes pct-only snapshots and skips packages with no summary', () => { + const root = tmp() + const out = join(root, 'out') + writePkg(root, 'ai', vitestSummary(90)) + mkdirSync(join(root, 'packages', 'empty'), { recursive: true }) + writeFileSync( + join(root, 'packages', 'empty', 'package.json'), + JSON.stringify({ name: '@tanstack/empty' }), + ) + const r = run(['--collect', out], { cwd: root }) + expect(r.status).toBe(0) + const written = JSON.parse( + require('node:fs').readFileSync(join(out, 'ai.json'), 'utf8'), + ) + expect(written).toEqual(metrics(90)) + expect(r.stdout).toMatch(/Collected 1 /) + }) + + it('fails --expect when a named package has no summary', () => { + const root = tmp() + const out = join(root, 'out') + writePkg(root, 'ai', vitestSummary(90)) + const r = run(['--collect', out, '--expect', 'ai,missing-pkg'], { + cwd: root, + }) + expect(r.status).toBe(1) + expect(r.stderr).toMatch(/missing-pkg/) + }) + + it('fails --expect when statements.total is 0', () => { + const root = tmp() + const out = join(root, 'out') + writePkg(root, 'empty', vitestSummary(100, 0)) + const r = run(['--collect', out, '--expect', 'empty'], { cwd: root }) + expect(r.status).toBe(1) + expect(r.stderr).toMatch(/0 statements/) + }) + + it('resolves @tanstack scoped names to the package directory', () => { + const root = tmp() + const out = join(root, 'out') + writePkg( + root, + 'ai-devtools', + vitestSummary(81), + '@tanstack/ai-devtools-core', + ) + const r = run( + ['--collect', out, '--expect', '@tanstack/ai-devtools-core'], + { cwd: root }, + ) + expect(r.status).toBe(0) + expect(require('node:fs').existsSync(join(out, 'ai-devtools.json'))).toBe( + true, + ) + }) +})