From ae3bec9e211ee7b3add549a963ff747694791a97 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 08:37:13 +0000 Subject: [PATCH] ci(ui-coverage): run objectui's coverage lane nightly from the root, thresholds unchanged The ui/ cutover did not port objectui's coverage lane (roster row 7 of #25697, "kept, follow-up port"), and the nested ui/.github copy runs nowhere, so the ui coverage thresholds (lines 40, functions 33, branches 30, statements 40) have been evaluated by nothing since the cutover. objectui ran the lane on every push and hourly: 8 legs of ~22 min, about 31 % of its CI minutes (#25924). It comes back as .github/workflows/ui-coverage-nightly.yml (`UI Coverage`), on schedule (06:13 UTC) plus workflow_dispatch only: eight blob-reporting shards, then one merge job that refuses a partial set, enforces the configured thresholds over the merged map, publishes the report and states whether the gate ran. Not a merge gate and not a release gate. scheduled-full-run-card.yml now also watches `UI Coverage`, with a `nightly` cadence in CARD_CADENCES, so a red night files or refreshes one card; the self-test pins the identity, the nightly body, the workflow's single daily cron and that the watcher list and the cadence map name the same workflows. The three ui pins that retired with the lane come back against the new file through helpers/root-ci.ts: the width said once, the shard reporter set, and the headroom reading's matrix and cap. Part of #25924 Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01WKpunbiE8c9w2i9FsEvQas --- .github/workflows/scheduled-full-run-card.yml | 9 +- .github/workflows/ui-coverage-nightly.yml | 370 ++++++++++++++++++ scripts/ci/scheduled-full-run.mjs | 38 +- ...overage-shard-reporter-readability.test.ts | 164 ++++++++ .../__tests__/coverage-shard-width.test.ts | 121 ++++++ ui/scripts/__tests__/helpers/root-ci.ts | 24 +- ui/scripts/__tests__/shard-headroom.test.ts | 44 ++- ui/scripts/shard-headroom.mjs | 5 +- 8 files changed, 755 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/ui-coverage-nightly.yml create mode 100644 ui/scripts/__tests__/coverage-shard-reporter-readability.test.ts create mode 100644 ui/scripts/__tests__/coverage-shard-width.test.ts diff --git a/.github/workflows/scheduled-full-run-card.yml b/.github/workflows/scheduled-full-run-card.yml index 91c064504d5..c915e0d4ab6 100644 --- a/.github/workflows/scheduled-full-run-card.yml +++ b/.github/workflows/scheduled-full-run-card.yml @@ -87,8 +87,13 @@ on: # # Every name listed here must have an entry in `CARD_CADENCES` # (scripts/ci/scheduled-full-run.mjs), which refuses a name it does not - # hold rather than default one, so a card never prints the wrong cadence. - workflows: ['CI', 'Lint & Type Check'] + # hold rather than default one, so a card never prints the wrong cadence; + # its self-test holds this list and that map to the same names. + # + # `UI Coverage` (ui-coverage-nightly.yml, #25924) is the nightly one: the + # ui coverage thresholds are enforced there and nowhere else, so a red + # night files `nightly full run: red on main (UI Coverage)`. + workflows: ['CI', 'Lint & Type Check', 'UI Coverage'] types: [completed] permissions: {} diff --git a/.github/workflows/ui-coverage-nightly.yml b/.github/workflows/ui-coverage-nightly.yml new file mode 100644 index 00000000000..de0bab94cb2 --- /dev/null +++ b/.github/workflows/ui-coverage-nightly.yml @@ -0,0 +1,370 @@ +# UI coverage, nightly (#25924). objectui's coverage lane, ported to the root. +# +# objectui's `ci.yml` ran this lane (`Test (coverage shard n/8)` and the merge +# job `Test (coverage)`) on every push to `main` and on its hourly schedule: +# 8 legs of ~22 min, about 31 % of that repository's CI minutes (#25924's +# measurement, 2026-10-17..24), to enforce four thresholds. The ui/ cutover +# (#25752) did not port it: it is row 7 "kept, follow-up port" of the +# objectstack#25697 workflow roster, and the nested `ui/.github/workflows/` +# copy runs nowhere, so from the cutover until this file the thresholds were +# evaluated by nothing. +# +# It comes back here on `schedule` plus `workflow_dispatch` only, beside +# `coverage-nightly.yml` (spec's coverage report), with the thresholds +# unchanged: `ui/vitest.config.mts` `coverage.thresholds` (lines 40, +# functions 33, branches 30, statements 40). The cost accepted with it, stated +# on #25924: a change that drops coverage under a threshold is reported the +# night after it merged, not on the merge. `scheduled-full-run-card.yml` +# watches this workflow, so a red night files or refreshes one card +# (`nightly full run: red on main (UI Coverage)`); without that reader a red +# scheduled run is seen by nobody (objectui#5402: four dark days). +# +# ⛔ Not a merge gate and not a release gate: no pull request or queue build +# runs it, branch protection reads none of its contexts, and +# `scripts/ci/scheduled-full-run.mjs`'s RELEASE_GATE_WORKFLOWS is `CI` and +# `Lint & Type Check` only. +# +# The pins: `ui/scripts/__tests__/coverage-shard-width.test.ts` (the width, +# said once), `coverage-shard-reporter-readability.test.ts` (the shard leg's +# reporter set) and `shard-headroom.test.ts` (the headroom reading's matrix +# and cap). They read this file through `ui/scripts/__tests__/helpers/root-ci.ts`. +name: UI Coverage + +on: + workflow_dispatch: + schedule: + # 06:13 UTC: off the `:00` minute every other repository's cron queues + # on, after the 04:00 four-hourly full run and the 05:xx nightlies. + - cron: '13 6 * * *' + +permissions: + contents: read + +# One run at a time per ref; a dispatch while the nightly runs waits rather +# than cancelling a run that has already paid for its shards. +concurrency: + group: ui-coverage-${{ github.ref }} + cancel-in-progress: false + +jobs: + ui-coverage: + name: Test UI (coverage shard ${{ matrix.shard }}/8) + runs-on: ubuntu-latest + # objectui#12256 measured the 40-minute margin gone at four shards, which + # is why the lane is eight wide; the report job prints the longest shard's + # headroom against this cap on every run. + timeout-minutes: 40 + permissions: + contents: read + + strategy: + # Report every shard's failures, not just the first one to break; the + # merge below refuses to publish anything unless every shard is green. + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Setup pnpm + uses: ./.github/actions/setup-pnpm + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + # Restore-only, here and below: a nightly reads `main`'s caches and the + # per-push workflows own saving them. + - name: Restore pnpm cache + uses: actions/cache/restore@v6 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + # ci.yml's Build UI Deps saves its turbo entries for exactly the build + # below on every `main` push, so the build replays them. + - name: Restore Turbo cache + uses: actions/cache/restore@v6 + with: + path: .turbo/cache + key: ${{ runner.os }}-turbo-ui-deps-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-turbo-ui-deps-${{ github.ref_name }}- + ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # The same command as the step of this name in ci.yml's ui jobs: + # objectui's suite resolves the seven @objectstack packages from their + # built dist. + - name: Build the @objectstack packages objectui consumes + run: | + pnpm exec turbo run build --concurrency=4 \ + --filter=@objectstack/spec... \ + --filter=@objectstack/client... \ + --filter=@objectstack/core... \ + --filter=@objectstack/formula... \ + --filter=@objectstack/lint... \ + --filter=@objectstack/sdui-parser... \ + --filter=@objectstack/types... + + # `--reporter=blob` writes `ui/.vitest-reports/blob-N-8.json` (the shard + # index is in the FILE NAME, so every shard's artifact downloads into one + # directory below without colliding). The blob carries this shard's raw + # coverage and test results; the report itself is produced once, by the + # merge. + # + # `--reporter=default --reporter=github-actions` because a CLI + # `--reporter` REPLACES the reporter set rather than adding to it: with + # the blob alone a red shard's log ends at `blob report written to …` with + # no failing test name, and the failure lives only in the download-only + # artifact (objectui#9177; objectui#8545 priced it at 84 and 87 pushes of + # an unevaluated gate). ⛔ Drop neither half: without the blob the merge + # has nothing to enforce the thresholds over. + # + # `--testTimeout=60000` (objectui#9271): `testTimeout: 15000` is + # calibrated for the uninstrumented Test UI lane, and the repo-sweeping + # rows under `ui/scripts/__tests__/` take several times longer under v8 + # instrumentation. A timed-out row kills the shard, and a dead shard is + # NO gate: the report job merges only when every shard is green. This is + # a hung-test detector for this lane, not a performance budget; Test UI's + # 15 s one is unchanged. + # + # The legs zero the thresholds: they are enforced once, by the merge, + # over the whole map. + - name: Run the ui tests with coverage (shard ${{ matrix.shard }}/8) + run: >- + pnpm --dir ui run test:coverage --reporter=blob --reporter=default + --reporter=github-actions --shard=${{ matrix.shard }}/8 + --testTimeout=60000 + --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 + --coverage.thresholds.branches=0 --coverage.thresholds.statements=0 + + # `always()`: a red shard is exactly when someone wants its blob. + # `retention-days: 7`: a Monday question about a Friday-night red run + # stays answerable. `if-no-files-found: error`: "the reporter wrote + # nothing" is a finding. + - name: Upload this shard's blob report + if: always() + uses: actions/upload-artifact@v7 + with: + name: ui-coverage-blob-${{ matrix.shard }} + path: ui/.vitest-reports/* + include-hidden-files: true + retention-days: 7 + if-no-files-found: error + + # Merges every shard's blob into ONE coverage report, enforces the + # configured thresholds over it, publishes it, and -- whatever happens -- + # says whether the coverage gate ran (objectui#5403, objectui#5436). + # + # Green ONLY when every shard passed, every blob arrived, they merged into + # one report, the thresholds held over the whole merged map and the report + # was published. Red, with an annotation naming which of those did not + # happen, every other time. ⛔ The merge is NOT unconditional: a report + # built from fewer than every shard understates coverage on every file the + # missing shard exercised, and a wrong number is worse than a missing one. + ui-coverage-report: + name: Test UI (coverage) + needs: ui-coverage + if: always() + runs-on: ubuntu-latest + timeout-minutes: 20 + # The matrix width, said once for this job: the blob count the merge + # refuses to go below and the verdict's wording both read it. + # `ui/scripts/__tests__/coverage-shard-width.test.ts` pins it to the + # `ui-coverage` matrix above. + env: + COVERAGE_SHARDS: '8' + # `actions: read` for the headroom reading's jobs API call; nothing here + # writes. + permissions: + contents: read + actions: read + + steps: + # Every step up to the verdict is gated on the shards having passed: on + # a red run this job checks out `ui/scripts` alone and reports. + - name: Checkout repository + if: needs.ui-coverage.result == 'success' + uses: actions/checkout@v7 + + - name: Setup Node.js + if: needs.ui-coverage.result == 'success' + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Setup pnpm + if: needs.ui-coverage.result == 'success' + uses: ./.github/actions/setup-pnpm + + - name: Get pnpm store directory + if: needs.ui-coverage.result == 'success' + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Restore pnpm cache + if: needs.ui-coverage.result == 'success' + uses: actions/cache/restore@v6 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + # No build: `--merge-reports` runs no test file, and the vitest config + # imports nothing from the @objectstack packages. + - name: Install dependencies + if: needs.ui-coverage.result == 'success' + run: pnpm install --frozen-lockfile + + - name: Download every shard's blob report + if: needs.ui-coverage.result == 'success' + uses: actions/download-artifact@v8 + with: + pattern: ui-coverage-blob-* + path: ui/.vitest-reports + merge-multiple: true + + # A zero is not a reading: `--merge-reports` happily merges three blobs + # and prints a confident, wrong total. Count them first. + - name: Refuse to merge a partial set of shard reports + id: blobs + if: needs.ui-coverage.result == 'success' + run: | + # `mkdir -p` first, and no `2>/dev/null`: a missing directory must + # not abort this step before it prints the annotation below. + mkdir -p ui/.vitest-reports + ls -la ui/.vitest-reports/ + blobs=$(find ui/.vitest-reports -maxdepth 1 -type f -name "blob-*-${COVERAGE_SHARDS}.json" | wc -l) + if [ "$blobs" -ne "$COVERAGE_SHARDS" ]; then + echo "::error title=Coverage merge::expected $COVERAGE_SHARDS shard blob reports, found $blobs — a merged report built from a partial set understates coverage, refusing to merge" + exit 1 + fi + echo "all $COVERAGE_SHARDS shard blob reports present" + + # No tests run here. `--merge-reports` reads the blobs, writes the one + # complete report and is ALSO the gate: the configured + # `coverage.thresholds` are enforced over the whole merged map, and a + # breach exits this step non-zero (`mergeReports()` ends in + # `generateReports(coverageMap, true)`, which calls `reportThresholds`; + # measured on objectui#5436: same blob set at 38.4 % lines, + # `--coverage.thresholds.lines=38` exits 0 and `=39` exits 1). + # `generateReports()` writes the json reporter BEFORE it evaluates the + # thresholds, which is what lets the publish step below still capture a + # report that breached. + - name: Merge the shard reports into one coverage report + id: merge + if: needs.ui-coverage.result == 'success' + run: pnpm --dir ui run test:coverage --merge-reports --coverage.reporter=json --coverage.reporter=text + + # `!cancelled()` plus `outcome != 'skipped'`: when the merge fails on the + # THRESHOLDS the report exists and is precisely what someone wants to + # open, so a red gate still publishes its evidence. Kept as long as the + # blobs it was derived from. + - name: Publish the merged coverage report + id: report + if: ${{ !cancelled() && needs.ui-coverage.result == 'success' && steps.merge.outcome != 'skipped' }} + uses: actions/upload-artifact@v7 + with: + name: ui-coverage-report + path: ui/coverage/ + retention-days: 7 + if-no-files-found: error + + # ⛔ REPORT-ONLY, not a gate (objectui#12208, ruling 6121816480): the + # longest coverage shard's wall against `ui-coverage`'s + # `timeout-minutes`, and the width this run's measured total implies at + # 20 minutes per shard. Before the verdict, so the verdict stays this + # job's last word; `always()` so a red lane still gets its reading; + # `continue-on-error` so the reading cannot change the verdict. On a red + # lane the checkout above was skipped, so this reads `ui/scripts` through + # a sparse checkout of its own. Pinned by + # `ui/scripts/__tests__/shard-headroom.test.ts`. + - name: Check out the headroom reading's script + if: always() && needs.ui-coverage.result != 'success' + uses: actions/checkout@v7 + with: + sparse-checkout: ui/scripts + + - name: The longest coverage shard's headroom against the cap (report-only) + if: always() + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node ui/scripts/shard-headroom.mjs --job 'Test UI (coverage shard {n}/8)' --shards 8 --cap-minutes 40 --target-minutes 20 + + # Runs on every path, including the one where every step above was + # skipped, which without this step would leave the job GREEN while the + # coverage gate never ran. + - name: Say out loud whether the coverage gate ran on this commit + if: always() + env: + SHARDS: ${{ needs.ui-coverage.result }} + BLOBS: ${{ steps.blobs.outcome }} + MERGE: ${{ steps.merge.outcome }} + REPORT: ${{ steps.report.outcome }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + + if [ "$SHARDS" = 'success' ] && [ "$BLOBS" = 'success' ] && [ "$MERGE" = 'success' ] && [ "$REPORT" = 'success' ]; then + echo "::notice title=Coverage gate ran::the merged $COVERAGE_SHARDS-shard report was produced for $GITHUB_SHA, the thresholds passed over the whole map, and the report is attached as the ui-coverage-report artifact" + { + echo "### UI coverage: the gate ran on the complete merged report" + echo + echo "- commit: \`$GITHUB_SHA\`" + echo "- shards: all $COVERAGE_SHARDS green, all $COVERAGE_SHARDS blob reports merged" + echo "- thresholds: enforced over the whole merged map, passed" + echo "- report: the \`ui-coverage-report\` artifact on [this run]($run_url), kept 7 days" + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # The gate RAN and said no: a coverage regression to fix, the + # opposite action from the broken lane below. `report=success` + # separates them: a threshold breach leaves a complete report behind, + # a merge that dies earlier writes none and the publish step reds. + if [ "$MERGE" = 'failure' ] && [ "$REPORT" = 'success' ]; then + echo "::error title=Coverage thresholds breached::shards=$SHARDS blobs=$BLOBS merge=failure report=$REPORT — the merged report for $GITHUB_SHA was produced and the configured thresholds were enforced over it, and they FAILED" + { + echo "### UI coverage: the thresholds were enforced and BREACHED" + echo + echo "- commit: \`$GITHUB_SHA\`" + echo "- shards: all $COVERAGE_SHARDS green, all $COVERAGE_SHARDS blob reports merged" + echo "- thresholds: enforced over the whole merged map, **failed**" + echo "- report published: \`$REPORT\`" + echo + echo "This is a coverage regression, not a broken lane. The merged" + echo "report is the \`ui-coverage-report\` artifact on [this run]($run_url);" + echo "the merge step's log names the metric and the numbers." + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + echo "::error title=Coverage gate did NOT run::shards=$SHARDS blobs=$BLOBS merge=$MERGE report=$REPORT — no complete merged report was produced for $GITHUB_SHA, so the coverage thresholds were never evaluated on this commit" + { + echo "### UI coverage: the gate did NOT run on this commit" + echo + echo "- commit: \`$GITHUB_SHA\`" + echo "- shard jobs: \`$SHARDS\`" + echo "- all $COVERAGE_SHARDS blobs present: \`$BLOBS\`" + echo "- merge + thresholds: \`$MERGE\`" + echo "- report published: \`$REPORT\`" + echo + echo "The thresholds are enforced on the MERGED report and nowhere" + echo "else; the shard legs override them to zero." + } >> "$GITHUB_STEP_SUMMARY" + exit 1 diff --git a/scripts/ci/scheduled-full-run.mjs b/scripts/ci/scheduled-full-run.mjs index 64b25636746..3e6b5e5e5ba 100644 --- a/scripts/ci/scheduled-full-run.mjs +++ b/scripts/ci/scheduled-full-run.mjs @@ -289,6 +289,8 @@ const titlePrefix = (cadence) => `${cadence} full run: red on main`; export const CARD_CADENCES = Object.freeze({ CI: '4-hourly', 'Lint & Type Check': '4-hourly', + // ui-coverage-nightly.yml (#25924): objectui's coverage lane, nightly. + 'UI Coverage': 'nightly', }); // THE CADENCES A CADENCE REPLACED, newest first. `CI` and `Lint & Type Check` @@ -1072,7 +1074,7 @@ async function releaseGate(argv) { // remedy is to find what stopped registering, never to lower the number. const SELF_TEST_BATTERIES = Object.freeze({ 'scheduled-full-run concurrency isolation': 32, - 'scheduled-full-run card identity and de-dup': 42, + 'scheduled-full-run card identity and de-dup': 46, 'card refresh: a stale pm:blocked (the measured #23885 sequence)': 22, 'release gate: the selection rule on the recorded 2026-10-12 listing': 8, 'release gate: the selection rule on synthetic runs': 17, @@ -1525,6 +1527,40 @@ async function selfTest() { } }); } + // UI Coverage is the one NIGHTLY workflow the card watches (#25924): its card + // says nightly, never 4-hourly, and the workflow runs exactly one daily cron. + const uiCoverage = cardIdentity('UI Coverage'); + await check(() => { + if (uiCoverage.cadence !== 'nightly' || uiCoverage.title !== 'nightly full run: red on main (UI Coverage)' || uiCoverage.marker !== 'os-nightly-full-run-ui-coverage') { + throw new Error(`cadence: the UI Coverage identity is (${uiCoverage.cadence}, '${uiCoverage.title}', '${uiCoverage.marker}'), not the nightly one`); + } + }); + await check(() => { + for (const b of bodiesOf(uiCoverage)) { + if (!/every night/.test(b)) throw new Error(`cadence: a nightly card body does not say nightly:\n${b}`); + if (/every 4 hours|4 hours ago|every hour\b|an hour ago/.test(b)) throw new Error(`cadence: a nightly card body names another cadence:\n${b}`); + } + }); + await check(() => { + const file = '.github/workflows/ui-coverage-nightly.yml'; + const text = readFileSync(file, 'utf8'); + const name = /^name:\s*(.+?)\s*$/m.exec(text)?.[1]; + if (name !== 'UI Coverage') throw new Error(`cadence: ${file} is named '${name}', not the 'UI Coverage' CARD_CADENCES holds`); + const crons = [...text.matchAll(/^\s*-\s*cron:\s*'([^']+)'/gm)].map((m) => m[1]); + if (crons.length !== 1 || !/^\d{1,2} \d{1,2} \* \* \*$/.test(crons[0])) { + throw new Error(`cadence: ${file} declares ${JSON.stringify(crons)}, but CARD_CADENCES says nightly (one daily cron)`); + } + }); + await check(() => { + // Every name the watcher lists has a cadence, and every cadence is watched. + const text = readFileSync('.github/workflows/scheduled-full-run-card.yml', 'utf8'); + const line = /^\s*workflows:\s*\[(.*)\]\s*$/m.exec(text)?.[1] ?? ''; + const watched = [...line.matchAll(/'([^']+)'/g)].map((m) => m[1]).sort(); + const held = Object.keys(CARD_CADENCES).sort(); + if (JSON.stringify(watched) !== JSON.stringify(held)) { + throw new Error(`cadence: scheduled-full-run-card.yml watches ${JSON.stringify(watched)}, CARD_CADENCES holds ${JSON.stringify(held)}`); + } + }); await check(() => { let message = ''; try { diff --git a/ui/scripts/__tests__/coverage-shard-reporter-readability.test.ts b/ui/scripts/__tests__/coverage-shard-reporter-readability.test.ts new file mode 100644 index 00000000000..074590b8589 --- /dev/null +++ b/ui/scripts/__tests__/coverage-shard-reporter-readability.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest'; +import { parse as parseYaml } from 'yaml'; + +import { UI_COVERAGE, rootWorkflow } from './helpers/root-ci'; + +/** + * The pin on the coverage shard leg's reporter set (objectui#9177). + * + * ## What this guards, and why nothing else could + * + * The root `ui-coverage-nightly.yml`'s `ui-coverage` job (objectui's `ci.yml` + * `test-coverage`, ported nightly by objectstack#25924) runs vitest as a + * sharded matrix and merges + * every shard's blob report in the job below it, which is where the configured + * coverage thresholds are enforced. A CLI `--reporter` **replaces** the + * reporter set rather than adding to it, and vitest's resolution is: + * + * if (!resolved.reporters.length) { + * resolved.reporters.push([isAgent ? "agent" : "default", {}]); + * if (process.env.GITHUB_ACTIONS === "true") + * resolved.reporters.push(["github-actions", {}]); + * } + * + * so `--reporter=blob` on its own displaced BOTH halves of that pair. A red + * shard's log then ended at `blob report written to …` — no failing test name, + * no assertion text, no timeout message — and the shard job's only annotation + * was the generic `Process completed with exit code 1.` The failures survived + * only inside the `coverage-blob-N` artifact, which is download-only. + * + * objectui#8545 measured what that costs: two episodes in which one test file + * held `main`'s coverage gate unevaluated for **84** and **87** consecutive + * pushes, both found the same way — a person reading a job log by hand, days + * later. + * + * Nothing else in this repository reads what that step runs. + * `ci-cd-pipeline-doc.test.ts` pins the pairing between `ci.yml` and the + * pipeline page, but it compares COMMANDS, not flags — its extractor is + * + * for (const m of text.matchAll(/\bpnpm\s+([\w:.-]+)/g)) { + * if (rootScripts.has(m[1])) found.add(`pnpm ${m[1]}`); + * } + * + * whose capture stops at the first space, so `pnpm test:coverage` is all it + * ever sees. Every flag on that line could be deleted and that file stays + * green. + * + * ## Both directions, because the failure has two shapes + * + * Dropping `--reporter=default` / `--reporter=github-actions` restores + * objectui#9177. Dropping `--reporter=blob` to "simplify" the line is the same + * defect wearing the other hat: the merge job loses the only input it has, so + * a readable failure is bought with an unevaluated coverage floor + * (objectui#5403). Neither is caught by CI going green, because the shard leg + * passing is exactly the state in which nobody looks at its log. + * + * ⛔ If you are deleting this file, you are deleting the only mechanical guard + * on that reporter set. The measurement behind it is in the ⚠️ comment above + * the step in `ui-coverage-nightly.yml`; re-run it before you change the line. + */ +const workflowSource = rootWorkflow(UI_COVERAGE.file); + +/** The reporter the merge job needs, and the two that keep the log readable. */ +const BLOB_REPORTER = '--reporter=blob'; +const READABLE_REPORTERS = ['--reporter=default', '--reporter=github-actions']; + +interface Step { + name?: string; + run?: unknown; +} + +/** + * Every `run:` body in the `ui-coverage` job that invokes the coverage suite. + * + * Read through the YAML parser rather than off the raw text: the step is a + * folded block scalar (`run: >-`), so the flags are spread over four source + * lines and a regex over the file would have to re-implement the folding to see + * the command the runner actually executes. Matching on `test:coverage` rather + * than on the full string is what makes a rewrite FAIL here instead of + * vanishing — a pin that searched for today's exact line would simply stop + * finding anything and report a healthy green over a step that no longer + * contains it. + */ +function coverageRunSteps(source: string): Array<{ step: string; run: string }> { + const workflow = parseYaml(source) as { + jobs?: Record; + }; + const job = workflow.jobs?.[UI_COVERAGE.shards]; + expect( + job, + `${UI_COVERAGE.file} no longer defines a \`${UI_COVERAGE.shards}\` job. If the coverage lane was restructured, this pin ` + + 'needs a new referent — do not delete it without giving the reporter set one.', + ).toBeDefined(); + const found: Array<{ step: string; run: string }> = []; + for (const step of job?.steps ?? []) { + if (typeof step.run !== 'string') continue; + if (!step.run.includes('test:coverage')) continue; + found.push({ step: step.name ?? '(unnamed)', run: step.run }); + } + return found; +} + +describe("the ui coverage shards keep a red run readable", () => { + it('still runs the coverage suite at all — the pin is not vacuous', () => { + // Without this, every assertion below passes over a job that stopped running + // vitest, which is the one way a `toContain` pin lies. + const steps = coverageRunSteps(workflowSource); + expect( + steps.map((s) => s.step), + `no step in ${UI_COVERAGE.file}'s \`${UI_COVERAGE.shards}\` job runs \`test:coverage\` any more. If that is ` + + 'deliberate, this pin is obsolete and should be deleted WITH the reasoning that made it ' + + 'obsolete written down. If it is not, the nightly lane just lost its coverage measurement.', + ).not.toEqual([]); + }); + + it('is reading the shard leg, not some other coverage invocation', () => { + // Names the population the two assertions below judge: the sharded legs. + // `--merge-reports` in the job below is a different step in a different job + // and is deliberately out of reach here. + const sharded = coverageRunSteps(workflowSource).filter((s) => s.run.includes('--shard=')); + expect( + sharded.map((s) => s.step), + `the \`${UI_COVERAGE.shards}\` job no longer passes \`--shard=\` to the coverage suite. The reporter ` + + 'rule below is about the SHARDED legs specifically — an unsharded lane has no blob to ' + + 'keep and a different set of reporters is right for it.', + ).not.toEqual([]); + }); + + it('passes a reporter that prints failing test names alongside the blob', () => { + const offenders = coverageRunSteps(workflowSource) + .filter((s) => s.run.includes(BLOB_REPORTER)) + .flatMap((s) => + READABLE_REPORTERS.filter((r) => !s.run.includes(r)).map((r) => `${s.step}: missing ${r}`), + ); + + expect( + offenders, + 'a coverage shard passes `--reporter=blob` without the reporters that keep its log readable.\n' + + offenders.map((o) => ` - ${o}`).join('\n') + + '\n\nA CLI `--reporter` REPLACES the default set rather than adding to it, so blob alone ends a ' + + 'failing shard at `blob report written to …` with no test name and no assertion text, and leaves ' + + 'the download-only artifact as the only copy of the failure. `--reporter=default` restores the ' + + 'job log; `--reporter=github-actions` restores the per-test `::error` annotation, which is the ' + + 'only form an API reader can get without downloading anything. objectui#9177; the cost is ' + + 'objectui#8545, where one test file held the coverage gate unevaluated for 84 and 87 pushes.', + ).toEqual([]); + }); + + it('keeps the blob reporter the merge job depends on', () => { + const offenders = coverageRunSteps(workflowSource) + .filter((s) => s.run.includes('--shard=')) + .filter((s) => !s.run.includes(BLOB_REPORTER)) + .map((s) => s.step); + + expect( + offenders, + 'a sharded coverage leg no longer writes a blob report:\n' + + offenders.map((o) => ` - ${o}`).join('\n') + + `\n\nThe \`${UI_COVERAGE.report}\` job merges every shard's blob and enforces the configured thresholds over ` + + 'the merged report — that is the whole reason the lane is sharded (objectui#5403). Restoring log ' + + 'readability by dropping the blob would trade an unreadable failure for an unevaluated coverage ' + + 'floor, which is the same defect wearing the other hat (objectui#9177).', + ).toEqual([]); + }); +}); diff --git a/ui/scripts/__tests__/coverage-shard-width.test.ts b/ui/scripts/__tests__/coverage-shard-width.test.ts new file mode 100644 index 00000000000..d0cff75a6cc --- /dev/null +++ b/ui/scripts/__tests__/coverage-shard-width.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest'; +import { parse as parseYaml } from 'yaml'; + +import { UI_COVERAGE, rootWorkflow } from './helpers/root-ci'; + +/** + * The coverage lane's width, held to ONE number (objectui#12256, folded into + * objectui#12208). + * + * The lane runs in the root `.github/workflows/ui-coverage-nightly.yml` + * (objectstack#25924), where objectui's `ci.yml` jobs `test-coverage` and + * `coverage-report` became `ui-coverage` and `ui-coverage-report`. + * + * The lane spells its shard count in two jobs. `ui-coverage` + * declares the matrix and repeats the count in its job name, its step name and + * its `--shard=N/M` flag; `ui-coverage-report` counts the blob reports the shards + * wrote and refuses to merge a partial set. The blob's FILE NAME carries the + * count too (`blob-N-M.json`, written by vitest from the `--shard` flag), so a + * width moved in the matrix alone leaves the merge looking for blobs that no + * shard writes: the count step answers zero, the merge never runs, and the + * coverage gate is unevaluated on every night -- the lane's own worst failure, + * one edit away. Nothing about that is visible on a pull request, because the + * lane runs on `schedule` and `workflow_dispatch` only. + * + * So every spelling is read here and held to the matrix: + * + * - the matrix is the contiguous run 1..M; + * - the job name, the step name and `--shard=` all say M; + * - `ui-coverage-report` declares `COVERAGE_SHARDS: 'M'`, and its count step + * reads THAT variable for both the blob glob and the expected count, with + * no literal width left in the step; + * - the merge step is a `--merge-reports` that overrides no threshold, so + * the configured `coverage.thresholds` are what is enforced over the merge + * of all M blobs -- the one place the gate runs, since the shard legs + * zero the thresholds. + */ +const workflowSource = rootWorkflow(UI_COVERAGE.file); + +interface Step { + name?: string; + run?: unknown; + id?: string; +} +interface Job { + name?: string; + env?: Record; + strategy?: { matrix?: { shard?: unknown } }; + steps?: Step[]; +} + +const workflow = parseYaml(workflowSource) as { jobs?: Record }; + +function job(key: string): Job { + const found = workflow.jobs?.[key]; + expect(found, `${UI_COVERAGE.file} no longer defines a \`${key}\` job; this pin needs a new referent, not deletion`).toBeDefined(); + return found as Job; +} + +/** The matrix width, after checking the matrix is the contiguous run 1..M. */ +function matrixWidth(): number { + const shards = job(UI_COVERAGE.shards).strategy?.matrix?.shard; + expect(Array.isArray(shards), `\`${UI_COVERAGE.shards}\` must declare a \`shard:\` matrix list`).toBe(true); + const list = shards as unknown[]; + expect(list.length, 'a zero-wide matrix is a broken reader, not a lane').toBeGreaterThan(0); + expect(list, 'the coverage matrix must be the contiguous run 1..M').toEqual(list.map((_, i) => i + 1)); + return list.length; +} + +function runOf(step: Step | undefined): string { + return typeof step?.run === 'string' ? step.run : ''; +} + +describe("the ui coverage lane says its width once (objectui#12256)", () => { + const width = matrixWidth(); + + it('names each shard job with the matrix width', () => { + expect(job(UI_COVERAGE.shards).name).toBe(`Test UI (coverage shard \${{ matrix.shard }}/${width})`); + }); + + it('passes the matrix width to vitest, in the flag and in the step name', () => { + const steps = (job(UI_COVERAGE.shards).steps ?? []).filter((s) => runOf(s).includes('test:coverage')); + expect(steps.length, `no step in \`${UI_COVERAGE.shards}\` runs \`test:coverage\` any more`).toBeGreaterThan(0); + for (const step of steps) { + const flags = [...runOf(step).matchAll(/--shard=\$\{\{ matrix\.shard \}\}\/(\d+)/g)].map((m) => Number(m[1])); + expect(flags, `${step.name}: one --shard flag, carrying the matrix width`).toEqual([width]); + expect(step.name, `${step.name}: the step name states a different width`).toContain(`/${width})`); + } + }); + + it('declares the same width as the count the merge refuses to go below', () => { + expect(job(UI_COVERAGE.report).env?.COVERAGE_SHARDS, `\`${UI_COVERAGE.report}\` must declare \`COVERAGE_SHARDS\``).toBe( + String(width), + ); + }); + + it('counts the blobs through that variable, with no literal width left in the step', () => { + const count = runOf((job(UI_COVERAGE.report).steps ?? []).find((s) => s.id === 'blobs')); + expect(count, `\`${UI_COVERAGE.report}\` must keep its \`blobs\` count step`).not.toBe(''); + expect(count).toContain('-name "blob-*-${COVERAGE_SHARDS}.json"'); + expect(count).toContain('-ne "$COVERAGE_SHARDS"'); + // A digit in the blob glob or the comparison is a second spelling of the + // width, and the one that goes stale first. + expect(count).not.toMatch(/blob-\*-\d/); + expect(count).not.toMatch(/-ne\s+\d/); + }); + + it('enforces the configured thresholds over the merge of every blob, overriding none', () => { + const merge = runOf((job(UI_COVERAGE.report).steps ?? []).find((s) => s.id === 'merge')); + expect(merge, `\`${UI_COVERAGE.report}\` must keep its \`merge\` step`).toContain('--merge-reports'); + expect(merge, 'the merge is where the thresholds are enforced; it must not override them').not.toContain( + 'thresholds', + ); + const legs = (job(UI_COVERAGE.shards).steps ?? []).filter((s) => runOf(s).includes('--shard=')); + expect(legs.length, `no \`${UI_COVERAGE.shards}\` step passes \`--shard=\``).toBeGreaterThan(0); + for (const leg of legs) { + // The other half of the same contract: the legs zero the thresholds, so + // the merge above is the only evaluation, over all of them. + expect(runOf(leg)).toContain('--coverage.thresholds.lines=0'); + } + }); +}); diff --git a/ui/scripts/__tests__/helpers/root-ci.ts b/ui/scripts/__tests__/helpers/root-ci.ts index 1709901e605..f4475a8390d 100644 --- a/ui/scripts/__tests__/helpers/root-ci.ts +++ b/ui/scripts/__tests__/helpers/root-ci.ts @@ -12,12 +12,13 @@ * the root `.github/workflows/` directory through `rootWorkflowFiles()`. It * never reads a copy under `ui/.github/`: nothing runs one. * - * What the port did not carry (the coverage lane, the docs job, the shard - * aggregate, the type-check-scope selector, the timings upload) has no job - * here, and the pins on those retired with a disposition beside them rather - * than being pointed at a job that does something else. The affected-test - * selector was wired into `test-ui` later (objectstack#25868), and its pins - * read that job. + * What the port did not carry (the docs job, the shard aggregate, the + * type-check-scope selector, the timings upload) has no job here, and the pins + * on those retired with a disposition beside them rather than being pointed at + * a job that does something else. The affected-test selector was wired into + * `test-ui` later (objectstack#25868), and its pins read that job. The + * coverage lane came back as a nightly workflow of its own + * (objectstack#25924), named by `UI_COVERAGE` below. */ import fs from 'node:fs'; import path from 'node:path'; @@ -32,6 +33,17 @@ export const UI_JOBS = { buildE2e: 'build-e2e-ui', } as const; +/** + * objectui's coverage lane, ported to the root as a nightly workflow + * (objectstack#25924): the file, the sharded job and the merge job that + * enforces the thresholds. + */ +export const UI_COVERAGE = { + file: 'ui-coverage-nightly.yml', + shards: 'ui-coverage', + report: 'ui-coverage-report', +} as const; + /** The directory GitHub runs: `.github/workflows/` at the workspace root. */ export const rootWorkflowsDir = path.join(workspaceRoot(), '.github/workflows'); diff --git a/ui/scripts/__tests__/shard-headroom.test.ts b/ui/scripts/__tests__/shard-headroom.test.ts index c240dd5d080..1cc8eacd99c 100644 --- a/ui/scripts/__tests__/shard-headroom.test.ts +++ b/ui/scripts/__tests__/shard-headroom.test.ts @@ -2,10 +2,12 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { parse as parseYaml } from 'yaml'; // Plain-JS CI helper; its types are INFERRED from the .mjs source by // `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here. import { EXIT_OK, EXIT_USAGE, headroom, main, render, shardNames } from '../shard-headroom.mjs'; +import { UI_COVERAGE, rootWorkflow } from './helpers/root-ci'; /** * The report-only headroom reading of the two test aggregators (objectui#12208, @@ -92,12 +94,36 @@ describe('⛔ not a gate', () => { }); }); -// `ci.yml wires it as a reading of the right matrix and the right cap` retired -// with objectstack#25712. It held the headroom step's `--job`, `--shards` and -// `--cap-minutes` to the two objectui ci.yml aggregators that ran it, -// `test-aggregate` (over `test`) and `coverage-report` (over `test-coverage`), -// and that file runs nowhere under the monorepo. Both are row 7 of the -// objectstack#25697 workflow roster: `Test (the shard aggregate)`, "dropped -// until cutover", and `Test (coverage shard n/8) · Test (coverage)`, "kept, -// follow-up port". No root job runs this reading until one of them is ported; -// the port brings the wiring pin back against its job (`helpers/root-ci.ts`). +// `ci.yml wires it as a reading of the right matrix and the right cap` held the +// headroom step's `--job`, `--shards` and `--cap-minutes` to the two objectui +// ci.yml aggregators that ran it, and that file runs nowhere under the +// monorepo (objectstack#25712). Both are row 7 of the objectstack#25697 +// workflow roster. The coverage half came back with the lane's nightly port +// (objectstack#25924) and is pinned below against its root job. The other half, +// `test-aggregate` over `test` (`Test (the shard aggregate)`, "dropped until +// cutover"), has no root job, so no root job runs that reading until it is +// ported; that port brings its wiring pin back the same way. +describe('ui-coverage-nightly.yml wires it as a reading of the right matrix and the right cap', () => { + interface Step { name?: string; run?: unknown; if?: string; 'continue-on-error'?: boolean } + interface Job { name?: string; 'timeout-minutes'?: number; steps?: Step[]; strategy?: { matrix?: { shard?: unknown[] } } } + const workflow = parseYaml(rootWorkflow(UI_COVERAGE.file)) as { jobs: Record }; + + it(`${UI_COVERAGE.report} reads ${UI_COVERAGE.shards}: its job name, its width and its timeout-minutes`, () => { + const steps = (workflow.jobs[UI_COVERAGE.report]?.steps ?? []).filter( + (s) => typeof s.run === 'string' && s.run.includes('scripts/shard-headroom.mjs'), + ); + expect(steps, `${UI_COVERAGE.report} must run the headroom reading exactly once`).toHaveLength(1); + const [step] = steps; + const run = step.run as string; + const target = workflow.jobs[UI_COVERAGE.shards]; + const width = (target.strategy?.matrix?.shard ?? []).length; + expect(width, `${UI_COVERAGE.shards} must declare a shard matrix`).toBeGreaterThan(0); + expect(run).toContain(`--job '${String(target.name).replace('${{ matrix.shard }}', '{n}')}'`); + expect(run).toContain(`--shards ${width} `); + expect(run).toContain(`--cap-minutes ${target['timeout-minutes']} `); + // ⛔ Not a gate: it runs whatever happened above it, and nothing it + // does can change the aggregator's conclusion. + expect(step.if).toBe('always()'); + expect(step['continue-on-error']).toBe(true); + }); +}); diff --git a/ui/scripts/shard-headroom.mjs b/ui/scripts/shard-headroom.mjs index 4793dbf6de0..fe245afe5dd 100644 --- a/ui/scripts/shard-headroom.mjs +++ b/ui/scripts/shard-headroom.mjs @@ -11,8 +11,9 @@ * node scripts/shard-headroom.mjs --job 'Test (shard {n}/16)' --shards 16 --cap-minutes 20 --target-minutes 10 * node scripts/shard-headroom.mjs ... --fixture * - * `ci.yml`'s two aggregators run it: `test-aggregate` (`Test`) for the `test` - * matrix and `coverage-report` (`Test (coverage)`) for the coverage matrix. + * The root `ui-coverage-nightly.yml`'s `ui-coverage-report` (`Test UI + * (coverage)`) runs it for the coverage matrix (objectstack#25924). objectui's + * other aggregator, `test-aggregate` (`Test`), has no root job yet. * `scripts/__tests__/shard-headroom.test.ts` pins each invocation's * `--shards` and `--cap-minutes` to the matrix and the `timeout-minutes` of * the job it reads, so the reading cannot quietly describe another cap.