diff --git a/.github/workflows/scheduled-full-run-card.yml b/.github/workflows/scheduled-full-run-card.yml index 91c064504d5..c915e0d4ab6 100644 --- a/.github/workflows/scheduled-full-run-card.yml +++ b/.github/workflows/scheduled-full-run-card.yml @@ -87,8 +87,13 @@ on: # # Every name listed here must have an entry in `CARD_CADENCES` # (scripts/ci/scheduled-full-run.mjs), which refuses a name it does not - # hold rather than default one, so a card never prints the wrong cadence. - workflows: ['CI', 'Lint & Type Check'] + # hold rather than default one, so a card never prints the wrong cadence; + # its self-test holds this list and that map to the same names. + # + # `UI Coverage` (ui-coverage-nightly.yml, #25924) is the nightly one: the + # ui coverage thresholds are enforced there and nowhere else, so a red + # night files `nightly full run: red on main (UI Coverage)`. + workflows: ['CI', 'Lint & Type Check', 'UI Coverage'] types: [completed] permissions: {} diff --git a/.github/workflows/ui-coverage-nightly.yml b/.github/workflows/ui-coverage-nightly.yml new file mode 100644 index 00000000000..de0bab94cb2 --- /dev/null +++ b/.github/workflows/ui-coverage-nightly.yml @@ -0,0 +1,370 @@ +# UI coverage, nightly (#25924). objectui's coverage lane, ported to the root. +# +# objectui's `ci.yml` ran this lane (`Test (coverage shard n/8)` and the merge +# job `Test (coverage)`) on every push to `main` and on its hourly schedule: +# 8 legs of ~22 min, about 31 % of that repository's CI minutes (#25924's +# measurement, 2026-10-17..24), to enforce four thresholds. The ui/ cutover +# (#25752) did not port it: it is row 7 "kept, follow-up port" of the +# objectstack#25697 workflow roster, and the nested `ui/.github/workflows/` +# copy runs nowhere, so from the cutover until this file the thresholds were +# evaluated by nothing. +# +# It comes back here on `schedule` plus `workflow_dispatch` only, beside +# `coverage-nightly.yml` (spec's coverage report), with the thresholds +# unchanged: `ui/vitest.config.mts` `coverage.thresholds` (lines 40, +# functions 33, branches 30, statements 40). The cost accepted with it, stated +# on #25924: a change that drops coverage under a threshold is reported the +# night after it merged, not on the merge. `scheduled-full-run-card.yml` +# watches this workflow, so a red night files or refreshes one card +# (`nightly full run: red on main (UI Coverage)`); without that reader a red +# scheduled run is seen by nobody (objectui#5402: four dark days). +# +# ⛔ Not a merge gate and not a release gate: no pull request or queue build +# runs it, branch protection reads none of its contexts, and +# `scripts/ci/scheduled-full-run.mjs`'s RELEASE_GATE_WORKFLOWS is `CI` and +# `Lint & Type Check` only. +# +# The pins: `ui/scripts/__tests__/coverage-shard-width.test.ts` (the width, +# said once), `coverage-shard-reporter-readability.test.ts` (the shard leg's +# reporter set) and `shard-headroom.test.ts` (the headroom reading's matrix +# and cap). They read this file through `ui/scripts/__tests__/helpers/root-ci.ts`. +name: UI Coverage + +on: + workflow_dispatch: + schedule: + # 06:13 UTC: off the `:00` minute every other repository's cron queues + # on, after the 04:00 four-hourly full run and the 05:xx nightlies. + - cron: '13 6 * * *' + +permissions: + contents: read + +# One run at a time per ref; a dispatch while the nightly runs waits rather +# than cancelling a run that has already paid for its shards. +concurrency: + group: ui-coverage-${{ github.ref }} + cancel-in-progress: false + +jobs: + ui-coverage: + name: Test UI (coverage shard ${{ matrix.shard }}/8) + runs-on: ubuntu-latest + # objectui#12256 measured the 40-minute margin gone at four shards, which + # is why the lane is eight wide; the report job prints the longest shard's + # headroom against this cap on every run. + timeout-minutes: 40 + permissions: + contents: read + + strategy: + # Report every shard's failures, not just the first one to break; the + # merge below refuses to publish anything unless every shard is green. + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Setup pnpm + uses: ./.github/actions/setup-pnpm + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + # Restore-only, here and below: a nightly reads `main`'s caches and the + # per-push workflows own saving them. + - name: Restore pnpm cache + uses: actions/cache/restore@v6 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + # ci.yml's Build UI Deps saves its turbo entries for exactly the build + # below on every `main` push, so the build replays them. + - name: Restore Turbo cache + uses: actions/cache/restore@v6 + with: + path: .turbo/cache + key: ${{ runner.os }}-turbo-ui-deps-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-turbo-ui-deps-${{ github.ref_name }}- + ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # The same command as the step of this name in ci.yml's ui jobs: + # objectui's suite resolves the seven @objectstack packages from their + # built dist. + - name: Build the @objectstack packages objectui consumes + run: | + pnpm exec turbo run build --concurrency=4 \ + --filter=@objectstack/spec... \ + --filter=@objectstack/client... \ + --filter=@objectstack/core... \ + --filter=@objectstack/formula... \ + --filter=@objectstack/lint... \ + --filter=@objectstack/sdui-parser... \ + --filter=@objectstack/types... + + # `--reporter=blob` writes `ui/.vitest-reports/blob-N-8.json` (the shard + # index is in the FILE NAME, so every shard's artifact downloads into one + # directory below without colliding). The blob carries this shard's raw + # coverage and test results; the report itself is produced once, by the + # merge. + # + # `--reporter=default --reporter=github-actions` because a CLI + # `--reporter` REPLACES the reporter set rather than adding to it: with + # the blob alone a red shard's log ends at `blob report written to …` with + # no failing test name, and the failure lives only in the download-only + # artifact (objectui#9177; objectui#8545 priced it at 84 and 87 pushes of + # an unevaluated gate). ⛔ Drop neither half: without the blob the merge + # has nothing to enforce the thresholds over. + # + # `--testTimeout=60000` (objectui#9271): `testTimeout: 15000` is + # calibrated for the uninstrumented Test UI lane, and the repo-sweeping + # rows under `ui/scripts/__tests__/` take several times longer under v8 + # instrumentation. A timed-out row kills the shard, and a dead shard is + # NO gate: the report job merges only when every shard is green. This is + # a hung-test detector for this lane, not a performance budget; Test UI's + # 15 s one is unchanged. + # + # The legs zero the thresholds: they are enforced once, by the merge, + # over the whole map. + - name: Run the ui tests with coverage (shard ${{ matrix.shard }}/8) + run: >- + pnpm --dir ui run test:coverage --reporter=blob --reporter=default + --reporter=github-actions --shard=${{ matrix.shard }}/8 + --testTimeout=60000 + --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 + --coverage.thresholds.branches=0 --coverage.thresholds.statements=0 + + # `always()`: a red shard is exactly when someone wants its blob. + # `retention-days: 7`: a Monday question about a Friday-night red run + # stays answerable. `if-no-files-found: error`: "the reporter wrote + # nothing" is a finding. + - name: Upload this shard's blob report + if: always() + uses: actions/upload-artifact@v7 + with: + name: ui-coverage-blob-${{ matrix.shard }} + path: ui/.vitest-reports/* + include-hidden-files: true + retention-days: 7 + if-no-files-found: error + + # Merges every shard's blob into ONE coverage report, enforces the + # configured thresholds over it, publishes it, and -- whatever happens -- + # says whether the coverage gate ran (objectui#5403, objectui#5436). + # + # Green ONLY when every shard passed, every blob arrived, they merged into + # one report, the thresholds held over the whole merged map and the report + # was published. Red, with an annotation naming which of those did not + # happen, every other time. ⛔ The merge is NOT unconditional: a report + # built from fewer than every shard understates coverage on every file the + # missing shard exercised, and a wrong number is worse than a missing one. + ui-coverage-report: + name: Test UI (coverage) + needs: ui-coverage + if: always() + runs-on: ubuntu-latest + timeout-minutes: 20 + # The matrix width, said once for this job: the blob count the merge + # refuses to go below and the verdict's wording both read it. + # `ui/scripts/__tests__/coverage-shard-width.test.ts` pins it to the + # `ui-coverage` matrix above. + env: + COVERAGE_SHARDS: '8' + # `actions: read` for the headroom reading's jobs API call; nothing here + # writes. + permissions: + contents: read + actions: read + + steps: + # Every step up to the verdict is gated on the shards having passed: on + # a red run this job checks out `ui/scripts` alone and reports. + - name: Checkout repository + if: needs.ui-coverage.result == 'success' + uses: actions/checkout@v7 + + - name: Setup Node.js + if: needs.ui-coverage.result == 'success' + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Setup pnpm + if: needs.ui-coverage.result == 'success' + uses: ./.github/actions/setup-pnpm + + - name: Get pnpm store directory + if: needs.ui-coverage.result == 'success' + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Restore pnpm cache + if: needs.ui-coverage.result == 'success' + uses: actions/cache/restore@v6 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + # No build: `--merge-reports` runs no test file, and the vitest config + # imports nothing from the @objectstack packages. + - name: Install dependencies + if: needs.ui-coverage.result == 'success' + run: pnpm install --frozen-lockfile + + - name: Download every shard's blob report + if: needs.ui-coverage.result == 'success' + uses: actions/download-artifact@v8 + with: + pattern: ui-coverage-blob-* + path: ui/.vitest-reports + merge-multiple: true + + # A zero is not a reading: `--merge-reports` happily merges three blobs + # and prints a confident, wrong total. Count them first. + - name: Refuse to merge a partial set of shard reports + id: blobs + if: needs.ui-coverage.result == 'success' + run: | + # `mkdir -p` first, and no `2>/dev/null`: a missing directory must + # not abort this step before it prints the annotation below. + mkdir -p ui/.vitest-reports + ls -la ui/.vitest-reports/ + blobs=$(find ui/.vitest-reports -maxdepth 1 -type f -name "blob-*-${COVERAGE_SHARDS}.json" | wc -l) + if [ "$blobs" -ne "$COVERAGE_SHARDS" ]; then + echo "::error title=Coverage merge::expected $COVERAGE_SHARDS shard blob reports, found $blobs — a merged report built from a partial set understates coverage, refusing to merge" + exit 1 + fi + echo "all $COVERAGE_SHARDS shard blob reports present" + + # No tests run here. `--merge-reports` reads the blobs, writes the one + # complete report and is ALSO the gate: the configured + # `coverage.thresholds` are enforced over the whole merged map, and a + # breach exits this step non-zero (`mergeReports()` ends in + # `generateReports(coverageMap, true)`, which calls `reportThresholds`; + # measured on objectui#5436: same blob set at 38.4 % lines, + # `--coverage.thresholds.lines=38` exits 0 and `=39` exits 1). + # `generateReports()` writes the json reporter BEFORE it evaluates the + # thresholds, which is what lets the publish step below still capture a + # report that breached. + - name: Merge the shard reports into one coverage report + id: merge + if: needs.ui-coverage.result == 'success' + run: pnpm --dir ui run test:coverage --merge-reports --coverage.reporter=json --coverage.reporter=text + + # `!cancelled()` plus `outcome != 'skipped'`: when the merge fails on the + # THRESHOLDS the report exists and is precisely what someone wants to + # open, so a red gate still publishes its evidence. Kept as long as the + # blobs it was derived from. + - name: Publish the merged coverage report + id: report + if: ${{ !cancelled() && needs.ui-coverage.result == 'success' && steps.merge.outcome != 'skipped' }} + uses: actions/upload-artifact@v7 + with: + name: ui-coverage-report + path: ui/coverage/ + retention-days: 7 + if-no-files-found: error + + # ⛔ REPORT-ONLY, not a gate (objectui#12208, ruling 6121816480): the + # longest coverage shard's wall against `ui-coverage`'s + # `timeout-minutes`, and the width this run's measured total implies at + # 20 minutes per shard. Before the verdict, so the verdict stays this + # job's last word; `always()` so a red lane still gets its reading; + # `continue-on-error` so the reading cannot change the verdict. On a red + # lane the checkout above was skipped, so this reads `ui/scripts` through + # a sparse checkout of its own. Pinned by + # `ui/scripts/__tests__/shard-headroom.test.ts`. + - name: Check out the headroom reading's script + if: always() && needs.ui-coverage.result != 'success' + uses: actions/checkout@v7 + with: + sparse-checkout: ui/scripts + + - name: The longest coverage shard's headroom against the cap (report-only) + if: always() + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node ui/scripts/shard-headroom.mjs --job 'Test UI (coverage shard {n}/8)' --shards 8 --cap-minutes 40 --target-minutes 20 + + # Runs on every path, including the one where every step above was + # skipped, which without this step would leave the job GREEN while the + # coverage gate never ran. + - name: Say out loud whether the coverage gate ran on this commit + if: always() + env: + SHARDS: ${{ needs.ui-coverage.result }} + BLOBS: ${{ steps.blobs.outcome }} + MERGE: ${{ steps.merge.outcome }} + REPORT: ${{ steps.report.outcome }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + + if [ "$SHARDS" = 'success' ] && [ "$BLOBS" = 'success' ] && [ "$MERGE" = 'success' ] && [ "$REPORT" = 'success' ]; then + echo "::notice title=Coverage gate ran::the merged $COVERAGE_SHARDS-shard report was produced for $GITHUB_SHA, the thresholds passed over the whole map, and the report is attached as the ui-coverage-report artifact" + { + echo "### UI coverage: the gate ran on the complete merged report" + echo + echo "- commit: \`$GITHUB_SHA\`" + echo "- shards: all $COVERAGE_SHARDS green, all $COVERAGE_SHARDS blob reports merged" + echo "- thresholds: enforced over the whole merged map, passed" + echo "- report: the \`ui-coverage-report\` artifact on [this run]($run_url), kept 7 days" + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # The gate RAN and said no: a coverage regression to fix, the + # opposite action from the broken lane below. `report=success` + # separates them: a threshold breach leaves a complete report behind, + # a merge that dies earlier writes none and the publish step reds. + if [ "$MERGE" = 'failure' ] && [ "$REPORT" = 'success' ]; then + echo "::error title=Coverage thresholds breached::shards=$SHARDS blobs=$BLOBS merge=failure report=$REPORT — the merged report for $GITHUB_SHA was produced and the configured thresholds were enforced over it, and they FAILED" + { + echo "### UI coverage: the thresholds were enforced and BREACHED" + echo + echo "- commit: \`$GITHUB_SHA\`" + echo "- shards: all $COVERAGE_SHARDS green, all $COVERAGE_SHARDS blob reports merged" + echo "- thresholds: enforced over the whole merged map, **failed**" + echo "- report published: \`$REPORT\`" + echo + echo "This is a coverage regression, not a broken lane. The merged" + echo "report is the \`ui-coverage-report\` artifact on [this run]($run_url);" + echo "the merge step's log names the metric and the numbers." + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + echo "::error title=Coverage gate did NOT run::shards=$SHARDS blobs=$BLOBS merge=$MERGE report=$REPORT — no complete merged report was produced for $GITHUB_SHA, so the coverage thresholds were never evaluated on this commit" + { + echo "### UI coverage: the gate did NOT run on this commit" + echo + echo "- commit: \`$GITHUB_SHA\`" + echo "- shard jobs: \`$SHARDS\`" + echo "- all $COVERAGE_SHARDS blobs present: \`$BLOBS\`" + echo "- merge + thresholds: \`$MERGE\`" + echo "- report published: \`$REPORT\`" + echo + echo "The thresholds are enforced on the MERGED report and nowhere" + echo "else; the shard legs override them to zero." + } >> "$GITHUB_STEP_SUMMARY" + exit 1 diff --git a/scripts/ci/scheduled-full-run.mjs b/scripts/ci/scheduled-full-run.mjs index 64b25636746..3e6b5e5e5ba 100644 --- a/scripts/ci/scheduled-full-run.mjs +++ b/scripts/ci/scheduled-full-run.mjs @@ -289,6 +289,8 @@ const titlePrefix = (cadence) => `${cadence} full run: red on main`; export const CARD_CADENCES = Object.freeze({ CI: '4-hourly', 'Lint & Type Check': '4-hourly', + // ui-coverage-nightly.yml (#25924): objectui's coverage lane, nightly. + 'UI Coverage': 'nightly', }); // THE CADENCES A CADENCE REPLACED, newest first. `CI` and `Lint & Type Check` @@ -1072,7 +1074,7 @@ async function releaseGate(argv) { // remedy is to find what stopped registering, never to lower the number. const SELF_TEST_BATTERIES = Object.freeze({ 'scheduled-full-run concurrency isolation': 32, - 'scheduled-full-run card identity and de-dup': 42, + 'scheduled-full-run card identity and de-dup': 46, 'card refresh: a stale pm:blocked (the measured #23885 sequence)': 22, 'release gate: the selection rule on the recorded 2026-10-12 listing': 8, 'release gate: the selection rule on synthetic runs': 17, @@ -1525,6 +1527,40 @@ async function selfTest() { } }); } + // UI Coverage is the one NIGHTLY workflow the card watches (#25924): its card + // says nightly, never 4-hourly, and the workflow runs exactly one daily cron. + const uiCoverage = cardIdentity('UI Coverage'); + await check(() => { + if (uiCoverage.cadence !== 'nightly' || uiCoverage.title !== 'nightly full run: red on main (UI Coverage)' || uiCoverage.marker !== 'os-nightly-full-run-ui-coverage') { + throw new Error(`cadence: the UI Coverage identity is (${uiCoverage.cadence}, '${uiCoverage.title}', '${uiCoverage.marker}'), not the nightly one`); + } + }); + await check(() => { + for (const b of bodiesOf(uiCoverage)) { + if (!/every night/.test(b)) throw new Error(`cadence: a nightly card body does not say nightly:\n${b}`); + if (/every 4 hours|4 hours ago|every hour\b|an hour ago/.test(b)) throw new Error(`cadence: a nightly card body names another cadence:\n${b}`); + } + }); + await check(() => { + const file = '.github/workflows/ui-coverage-nightly.yml'; + const text = readFileSync(file, 'utf8'); + const name = /^name:\s*(.+?)\s*$/m.exec(text)?.[1]; + if (name !== 'UI Coverage') throw new Error(`cadence: ${file} is named '${name}', not the 'UI Coverage' CARD_CADENCES holds`); + const crons = [...text.matchAll(/^\s*-\s*cron:\s*'([^']+)'/gm)].map((m) => m[1]); + if (crons.length !== 1 || !/^\d{1,2} \d{1,2} \* \* \*$/.test(crons[0])) { + throw new Error(`cadence: ${file} declares ${JSON.stringify(crons)}, but CARD_CADENCES says nightly (one daily cron)`); + } + }); + await check(() => { + // Every name the watcher lists has a cadence, and every cadence is watched. + const text = readFileSync('.github/workflows/scheduled-full-run-card.yml', 'utf8'); + const line = /^\s*workflows:\s*\[(.*)\]\s*$/m.exec(text)?.[1] ?? ''; + const watched = [...line.matchAll(/'([^']+)'/g)].map((m) => m[1]).sort(); + const held = Object.keys(CARD_CADENCES).sort(); + if (JSON.stringify(watched) !== JSON.stringify(held)) { + throw new Error(`cadence: scheduled-full-run-card.yml watches ${JSON.stringify(watched)}, CARD_CADENCES holds ${JSON.stringify(held)}`); + } + }); await check(() => { let message = ''; try { diff --git a/ui/scripts/__tests__/coverage-shard-reporter-readability.test.ts b/ui/scripts/__tests__/coverage-shard-reporter-readability.test.ts new file mode 100644 index 00000000000..074590b8589 --- /dev/null +++ b/ui/scripts/__tests__/coverage-shard-reporter-readability.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest'; +import { parse as parseYaml } from 'yaml'; + +import { UI_COVERAGE, rootWorkflow } from './helpers/root-ci'; + +/** + * The pin on the coverage shard leg's reporter set (objectui#9177). + * + * ## What this guards, and why nothing else could + * + * The root `ui-coverage-nightly.yml`'s `ui-coverage` job (objectui's `ci.yml` + * `test-coverage`, ported nightly by objectstack#25924) runs vitest as a + * sharded matrix and merges + * every shard's blob report in the job below it, which is where the configured + * coverage thresholds are enforced. A CLI `--reporter` **replaces** the + * reporter set rather than adding to it, and vitest's resolution is: + * + * if (!resolved.reporters.length) { + * resolved.reporters.push([isAgent ? "agent" : "default", {}]); + * if (process.env.GITHUB_ACTIONS === "true") + * resolved.reporters.push(["github-actions", {}]); + * } + * + * so `--reporter=blob` on its own displaced BOTH halves of that pair. A red + * shard's log then ended at `blob report written to …` — no failing test name, + * no assertion text, no timeout message — and the shard job's only annotation + * was the generic `Process completed with exit code 1.` The failures survived + * only inside the `coverage-blob-N` artifact, which is download-only. + * + * objectui#8545 measured what that costs: two episodes in which one test file + * held `main`'s coverage gate unevaluated for **84** and **87** consecutive + * pushes, both found the same way — a person reading a job log by hand, days + * later. + * + * Nothing else in this repository reads what that step runs. + * `ci-cd-pipeline-doc.test.ts` pins the pairing between `ci.yml` and the + * pipeline page, but it compares COMMANDS, not flags — its extractor is + * + * for (const m of text.matchAll(/\bpnpm\s+([\w:.-]+)/g)) { + * if (rootScripts.has(m[1])) found.add(`pnpm ${m[1]}`); + * } + * + * whose capture stops at the first space, so `pnpm test:coverage` is all it + * ever sees. Every flag on that line could be deleted and that file stays + * green. + * + * ## Both directions, because the failure has two shapes + * + * Dropping `--reporter=default` / `--reporter=github-actions` restores + * objectui#9177. Dropping `--reporter=blob` to "simplify" the line is the same + * defect wearing the other hat: the merge job loses the only input it has, so + * a readable failure is bought with an unevaluated coverage floor + * (objectui#5403). Neither is caught by CI going green, because the shard leg + * passing is exactly the state in which nobody looks at its log. + * + * ⛔ If you are deleting this file, you are deleting the only mechanical guard + * on that reporter set. The measurement behind it is in the ⚠️ comment above + * the step in `ui-coverage-nightly.yml`; re-run it before you change the line. + */ +const workflowSource = rootWorkflow(UI_COVERAGE.file); + +/** The reporter the merge job needs, and the two that keep the log readable. */ +const BLOB_REPORTER = '--reporter=blob'; +const READABLE_REPORTERS = ['--reporter=default', '--reporter=github-actions']; + +interface Step { + name?: string; + run?: unknown; +} + +/** + * Every `run:` body in the `ui-coverage` job that invokes the coverage suite. + * + * Read through the YAML parser rather than off the raw text: the step is a + * folded block scalar (`run: >-`), so the flags are spread over four source + * lines and a regex over the file would have to re-implement the folding to see + * the command the runner actually executes. Matching on `test:coverage` rather + * than on the full string is what makes a rewrite FAIL here instead of + * vanishing — a pin that searched for today's exact line would simply stop + * finding anything and report a healthy green over a step that no longer + * contains it. + */ +function coverageRunSteps(source: string): Array<{ step: string; run: string }> { + const workflow = parseYaml(source) as { + jobs?: Record; + }; + const job = workflow.jobs?.[UI_COVERAGE.shards]; + expect( + job, + `${UI_COVERAGE.file} no longer defines a \`${UI_COVERAGE.shards}\` job. If the coverage lane was restructured, this pin ` + + 'needs a new referent — do not delete it without giving the reporter set one.', + ).toBeDefined(); + const found: Array<{ step: string; run: string }> = []; + for (const step of job?.steps ?? []) { + if (typeof step.run !== 'string') continue; + if (!step.run.includes('test:coverage')) continue; + found.push({ step: step.name ?? '(unnamed)', run: step.run }); + } + return found; +} + +describe("the ui coverage shards keep a red run readable", () => { + it('still runs the coverage suite at all — the pin is not vacuous', () => { + // Without this, every assertion below passes over a job that stopped running + // vitest, which is the one way a `toContain` pin lies. + const steps = coverageRunSteps(workflowSource); + expect( + steps.map((s) => s.step), + `no step in ${UI_COVERAGE.file}'s \`${UI_COVERAGE.shards}\` job runs \`test:coverage\` any more. If that is ` + + 'deliberate, this pin is obsolete and should be deleted WITH the reasoning that made it ' + + 'obsolete written down. If it is not, the nightly lane just lost its coverage measurement.', + ).not.toEqual([]); + }); + + it('is reading the shard leg, not some other coverage invocation', () => { + // Names the population the two assertions below judge: the sharded legs. + // `--merge-reports` in the job below is a different step in a different job + // and is deliberately out of reach here. + const sharded = coverageRunSteps(workflowSource).filter((s) => s.run.includes('--shard=')); + expect( + sharded.map((s) => s.step), + `the \`${UI_COVERAGE.shards}\` job no longer passes \`--shard=\` to the coverage suite. The reporter ` + + 'rule below is about the SHARDED legs specifically — an unsharded lane has no blob to ' + + 'keep and a different set of reporters is right for it.', + ).not.toEqual([]); + }); + + it('passes a reporter that prints failing test names alongside the blob', () => { + const offenders = coverageRunSteps(workflowSource) + .filter((s) => s.run.includes(BLOB_REPORTER)) + .flatMap((s) => + READABLE_REPORTERS.filter((r) => !s.run.includes(r)).map((r) => `${s.step}: missing ${r}`), + ); + + expect( + offenders, + 'a coverage shard passes `--reporter=blob` without the reporters that keep its log readable.\n' + + offenders.map((o) => ` - ${o}`).join('\n') + + '\n\nA CLI `--reporter` REPLACES the default set rather than adding to it, so blob alone ends a ' + + 'failing shard at `blob report written to …` with no test name and no assertion text, and leaves ' + + 'the download-only artifact as the only copy of the failure. `--reporter=default` restores the ' + + 'job log; `--reporter=github-actions` restores the per-test `::error` annotation, which is the ' + + 'only form an API reader can get without downloading anything. objectui#9177; the cost is ' + + 'objectui#8545, where one test file held the coverage gate unevaluated for 84 and 87 pushes.', + ).toEqual([]); + }); + + it('keeps the blob reporter the merge job depends on', () => { + const offenders = coverageRunSteps(workflowSource) + .filter((s) => s.run.includes('--shard=')) + .filter((s) => !s.run.includes(BLOB_REPORTER)) + .map((s) => s.step); + + expect( + offenders, + 'a sharded coverage leg no longer writes a blob report:\n' + + offenders.map((o) => ` - ${o}`).join('\n') + + `\n\nThe \`${UI_COVERAGE.report}\` job merges every shard's blob and enforces the configured thresholds over ` + + 'the merged report — that is the whole reason the lane is sharded (objectui#5403). Restoring log ' + + 'readability by dropping the blob would trade an unreadable failure for an unevaluated coverage ' + + 'floor, which is the same defect wearing the other hat (objectui#9177).', + ).toEqual([]); + }); +}); diff --git a/ui/scripts/__tests__/coverage-shard-width.test.ts b/ui/scripts/__tests__/coverage-shard-width.test.ts new file mode 100644 index 00000000000..d0cff75a6cc --- /dev/null +++ b/ui/scripts/__tests__/coverage-shard-width.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest'; +import { parse as parseYaml } from 'yaml'; + +import { UI_COVERAGE, rootWorkflow } from './helpers/root-ci'; + +/** + * The coverage lane's width, held to ONE number (objectui#12256, folded into + * objectui#12208). + * + * The lane runs in the root `.github/workflows/ui-coverage-nightly.yml` + * (objectstack#25924), where objectui's `ci.yml` jobs `test-coverage` and + * `coverage-report` became `ui-coverage` and `ui-coverage-report`. + * + * The lane spells its shard count in two jobs. `ui-coverage` + * declares the matrix and repeats the count in its job name, its step name and + * its `--shard=N/M` flag; `ui-coverage-report` counts the blob reports the shards + * wrote and refuses to merge a partial set. The blob's FILE NAME carries the + * count too (`blob-N-M.json`, written by vitest from the `--shard` flag), so a + * width moved in the matrix alone leaves the merge looking for blobs that no + * shard writes: the count step answers zero, the merge never runs, and the + * coverage gate is unevaluated on every night -- the lane's own worst failure, + * one edit away. Nothing about that is visible on a pull request, because the + * lane runs on `schedule` and `workflow_dispatch` only. + * + * So every spelling is read here and held to the matrix: + * + * - the matrix is the contiguous run 1..M; + * - the job name, the step name and `--shard=` all say M; + * - `ui-coverage-report` declares `COVERAGE_SHARDS: 'M'`, and its count step + * reads THAT variable for both the blob glob and the expected count, with + * no literal width left in the step; + * - the merge step is a `--merge-reports` that overrides no threshold, so + * the configured `coverage.thresholds` are what is enforced over the merge + * of all M blobs -- the one place the gate runs, since the shard legs + * zero the thresholds. + */ +const workflowSource = rootWorkflow(UI_COVERAGE.file); + +interface Step { + name?: string; + run?: unknown; + id?: string; +} +interface Job { + name?: string; + env?: Record; + strategy?: { matrix?: { shard?: unknown } }; + steps?: Step[]; +} + +const workflow = parseYaml(workflowSource) as { jobs?: Record }; + +function job(key: string): Job { + const found = workflow.jobs?.[key]; + expect(found, `${UI_COVERAGE.file} no longer defines a \`${key}\` job; this pin needs a new referent, not deletion`).toBeDefined(); + return found as Job; +} + +/** The matrix width, after checking the matrix is the contiguous run 1..M. */ +function matrixWidth(): number { + const shards = job(UI_COVERAGE.shards).strategy?.matrix?.shard; + expect(Array.isArray(shards), `\`${UI_COVERAGE.shards}\` must declare a \`shard:\` matrix list`).toBe(true); + const list = shards as unknown[]; + expect(list.length, 'a zero-wide matrix is a broken reader, not a lane').toBeGreaterThan(0); + expect(list, 'the coverage matrix must be the contiguous run 1..M').toEqual(list.map((_, i) => i + 1)); + return list.length; +} + +function runOf(step: Step | undefined): string { + return typeof step?.run === 'string' ? step.run : ''; +} + +describe("the ui coverage lane says its width once (objectui#12256)", () => { + const width = matrixWidth(); + + it('names each shard job with the matrix width', () => { + expect(job(UI_COVERAGE.shards).name).toBe(`Test UI (coverage shard \${{ matrix.shard }}/${width})`); + }); + + it('passes the matrix width to vitest, in the flag and in the step name', () => { + const steps = (job(UI_COVERAGE.shards).steps ?? []).filter((s) => runOf(s).includes('test:coverage')); + expect(steps.length, `no step in \`${UI_COVERAGE.shards}\` runs \`test:coverage\` any more`).toBeGreaterThan(0); + for (const step of steps) { + const flags = [...runOf(step).matchAll(/--shard=\$\{\{ matrix\.shard \}\}\/(\d+)/g)].map((m) => Number(m[1])); + expect(flags, `${step.name}: one --shard flag, carrying the matrix width`).toEqual([width]); + expect(step.name, `${step.name}: the step name states a different width`).toContain(`/${width})`); + } + }); + + it('declares the same width as the count the merge refuses to go below', () => { + expect(job(UI_COVERAGE.report).env?.COVERAGE_SHARDS, `\`${UI_COVERAGE.report}\` must declare \`COVERAGE_SHARDS\``).toBe( + String(width), + ); + }); + + it('counts the blobs through that variable, with no literal width left in the step', () => { + const count = runOf((job(UI_COVERAGE.report).steps ?? []).find((s) => s.id === 'blobs')); + expect(count, `\`${UI_COVERAGE.report}\` must keep its \`blobs\` count step`).not.toBe(''); + expect(count).toContain('-name "blob-*-${COVERAGE_SHARDS}.json"'); + expect(count).toContain('-ne "$COVERAGE_SHARDS"'); + // A digit in the blob glob or the comparison is a second spelling of the + // width, and the one that goes stale first. + expect(count).not.toMatch(/blob-\*-\d/); + expect(count).not.toMatch(/-ne\s+\d/); + }); + + it('enforces the configured thresholds over the merge of every blob, overriding none', () => { + const merge = runOf((job(UI_COVERAGE.report).steps ?? []).find((s) => s.id === 'merge')); + expect(merge, `\`${UI_COVERAGE.report}\` must keep its \`merge\` step`).toContain('--merge-reports'); + expect(merge, 'the merge is where the thresholds are enforced; it must not override them').not.toContain( + 'thresholds', + ); + const legs = (job(UI_COVERAGE.shards).steps ?? []).filter((s) => runOf(s).includes('--shard=')); + expect(legs.length, `no \`${UI_COVERAGE.shards}\` step passes \`--shard=\``).toBeGreaterThan(0); + for (const leg of legs) { + // The other half of the same contract: the legs zero the thresholds, so + // the merge above is the only evaluation, over all of them. + expect(runOf(leg)).toContain('--coverage.thresholds.lines=0'); + } + }); +}); diff --git a/ui/scripts/__tests__/helpers/root-ci.ts b/ui/scripts/__tests__/helpers/root-ci.ts index 1709901e605..f4475a8390d 100644 --- a/ui/scripts/__tests__/helpers/root-ci.ts +++ b/ui/scripts/__tests__/helpers/root-ci.ts @@ -12,12 +12,13 @@ * the root `.github/workflows/` directory through `rootWorkflowFiles()`. It * never reads a copy under `ui/.github/`: nothing runs one. * - * What the port did not carry (the coverage lane, the docs job, the shard - * aggregate, the type-check-scope selector, the timings upload) has no job - * here, and the pins on those retired with a disposition beside them rather - * than being pointed at a job that does something else. The affected-test - * selector was wired into `test-ui` later (objectstack#25868), and its pins - * read that job. + * What the port did not carry (the docs job, the shard aggregate, the + * type-check-scope selector, the timings upload) has no job here, and the pins + * on those retired with a disposition beside them rather than being pointed at + * a job that does something else. The affected-test selector was wired into + * `test-ui` later (objectstack#25868), and its pins read that job. The + * coverage lane came back as a nightly workflow of its own + * (objectstack#25924), named by `UI_COVERAGE` below. */ import fs from 'node:fs'; import path from 'node:path'; @@ -32,6 +33,17 @@ export const UI_JOBS = { buildE2e: 'build-e2e-ui', } as const; +/** + * objectui's coverage lane, ported to the root as a nightly workflow + * (objectstack#25924): the file, the sharded job and the merge job that + * enforces the thresholds. + */ +export const UI_COVERAGE = { + file: 'ui-coverage-nightly.yml', + shards: 'ui-coverage', + report: 'ui-coverage-report', +} as const; + /** The directory GitHub runs: `.github/workflows/` at the workspace root. */ export const rootWorkflowsDir = path.join(workspaceRoot(), '.github/workflows'); diff --git a/ui/scripts/__tests__/shard-headroom.test.ts b/ui/scripts/__tests__/shard-headroom.test.ts index c240dd5d080..1cc8eacd99c 100644 --- a/ui/scripts/__tests__/shard-headroom.test.ts +++ b/ui/scripts/__tests__/shard-headroom.test.ts @@ -2,10 +2,12 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { parse as parseYaml } from 'yaml'; // Plain-JS CI helper; its types are INFERRED from the .mjs source by // `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here. import { EXIT_OK, EXIT_USAGE, headroom, main, render, shardNames } from '../shard-headroom.mjs'; +import { UI_COVERAGE, rootWorkflow } from './helpers/root-ci'; /** * The report-only headroom reading of the two test aggregators (objectui#12208, @@ -92,12 +94,36 @@ describe('⛔ not a gate', () => { }); }); -// `ci.yml wires it as a reading of the right matrix and the right cap` retired -// with objectstack#25712. It held the headroom step's `--job`, `--shards` and -// `--cap-minutes` to the two objectui ci.yml aggregators that ran it, -// `test-aggregate` (over `test`) and `coverage-report` (over `test-coverage`), -// and that file runs nowhere under the monorepo. Both are row 7 of the -// objectstack#25697 workflow roster: `Test (the shard aggregate)`, "dropped -// until cutover", and `Test (coverage shard n/8) · Test (coverage)`, "kept, -// follow-up port". No root job runs this reading until one of them is ported; -// the port brings the wiring pin back against its job (`helpers/root-ci.ts`). +// `ci.yml wires it as a reading of the right matrix and the right cap` held the +// headroom step's `--job`, `--shards` and `--cap-minutes` to the two objectui +// ci.yml aggregators that ran it, and that file runs nowhere under the +// monorepo (objectstack#25712). Both are row 7 of the objectstack#25697 +// workflow roster. The coverage half came back with the lane's nightly port +// (objectstack#25924) and is pinned below against its root job. The other half, +// `test-aggregate` over `test` (`Test (the shard aggregate)`, "dropped until +// cutover"), has no root job, so no root job runs that reading until it is +// ported; that port brings its wiring pin back the same way. +describe('ui-coverage-nightly.yml wires it as a reading of the right matrix and the right cap', () => { + interface Step { name?: string; run?: unknown; if?: string; 'continue-on-error'?: boolean } + interface Job { name?: string; 'timeout-minutes'?: number; steps?: Step[]; strategy?: { matrix?: { shard?: unknown[] } } } + const workflow = parseYaml(rootWorkflow(UI_COVERAGE.file)) as { jobs: Record }; + + it(`${UI_COVERAGE.report} reads ${UI_COVERAGE.shards}: its job name, its width and its timeout-minutes`, () => { + const steps = (workflow.jobs[UI_COVERAGE.report]?.steps ?? []).filter( + (s) => typeof s.run === 'string' && s.run.includes('scripts/shard-headroom.mjs'), + ); + expect(steps, `${UI_COVERAGE.report} must run the headroom reading exactly once`).toHaveLength(1); + const [step] = steps; + const run = step.run as string; + const target = workflow.jobs[UI_COVERAGE.shards]; + const width = (target.strategy?.matrix?.shard ?? []).length; + expect(width, `${UI_COVERAGE.shards} must declare a shard matrix`).toBeGreaterThan(0); + expect(run).toContain(`--job '${String(target.name).replace('${{ matrix.shard }}', '{n}')}'`); + expect(run).toContain(`--shards ${width} `); + expect(run).toContain(`--cap-minutes ${target['timeout-minutes']} `); + // ⛔ Not a gate: it runs whatever happened above it, and nothing it + // does can change the aggregator's conclusion. + expect(step.if).toBe('always()'); + expect(step['continue-on-error']).toBe(true); + }); +}); diff --git a/ui/scripts/shard-headroom.mjs b/ui/scripts/shard-headroom.mjs index 4793dbf6de0..fe245afe5dd 100644 --- a/ui/scripts/shard-headroom.mjs +++ b/ui/scripts/shard-headroom.mjs @@ -11,8 +11,9 @@ * node scripts/shard-headroom.mjs --job 'Test (shard {n}/16)' --shards 16 --cap-minutes 20 --target-minutes 10 * node scripts/shard-headroom.mjs ... --fixture * - * `ci.yml`'s two aggregators run it: `test-aggregate` (`Test`) for the `test` - * matrix and `coverage-report` (`Test (coverage)`) for the coverage matrix. + * The root `ui-coverage-nightly.yml`'s `ui-coverage-report` (`Test UI + * (coverage)`) runs it for the coverage matrix (objectstack#25924). objectui's + * other aggregator, `test-aggregate` (`Test`), has no root job yet. * `scripts/__tests__/shard-headroom.test.ts` pins each invocation's * `--shards` and `--cap-minutes` to the matrix and the `timeout-minutes` of * the job it reads, so the reading cannot quietly describe another cap.