From b07ef8cc2c4aea8c83ed4a4a2da9afdc9b71bb89 Mon Sep 17 00:00:00 2001 From: Marcel Rebro Date: Thu, 17 Sep 2026 16:07:41 +0200 Subject: [PATCH] docs: add an advisory PR-time doc-integrity check to docs-ui-tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the workflow into two jobs so doc-side drift is caught in review instead of up to a week later. doc-integrity runs on pull requests touching sources/platform/** or docs-tests/**. It runs only the `integrity` project — static checks that each assertion's source_file and source_quote still resolve. No browser and no credentials, so it needs no Playwright install and works on fork PRs, where secrets are unavailable. The check is informative only: `continue-on-error` keeps a failure from blocking the merge, and findings are written to the job summary as a table of assertion IDs and their doc sources, so a reviewer sees them without opening logs. Promoting it to a gate later means removing `continue-on-error` and marking it required in branch protection. ui-drift is unchanged except for an `if` guard keeping it off pull requests. It still runs weekly and on demand, against staging, and still files a drift issue on failure. The motivation is #2951: #2919 moved account/settings.md on 09-01, the baseline went stale, and nothing surfaced it until the scheduled run on 09-07 — into an unassigned issue that sat another 10 days. Run against that state locally, this job reports all 16 stale back-references in 4.3 seconds with no browser and no credentials. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FSj1mTUbMKgmu2JoGJLSKD --- .github/workflows/docs-ui-tests.yaml | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/.github/workflows/docs-ui-tests.yaml b/.github/workflows/docs-ui-tests.yaml index 8156138b26..65580542e6 100644 --- a/.github/workflows/docs-ui-tests.yaml +++ b/.github/workflows/docs-ui-tests.yaml @@ -8,18 +8,106 @@ name: Docs UI drift tests # Credentials come from repo Actions secrets (CONSOLE_STAGING_*). The seeded # staging user is low-privilege and has no 2FA; login is scripted fresh each run # and the session is held in memory (no auth file). +# +# Two jobs, deliberately split: +# +# doc-integrity runs on PULL REQUESTS that touch the docs or the harness. +# Only the `integrity` project: pure static checks that each +# assertion's source_file/source_quote still resolves. No +# browser, no credentials, a few seconds — so it also works on +# fork PRs, where secrets are unavailable. INFORMATIVE ONLY: it +# never fails the PR; findings go to the job summary. +# +# ui-drift runs WEEKLY and on demand, never on a PR. The full suite +# against Console staging, and it files a drift issue on +# failure. +# +# The split exists because a doc can move mid-week and stay undetected until the +# next Monday — which is exactly how #2951 happened. The PR-time check catches +# that class of drift in review, when the move is still one edit away from fixed. on: schedule: - cron: '0 6 * * 1' # Every Monday 06:00 UTC workflow_dispatch: # Manual on-demand run + pull_request: + paths: + - 'sources/platform/**' # a doc move/rewrite can stale the baseline + - 'docs-tests/**' # harness or baseline changes defaults: run: working-directory: docs-tests jobs: + # PR-time doc-side check. Advisory: `continue-on-error` keeps a failure from + # blocking the merge, so this is an additional signal rather than a new gate. + # Findings are written to the job summary, where a reviewer sees them without + # opening logs. Promote it to a required check later by removing + # `continue-on-error` and marking it required in branch protection. + doc-integrity: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Use Node.js 24 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + + - name: Install pnpm and docs-tests dependencies + uses: apify/actions/pnpm-install@v1.4.0 + with: + working-directory: docs-tests + + # No `playwright install`: the integrity project never requests a + # browser fixture, so it runs on the bare Node install. + - name: Check doc back-references resolve + id: integrity + continue-on-error: true + run: pnpm test:integrity + + - name: Summarize + if: always() + env: + OUTCOME: ${{ steps.integrity.outcome }} + run: | + set -euo pipefail + REPORT=output/issues.json + + if [[ "$OUTCOME" == "success" ]]; then + echo "### Docs-tests: doc back-references OK" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Every assertion in \`docs-tests/assertions/\` still points at a doc that exists, with its quoted text intact." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + { + echo "### Docs-tests: stale doc back-references" + echo "" + echo "This is **informative only** and does not block the merge." + echo "" + } >> "$GITHUB_STEP_SUMMARY" + + if [[ -f "$REPORT" ]]; then + { + echo "Assertions in \`docs-tests/assertions/\` point at documentation that this PR moved, renamed or rewrote:" + echo "" + echo "| Assertion | Doc source |" + echo "| --- | --- |" + jq -r '.issues[] | "| `\(.id)` | `\(.source_file):\(.source_line)` |"' "$REPORT" + echo "" + echo "Fix by repointing \`docs-tests/pages.json\` and the affected \`assertions/*.json\` at the new path, or re-extracting the page. Left unfixed, the weekly drift check goes red and files an issue." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "The check failed before producing a report — see the job logs." >> "$GITHUB_STEP_SUMMARY" + fi + ui-drift: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: read