diff --git a/.github/workflows/_visual-regression.yaml b/.github/workflows/_visual-regression.yaml new file mode 100644 index 00000000..aa9e0c57 --- /dev/null +++ b/.github/workflows/_visual-regression.yaml @@ -0,0 +1,97 @@ +name: Visual regression + +on: + workflow_call: + +permissions: + contents: read + +jobs: + visual-regression: + name: Playwright visual snapshots + runs-on: ubuntu-latest + timeout-minutes: 15 + # Pinned to the exact @playwright/test version in package.json — the + # image and npm package must match exactly, or browser launch breaks. + # Screenshot comparison is sensitive to font rendering and software + # rasterization, which differ by host OS/GPU — running inside this + # exact image keeps this job and the /update-snapshots regeneration + # job (update-visual-baselines.yaml, same digest) pixel-consistent + # with each other regardless of which runner picks up the job. + container: + image: mcr.microsoft.com/playwright@sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27 # v1.63.0-noble + options: --ipc=host + env: + # Keep in sync with the image tag in the comment above — the + # "Verify Playwright version" step below checks the installed + # @playwright/test against this, so a dependency bump without a + # matching image bump fails loud and named instead of as a + # confusing browser-launch error on an unrelated PR. + PLAYWRIGHT_IMAGE_VERSION: '1.63.0' + # Keep the build-time MCP metadata format stable. The regular site + # build still validates the current ToolHive release independently. + TOOLHIVE_VERSION: '0.49.0' + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # actions/checkout marks the repo safe in the RUNNER's git config, + # but a run: step inside `container:` executes via `docker exec` + # into a separate environment that never got that exception — + # without this, Docusaurus's last-update-date git log call (and + # anything else here shelling out to git) fails with "detected + # dubious ownership in repository". + - name: Mark workspace as a safe git directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + # The Playwright image is minimal — it ships neither jq, which + # scripts/install-thv.sh requires, nor a Node version manager, since + # it already bundles the Node build this image's Chromium was + # tested against. A pinned static binary (checksum-verified) avoids + # `apt-get update`, which syncs the full Ubuntu archive index just + # to install one small tool. + - name: Install jq + run: | + curl -sSL -o /usr/local/bin/jq https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-amd64 + echo "b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f /usr/local/bin/jq" | sha256sum -c - + chmod +x /usr/local/bin/jq + + - name: Cache dependencies + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ./node_modules + key: modules-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: npm ci + + - name: Verify Playwright version matches pinned image + run: | + installed="$(npx playwright --version | sed -n 's/^Version \([0-9.]*\)$/\1/p')" + if [ "$installed" != "$PLAYWRIGHT_IMAGE_VERSION" ]; then + echo "::error::@playwright/test resolved to $installed but the pinned image is for $PLAYWRIGHT_IMAGE_VERSION — bump the image digest (see container.image comment) to match." + exit 1 + fi + + - name: Install ToolHive CLI + run: ./scripts/install-thv.sh + + - name: Build site + run: npm run build + + - name: Run visual regression tests + run: npm run test:visual + + - name: Upload test artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: playwright-report + path: | + test-results/ + playwright-report/ + retention-days: 7 diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 692b47d0..18c2fa66 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -21,3 +21,7 @@ jobs: static-checks: name: Static checks uses: $/.github/workflows/_static-checks.yaml + + visual-regression: + name: Visual regression + uses: $/.github/workflows/_visual-regression.yaml diff --git a/.github/workflows/pr-screenshot-summary.yaml b/.github/workflows/pr-screenshot-summary.yaml new file mode 100644 index 00000000..ef942bdc --- /dev/null +++ b/.github/workflows/pr-screenshot-summary.yaml @@ -0,0 +1,82 @@ +name: PR Screenshot Summary + +# workflow_run always loads this workflow from the default branch. That makes +# it a trusted follow-up to the read-only PR workflow even though this job can +# update pull request descriptions. +# The privileged job checks out only the API-resolved base SHA and treats the +# PR head as inert Git data. Zizmor cannot infer that cross-step trust boundary. +on: # zizmor: ignore[dangerous-triggers] + workflow_run: + workflows: ['On PR'] + types: [completed] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number }} + cancel-in-progress: true + +permissions: {} + +jobs: + update-description: + name: Update PR description with screenshot summary + if: | + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.pull_requests[0] != null + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Resolve current PR context + id: pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + REPO: ${{ github.repository }} + RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + PR_JSON="$(gh api "repos/$REPO/pulls/$PR_NUMBER")" + BASE_SHA="$(echo "$PR_JSON" | jq -r '.base.sha')" + HEAD_SHA="$(echo "$PR_JSON" | jq -r '.head.sha')" + if [ "$HEAD_SHA" != "$RUN_HEAD_SHA" ]; then + echo "The PR advanced after this run; its newer run will update the summary." + echo "current=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + { + echo "current=true" + echo "base_sha=$BASE_SHA" + echo "head_sha=$HEAD_SHA" + } >> "$GITHUB_OUTPUT" + + - name: Checkout trusted base revision + if: steps.pr.outputs.current == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ steps.pr.outputs.base_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Fetch PR head for read-only inspection + if: steps.pr.outputs.current == 'true' + env: + EXPECTED_HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + run: | + git fetch --no-tags origin \ + "+refs/pull/$PR_NUMBER/head:refs/remotes/origin/pr/$PR_NUMBER/head" + ACTUAL_HEAD_SHA="$(git rev-parse "refs/remotes/origin/pr/$PR_NUMBER/head")" + if [ "$ACTUAL_HEAD_SHA" != "$EXPECTED_HEAD_SHA" ]; then + echo "::error::PR head changed while preparing the summary; retry against the current head." + exit 1 + fi + + - name: Update PR description with screenshot summary + if: steps.pr.outputs.current == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + BASE_SHA: ${{ steps.pr.outputs.base_sha }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + # This script comes from the checked-out base revision, never the PR. + run: node scripts/update-pr-screenshot-summary.mjs diff --git a/.github/workflows/update-visual-baselines.yaml b/.github/workflows/update-visual-baselines.yaml new file mode 100644 index 00000000..f14ddf80 --- /dev/null +++ b/.github/workflows/update-visual-baselines.yaml @@ -0,0 +1,244 @@ +name: Update visual baselines + +# issue_comment workflows always come from the default branch. Keep all write +# operations in jobs that execute only this trusted workflow code; the PR job +# receives no write token and communicates solely through a validated artifact. +on: + issue_comment: + types: [created] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # Must match _visual-regression.yaml so baseline generation and comparison + # use the same build-time MCP metadata format. + TOOLHIVE_VERSION: '0.49.0' + +jobs: + authorize: + name: Authorize the request + if: | + github.event.issue.pull_request != null && + startsWith(github.event.comment.body, '/update-snapshots') + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + head_ref: ${{ steps.pr.outputs.head_ref }} + head_sha: ${{ steps.pr.outputs.head_sha }} + steps: + - name: Check commenter has write access + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMENTER: ${{ github.event.comment.user.login }} + run: | + PERMISSION="$(gh api "repos/${{ github.repository }}/collaborators/$COMMENTER/permission" --jq '.permission')" + if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "write" ]]; then + echo "::error::@$COMMENTER does not have write access to this repo - refusing to run." + exit 1 + fi + + - name: Resolve PR head + id: pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + PR_JSON="$(gh api "repos/${{ github.repository }}/pulls/$ISSUE_NUMBER")" + HEAD_REF="$(echo "$PR_JSON" | jq -r '.head.ref')" + HEAD_SHA="$(echo "$PR_JSON" | jq -r '.head.sha')" + HEAD_REPO="$(echo "$PR_JSON" | jq -r '.head.repo.full_name')" + if [ "$HEAD_REPO" != "${{ github.repository }}" ]; then + echo "::error::This workflow can update only branches in ${{ github.repository }}." + exit 1 + fi + { + echo "head_ref=$HEAD_REF" + echo "head_sha=$HEAD_SHA" + } >> "$GITHUB_OUTPUT" + + generate: + name: Generate baselines without write access + needs: authorize + runs-on: ubuntu-latest + permissions: + contents: read + container: + image: mcr.microsoft.com/playwright@sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27 # v1.63.0-noble + options: --ipc=host + timeout-minutes: 15 + steps: + - name: Checkout the untrusted PR revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.authorize.outputs.head_sha }} + persist-credentials: false + + - name: Mark workspace as a safe git directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Install jq + run: | + curl -sSL -o /usr/local/bin/jq https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-amd64 + echo "b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f /usr/local/bin/jq" | sha256sum -c - + chmod +x /usr/local/bin/jq + + - name: Install dependencies + run: npm ci + + - name: Install ToolHive CLI + run: ./scripts/install-thv.sh + + - name: Build site + run: npm run build + + - name: Regenerate baselines + # A functional assertion may fail before one screenshot is reached. + # Preserve any other regenerated baselines for review, as before. + run: npm run test:visual:update || true + + - name: Upload untrusted baseline data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: visual-baselines-${{ needs.authorize.outputs.head_sha }} + path: | + tests/visual/*.spec.ts-snapshots/*.json + tests/visual/*.spec.ts-snapshots/*.png + if-no-files-found: error + retention-days: 1 + + publish: + name: Validate and commit baselines + needs: [authorize, generate] + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + updated: ${{ steps.commit.outputs.updated }} + steps: + - name: Download untrusted baseline data + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: visual-baselines-${{ needs.authorize.outputs.head_sha }} + path: ${{ runner.temp }}/visual-baselines + + - name: Validate artifact paths and formats + env: + ARTIFACT_DIR: ${{ runner.temp }}/visual-baselines + run: | + if find "$ARTIFACT_DIR" -type l -print -quit | grep -q .; then + echo "::error::Baseline artifact contains a symbolic link." + exit 1 + fi + + FILE_COUNT="$(find "$ARTIFACT_DIR" -type f | wc -l)" + TOTAL_BYTES="$(du -sb "$ARTIFACT_DIR" | cut -f1)" + if [ "$FILE_COUNT" -eq 0 ] || [ "$FILE_COUNT" -gt 200 ]; then + echo "::error::Unexpected baseline file count: $FILE_COUNT" + exit 1 + fi + if [ "$TOTAL_BYTES" -gt 104857600 ]; then + echo "::error::Baseline artifact exceeds 100 MiB." + exit 1 + fi + + while IFS= read -r -d '' FILE; do + RELATIVE="${FILE#"$ARTIFACT_DIR"/}" + if [[ ! "$RELATIVE" =~ ^[^/]+\.spec\.ts-snapshots/[^/]+\.(json|png)$ ]]; then + echo "::error::Unexpected artifact path: $RELATIVE" + exit 1 + fi + case "$FILE" in + *.json) + jq -e ' + type == "object" and + (.titlePath | type == "array") and + (.testFile | type == "string") and + (.url.path | type == "string") + ' "$FILE" >/dev/null + ;; + *.png) + SIGNATURE="$(head -c 8 "$FILE" | od -An -tx1 | tr -d ' \n')" + if [ "$SIGNATURE" != "89504e470d0a1a0a" ]; then + echo "::error::Invalid PNG signature: $RELATIVE" + exit 1 + fi + ;; + esac + done < <(find "$ARTIFACT_DIR" -type f -print0) + + - name: Checkout exact PR revision for publishing + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.authorize.outputs.head_sha }} + fetch-depth: 0 + persist-credentials: true + + - name: Commit validated baseline data + id: commit + env: + ARTIFACT_DIR: ${{ runner.temp }}/visual-baselines + HEAD_REF: ${{ needs.authorize.outputs.head_ref }} + HEAD_SHA: ${{ needs.authorize.outputs.head_sha }} + run: | + if [ -L tests ] || [ -L tests/visual ] || \ + find tests/visual -type l -print -quit | grep -q .; then + echo "::error::PR snapshot paths must not contain symbolic links." + exit 1 + fi + + while IFS= read -r -d '' FILE; do + RELATIVE="${FILE#"$ARTIFACT_DIR"/}" + DESTINATION="tests/visual/$RELATIVE" + mkdir -p "$(dirname "$DESTINATION")" + cp "$FILE" "$DESTINATION" + done < <(find "$ARTIFACT_DIR" -type f -print0) + + git add -- 'tests/visual/*.spec.ts-snapshots/*.json' \ + 'tests/visual/*.spec.ts-snapshots/*.png' + if git diff --cached --quiet; then + echo "No snapshot diff reproduced against this branch." + echo "updated=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local core.hooksPath /dev/null + git commit --no-verify -m "Update visual baselines for ${HEAD_SHA:0:12}" + git push origin "HEAD:$HEAD_REF" + echo "updated=true" >> "$GITHUB_OUTPUT" + + comment-result: + name: Comment result + needs: [authorize, generate, publish] + if: always() && needs.authorize.result == 'success' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Comment success or no-op + if: needs.publish.result == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + UPDATED: ${{ needs.publish.outputs.updated }} + run: | + if [ "$UPDATED" == "true" ]; then + BODY="📸 Regenerated visual baselines per your \`/update-snapshots\` request and pushed a new commit." + else + BODY="No snapshot diff reproduced against this branch's current state. If the visual check is red, inspect the job log for a functional failure." + fi + gh pr comment "$ISSUE_NUMBER" --repo "${{ github.repository }}" --body "$BODY" + + - name: Comment failure + if: needs.publish.result != 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + run: | + gh pr comment "$ISSUE_NUMBER" --repo "${{ github.repository }}" --body "❌ \`/update-snapshots\` failed before it could push anything. See the run log: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.gitignore b/.gitignore index 6980cd14..c7c22ef0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ # Production /build +# Playwright visual regression test output +/playwright-report +/test-results + # Generated files .docusaurus .cache-loader diff --git a/package-lock.json b/package-lock.json index 7d96302b..14b82941 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "@docusaurus/types": "3.10.2", "@eslint/compat": "^2.1.1", "@eslint/js": "^9.39.5", + "@playwright/test": "1.63.0", "eslint": "^9.39.5", "eslint-config-prettier": "^10.1.8", "eslint-plugin-mdx": "^3.8.1", @@ -6045,6 +6046,22 @@ "url": "https://opencollective.com/pkgr" } }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@pnpm/config.env-replace": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz", @@ -20009,6 +20026,35 @@ "node": ">=16.0.0" } }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", diff --git a/package.json b/package.json index b42ca173..b7713709 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "serve": "docusaurus serve", "start": "docusaurus start --host 0.0.0.0", "swizzle": "docusaurus swizzle", + "test:visual": "playwright test", + "test:visual:update": "playwright test --update-snapshots", "typecheck": "tsc", "write-heading-ids": "docusaurus write-heading-ids", "write-translations": "docusaurus write-translations" @@ -43,6 +45,7 @@ "@docusaurus/types": "3.10.2", "@eslint/compat": "^2.1.1", "@eslint/js": "^9.39.5", + "@playwright/test": "1.63.0", "eslint": "^9.39.5", "eslint-config-prettier": "^10.1.8", "eslint-plugin-mdx": "^3.8.1", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 00000000..af0f2987 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import { defineConfig, devices } from '@playwright/test'; + +// Fixed port so `webServer` and `use.baseURL` agree without extra +// plumbing. Not the default 3000 that `npm start`/`npm run serve` use, so +// a visual test run never collides with a dev server already running on +// the default port. +const PORT = 3005; +const BASE_URL = `http://localhost:${PORT}`; + +export default defineConfig({ + testDir: './tests/visual', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [['html'], ['github']] : 'list', + timeout: 30_000, + use: { + baseURL: BASE_URL, + trace: 'on-first-retry', + }, + projects: [ + { + name: 'desktop', + use: { + ...devices['Desktop Chrome'], + viewport: { width: 1280, height: 800 }, + // The PR screenshot summary embeds these at whatever width + // GitHub's PR body happens to be (~700-900px, well under 1280) — + // capturing at 2x means that downscale is supersampling real + // extra resolution rather than stretching a 1x image, which is + // the difference between crisp and visibly soft body text for a + // page this text-dense. Doubles PNG size; still small enough not + // to matter for a doc site's screenshot count. + deviceScaleFactor: 2, + }, + testIgnore: /mobile\.spec\.ts$/, + }, + { + name: 'mobile', + // Viewport-only, deliberately not a full `devices['iPhone ...']` + // preset: the layout breakpoint is what we're testing, not + // touch/UA-driven rendering differences on top of it. + use: { + ...devices['Desktop Chrome'], + viewport: { width: 390, height: 844 }, + // Real mobile hardware is almost universally >1x DPR too — same + // supersampling reasoning as the desktop project above. + deviceScaleFactor: 2, + }, + testMatch: /mobile\.spec\.ts$/, + }, + ], + webServer: { + // Production build, not the dev server: matches what's actually + // deployed (Vercel serves the build) and avoids dev-mode HMR/overlay + // noise in the screenshots. + command: `npm run build && npm run serve -- --port ${PORT} --no-open`, + url: BASE_URL, + reuseExistingServer: !process.env.CI, + timeout: 180_000, + }, +}); diff --git a/scripts/install-thv.sh b/scripts/install-thv.sh index f6444b4d..9df8aa99 100755 --- a/scripts/install-thv.sh +++ b/scripts/install-thv.sh @@ -13,7 +13,11 @@ if ! command -v jq >/dev/null 2>&1; then exit 1 fi -API_ENDPOINT="https://api.github.com/repos/stacklok/toolhive/releases/latest" +if [[ -n "${TOOLHIVE_VERSION:-}" ]]; then + API_ENDPOINT="https://api.github.com/repos/stacklok/toolhive/releases/tags/v${TOOLHIVE_VERSION#v}" +else + API_ENDPOINT="https://api.github.com/repos/stacklok/toolhive/releases/latest" +fi # Fetch release information RELEASE_JSON=$(curl -sf "$API_ENDPOINT" || { diff --git a/scripts/pr-screenshot-summary.mjs b/scripts/pr-screenshot-summary.mjs new file mode 100644 index 00000000..7c5518a2 --- /dev/null +++ b/scripts/pr-screenshot-summary.mjs @@ -0,0 +1,258 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Generates an HTML-table summary of visual-regression screenshot changes +// (tests/visual/*.spec.ts-snapshots/*.png) between two git refs, for +// pasting into a PR description. Grouped into New / Changed / Deleted +// (git's A/M/D status), one HTML table per snapshot group (a light/dark +// pair, or a singleton when only one scheme changed) so each group reads +// as a self-contained card. Raw HTML tables are used (not GFM pipe +// tables) because only HTML supports colspan, and GitHub renders HTML +// tables fine inside markdown. +// +// Images are embedded via GitHub's own raw-blob URLs +// (github.com///raw//) — baselines are already +// committed PNGs, so this needs no separate image hosting. Changed +// entries additionally link into GitHub's own rich image diff via its +// undocumented but empirically-verified #diff- anchor +// convention (this has changed once before per public discussion; treat +// it as best-effort, not a stable contract). +// +// Usage: +// node scripts/pr-screenshot-summary.mjs [--base ] [--head ] +// Defaults: --base origin/main, --head HEAD. Run `git fetch origin` first +// if --base hasn't been fetched recently — this script never fetches on +// its own. + +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; + +function parseArgs(argv) { + const args = { base: 'origin/main', head: 'HEAD' }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--base' && argv[i + 1]) args.base = argv[++i]; + else if (argv[i] === '--head' && argv[i + 1]) args.head = argv[++i]; + } + return args; +} + +function git(...gitArgs) { + return execFileSync('git', gitArgs, { encoding: 'utf-8' }).trim(); +} + +function repoSlug() { + const url = git('remote', 'get-url', 'origin'); + const match = url.match(/github\.com[:/]([^/]+\/[^/.]+?)(\.git)?$/); + if (!match) { + throw new Error(`Could not parse an owner/repo from remote url: ${url}`); + } + return match[1]; +} + +const SNAPSHOT_GLOB = 'tests/visual/*.spec.ts-snapshots/*.png'; +const SCHEME_RE = /-(light|dark)-(?:desktop|mobile)-linux\.png$/; + +/** @typedef {{ path: string, status: 'A'|'M'|'D', sha: string }} Entry */ + +function lastCommitWith(filePath, beforeRef) { + return git('log', '-1', '--format=%H', beforeRef, '--', filePath); +} + +/** @returns {Entry[]} */ +function diffEntries(base, head) { + const out = git( + 'diff', + '--name-status', + `${base}...${head}`, + '--', + SNAPSHOT_GLOB + ); + if (!out) return []; + return out.split('\n').map((line) => { + const [rawStatus, filePath] = line.split('\t'); + // Collapse rename-detection scores (e.g. "R100") to a single letter — + // PNGs don't meaningfully rename-detect against each other, but git + // diff still tags binary adds/deletes with a bare status letter, so + // this only ever needs the first character in practice. + const status = rawStatus[0]; + const sha = status === 'D' ? lastCommitWith(filePath, base) : head; + return { path: filePath, status, sha }; + }); +} + +function titleize(pngPath) { + const base = pngPath.split('/').pop() ?? pngPath; + const stripped = base.replace(SCHEME_RE, ''); + return stripped + .split('-') + .map((w) => (w ? w[0].toUpperCase() + w.slice(1) : w)) + .join(' '); +} + +function schemeOf(pngPath) { + return pngPath.match(SCHEME_RE)?.[1] ?? 'unknown'; +} + +const SCHEME_EMOJI = { light: '☀️', dark: '🌙' }; +// Fixed order (not alphabetical — "dark" < "light" lexically) so light +// always renders above dark within a group. +const SCHEME_ORDER = ['light', 'dark']; + +/** Same PNG path with the "---linux.png" suffix stripped + * — the key that groups a light/dark pair (or singleton) back into one + * logical snapshot for the merged-cell table layout. */ +function baseKeyOf(pngPath) { + return pngPath.replace(SCHEME_RE, ''); +} + +function groupByBaseSnapshot(entries) { + const order = []; + const groups = new Map(); + for (const e of entries) { + const key = baseKeyOf(e.path); + if (!groups.has(key)) { + groups.set(key, []); + order.push(key); + } + groups.get(key)?.push(e); + } + return order.map((key) => { + const g = groups.get(key) ?? []; + return [...g].sort( + (a, b) => + SCHEME_ORDER.indexOf(schemeOf(a.path)) - + SCHEME_ORDER.indexOf(schemeOf(b.path)) + ); + }); +} + +function rawUrl(slug, sha, filePath) { + return `https://github.com/${slug}/raw/${sha}/${filePath}`; +} + +/** GitHub's file-diff anchor: sha256 of the repo-relative path, hex — + * verified against a known-good example, not officially documented. */ +function diffAnchor(filePath) { + return createHash('sha256').update(filePath).digest('hex'); +} + +function compareUrl(slug, baseSha, headSha, filePath) { + return `https://github.com/${slug}/compare/${baseSha}...${headSha}#diff-${diffAnchor(filePath)}`; +} + +function blobUrl(slug, sha, testFile) { + return `https://github.com/${slug}/blob/${sha}/${testFile}`; +} + +const UNKNOWN_METADATA = { urlPath: '(unknown URL path)', testFile: null }; + +/** Reads the PNG's same-named .json sidecar at the given commit (not off + * disk) so this works uniformly for New/Changed entries (sha = head) and + * Deleted ones (sha = the last commit that still had the file) alike. */ +function readMetadata(pngPath, sha) { + const jsonPath = pngPath.replace(/\.png$/, '.json'); + try { + const raw = git('show', `${sha}:${jsonPath}`); + const metadata = JSON.parse(raw); + return { + urlPath: metadata.url?.path ?? UNKNOWN_METADATA.urlPath, + testFile: metadata.testFile ?? null, + }; + } catch { + return UNKNOWN_METADATA; + } +} + +function titleCell(title, slug, sha, meta) { + if (!meta.testFile) return `${title}`; + return `${title}`; +} + +const STATUS_HEADING = { A: '🟢 New', M: '🟡 Changed', D: '🔴 Deleted' }; +const STATUS_TITLE_SUFFIX = { A: '✨', M: '🔀', D: '🗑️' }; + +/** New/Deleted entries: one HTML table per snapshot group. */ +function buildSection(status, entries, slug) { + const group = entries.filter((e) => e.status === status); + if (group.length === 0) return ''; + + const lines = [`### ${STATUS_HEADING[status]}`, '']; + for (const rowGroup of groupByBaseSnapshot(group)) { + const title = `${titleize(rowGroup[0].path)} ${STATUS_TITLE_SUFFIX[status]}`; + const meta = readMetadata(rowGroup[0].path, rowGroup[0].sha); + lines.push( + '', + ``, + '' + ); + for (const e of rowGroup) { + const emoji = SCHEME_EMOJI[schemeOf(e.path)] ?? ''; + const img = `${title} (${schemeOf(e.path)})`; + lines.push(``); + } + lines.push('
${titleCell(title, slug, rowGroup[0].sha, meta)}
${meta.urlPath}
Preview
${emoji}${img}
', ''); + } + return lines.join('\n'); +} + +const CHANGE_ARROW = '➡️'; + +/** Changed entries get a before/after layout plus a link into GitHub's + * own rich diff for pixel-level comparison. */ +function buildChangedSection(entries, slug, baseSha, headSha) { + const group = entries.filter((e) => e.status === 'M'); + if (group.length === 0) return ''; + + const lines = [`### ${STATUS_HEADING.M}`, '']; + for (const rowGroup of groupByBaseSnapshot(group)) { + const title = `${titleize(rowGroup[0].path)} ${STATUS_TITLE_SUFFIX.M}`; + const meta = readMetadata(rowGroup[0].path, rowGroup[0].sha); + lines.push( + '', + ``, + '' + ); + for (const e of rowGroup) { + const scheme = schemeOf(e.path); + const emoji = SCHEME_EMOJI[scheme] ?? ''; + const diff = compareUrl(slug, baseSha, headSha, e.path); + const schemeCell = `${emoji}
diff`; + const beforeImg = `Before (${scheme})`; + const afterImg = `After (${scheme})`; + lines.push( + `` + ); + } + lines.push('
${titleCell(title, slug, rowGroup[0].sha, meta)}
${meta.urlPath}
BeforeAfter
${schemeCell}${beforeImg}${CHANGE_ARROW}${afterImg}
', ''); + } + return lines.join('\n'); +} + +function main() { + const { base, head } = parseArgs(process.argv.slice(2)); + const slug = repoSlug(); + // Resolve refs to full commit SHAs up front — entries store a SHA for + // the raw-URL link, and a bare ref name like "HEAD" or "origin/main" + // isn't stable/dereferenceable in a raw-blob-style URL. + const baseSha = git('rev-parse', base); + const headSha = git('rev-parse', head); + const entries = diffEntries(baseSha, headSha); + + if (entries.length === 0) { + console.log( + `No visual-regression snapshot changes between ${base} and ${head}.` + ); + return; + } + + const sections = [ + buildSection('A', entries, slug), + buildChangedSection(entries, slug, baseSha, headSha), + buildSection('D', entries, slug), + ].filter(Boolean); + + console.log(sections.join('\n')); +} + +main(); diff --git a/scripts/update-pr-screenshot-summary.mjs b/scripts/update-pr-screenshot-summary.mjs new file mode 100644 index 00000000..d1c8f1b1 --- /dev/null +++ b/scripts/update-pr-screenshot-summary.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// CI-only: regenerates the visual-regression screenshot summary (see +// pr-screenshot-summary.mjs) and folds it into the pull request's +// description, inside a fenced marker block. Replaces just that block if +// one already exists (e.g. from a previous push); otherwise prepends it, +// leaving the rest of the description exactly as the author wrote it. +// +// Requires GH_TOKEN (or GITHUB_TOKEN) with pull-requests: write, plus +// PR_NUMBER / BASE_SHA / HEAD_SHA in the environment — see +// .github/workflows/pr-screenshot-summary.yaml, the only intended caller. + +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const FENCE_START = ''; +const FENCE_END = ''; +const FENCE_HEADING = '## Visual regression screenshots'; +const FENCE_NOTE = + '_Auto-generated by CI from `scripts/pr-screenshot-summary.mjs` — edits inside this block are overwritten on the next push._'; + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} must be set`); + return value; +} + +function gh(args) { + return execFileSync('gh', args, { encoding: 'utf-8' }); +} + +/** Splices `fenced` into `body`: replaces an existing fenced block in + * place, or prepends one if none is present yet. */ +function withFencedSection(body, fenced) { + const startIdx = body.indexOf(FENCE_START); + const endIdx = body.indexOf(FENCE_END); + if (startIdx !== -1 && endIdx !== -1 && endIdx > startIdx) { + return ( + body.slice(0, startIdx) + fenced + body.slice(endIdx + FENCE_END.length) + ); + } + return body.trim().length > 0 ? `${fenced}\n\n${body}` : fenced; +} + +/** Reverses `withFencedSection`'s prepend: drops an existing fenced block + * (and the blank line that used to separate it from the rest) so a push + * that removes all snapshot changes (e.g. a rebase or revert) doesn't + * leave a stale summary behind. No-op if there's no fenced block. */ +function removeFencedSection(body) { + const startIdx = body.indexOf(FENCE_START); + const endIdx = body.indexOf(FENCE_END); + if (startIdx === -1 || endIdx === -1 || endIdx <= startIdx) return body; + const before = body.slice(0, startIdx); + const after = body.slice(endIdx + FENCE_END.length).replace(/^\n+/, ''); + return before + after; +} + +function main() { + const prNumber = requiredEnv('PR_NUMBER'); + const baseSha = requiredEnv('BASE_SHA'); + const headSha = requiredEnv('HEAD_SHA'); + + const summary = execFileSync( + 'node', + ['scripts/pr-screenshot-summary.mjs', '--base', baseSha, '--head', headSha], + { encoding: 'utf-8' } + ).trim(); + + const currentBody = gh([ + 'pr', + 'view', + prNumber, + '--json', + 'body', + '-q', + '.body', + ]); + + let newBody; + if (summary.startsWith('No visual-regression')) { + newBody = removeFencedSection(currentBody); + if (newBody === currentBody) { + console.log('No snapshot changes — leaving PR description untouched.'); + return; + } + console.log( + 'No snapshot changes remain — removing the stale screenshot summary.' + ); + } else { + const fenced = `${FENCE_START}\n${FENCE_HEADING}\n\n${FENCE_NOTE}\n\n${summary}\n${FENCE_END}`; + newBody = withFencedSection(currentBody, fenced); + if (newBody === currentBody) { + console.log('Screenshot summary unchanged — skipping PR body update.'); + return; + } + } + + const tmpFile = path.join(os.tmpdir(), `pr-${prNumber}-body.md`); + fs.writeFileSync(tmpFile, newBody); + gh(['pr', 'edit', prNumber, '--body-file', tmpFile]); + console.log(`Updated PR #${prNumber} description with screenshot summary.`); +} + +main(); diff --git a/tests/visual/README.md b/tests/visual/README.md new file mode 100644 index 00000000..7a0a4686 --- /dev/null +++ b/tests/visual/README.md @@ -0,0 +1,48 @@ +# Visual regression tests + +Playwright screenshot tests covering docs-website's shared layout, theme, and +navigation — see `pages.spec.ts` and `mobile.spec.ts` for the exact +page/viewport matrix (issue +[#1163](https://github.com/stacklok/docs-website/issues/1163)). + +## Running locally + +```bash +npm run test:visual # compare against the committed baselines +npm run test:visual:update # regenerate them +``` + +This builds the site and serves it in production mode before running the suite +(see `playwright.config.ts`'s `webServer`). + +**Local baselines won't match CI.** `toHaveScreenshot()` compares rendered +pixels, and font rendering differs by host OS/GPU even on the same Chromium +build. CI runs inside a pinned `mcr.microsoft.com/playwright` image for exactly +this reason — a locally-generated PNG committed by hand will just fail again on +the next CI run. Use `npm run test:visual:update` locally only to sanity-check +that a scenario renders correctly (the metadata panel opens, the mobile menu +doesn't overflow, mermaid diagrams finish rendering) — never to produce the +baseline you commit. + +## Updating baselines + +When a visual difference is intentional, comment **`/update-snapshots`** on the +pull request. Any repository collaborator with write access can trigger it. The +workflow regenerates the baselines inside the same pinned container used by the +check and pushes them to the pull request branch. Review the resulting +before-and-after screenshot summary before merging. A passing check only proves +that the render is consistent with the approved baseline, not that it is +correct. + +The command refuses to run on forked pull requests because it cannot push to +their branches. For a fork, regenerate locally with +`npm run test:visual:update`, then push the changed snapshot files yourself. + +## Adding a new case + +Add to the existing `NAV_PAGES` array or as its own `test(...)` in +`pages.spec.ts`/`mobile.spec.ts`, calling `captureColorScheme` (single theme) or +`captureBothThemes` (light + dark) from `fixtures.ts` once the test has already +asserted the state being snapshotted. Keep the matrix small — a new entry should +exercise shared layout/theme/nav that nothing else here already covers, not add +per-page content coverage (that's what regular review is for, not this suite). diff --git a/tests/visual/fixtures.ts b/tests/visual/fixtures.ts new file mode 100644 index 00000000..99ea4876 --- /dev/null +++ b/tests/visual/fixtures.ts @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import fs from 'node:fs'; +import path from 'node:path'; +import { + expect, + type Locator, + type Page, + type TestInfo, +} from '@playwright/test'; + +/** + * Waits out generic sources of false-positive visual diffs: in-flight + * client work (networkidle) and web fonts still loading. Nothing here has + * a client-side loading skeleton (the MCP metadata panel's content is + * baked in at build time), so unlike a stateful app there's no third + * generic thing to wait for. + */ +async function waitForVisualStability(page: Page): Promise { + await page.waitForLoadState('networkidle'); + await page.evaluate(() => document.fonts.ready); +} + +function slugifySnapshotName(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * Writes a `.json` next to each baseline PNG: which spec produced + * it and the page URL, plus a ready-to-paste repro command. Not debug + * fluff — `scripts/pr-screenshot-summary.mjs` reads this to label and link + * each screenshot in the PR description. + */ +async function writeSnapshotMetadata( + page: Page, + testInfo: TestInfo, + pngName: string +): Promise { + const pngPath = testInfo.snapshotPath(pngName); + const jsonPath = pngPath.replace(/\.png$/, '.json'); + const testFile = path.relative(process.cwd(), testInfo.file); + const pageUrl = new URL(page.url()); + + await fs.promises.writeFile( + jsonPath, + `${JSON.stringify( + { + titlePath: testInfo.titlePath, + testFile, + url: { + origin: pageUrl.origin, + path: pageUrl.pathname + pageUrl.search + pageUrl.hash, + }, + reproCommand: `npx playwright test ${testFile} -g ${JSON.stringify(testInfo.title)}`, + }, + null, + 2 + )}\n` + ); +} + +async function fileMtimeMs(filePath: string): Promise { + try { + return (await fs.promises.stat(filePath)).mtimeMs; + } catch { + return null; + } +} + +export async function captureColorScheme( + page: Page, + testInfo: TestInfo, + name: string, + scheme: 'light' | 'dark', + options: { mask?: Locator[] } = {} +): Promise { + await page.emulateMedia({ colorScheme: scheme }); + await waitForVisualStability(page); + + const pngName = `${slugifySnapshotName(name)}-${scheme}.png`; + const pngPath = testInfo.snapshotPath(pngName); + const before = await fileMtimeMs(pngPath); + // scale: 'device' — toHaveScreenshot() defaults to 'css' (downsamples + // back to CSS pixel dimensions regardless of deviceScaleFactor). The + // whole point of the project config's deviceScaleFactor: 2 is a real + // higher-resolution PNG, since these get embedded in the PR + // description at whatever width GitHub's body happens to be (usually + // well under 1280px) — 'css' would throw away exactly the extra + // resolution that downscale needs to stay crisp. + await expect(page).toHaveScreenshot(pngName, { + fullPage: true, + scale: 'device', + mask: options.mask, + maskColor: scheme === 'dark' ? '#282a36' : '#f6f8fa', + }); + const after = await fileMtimeMs(pngPath); + if (after !== before) { + await writeSnapshotMetadata(page, testInfo, pngName); + } +} + +/** + * Snapshots `page` in both light and dark color schemes. Call only after + * the test has already asserted the state being snapshotted (e.g. the + * metadata panel is open) — those assertions are what make the page + * deterministic, not this function. + */ +export async function captureBothThemes( + page: Page, + testInfo: TestInfo, + name: string, + options: { mask?: Locator[] } = {} +): Promise { + await captureColorScheme(page, testInfo, name, 'light', options); + await captureColorScheme(page, testInfo, name, 'dark', options); + await page.emulateMedia({ colorScheme: 'light' }); +} + +export async function captureElementBothThemes( + page: Page, + element: Locator, + testInfo: TestInfo, + name: string +): Promise { + for (const scheme of ['light', 'dark'] as const) { + await page.emulateMedia({ colorScheme: scheme }); + await waitForVisualStability(page); + + const pngName = `${slugifySnapshotName(name)}-${scheme}.png`; + const pngPath = testInfo.snapshotPath(pngName); + const before = await fileMtimeMs(pngPath); + await expect(element).toHaveScreenshot(pngName, { scale: 'device' }); + const after = await fileMtimeMs(pngPath); + if (after !== before) { + await writeSnapshotMetadata(page, testInfo, pngName); + } + } + + await page.emulateMedia({ colorScheme: 'light' }); +} diff --git a/tests/visual/mobile.spec.ts b/tests/visual/mobile.spec.ts new file mode 100644 index 00000000..23e42570 --- /dev/null +++ b/tests/visual/mobile.spec.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, test } from '@playwright/test'; +import { captureBothThemes } from './fixtures'; + +test('home page - mobile', async ({ page }, testInfo) => { + const response = await page.goto('/'); + expect(response?.ok()).toBe(true); + await expect(page.locator('body')).toBeVisible(); + await captureBothThemes(page, testInfo, 'Home page - mobile'); +}); + +test('mobile navigation opens without overflow', async ({ page }, testInfo) => { + const response = await page.goto('/toolhive/guides-cli'); + expect(response?.ok()).toBe(true); + + const toggle = page.getByRole('button', { name: /toggle navigation bar/i }); + await toggle.click(); + + const sidebar = page.locator('.navbar-sidebar--show'); + await expect(sidebar).toBeVisible(); + + const hasOverflow = await page.evaluate( + () => document.documentElement.scrollWidth > window.innerWidth + ); + expect(hasOverflow).toBe(false); + + await captureBothThemes(page, testInfo, 'Mobile navigation open'); +}); diff --git a/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-dark-mobile-linux.json b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-dark-mobile-linux.json new file mode 100644 index 00000000..34840845 --- /dev/null +++ b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-dark-mobile-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["mobile.spec.ts", "home page - mobile"], + "testFile": "tests/visual/mobile.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/" + }, + "reproCommand": "npx playwright test tests/visual/mobile.spec.ts -g \"home page - mobile\"" +} diff --git a/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-dark-mobile-linux.png b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-dark-mobile-linux.png new file mode 100644 index 00000000..1b35ccc5 Binary files /dev/null and b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-dark-mobile-linux.png differ diff --git a/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-light-mobile-linux.json b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-light-mobile-linux.json new file mode 100644 index 00000000..34840845 --- /dev/null +++ b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-light-mobile-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["mobile.spec.ts", "home page - mobile"], + "testFile": "tests/visual/mobile.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/" + }, + "reproCommand": "npx playwright test tests/visual/mobile.spec.ts -g \"home page - mobile\"" +} diff --git a/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-light-mobile-linux.png b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-light-mobile-linux.png new file mode 100644 index 00000000..65153377 Binary files /dev/null and b/tests/visual/mobile.spec.ts-snapshots/home-page-mobile-light-mobile-linux.png differ diff --git a/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-dark-mobile-linux.json b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-dark-mobile-linux.json new file mode 100644 index 00000000..f572684a --- /dev/null +++ b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-dark-mobile-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["mobile.spec.ts", "mobile navigation opens without overflow"], + "testFile": "tests/visual/mobile.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive/guides-cli" + }, + "reproCommand": "npx playwright test tests/visual/mobile.spec.ts -g \"mobile navigation opens without overflow\"" +} diff --git a/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-dark-mobile-linux.png b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-dark-mobile-linux.png new file mode 100644 index 00000000..47934103 Binary files /dev/null and b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-dark-mobile-linux.png differ diff --git a/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-light-mobile-linux.json b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-light-mobile-linux.json new file mode 100644 index 00000000..f572684a --- /dev/null +++ b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-light-mobile-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["mobile.spec.ts", "mobile navigation opens without overflow"], + "testFile": "tests/visual/mobile.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive/guides-cli" + }, + "reproCommand": "npx playwright test tests/visual/mobile.spec.ts -g \"mobile navigation opens without overflow\"" +} diff --git a/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-light-mobile-linux.png b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-light-mobile-linux.png new file mode 100644 index 00000000..2a2a7a2c Binary files /dev/null and b/tests/visual/mobile.spec.ts-snapshots/mobile-navigation-open-light-mobile-linux.png differ diff --git a/tests/visual/pages.spec.ts b/tests/visual/pages.spec.ts new file mode 100644 index 00000000..2af2595f --- /dev/null +++ b/tests/visual/pages.spec.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, test, type Page } from '@playwright/test'; +import { captureBothThemes, captureElementBothThemes } from './fixtures'; + +/** + * The documented representative-page matrix for issue #1163. Keep this + * list small, stable, and reviewable — it IS the "documented matrix" the + * issue asks for, not a stand-in for one. Add a page here only when it + * exercises shared layout/theme/nav that nothing else in the list already + * covers. + */ +const NAV_PAGES: Array<{ section: string; path: string }> = [ + { section: 'Stacklok Platform', path: '/platform' }, + { section: 'ToolHive', path: '/toolhive' }, + { section: 'AI Gateway', path: '/ai-gateway' }, + { section: 'Connector Gateway', path: '/connector-gateway' }, + { section: 'Resources', path: '/toolhive/concepts' }, +]; + +async function gotoSuccessful(page: Page, path: string) { + const response = await page.goto(path); + expect(response?.ok()).toBe(true); +} + +test('home page', async ({ page }, testInfo) => { + await gotoSuccessful(page, '/'); + await expect(page.locator('body')).toBeVisible(); + await captureBothThemes(page, testInfo, 'Home page'); +}); + +test('theme preview page', async ({ page }, testInfo) => { + await gotoSuccessful(page, '/theme-preview'); + // Mermaid diagrams render client-side and asynchronously — wait for + // both of the page's diagrams to finish before capturing, or the + // snapshot flakes between "still rendering" and "done". + await expect(page.locator('.docusaurus-mermaid-container svg')).toHaveCount( + 2 + ); + await captureBothThemes(page, testInfo, 'Theme preview page'); +}); + +for (const { section, path } of NAV_PAGES) { + test(`nav page - ${section}`, async ({ page }, testInfo) => { + await gotoSuccessful(page, path); + const sidebarViewport = page.locator('.theme-doc-sidebar-container > div'); + await expect(sidebarViewport).toBeVisible(); + await captureElementBothThemes( + page, + sidebarViewport, + testInfo, + `Nav page - ${section}` + ); + }); +} + +test('MCP guide - context7 metadata expanded', async ({ page }, testInfo) => { + await gotoSuccessful(page, '/toolhive/guides-mcp/context7'); + + const summary = page.getByText("Expand to view the MCP server's metadata"); + await summary.click(); + + const codeBlock = page.locator('details[open] pre'); + await expect(codeBlock).toBeVisible(); + // The plugin falls back to this comment when `thv registry info` fails + // (missing/broken `thv`, registry lookup failure) — assert real + // metadata rendered instead of that fallback, so a `thv`/registry + // failure surfaces as a clear assertion, not a mystery pixel diff. + await expect(codeBlock).not.toContainText('Error fetching data for'); + await expect(codeBlock).toContainText('Name: io.github.stacklok/context7'); + + await captureBothThemes( + page, + testInfo, + 'MCP guide - context7 metadata expanded', + { mask: [codeBlock] } + ); +}); diff --git a/tests/visual/pages.spec.ts-snapshots/home-page-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/home-page-dark-desktop-linux.json new file mode 100644 index 00000000..9d218960 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/home-page-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "home page"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"home page\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/home-page-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/home-page-dark-desktop-linux.png new file mode 100644 index 00000000..a5017526 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/home-page-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/home-page-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/home-page-light-desktop-linux.json new file mode 100644 index 00000000..9d218960 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/home-page-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "home page"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"home page\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/home-page-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/home-page-light-desktop-linux.png new file mode 100644 index 00000000..7312895b Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/home-page-light-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-dark-desktop-linux.json new file mode 100644 index 00000000..7aa4047e --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "MCP guide - context7 metadata expanded"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive/guides-mcp/context7" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"MCP guide - context7 metadata expanded\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-dark-desktop-linux.png new file mode 100644 index 00000000..a7cefbbd Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-light-desktop-linux.json new file mode 100644 index 00000000..7aa4047e --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "MCP guide - context7 metadata expanded"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive/guides-mcp/context7" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"MCP guide - context7 metadata expanded\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-light-desktop-linux.png new file mode 100644 index 00000000..952a13ff Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/mcp-guide-context7-metadata-expanded-light-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-dark-desktop-linux.json new file mode 100644 index 00000000..92fad2c4 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - AI Gateway"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/ai-gateway" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - AI Gateway\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-dark-desktop-linux.png new file mode 100644 index 00000000..89300328 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-light-desktop-linux.json new file mode 100644 index 00000000..92fad2c4 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - AI Gateway"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/ai-gateway" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - AI Gateway\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-light-desktop-linux.png new file mode 100644 index 00000000..be7a0630 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-ai-gateway-light-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-dark-desktop-linux.json new file mode 100644 index 00000000..d3cff5ec --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - Connector Gateway"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/connector-gateway" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - Connector Gateway\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-dark-desktop-linux.png new file mode 100644 index 00000000..21f90bb0 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-light-desktop-linux.json new file mode 100644 index 00000000..d3cff5ec --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - Connector Gateway"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/connector-gateway" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - Connector Gateway\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-light-desktop-linux.png new file mode 100644 index 00000000..08a2a0ef Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-connector-gateway-light-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-resources-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-dark-desktop-linux.json new file mode 100644 index 00000000..36463d66 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - Resources"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive/concepts" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - Resources\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-resources-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-dark-desktop-linux.png new file mode 100644 index 00000000..837a4beb Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-resources-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-light-desktop-linux.json new file mode 100644 index 00000000..36463d66 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - Resources"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive/concepts" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - Resources\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-resources-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-light-desktop-linux.png new file mode 100644 index 00000000..c371049e Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-resources-light-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-dark-desktop-linux.json new file mode 100644 index 00000000..e50fcc72 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - Stacklok Platform"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/platform" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - Stacklok Platform\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-dark-desktop-linux.png new file mode 100644 index 00000000..b5dddc35 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-light-desktop-linux.json new file mode 100644 index 00000000..e50fcc72 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - Stacklok Platform"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/platform" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - Stacklok Platform\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-light-desktop-linux.png new file mode 100644 index 00000000..46af5bd4 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-stacklok-platform-light-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-dark-desktop-linux.json new file mode 100644 index 00000000..0b4f6840 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - ToolHive"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - ToolHive\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-dark-desktop-linux.png new file mode 100644 index 00000000..941581d3 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-light-desktop-linux.json new file mode 100644 index 00000000..0b4f6840 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "nav page - ToolHive"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/toolhive" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"nav page - ToolHive\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-light-desktop-linux.png new file mode 100644 index 00000000..97b396fc Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/nav-page-toolhive-light-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/theme-preview-page-dark-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-dark-desktop-linux.json new file mode 100644 index 00000000..2df754a9 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-dark-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "theme preview page"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/theme-preview" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"theme preview page\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/theme-preview-page-dark-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-dark-desktop-linux.png new file mode 100644 index 00000000..e8770a75 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-dark-desktop-linux.png differ diff --git a/tests/visual/pages.spec.ts-snapshots/theme-preview-page-light-desktop-linux.json b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-light-desktop-linux.json new file mode 100644 index 00000000..2df754a9 --- /dev/null +++ b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-light-desktop-linux.json @@ -0,0 +1,9 @@ +{ + "titlePath": ["pages.spec.ts", "theme preview page"], + "testFile": "tests/visual/pages.spec.ts", + "url": { + "origin": "http://localhost:3005", + "path": "/theme-preview" + }, + "reproCommand": "npx playwright test tests/visual/pages.spec.ts -g \"theme preview page\"" +} diff --git a/tests/visual/pages.spec.ts-snapshots/theme-preview-page-light-desktop-linux.png b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-light-desktop-linux.png new file mode 100644 index 00000000..898ebe02 Binary files /dev/null and b/tests/visual/pages.spec.ts-snapshots/theme-preview-page-light-desktop-linux.png differ