From f0e1c1bad4515caf1b250cac5f5eaab149c22434 Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:27:48 +0200 Subject: [PATCH] Add a GitHub Action: preview or build vs production on every PR Composite action at the repo root: runs src/cli.ts from the action's ref, caches Chromium, waits for left, writes the compact summary to the job summary, uploads the out dir, and fails on renderdiff's exit code. accept records differences on the runner, since pixel hashes don't match across macOS and Linux. Releases move the v0 tag. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/action.yml | 66 ++++++++++++++ .github/workflows/release.yml | 5 +- README.md | 6 ++ action.yml | 156 ++++++++++++++++++++++++++++++++++ docs/accepted.md | 4 + docs/github-action.md | 111 ++++++++++++++++++++++++ tests/helpers/serve.ts | 5 ++ 7 files changed, 351 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/action.yml create mode 100644 action.yml create mode 100644 docs/github-action.md create mode 100644 tests/helpers/serve.ts diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml new file mode 100644 index 0000000..cbfbbac --- /dev/null +++ b/.github/workflows/action.yml @@ -0,0 +1,66 @@ +name: Action + +on: + push: + branches: [main] + pull_request: + +jobs: + self-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 22 + - id: site + run: | + node tests/helpers/serve.ts > "$RUNNER_TEMP/origins" & + until grep -q '^right=' "$RUNNER_TEMP/origins"; do sleep 1; done + cat "$RUNNER_TEMP/origins" >> "$GITHUB_OUTPUT" + + - id: clean + uses: ./ + with: + left: ${{ steps.site.outputs.left }} + right: ${{ steps.site.outputs.right }} + routes: / /flex-vs-grid + widths: 1440,375 + threshold: "0.999" + artifact-name: clean + + - id: differ + continue-on-error: true + uses: ./ + with: + left: ${{ steps.site.outputs.left }} + right: ${{ steps.site.outputs.right }} + routes: | + /color + /tint + widths: "1440" + threshold: "0.999" + out: differ-out + artifact-name: differ + + - id: accept + uses: ./ + with: + left: ${{ steps.site.outputs.left }} + right: ${{ steps.site.outputs.right }} + routes: /color + widths: "1440" + accept: self-test + out: accept-out + artifact-name: accept + + - env: + CLEAN: ${{ steps.clean.outputs.exit-code }} + DIFFER: ${{ steps.differ.outputs.exit-code }} + OUTCOME: ${{ steps.differ.outcome }} + run: | + test "$CLEAN" = 0 + test "$DIFFER" = 1 + test "$OUTCOME" = failure + test -f differ-out/results.json + test -f accept-out/renderdiff.accepted.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b7da95d..a68eade 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,10 +2,10 @@ name: Release on: push: - tags: ["v*"] + tags: ["v*.*.*"] permissions: - contents: read + contents: write id-token: write jobs: @@ -21,3 +21,4 @@ jobs: - run: test "v$(node -p 'require("./package.json").version')" = "$GITHUB_REF_NAME" - run: npm test - run: npm publish --provenance --access public + - run: git tag -f v0 && git push -f origin v0 diff --git a/README.md b/README.md index 4fd266a..2189365 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,12 @@ Install the skill so the agent runs the loop and reports a verdict, not a bare s npx skills add tmashara/renderdiff # Codex, Cursor, others (skills.sh) ``` +### In CI + +`uses: tmashara/renderdiff@v0` checks a preview deployment or a build in the job against +production on every pull request, with the result in the job summary: +[docs/github-action.md](https://github.com/tmashara/renderdiff/blob/main/docs/github-action.md). + ### Requirements Node >= 22.18 and Chromium (`npx playwright install chromium`). `npm install -g renderdiff` diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..76bc10a --- /dev/null +++ b/action.yml @@ -0,0 +1,156 @@ +name: renderdiff visual parity +description: Screenshot-diff a preview or local build against production and fail the job on a visual change. +author: Timur Mashara +branding: + icon: eye + color: purple + +inputs: + left: + description: Your change, e.g. the preview deployment URL or http://localhost:3000. + required: true + right: + description: The reference, e.g. https://example.com. + required: true + routes: + description: Routes separated by spaces or newlines, e.g. "/ /about /pricing::/plans". Optional when the config lists them. + default: "" + config: + description: Path to a renderdiff config file. Default renderdiff.config.{ts,mts,js,mjs} in the workspace. + default: "" + widths: + description: Viewport widths separated by spaces or commas. Default renderdiff's five. + default: "" + threshold: + description: Fail when any comparison scores below this (0..1). Without it only errors fail the job. + default: "" + concurrency: + description: Parallel workers, which is also the most loads production sees at once. + default: "" + storage-state: + description: Contents of a Playwright storage-state JSON for both sides, from a secret. + default: "" + accept: + description: Record this run's differences as accepted, with this reason. Commit the renderdiff.accepted.json from the artifact. + default: "" + wait-timeout: + description: Seconds to wait for left to answer before running. + default: "300" + out: + description: Output directory. + default: renderdiff-out + artifact-name: + description: Name of the uploaded artifact with the report, results.json and crops. + default: renderdiff + +outputs: + exit-code: + description: renderdiff's exit code. 0 pass, 1 below threshold, 2 bad input or origin down, 3 a route errored. + value: ${{ steps.run.outputs.exit-code }} + results: + description: Path to results.json. + value: ${{ steps.run.outputs.results }} + +runs: + using: composite + steps: + - uses: actions/setup-node@v5 + with: + node-version: 22 + + - id: pw + shell: bash + working-directory: ${{ github.action_path }} + run: echo "version=$(node -p 'require("./package-lock.json").packages["node_modules/playwright"].version')" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: renderdiff-chromium-${{ runner.os }}-${{ runner.arch }}-${{ steps.pw.outputs.version }} + + - shell: bash + working-directory: ${{ github.action_path }} + run: | + npm ci --omit=dev --ignore-scripts --no-audit --no-fund + npx playwright install --with-deps chromium + + - shell: bash + env: + LEFT: ${{ inputs.left }} + TIMEOUT: ${{ inputs.wait-timeout }} + run: | + end=$((SECONDS + TIMEOUT)) + until code=$(curl -s -o /dev/null -w '%{http_code}' "$LEFT") && [ "$code" -gt 0 ] && [ "$code" -lt 500 ]; do + if [ "$SECONDS" -ge "$end" ]; then echo "::error::$LEFT did not answer within ${TIMEOUT}s"; exit 2; fi + sleep 5 + done + + - id: run + shell: bash + env: + CLI: ${{ github.action_path }}/src/cli.ts + LEFT: ${{ inputs.left }} + RIGHT: ${{ inputs.right }} + ROUTES: ${{ inputs.routes }} + CONFIG: ${{ inputs.config }} + WIDTHS: ${{ inputs.widths }} + THRESHOLD: ${{ inputs.threshold }} + CONCURRENCY: ${{ inputs.concurrency }} + STORAGE_STATE: ${{ inputs.storage-state }} + ACCEPT: ${{ inputs.accept }} + OUT: ${{ inputs.out }} + run: | + args=(--left "$LEFT" --right "$RIGHT" --out "$OUT" --no-progress) + read -ra routes <<< "$(echo "$ROUTES" | tr '\n' ' ')" + args+=("${routes[@]}") + for w in ${WIDTHS//,/ }; do args+=(--width "$w"); done + [ -n "$CONFIG" ] && args+=(--config "$CONFIG") + [ -n "$THRESHOLD" ] && args+=(--threshold "$THRESHOLD") + [ -n "$CONCURRENCY" ] && args+=(--concurrency "$CONCURRENCY") + [ -n "$ACCEPT" ] && args+=(--accept "$ACCEPT") + if [ -n "$STORAGE_STATE" ]; then + state="$RUNNER_TEMP/renderdiff-state.json" + (umask 077; printf '%s' "$STORAGE_STATE" > "$state") + args+=(--storage-state "$state") + fi + + set +e + node "$CLI" "${args[@]}" 2>&1 | tee "$RUNNER_TEMP/renderdiff.txt" + code=${PIPESTATUS[0]} + set -e + rm -f "${state:-}" + + echo "exit-code=$code" >> "$GITHUB_OUTPUT" + echo "results=$OUT/results.json" >> "$GITHUB_OUTPUT" + accepted="$(dirname "${CONFIG:-./_}")/renderdiff.accepted.json" + if [ -n "$ACCEPT" ] && [ -f "$accepted" ] && [ -d "$OUT" ]; then cp "$accepted" "$OUT/"; fi + + case $code in + 0) verdict="✅ No visual change" ;; + 1) verdict="❌ Below threshold" ;; + 2) verdict="⚠️ Bad input or an origin is down" ;; + *) verdict="⚠️ A route errored or the run crashed" ;; + esac + { + echo "### renderdiff: $verdict" + echo + echo "\`$LEFT\` (your change) vs \`$RIGHT\` (reference). Report, results.json and crops are in the \`${{ inputs.artifact-name }}\` artifact." + echo + echo '```text' + head -c 900000 "$RUNNER_TEMP/renderdiff.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - if: always() && steps.run.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.out }} + if-no-files-found: ignore + retention-days: 14 + + - if: steps.run.outputs.exit-code != '0' + shell: bash + env: + CODE: ${{ steps.run.outputs.exit-code }} + run: exit "$CODE" diff --git a/docs/accepted.md b/docs/accepted.md index 0f3fb1c..372cb85 100644 --- a/docs/accepted.md +++ b/docs/accepted.md @@ -42,6 +42,10 @@ differently (turning an accepted region into bands) makes an accepted change sho accept it again once you've looked. What can still hide is a shade change under one step (e.g. `#000` to `#080808`) that doesn't cross a step boundary anywhere in the change. +Accept on the platform that checks. The pixel hash is not portable: macOS and Linux render +the same page with different fonts and anti-aliasing, so entries accepted on a Mac mostly come +back stale on a Linux CI runner. For CI, accept in CI ([github-action.md](github-action.md)). + A shot whose pages differ in width, or in height without the rows aligning into bands, is never accepted: the padding past the shorter page belongs to no band or region, so no fingerprint covers it. diff --git a/docs/github-action.md b/docs/github-action.md new file mode 100644 index 0000000..41e6a2b --- /dev/null +++ b/docs/github-action.md @@ -0,0 +1,111 @@ +# GitHub Action + +Run renderdiff on every pull request: your preview deployment (or a build started in the job) +on the left, production on the right. The job summary shows the same compact output an agent +reads, the report and crops are uploaded as an artifact, and the step fails when renderdiff +exits non-zero. + +```yaml +- uses: tmashara/renderdiff@v0 + with: + left: ${{ github.event.deployment_status.environment_url }} + right: https://example.com + routes: / /about /pricing + threshold: "0.999" +``` + +`v0` follows the latest 0.x release. Pin a full tag (`@v0.3.2`) to stay put. + +## Inputs + +| Input | | +| --- | --- | +| `left` | Your change: a preview URL or `http://localhost:3000`. Required. | +| `right` | The reference, usually production. Required. | +| `routes` | Separated by spaces or newlines; pairs (`/pricing::/plans`) work. Optional when the config lists routes. | +| `config` | Config file path. Default: `renderdiff.config.{ts,mts,js,mjs}` in the workspace. | +| `widths` | Spaces or commas. Default renderdiff's five. | +| `threshold` | Fail below this score. Without it, only errors (exit 2 or 3) fail the job. | +| `concurrency` | Parallel workers, which is also the most page loads production sees at once. Default 2 on a hosted runner. | +| `storage-state` | Contents of a Playwright storage-state JSON, from a secret. Written to a 0600 temp file and deleted after the run. | +| `accept` | Record this run's differences as accepted, with this reason. See below. | +| `wait-timeout` | Seconds to wait for `left` to answer (any status under 500). Default 300. | +| `out` | Output directory. Default `renderdiff-out`. | +| `artifact-name` | Default `renderdiff`. Set it when one job runs the action twice. | + +Outputs: `exit-code` (0 pass, 1 below threshold, 2 bad input or an origin down, 3 a route +errored) and `results` (the path to `results.json`). + +A Vercel preview behind Deployment Protection serves its login page to the runner, and +renderdiff will diff that. Turn protection off for previews or use a bypass first. + +## Recipes + +### Preview deployment vs production + +Vercel and Netlify report each preview as a GitHub deployment: + +```yaml +on: deployment_status +jobs: + renderdiff: + if: github.event.deployment_status.state == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: tmashara/renderdiff@v0 + with: + left: ${{ github.event.deployment_status.environment_url }} + right: https://example.com + threshold: "0.999" +``` + +The checkout is for a committed config and `renderdiff.accepted.json`. + +### A build in the job vs production + +```yaml +on: pull_request +jobs: + renderdiff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - run: npm ci && npm run build + - run: npm start & + - uses: tmashara/renderdiff@v0 + with: + left: http://localhost:3000 + right: https://example.com + threshold: "0.999" +``` + +The action waits for `left` to answer, so the server can still be starting. + +### Dependency upgrades from Renovate or Dependabot + +A dependency bump should change nothing, so fail on any difference: + +```yaml + if: github.actor == 'renovate[bot]' || github.actor == 'dependabot[bot]' + # …the build-in-the-job steps above, then: + - uses: tmashara/renderdiff@v0 + with: + left: http://localhost:3000 + right: https://example.com + threshold: "1" +``` + +## Accepted differences: accept in CI + +Accept where you check. Fingerprints include a hash of the pixels, and the same page renders +a few pixels differently on macOS and on the Linux runner (fonts, anti-aliasing), so entries +accepted on a Mac mostly don't match in CI. To record accepted differences for CI: + +1. Run the workflow once with `accept: ""` (for example through a + `workflow_dispatch` input). +2. Download the artifact; it holds `renderdiff.accepted.json`. +3. Commit it next to your config (or at the repo root without one). + +Later runs read the committed file and stop failing on those differences; anything new +still shows. Details: [accepted.md](accepted.md). diff --git a/tests/helpers/serve.ts b/tests/helpers/serve.ts new file mode 100644 index 0000000..196a4d4 --- /dev/null +++ b/tests/helpers/serve.ts @@ -0,0 +1,5 @@ +/** Serve the fixture site until killed, printing `left=` and `right=` origins for $GITHUB_OUTPUT. */ +import { startFixtureSite } from "./server.ts"; + +const site = await startFixtureSite(); +console.log(`left=${site.left}\nright=${site.right}`);