diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml new file mode 100644 index 0000000..cbfbbac --- /dev/null +++ b/.github/workflows/action.yml @@ -0,0 +1,66 @@ +name: Action + +on: + push: + branches: [main] + pull_request: + +jobs: + self-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 22 + - id: site + run: | + node tests/helpers/serve.ts > "$RUNNER_TEMP/origins" & + until grep -q '^right=' "$RUNNER_TEMP/origins"; do sleep 1; done + cat "$RUNNER_TEMP/origins" >> "$GITHUB_OUTPUT" + + - id: clean + uses: ./ + with: + left: ${{ steps.site.outputs.left }} + right: ${{ steps.site.outputs.right }} + routes: / /flex-vs-grid + widths: 1440,375 + threshold: "0.999" + artifact-name: clean + + - id: differ + continue-on-error: true + uses: ./ + with: + left: ${{ steps.site.outputs.left }} + right: ${{ steps.site.outputs.right }} + routes: | + /color + /tint + widths: "1440" + threshold: "0.999" + out: differ-out + artifact-name: differ + + - id: accept + uses: ./ + with: + left: ${{ steps.site.outputs.left }} + right: ${{ steps.site.outputs.right }} + routes: /color + widths: "1440" + accept: self-test + out: accept-out + artifact-name: accept + + - env: + CLEAN: ${{ steps.clean.outputs.exit-code }} + DIFFER: ${{ steps.differ.outputs.exit-code }} + OUTCOME: ${{ steps.differ.outcome }} + run: | + test "$CLEAN" = 0 + test "$DIFFER" = 1 + test "$OUTCOME" = failure + test -f differ-out/results.json + test -f accept-out/renderdiff.accepted.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b7da95d..a68eade 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,10 +2,10 @@ name: Release on: push: - tags: ["v*"] + tags: ["v*.*.*"] permissions: - contents: read + contents: write id-token: write jobs: @@ -21,3 +21,4 @@ jobs: - run: test "v$(node -p 'require("./package.json").version')" = "$GITHUB_REF_NAME" - run: npm test - run: npm publish --provenance --access public + - run: git tag -f v0 && git push -f origin v0 diff --git a/README.md b/README.md index 4fd266a..2189365 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,12 @@ Install the skill so the agent runs the loop and reports a verdict, not a bare s npx skills add tmashara/renderdiff # Codex, Cursor, others (skills.sh) ``` +### In CI + +`uses: tmashara/renderdiff@v0` checks a preview deployment or a build in the job against +production on every pull request, with the result in the job summary: +[docs/github-action.md](https://github.com/tmashara/renderdiff/blob/main/docs/github-action.md). + ### Requirements Node >= 22.18 and Chromium (`npx playwright install chromium`). `npm install -g renderdiff` diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..76bc10a --- /dev/null +++ b/action.yml @@ -0,0 +1,156 @@ +name: renderdiff visual parity +description: Screenshot-diff a preview or local build against production and fail the job on a visual change. +author: Timur Mashara +branding: + icon: eye + color: purple + +inputs: + left: + description: Your change, e.g. the preview deployment URL or http://localhost:3000. + required: true + right: + description: The reference, e.g. https://example.com. + required: true + routes: + description: Routes separated by spaces or newlines, e.g. "/ /about /pricing::/plans". Optional when the config lists them. + default: "" + config: + description: Path to a renderdiff config file. Default renderdiff.config.{ts,mts,js,mjs} in the workspace. + default: "" + widths: + description: Viewport widths separated by spaces or commas. Default renderdiff's five. + default: "" + threshold: + description: Fail when any comparison scores below this (0..1). Without it only errors fail the job. + default: "" + concurrency: + description: Parallel workers, which is also the most loads production sees at once. + default: "" + storage-state: + description: Contents of a Playwright storage-state JSON for both sides, from a secret. + default: "" + accept: + description: Record this run's differences as accepted, with this reason. Commit the renderdiff.accepted.json from the artifact. + default: "" + wait-timeout: + description: Seconds to wait for left to answer before running. + default: "300" + out: + description: Output directory. + default: renderdiff-out + artifact-name: + description: Name of the uploaded artifact with the report, results.json and crops. + default: renderdiff + +outputs: + exit-code: + description: renderdiff's exit code. 0 pass, 1 below threshold, 2 bad input or origin down, 3 a route errored. + value: ${{ steps.run.outputs.exit-code }} + results: + description: Path to results.json. + value: ${{ steps.run.outputs.results }} + +runs: + using: composite + steps: + - uses: actions/setup-node@v5 + with: + node-version: 22 + + - id: pw + shell: bash + working-directory: ${{ github.action_path }} + run: echo "version=$(node -p 'require("./package-lock.json").packages["node_modules/playwright"].version')" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: renderdiff-chromium-${{ runner.os }}-${{ runner.arch }}-${{ steps.pw.outputs.version }} + + - shell: bash + working-directory: ${{ github.action_path }} + run: | + npm ci --omit=dev --ignore-scripts --no-audit --no-fund + npx playwright install --with-deps chromium + + - shell: bash + env: + LEFT: ${{ inputs.left }} + TIMEOUT: ${{ inputs.wait-timeout }} + run: | + end=$((SECONDS + TIMEOUT)) + until code=$(curl -s -o /dev/null -w '%{http_code}' "$LEFT") && [ "$code" -gt 0 ] && [ "$code" -lt 500 ]; do + if [ "$SECONDS" -ge "$end" ]; then echo "::error::$LEFT did not answer within ${TIMEOUT}s"; exit 2; fi + sleep 5 + done + + - id: run + shell: bash + env: + CLI: ${{ github.action_path }}/src/cli.ts + LEFT: ${{ inputs.left }} + RIGHT: ${{ inputs.right }} + ROUTES: ${{ inputs.routes }} + CONFIG: ${{ inputs.config }} + WIDTHS: ${{ inputs.widths }} + THRESHOLD: ${{ inputs.threshold }} + CONCURRENCY: ${{ inputs.concurrency }} + STORAGE_STATE: ${{ inputs.storage-state }} + ACCEPT: ${{ inputs.accept }} + OUT: ${{ inputs.out }} + run: | + args=(--left "$LEFT" --right "$RIGHT" --out "$OUT" --no-progress) + read -ra routes <<< "$(echo "$ROUTES" | tr '\n' ' ')" + args+=("${routes[@]}") + for w in ${WIDTHS//,/ }; do args+=(--width "$w"); done + [ -n "$CONFIG" ] && args+=(--config "$CONFIG") + [ -n "$THRESHOLD" ] && args+=(--threshold "$THRESHOLD") + [ -n "$CONCURRENCY" ] && args+=(--concurrency "$CONCURRENCY") + [ -n "$ACCEPT" ] && args+=(--accept "$ACCEPT") + if [ -n "$STORAGE_STATE" ]; then + state="$RUNNER_TEMP/renderdiff-state.json" + (umask 077; printf '%s' "$STORAGE_STATE" > "$state") + args+=(--storage-state "$state") + fi + + set +e + node "$CLI" "${args[@]}" 2>&1 | tee "$RUNNER_TEMP/renderdiff.txt" + code=${PIPESTATUS[0]} + set -e + rm -f "${state:-}" + + echo "exit-code=$code" >> "$GITHUB_OUTPUT" + echo "results=$OUT/results.json" >> "$GITHUB_OUTPUT" + accepted="$(dirname "${CONFIG:-./_}")/renderdiff.accepted.json" + if [ -n "$ACCEPT" ] && [ -f "$accepted" ] && [ -d "$OUT" ]; then cp "$accepted" "$OUT/"; fi + + case $code in + 0) verdict="✅ No visual change" ;; + 1) verdict="❌ Below threshold" ;; + 2) verdict="⚠️ Bad input or an origin is down" ;; + *) verdict="⚠️ A route errored or the run crashed" ;; + esac + { + echo "### renderdiff: $verdict" + echo + echo "\`$LEFT\` (your change) vs \`$RIGHT\` (reference). Report, results.json and crops are in the \`${{ inputs.artifact-name }}\` artifact." + echo + echo '```text' + head -c 900000 "$RUNNER_TEMP/renderdiff.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - if: always() && steps.run.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.out }} + if-no-files-found: ignore + retention-days: 14 + + - if: steps.run.outputs.exit-code != '0' + shell: bash + env: + CODE: ${{ steps.run.outputs.exit-code }} + run: exit "$CODE" diff --git a/docs/accepted.md b/docs/accepted.md index 0f3fb1c..372cb85 100644 --- a/docs/accepted.md +++ b/docs/accepted.md @@ -42,6 +42,10 @@ differently (turning an accepted region into bands) makes an accepted change sho accept it again once you've looked. What can still hide is a shade change under one step (e.g. `#000` to `#080808`) that doesn't cross a step boundary anywhere in the change. +Accept on the platform that checks. The pixel hash is not portable: macOS and Linux render +the same page with different fonts and anti-aliasing, so entries accepted on a Mac mostly come +back stale on a Linux CI runner. For CI, accept in CI ([github-action.md](github-action.md)). + A shot whose pages differ in width, or in height without the rows aligning into bands, is never accepted: the padding past the shorter page belongs to no band or region, so no fingerprint covers it. diff --git a/docs/github-action.md b/docs/github-action.md new file mode 100644 index 0000000..41e6a2b --- /dev/null +++ b/docs/github-action.md @@ -0,0 +1,111 @@ +# GitHub Action + +Run renderdiff on every pull request: your preview deployment (or a build started in the job) +on the left, production on the right. The job summary shows the same compact output an agent +reads, the report and crops are uploaded as an artifact, and the step fails when renderdiff +exits non-zero. + +```yaml +- uses: tmashara/renderdiff@v0 + with: + left: ${{ github.event.deployment_status.environment_url }} + right: https://example.com + routes: / /about /pricing + threshold: "0.999" +``` + +`v0` follows the latest 0.x release. Pin a full tag (`@v0.3.2`) to stay put. + +## Inputs + +| Input | | +| --- | --- | +| `left` | Your change: a preview URL or `http://localhost:3000`. Required. | +| `right` | The reference, usually production. Required. | +| `routes` | Separated by spaces or newlines; pairs (`/pricing::/plans`) work. Optional when the config lists routes. | +| `config` | Config file path. Default: `renderdiff.config.{ts,mts,js,mjs}` in the workspace. | +| `widths` | Spaces or commas. Default renderdiff's five. | +| `threshold` | Fail below this score. Without it, only errors (exit 2 or 3) fail the job. | +| `concurrency` | Parallel workers, which is also the most page loads production sees at once. Default 2 on a hosted runner. | +| `storage-state` | Contents of a Playwright storage-state JSON, from a secret. Written to a 0600 temp file and deleted after the run. | +| `accept` | Record this run's differences as accepted, with this reason. See below. | +| `wait-timeout` | Seconds to wait for `left` to answer (any status under 500). Default 300. | +| `out` | Output directory. Default `renderdiff-out`. | +| `artifact-name` | Default `renderdiff`. Set it when one job runs the action twice. | + +Outputs: `exit-code` (0 pass, 1 below threshold, 2 bad input or an origin down, 3 a route +errored) and `results` (the path to `results.json`). + +A Vercel preview behind Deployment Protection serves its login page to the runner, and +renderdiff will diff that. Turn protection off for previews or use a bypass first. + +## Recipes + +### Preview deployment vs production + +Vercel and Netlify report each preview as a GitHub deployment: + +```yaml +on: deployment_status +jobs: + renderdiff: + if: github.event.deployment_status.state == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: tmashara/renderdiff@v0 + with: + left: ${{ github.event.deployment_status.environment_url }} + right: https://example.com + threshold: "0.999" +``` + +The checkout is for a committed config and `renderdiff.accepted.json`. + +### A build in the job vs production + +```yaml +on: pull_request +jobs: + renderdiff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - run: npm ci && npm run build + - run: npm start & + - uses: tmashara/renderdiff@v0 + with: + left: http://localhost:3000 + right: https://example.com + threshold: "0.999" +``` + +The action waits for `left` to answer, so the server can still be starting. + +### Dependency upgrades from Renovate or Dependabot + +A dependency bump should change nothing, so fail on any difference: + +```yaml + if: github.actor == 'renovate[bot]' || github.actor == 'dependabot[bot]' + # …the build-in-the-job steps above, then: + - uses: tmashara/renderdiff@v0 + with: + left: http://localhost:3000 + right: https://example.com + threshold: "1" +``` + +## Accepted differences: accept in CI + +Accept where you check. Fingerprints include a hash of the pixels, and the same page renders +a few pixels differently on macOS and on the Linux runner (fonts, anti-aliasing), so entries +accepted on a Mac mostly don't match in CI. To record accepted differences for CI: + +1. Run the workflow once with `accept: ""` (for example through a + `workflow_dispatch` input). +2. Download the artifact; it holds `renderdiff.accepted.json`. +3. Commit it next to your config (or at the repo root without one). + +Later runs read the committed file and stop failing on those differences; anything new +still shows. Details: [accepted.md](accepted.md). diff --git a/tests/helpers/serve.ts b/tests/helpers/serve.ts new file mode 100644 index 0000000..196a4d4 --- /dev/null +++ b/tests/helpers/serve.ts @@ -0,0 +1,5 @@ +/** Serve the fixture site until killed, printing `left=` and `right=` origins for $GITHUB_OUTPUT. */ +import { startFixtureSite } from "./server.ts"; + +const site = await startFixtureSite(); +console.log(`left=${site.left}\nright=${site.right}`);