diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index cbfbbac..13cba59 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -16,7 +16,7 @@ jobs: - id: site run: | node tests/helpers/serve.ts > "$RUNNER_TEMP/origins" & - until grep -q '^right=' "$RUNNER_TEMP/origins"; do sleep 1; done + timeout 30 bash -c 'until grep -q "^right=" "$0"; do sleep 1; done' "$RUNNER_TEMP/origins" cat "$RUNNER_TEMP/origins" >> "$GITHUB_OUTPUT" - id: clean @@ -54,7 +54,18 @@ jobs: out: accept-out artifact-name: accept + - id: down + continue-on-error: true + uses: ./ + with: + left: http://127.0.0.1:9 + right: ${{ steps.site.outputs.right }} + routes: / + wait-timeout: "1" + artifact-name: down + - env: + DOWN: ${{ steps.down.outputs.exit-code }} CLEAN: ${{ steps.clean.outputs.exit-code }} DIFFER: ${{ steps.differ.outputs.exit-code }} OUTCOME: ${{ steps.differ.outcome }} @@ -62,5 +73,6 @@ jobs: test "$CLEAN" = 0 test "$DIFFER" = 1 test "$OUTCOME" = failure - test -f differ-out/results.json - test -f accept-out/renderdiff.accepted.json + test "$DOWN" = 2 + node -e 'const r = require("./differ-out/results.json").results; for (const p of ["/color", "/tint"]) if (!(r.find((x) => x.url === p)?.score < 0.999)) throw p' + grep -q '"/color"' accept-out/renderdiff.accepted.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a68eade..d0a1e4c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ on: tags: ["v*.*.*"] permissions: - contents: write + contents: read id-token: write jobs: @@ -21,4 +21,13 @@ jobs: - run: test "v$(node -p 'require("./package.json").version')" = "$GITHUB_REF_NAME" - run: npm test - run: npm publish --provenance --access public + + move-v0: + needs: publish + if: startsWith(github.ref_name, 'v0.') && !contains(github.ref_name, '-') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v5 - run: git tag -f v0 && git push -f origin v0 diff --git a/action.yml b/action.yml index 76bc10a..4c8d4a4 100644 --- a/action.yml +++ b/action.yml @@ -37,7 +37,7 @@ inputs: description: Seconds to wait for left to answer before running. default: "300" out: - description: Output directory. + description: Output directory. Overrides the config's outDir. default: renderdiff-out artifact-name: description: Name of the uploaded artifact with the report, results.json and crops. @@ -56,7 +56,8 @@ runs: steps: - uses: actions/setup-node@v5 with: - node-version: 22 + node-version: 24 + package-manager-cache: false - id: pw shell: bash @@ -65,29 +66,21 @@ runs: - uses: actions/cache@v4 with: - path: ~/.cache/ms-playwright + path: ${{ runner.tool_cache }}/renderdiff-ms-playwright key: renderdiff-chromium-${{ runner.os }}-${{ runner.arch }}-${{ steps.pw.outputs.version }} - shell: bash working-directory: ${{ github.action_path }} + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ runner.tool_cache }}/renderdiff-ms-playwright run: | npm ci --omit=dev --ignore-scripts --no-audit --no-fund npx playwright install --with-deps chromium - - shell: bash - env: - LEFT: ${{ inputs.left }} - TIMEOUT: ${{ inputs.wait-timeout }} - run: | - end=$((SECONDS + TIMEOUT)) - until code=$(curl -s -o /dev/null -w '%{http_code}' "$LEFT") && [ "$code" -gt 0 ] && [ "$code" -lt 500 ]; do - if [ "$SECONDS" -ge "$end" ]; then echo "::error::$LEFT did not answer within ${TIMEOUT}s"; exit 2; fi - sleep 5 - done - - id: run shell: bash env: + PLAYWRIGHT_BROWSERS_PATH: ${{ runner.tool_cache }}/renderdiff-ms-playwright CLI: ${{ github.action_path }}/src/cli.ts LEFT: ${{ inputs.left }} RIGHT: ${{ inputs.right }} @@ -98,32 +91,55 @@ runs: CONCURRENCY: ${{ inputs.concurrency }} STORAGE_STATE: ${{ inputs.storage-state }} ACCEPT: ${{ inputs.accept }} + TIMEOUT: ${{ inputs.wait-timeout }} OUT: ${{ inputs.out }} + ARTIFACT: ${{ inputs.artifact-name }} run: | - args=(--left "$LEFT" --right "$RIGHT" --out "$OUT" --no-progress) - read -ra routes <<< "$(echo "$ROUTES" | tr '\n' ' ')" - args+=("${routes[@]}") - for w in ${WIDTHS//,/ }; do args+=(--width "$w"); done - [ -n "$CONFIG" ] && args+=(--config "$CONFIG") - [ -n "$THRESHOLD" ] && args+=(--threshold "$THRESHOLD") - [ -n "$CONCURRENCY" ] && args+=(--concurrency "$CONCURRENCY") - [ -n "$ACCEPT" ] && args+=(--accept "$ACCEPT") - if [ -n "$STORAGE_STATE" ]; then - state="$RUNNER_TEMP/renderdiff-state.json" - (umask 077; printf '%s' "$STORAGE_STATE" > "$state") - args+=(--storage-state "$state") + echo "results=$OUT/results.json" >> "$GITHUB_OUTPUT" + log="$RUNNER_TEMP/renderdiff-$ARTIFACT.txt" + : > "$log" + code=0 + + if ! [[ "$TIMEOUT" =~ ^[0-9]+$ ]]; then + echo "wait-timeout must be whole seconds, got '$TIMEOUT'" | tee "$log" + code=2 + else + end=$((SECONDS + TIMEOUT)) + until status=$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' "$LEFT") && [ "$status" -gt 0 ] && [ "$status" -lt 500 ]; do + if [ "$SECONDS" -ge "$end" ]; then + echo "$LEFT did not answer within ${TIMEOUT}s" | tee "$log" + code=2 + break + fi + sleep 5 + done fi - set +e - node "$CLI" "${args[@]}" 2>&1 | tee "$RUNNER_TEMP/renderdiff.txt" - code=${PIPESTATUS[0]} - set -e - rm -f "${state:-}" + if [ "$code" = 0 ]; then + args=("--left=$LEFT" "--right=$RIGHT" "--out=$OUT" --no-progress) + read -ra routes <<< "$(echo "$ROUTES" | tr '\n' ' ')" + args+=("${routes[@]}") + for w in ${WIDTHS//,/ }; do args+=("--width=$w"); done + [ -n "$CONFIG" ] && args+=("--config=$CONFIG") + [ -n "$THRESHOLD" ] && args+=("--threshold=$THRESHOLD") + [ -n "$CONCURRENCY" ] && args+=("--concurrency=$CONCURRENCY") + [ -n "$ACCEPT" ] && args+=("--accept=$ACCEPT") + if [ -n "$STORAGE_STATE" ]; then + state="$RUNNER_TEMP/renderdiff-state-$ARTIFACT.json" + (umask 077; printf '%s' "$STORAGE_STATE" > "$state") + args+=("--storage-state=$state") + fi + set +e + node "$CLI" "${args[@]}" 2>&1 | tee "$log" + code=${PIPESTATUS[0]} + set -e + rm -f "${state:-}" + + accepted="$(dirname "${CONFIG:-./_}")/renderdiff.accepted.json" + if [ -n "$ACCEPT" ] && [ -f "$accepted" ] && [ -d "$OUT" ]; then cp "$accepted" "$OUT/"; fi + fi echo "exit-code=$code" >> "$GITHUB_OUTPUT" - echo "results=$OUT/results.json" >> "$GITHUB_OUTPUT" - accepted="$(dirname "${CONFIG:-./_}")/renderdiff.accepted.json" - if [ -n "$ACCEPT" ] && [ -f "$accepted" ] && [ -d "$OUT" ]; then cp "$accepted" "$OUT/"; fi case $code in 0) verdict="✅ No visual change" ;; @@ -134,10 +150,10 @@ runs: { echo "### renderdiff: $verdict" echo - echo "\`$LEFT\` (your change) vs \`$RIGHT\` (reference). Report, results.json and crops are in the \`${{ inputs.artifact-name }}\` artifact." + echo "\`$LEFT\` (your change) vs \`$RIGHT\` (reference). Report, results.json and crops are in the \`$ARTIFACT\` artifact." echo echo '```text' - head -c 900000 "$RUNNER_TEMP/renderdiff.txt" + head -c 900000 "$log" echo '```' } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/accepted.md b/docs/accepted.md index 372cb85..4b158c1 100644 --- a/docs/accepted.md +++ b/docs/accepted.md @@ -45,6 +45,9 @@ accept it again once you've looked. What can still hide is a shade change under Accept on the platform that checks. The pixel hash is not portable: macOS and Linux render the same page with different fonts and anti-aliasing, so entries accepted on a Mac mostly come back stale on a Linux CI runner. For CI, accept in CI ([github-action.md](github-action.md)). +The same goes for renderdiff upgrades that change what counts as a changed pixel (0.3.2 +lowered the threshold): accepted regions can come back with a new box, so re-accept once +you've looked. A shot whose pages differ in width, or in height without the rows aligning into bands, is never accepted: the padding past the shorter page belongs to no band or region, so no diff --git a/docs/github-action.md b/docs/github-action.md index 41e6a2b..35f9701 100644 --- a/docs/github-action.md +++ b/docs/github-action.md @@ -27,17 +27,22 @@ exits non-zero. | `widths` | Spaces or commas. Default renderdiff's five. | | `threshold` | Fail below this score. Without it, only errors (exit 2 or 3) fail the job. | | `concurrency` | Parallel workers, which is also the most page loads production sees at once. Default 2 on a hosted runner. | -| `storage-state` | Contents of a Playwright storage-state JSON, from a secret. Written to a 0600 temp file and deleted after the run. | +| `storage-state` | Contents of a Playwright storage-state JSON, from a secret. Written to a 0600 file in the runner's temp dir and deleted when renderdiff exits. | | `accept` | Record this run's differences as accepted, with this reason. See below. | -| `wait-timeout` | Seconds to wait for `left` to answer (any status under 500). Default 300. | -| `out` | Output directory. Default `renderdiff-out`. | -| `artifact-name` | Default `renderdiff`. Set it when one job runs the action twice. | +| `wait-timeout` | Whole seconds to wait for `left` to answer (any status under 500). Default 300. Past it, `exit-code` is 2. | +| `out` | Output directory, over the config's `outDir`. Default `renderdiff-out`. | +| `artifact-name` | Default `renderdiff`. | + +To run the action twice in one job, give each run its own `out` and `artifact-name`; in a +matrix, put the matrix value in `artifact-name`. The action sets up Node 24, which later +steps in the job then use. Outputs: `exit-code` (0 pass, 1 below threshold, 2 bad input or an origin down, 3 a route errored) and `results` (the path to `results.json`). -A Vercel preview behind Deployment Protection serves its login page to the runner, and -renderdiff will diff that. Turn protection off for previews or use a bypass first. +A Vercel preview behind Deployment Protection answers the runner with its login page: a +401 makes every route an error, a login page served as 200 gets diffed. Turn protection off +for previews, or use a bypass, first. ## Recipes @@ -49,7 +54,9 @@ Vercel and Netlify report each preview as a GitHub deployment: on: deployment_status jobs: renderdiff: - if: github.event.deployment_status.state == 'success' + if: >- + github.event.deployment_status.state == 'success' && + github.event.deployment_status.environment != 'Production' runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -60,7 +67,8 @@ jobs: threshold: "0.999" ``` -The checkout is for a committed config and `renderdiff.accepted.json`. +The second condition skips production's own deployment, which would compare production to +itself. The checkout is for a committed config and `renderdiff.accepted.json`. ### A build in the job vs production diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 27da5d9..f562539 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -33,6 +33,11 @@ honestly. This is what renderdiff does to make sure of that. noise from that origin: they're painted magenta on both sides, reported per comparison as `unstable`, and excluded from the measurement — the score is over the remaining pixels. A route more than 25% unstable errors as `page did not settle`. +- **Counts a pixel as changed** when pixelmatch's YIQ colour distance is over threshold + 0.05 (a delta of about 88), with anti-aliased pixels left out. Its default of 0.1 passed + Tailwind v3 → v4 `pink-500` as identical; a recolour still smaller than that passes. The + same sensitivity sees content moved by half a pixel: a `translate(-50%)` that lands on + `.5` swapped for grid centring shows as a thin region, not clean. - **Isolates state** — one browser context per origin, so cookies and localStorage never cross (a storage state file seeds each context on its own). Both sides of a route load at the same time. diff --git a/tests/fixtures/pages/tint/left.html b/tests/fixtures/pages/tint/left.html index d6a534c..0247530 100644 --- a/tests/fixtures/pages/tint/left.html +++ b/tests/fixtures/pages/tint/left.html @@ -6,7 +6,7 @@