Accepted differences and --rerun-failed

README.md

@@ -96,6 +96,8 @@ renderdiff <left-origin> <right-origin> <route…> [options]
 | `--left-selector <sel>`, `--right-selector <sel>` | That side's selector, over `--selector`. One without the other side (or `--selector`) is an error. |
 | `--storage-state <file>` | Start both sides with this Playwright storage state (cookies, localStorage), e.g. signed in. Relative to cwd. Over the config's `storageState`. |
 | `--left-storage-state <file>`, `--right-storage-state <file>` | That side's storage state, over `--storage-state`. One side alone is fine. |
+| `--accept <reason>` | Record this run's differences as accepted, with a reason. See **Accepted differences and reruns**. |
+| `--rerun-failed` | Rerun only what the last run in `--out` didn't find clean or accepted, keeping the rest. |
 | `--dry-run` | Print the URL pairs and exit. No browser. |
 | `--open` | Open the report when the run finishes. |
 | `--progress` | A line per shot as it finishes, then the worst scores. The default on a terminal; piped, renderdiff prints one compact summary at the end. |
@@ -181,10 +183,31 @@ only.
 | `dom sel` | On both sides and unchanged in style or position, so its own pixels changed (an image or canvas). |
 | `crops …` | Up to 3 PNGs per comparison, bands first, relative to `results.json`. Left is on top or on the left, and the change is boxed in red. |
 | `ERROR …` | Unmeasured, which is neither a pass nor a fail. Exits 3. |
+| `accepted:` / `stale accepted:` | Shots whose every difference was accepted, and accepted entries the run no longer saw. See below. |
+| `rerun:` | With `--rerun-failed`: how many shots were rerun and how many reused. |
 
 Read text first, then `dom`, then the crops for anything `dom` doesn't explain. Full
 detail, including the `--progress` and report forms: [docs/output.md](https://github.com/tmashara/renderdiff/blob/main/docs/output.md).
 
+## Accepted differences and reruns
+
+A migration starts with differences that already exist between the untouched local build
+and production. Record them once, then change code and re-check only what's red:
+
+```
+renderdiff http://localhost:3000 https://example.com / /about --accept "differs before the migration"
+# …change code…
+renderdiff http://localhost:3000 https://example.com / /about --rerun-failed
+```
+
+`--accept "<reason>"` writes every difference the run saw to `renderdiff.accepted.json` (next
+to the config, or in cwd); commit it. Later, a shot whose every difference matches an entry is
+listed on an `accepted:` line and counts neither as differing nor toward `--threshold`;
+anything new on the same page still shows. `--rerun-failed` reruns only what the last run in
+`--out` didn't find clean or accepted and merges it back; it exits 2 if the run differs from
+the previous one. Matching rules, what can still hide, and stale entries:
+[docs/accepted.md](https://github.com/tmashara/renderdiff/blob/main/docs/accepted.md).
+
 ## Output
 
 Everything lands in the out dir (`./renderdiff-out` by default):
@@ -243,7 +266,9 @@ Everything lands in the out dir (`./renderdiff-out` by default):
   "summary": {
     "compared": 1, "errors": 1, "mean": 0.9714,
     "worst": { "url": "/about", "width": 1440, "score": 0.9714 },
-    "belowThreshold": 0
+    "belowThreshold": 0,          // accepted shots aside
+    "accepted": 0,                // shots whose every difference was accepted
+    "stale": []                   // accepted entries the run measured but didn't see
   }
 }
 ```

docs/accepted.md

@@ -0,0 +1,58 @@
+# Accepted differences and reruns
+
+A migration starts with differences that already exist between the untouched local build
+and production. Record them once, then change code and re-check only what's red:
+
+```
+renderdiff http://localhost:3000 https://example.com / /about --accept "differs before the migration"
+# …change code…
+renderdiff http://localhost:3000 https://example.com / /about --rerun-failed
+```
+
+`--accept "<reason>"` runs as usual, then writes every difference it saw that isn't accepted
+yet to `renderdiff.accepted.json`, next to the config file (or in cwd without one), each with
+the reason. Scope it the way you scope any run: pass only the routes and `--width`s to accept.
+Commit the file; it is one entry per line, so its diff reads as what was accepted.
+
+A shot whose every difference matches an accepted one is `accepted`: it is listed on an
+`accepted:` line like `clean:`, doesn't count as differing or toward `--threshold`, and has
+`"accepted": true` in `results.json` and a badge in the report. Anything else on the page
+still shows: a row with some changes accepted says `1 accepted`, and its `dom` line skips
+them. A shot with unstable pixels is never accepted, since part of it went unmeasured.
+`summary.mean` and `summary.worst` leave accepted shots out, like the verdict line.
+Accepted entries for a route and width the run measured but didn't see are **stale**: they
+print on a `stale accepted:` line, sit in `summary.stale` and the report, and the next
+`--accept` over that route and width prunes them.
+
+Each entry is one change, a band or a region, keyed by route (as the summary names it) and
+width, with a fingerprint. Two changes match when they have the same kind and side, the same
+`x`, width and height (regions sit on the 16px grid, bands are exact rows), the same hash of
+every pixel under the change on both sides at 4 bits a channel, and, when the entry named
+elements, the same `dom` elements with the same changed values. That makes a match:
+
+- robust to a change above it moving it down the page (`y` is left out), to unrelated
+  changes elsewhere (they are their own changes, and stay unaccepted), and to shade jitter
+  that stays within one of sixteen steps per channel;
+- strict about the spot itself: any other change to what is drawn there, a change touching
+  it (the region grows), or a second identical change (an entry matches once) all surface.
+
+The cost is false alarms rather than hidden changes. Noise that crosses a 4-bit step, an
+element probe that times out, or a new change that makes repeated identical rows align
+differently (turning an accepted region into bands) makes an accepted change show again;
+accept it again once you've looked. What can still hide is a shade change under one step
+(e.g. `#000` to `#080808`) that doesn't cross a step boundary anywhere in the change.
+
+A shot whose pages differ in width, or in height without the rows aligning into bands, is
+never accepted: the padding past the shorter page belongs to no band or region, so no
+fingerprint covers it.
+
+`--rerun-failed` reads `results.json` in `--out`, keeps every shot that was clean or is
+accepted (against the accepted file as it is now), reruns the rest — differences, unstable
+shots, errors — and merges them back into `results.json` and `report.html` in route and width
+order. The out dir is kept rather than wiped; only a rerun shot's own old files are replaced.
+A `rerun:` line says how many shots it reran on how many routes, and how many it reused.
+`meta` describes the rerun (its clock, browser and time); a reused shot was taken under the
+earlier run's. It must repeat the previous run: different origins, routes, widths, config
+(`meta.config` hashes masks, `before` hooks and `locate` sources, hidden selectors, block
+patterns, extra CSS and storage state file names) or renderdiff version exit 2, as does a
+missing `results.json`.

skills/renderdiff/SKILL.md

@@ -18,7 +18,8 @@ legitimate: [reference.md](reference.md).
 ## Loop
 
 1. **Baseline first.** Before changing code, run the unmodified build against the reference
-   (`--out renderdiff-baseline`). Differences already there are not yours.
+   with `--accept "differs before the migration"`: what already differs is recorded, not
+   yours. Accept only what you looked at, never to get green.
 2. Both origins up, with the same content and data. `--dry-run` shows the URL pairs.
 3. One route, one width. A row: score, `h±N`/`w±N` height/width delta,
    `band`, `aligned`, `N reg`, `unstable Npx` (moving pixels, unmeasured), `text`.
@@ -30,8 +31,10 @@ legitimate: [reference.md](reference.md).
    Bare `sel`: its own pixels changed (image, canvas).
 7. `Read` the crops under each row (next to results.json) for what `dom` doesn't explain.
    Left is on top or left; red box = change, red seam = missing rows.
-8. Fix or justify every band, region and text line. Re-run after each migration step, then
-   all routes at default widths; for a multi-width row, open the widest and narrowest crop.
+8. Fix or justify every band, region and text line. After each migration step,
+   `--rerun-failed` (same arguments) reruns only what isn't clean or accepted; finish with
+   all routes at default widths. For a multi-width row, open the widest and narrowest crop.
+   `accepted:` routes are known; a row saying `1 accepted` still has something new.
 
 Exit: 0 nothing below `--threshold` (**without it, a diff still exits 0**) · 1 below
 threshold · 2 bad args/config/origin down · 3 a route errored (unmeasured) or crash.

skills/renderdiff/reference.md

@@ -24,6 +24,9 @@ Left is your change, right is the reference: `+` is only/more on the left, `-` o
 | `unstable Npx` | Pixels that kept changing between shots of the same load on one origin (a ticker, a carousel, a video). Painted out on both sides and excluded from the score. A large number means a big area went unmeasured: look at the magenta on the full shots. Such a route is never on the `clean:` line. |
 | `ERROR selector "…" matched N elements on left (/a), expected 1` | A selector must match exactly one element per side. Make it more specific (`>> nth=0`, an id, `data-test`), or list the route once per element. |
 | `ERROR selector "…" has no box on …` | It matched an element that isn't rendered (`display: none`, zero size). |
+| `accepted: /a /b` | Every difference on those routes matches `renderdiff.accepted.json`. Known, not verified again: say so in the verdict. |
+| `N accepted` on a row | Some of that shot's changes are accepted; the rest are new. `dom` names only the new ones. |
+| `stale accepted: /a@1440` | An accepted difference wasn't seen. It was fixed, or the page changed under it; `--accept` over that route and width prunes it. |
 | `ERROR page did not settle: Npx unstable at …` | More than 25% of the page kept changing between shots, so too little was measured to score. Mask or stop the moving part. |
 
 Report colouring (`--open`, for a human): green above 99.5%, amber to 97%, red below.
@@ -38,6 +41,7 @@ Report colouring (`--open`, for a human): green above 99.5%, amber to 97%, red b
 | "`dom` says margin, done." | It's a computed value. Confirm the source rule changed on purpose, and still open the crop when the region is large or `dom` names nothing. |
 | "The crops looked fine, done." | There are at most three: bands, then the largest regions. Check the region count, any `clipped` crop in results.json, and open the diff PNG for the rest and for any height delta: padding is never cropped. |
 | "Masking the header gets this to 100%." | Every mask is coverage surrendered. Mask only content *proven* to vary per load (clocks, star counts, rotating quotes), with a `reason`, which the report prints verbatim. |
+| "I'll `--accept` it so the run is green." | Accept only differences you opened and that exist before your change (or are intended), with a reason that says which. |
 | "I'll lower `--threshold` until it passes." | Threshold is a CI gate, not a dial for reaching green. |
 | "That route errors, I'll drop it from the list." | Shrinking the route list shrinks what you verified. Fix it, or say it's unverified. |
 | "One width is enough." | Migrations break at one width and not others. That's why there are five defaults. |
@@ -91,7 +95,16 @@ cookie script ran wedges the renderer).
 
 - **stdout is compact when piped** (what an agent sees). `--progress` prints a line per
   shot instead; `--no-progress` forces the summary on a terminal.
-- **The out dir is wiped at the start of every run.** To keep a previous run's evidence
+- **Accepted differences** (`--accept "<reason>"`) go to `renderdiff.accepted.json` next to the
+  config (or in cwd), one entry per band or region, scoped by the run's routes and widths. A
+  match ignores `y` and needs the same box, the same pixel hash and the same named elements, so
+  anything else drawn in that spot still surfaces. A shot whose widths differ, or whose heights
+  differ without aligning, is never accepted. A new change can make an accepted one re-read
+  as different (repeated rows aligning differently): look, then accept again.
+- **`--rerun-failed`** keeps the out dir and reruns only what wasn't clean or accepted. Give it
+  the same origins, routes, widths and config (hooks included) as the run it continues, or it
+  exits 2.
+- **The out dir is wiped at the start of every run** (except `--rerun-failed`). To keep a previous run's evidence
   next to a new one, pass `--out renderdiff-before` / `--out renderdiff-after`. A
   directory holding files renderdiff didn't write is refused (exit 2).
 - **Routes normalize** to no trailing slash; `/` stays `/`. Globs are rejected.

src/accept.ts

@@ -0,0 +1,144 @@
+import fs from "node:fs";
+import type { Band, Region } from "./compare.ts";
+import { name } from "./config.ts";
+import { fmtElements } from "./format.ts";
+import type { ComparedResult, RouteResult } from "./report.ts";
+
+export const ACCEPTED_FILE = "renderdiff.accepted.json";
+
+/**
+ * One band or region, without its `y`: a change above it moves it down the page without
+ * changing it. `hash` and `elements` tell a new change in the same spot from the accepted one.
+ */
+export interface Fingerprint {
+  kind: "band" | "region";
+  side?: "left" | "right";
+  /** A region's box; a band's rows are in its hash. */
+  x?: number;
+  w?: number;
+  h: number;
+  /** The `dom` of each element under it, sorted; absent when they weren't looked at. */
+  elements?: string[];
+  /** Both sides' pixels under it, 4 bits a channel. */
+  hash?: string;
+}
+
+/** A difference recorded as already there or intended, so later runs report it as accepted. */
+export interface Accepted {
+  /** The route as the summary names it: `/a`, `/a::/b`, `/a [.sel]`. */
+  route: string;
+  width: number;
+  reason: string;
+  fingerprint: Fingerprint;
+}
+
+type Change = { fp: Fingerprint; target: Region | Band };
+
+function changes(r: ComparedResult): Change[] {
+  const els = (g: Region | Band) =>
+    g.elements && { elements: g.elements.map((e) => fmtElements([e], Infinity)).sort() };
+  return [
+    ...(r.bands ?? []).map((b) => ({
+      fp: { kind: "band" as const, side: b.side, h: b.h, ...els(b), hash: b.hash },
+      target: b,
+    })),
+    ...r.regions.map((g) => ({
+      fp: { kind: "region" as const, x: g.x, w: g.w, h: g.h, ...els(g), hash: g.hash },
+      target: g,
+    })),
+  ];
+}
+
+/** `a` is the accepted entry: elements it recorded must be named again, not merely unprobed. */
+const same = (a: Fingerprint, b: Fingerprint) =>
+  a.kind === b.kind &&
+  a.side === b.side &&
+  a.x === b.x &&
+  a.w === b.w &&
+  a.h === b.h &&
+  a.hash === b.hash &&
+  (!a.elements || a.elements.join("\n") === b.elements?.join("\n"));
+
+/**
+ * Padding no band or region covers: columns past the narrower page, or rows past the shorter
+ * one when they didn't align. A shot with any, or with unstable pixels, is never accepted.
+ */
+const acceptable = (r: ComparedResult) =>
+  r.unstablePixels === 0 && r.widthDelta === 0 && (r.bands !== null || r.heightDelta === 0);
+
+export function readAccepted(file: string): Accepted[] {
+  if (!fs.existsSync(file)) return [];
+  let list: unknown;
+  try {
+    list = JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (e) {
+    throw new Error(`${file} is not valid JSON: ${e instanceof Error ? e.message : e}`);
+  }
+  const ok = (a: Record<string, unknown>) =>
+    typeof a.route === "string" &&
+    typeof a.width === "number" &&
+    typeof a.reason === "string" &&
+    a.reason.trim() &&
+    typeof a.fingerprint === "object" &&
+    a.fingerprint !== null;
+  if (!Array.isArray(list) || !list.every(ok)) {
+    throw new Error(`${file} must be an array of { route, width, reason, fingerprint }`);
+  }
+  return list as Accepted[];
+}
+
+/**
+ * Mark each change an accepted entry matches, one entry per change, and a result `accepted`
+ * when every change matched and it is `acceptable`. Returns the changes of acceptable results
+ * nothing matched, and the entries for a measured route and width that matched nothing.
+ */
+export function applyAccepted(results: RouteResult[], list: Accepted[]) {
+  const used = new Set<Accepted>();
+  const unmatched: Omit<Accepted, "reason">[] = [];
+  for (const r of results) {
+    if (r.status !== "compared") continue;
+    const mine = list.filter((a) => a.route === name(r) && a.width === r.width);
+    const cs = r.diffPixels > 0 ? changes(r) : [];
+    let missed = 0;
+    for (const { fp, target } of cs) {
+      const hit = mine.find((a) => !used.has(a) && same(a.fingerprint, fp));
+      if (hit) {
+        used.add(hit);
+        target.accepted = hit.reason;
+      } else {
+        delete target.accepted;
+        missed++;
+        if (acceptable(r)) unmatched.push({ route: name(r), width: r.width, fingerprint: fp });
+      }
+    }
+    r.accepted = cs.length > 0 && !missed && acceptable(r);
+  }
+  const measured = new Set(
+    results.filter((r) => r.status === "compared").map((r) => `${name(r)}@${r.width}`),
+  );
+  const stale = list.filter((a) => !used.has(a) && measured.has(`${a.route}@${a.width}`));
+  return { unmatched, stale };
+}
+
+const key = (a: Accepted) => JSON.stringify([a.route, a.width, a.reason, a.fingerprint]);
+
+/**
+ * Drop `stale` and add `added` to the file as it is now, so a concurrent `--accept` isn't lost,
+ * sorted by route then width, through a rename so a crash never leaves it half written.
+ * Returns false, writing nothing, when there is nothing to change.
+ */
+export function updateAccepted(file: string, stale: Accepted[], added: Accepted[]): boolean {
+  if (!stale.length && !added.length) return false;
+  const drop = new Set(stale.map(key));
+  const kept = readAccepted(file).filter((a) => !drop.has(key(a)));
+  const have = new Set(kept.map(key));
+  const list = [...kept, ...added.filter((a) => !have.has(key(a)))].sort((a, b) =>
+    a.route < b.route ? -1 : a.route > b.route ? 1 : a.width - b.width,
+  );
+  // One entry per line, so a diff of the file reads as the differences it adds and drops.
+  const rows = list.map((a) => `  ${JSON.stringify(a)}`).join(",\n");
+  const tmp = `${file}.${process.pid}.tmp`;
+  fs.writeFileSync(tmp, rows ? `[\n${rows}\n]\n` : "[]\n");
+  fs.renameSync(tmp, file);
+  return true;
+}