diff --git a/.gitignore b/.gitignore index e6fd2bc07..9b1fedeb3 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ npm-debug.log* *.log *.swp *.stackdump + +# output of scripts/measure-tolerances +/tolerances.report.json diff --git a/package.json b/package.json index 5cd621be7..b3ce3bcbe 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "test-karma": "karma start ./karma.conf.cjs --auto-watch --single-run", "test-lint": "npm run lint", "test-types": "tsc -p types/tests/", - "test-integration": "mocha --full-trace test/integration/*-test.js" + "test-integration": "mocha --full-trace test/integration/*-test.js", + "measure-tolerances": "node scripts/measure-tolerances/measure.js" }, "devDependencies": { "@rollup/plugin-json": "^6.0.0", diff --git a/scripts/measure-tolerances/Dockerfile b/scripts/measure-tolerances/Dockerfile new file mode 100644 index 000000000..5c6151c62 --- /dev/null +++ b/scripts/measure-tolerances/Dockerfile @@ -0,0 +1,21 @@ +# Reproduces the browsers CI runs (ubuntu-latest, x86_64) so fixture deltas can +# be measured against the environment that actually gates the build. +# +# linux/amd64 is deliberate: Chrome is not published for linux/arm64, and native +# arm64 rendering differs from x86_64 (point/starShadow measures 1474px there vs +# 1476px on x86_64 and in CI), so arm64 numbers would not be CI-faithful. +FROM --platform=linux/amd64 ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates gnupg xvfb xz-utils \ + fonts-liberation fonts-dejavu-core fontconfig \ + && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && curl -fsSL -o /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \ + && apt-get install -y --no-install-recommends /tmp/chrome.deb \ + && curl -fsSL -o /tmp/ff.tar.xz "https://download.mozilla.org/?product=firefox-latest-ssl&os=linux64&lang=en-US" \ + && mkdir -p /opt && tar -xJf /tmp/ff.tar.xz -C /opt \ + && rm -rf /tmp/*.deb /tmp/*.tar.xz /var/lib/apt/lists/* +ENV CHROME_BIN=/usr/bin/google-chrome +ENV FIREFOX_BIN=/opt/firefox/firefox +WORKDIR /app diff --git a/scripts/measure-tolerances/README.md b/scripts/measure-tolerances/README.md new file mode 100644 index 000000000..63ed7a0be --- /dev/null +++ b/scripts/measure-tolerances/README.md @@ -0,0 +1,75 @@ +# Measuring fixture tolerances + +Fixture tests compare a rendered chart against a reference PNG and allow a +`tolerance` — the fraction of pixels that may differ. Browsers do not rasterise +text and shadows identically, so some difference is expected, and the tolerance +has to absorb it without hiding real regressions. + +Picking those numbers by hand does not work for long. Every browser update +shifts the deltas, a fixture starts failing, and the tolerance gets nudged up +based on the one number in the failure report — which is a guess about the +margin, not a measurement of it. Do that a few times and the suite is loose +everywhere and still failing somewhere. + +This tool measures instead. + +## Usage + +```sh +npm run measure-tolerances # report what would change +npm run measure-tolerances -- --apply # write the derived values +``` + +Requires Docker. `test/fixtures` must be clean — the fixtures are temporarily +rewritten and then restored with `git checkout`. + +Host browsers default to `chrome,firefox`; override with +`-- --host-browsers=firefox` if you only have one installed. + +## How it works + +Every fixture is forced to `tolerance: 0`, which makes each one report its real +pixel delta instead of just passing or failing. That runs in four environments: +your own Chrome and Firefox, and Linux Chrome and Firefox in a container built +to match the CI image. + +Tolerances are then derived: + +| condition | tolerance | +|---|---| +| delta is 0 in every environment | `0` | +| otherwise | `max(1.5 × linux, 1.05 × host)` | + +Linux gets a real margin because CI is what gates the build. The host gets only +enough to keep local runs green — macOS in particular renders text 3-4x further +from the reference than Linux does, and sizing every tolerance for it is what +makes them loose. + +The trade-off is deliberate: a host browser update will redden local runs +without affecting CI, and re-measuring is cheap. + +## Why the container is emulated + +It runs `linux/amd64` under emulation even on Apple Silicon. Chrome is not +published for `linux/arm64` at all, and native arm64 rendering genuinely +differs — `point/starShadow` measures 1474px there against 1476px on x86_64 and +in CI, and `line/labelShadowColors` differs by 74%. Faster, but wrong. + +Emulation was verified against three pixel counts observed in CI, all +reproduced exactly: + +| fixture | browser | CI | container | +|---|---|---|---| +| `point/crossShadow` | Firefox | 1565px | 1565px | +| `point/starShadow` | Firefox | 1476px | 1476px | +| `doughnutLabel/contentMultiline` | Chrome | 919px | 919px | + +If you change the image, re-check it against numbers from a real CI run before +trusting what it produces. + +Emulated Chrome needs two accommodations, both in `prepare.js`: karma's +timeouts are raised well past their defaults, and `--no-sandbox` is added +because the container does not grant the user namespaces Chrome's sandbox +wants. The flag was confirmed not to affect rendering — `contentMultiline` +still measures the same 919px with it. Without `--shm-size=2g` Chrome hangs +outright rather than running slowly. diff --git a/scripts/measure-tolerances/in-container.sh b/scripts/measure-tolerances/in-container.sh new file mode 100755 index 000000000..0867e234a --- /dev/null +++ b/scripts/measure-tolerances/in-container.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Runs inside the image built from the Dockerfile next to this file. +# Expects the repo mounted read-only at /src and a browser name as $1. +set -e +browser="$1" + +useradd -m runner 2>/dev/null || true +mkdir -p /app && cp -r /src/. /app/ && rm -rf /app/node_modules /app/dist +chown -R runner /app + +su runner -c "HOME=/home/runner bash -c ' +set -e +cd /app +npm ci --no-audit --no-fund >/dev/null 2>&1 +npm run build >/dev/null 2>&1 +node scripts/measure-tolerances/prepare.js +xvfb-run --auto-servernum npx karma start ./karma.conf.cjs \ + --single-run --no-auto-watch --browsers $browser 2>&1 +'" diff --git a/scripts/measure-tolerances/measure.js b/scripts/measure-tolerances/measure.js new file mode 100644 index 000000000..224589104 --- /dev/null +++ b/scripts/measure-tolerances/measure.js @@ -0,0 +1,174 @@ +// Measures every fixture's real pixel delta in each environment that matters, +// then derives tolerances from those measurements instead of guessing margins. +// +// npm run measure-tolerances report only +// npm run measure-tolerances -- --apply also write the derived tolerances +// +// Environments: the host's Chrome and Firefox, plus Linux Chrome and Firefox in +// a container reproducing the CI image. See README.md next to this file. +import fs from 'fs'; +import {execFileSync} from 'child_process'; +import path from 'path'; +import {fileURLToPath} from 'url'; +import {fixtureFiles, readTolerance, writeTolerance, forceZeroTolerance} from './prepare.js'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const IMAGE = 'chartjs-annotation-ci'; +const REPORT = 'tolerances.report.json'; + +// CI is the gate, so Linux gets a real margin. macOS renders text 3-4x further +// from the reference than Linux does; giving it the same margin is what made +// tolerances loose, so it gets only enough to keep local runs green. +const LINUX_MARGIN = 1.5; +const HOST_MARGIN = 1.05; +// A fixture that is pixel-perfect everywhere asserts exactly that. +const ZERO_STAYS_ZERO = true; + +const ceil = v => Math.ceil(v * 20000) / 20000; +const run = (cmd, args, opts = {}) => + execFileSync(cmd, args, {encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024, ...opts}); + +function pixelCount(fixture) { + const buf = fs.readFileSync(`${fixture.replace(/\.js$/, '')}.png`); + return buf.readUInt32BE(16) * buf.readUInt32BE(20); // IHDR width * height +} + +// karma prints ` ... /base/test/fixtures/.js FAILED` and then +// `Difference: px`, so deltas are read back from its output. +function parseDeltas(output) { + const deltas = {}; + let fixture = null; + for (const line of output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, '').split('\n')) { + const failed = line.match(/\/base\/(test\/fixtures\/.+?\.js) FAILED/); + if (failed) { + fixture = failed[1]; + continue; + } + const diff = line.match(/Difference: (\d+)px/); + if (diff && fixture) { + deltas[fixture] = Math.max(deltas[fixture] || 0, Number(diff[1])); + fixture = null; + } + } + return deltas; +} + +function assertFixturesClean() { + if (run('git', ['status', '--porcelain', 'test/fixtures']).trim()) { + throw new Error('test/fixtures has uncommitted changes; commit or stash them first ' + + '(they would be lost when the measured fixtures are restored)'); + } +} + +// A run that dies partway (karma DISCONNECTED, a crashed browser) still yields +// deltas for the specs it reached. Deriving tolerances from those would silently +// produce values that are too tight, so incomplete runs are rejected outright. +function assertComplete(output, label) { + const clean = output.replace(/\x1b\[[0-9;]*[A-Za-z]/g, ''); + if (/DISCONNECTED/.test(clean)) { + throw new Error(`${label}: browser disconnected, measurement incomplete`); + } + const executed = [...clean.matchAll(/Executed (\d+) of (\d+)/g)].pop(); + if (!executed) { + throw new Error(`${label}: karma reported no specs`); + } + if (executed[1] !== executed[2]) { + throw new Error(`${label}: only ${executed[1]} of ${executed[2]} specs ran`); + } +} + +function measureHost(browsers) { + console.log(`measuring host (${browsers})...`); + forceZeroTolerance(); + try { + const out = run('npx', ['karma', 'start', './karma.conf.cjs', + '--single-run', '--no-auto-watch', '--browsers', browsers], {stdio: 'pipe'}); + assertComplete(out, 'host'); + return parseDeltas(out); + } catch (err) { + // karma exits non-zero because every spec fails at tolerance 0; that is the + // point of the run, so the deltas still have to be read out of its output. + if (err.stdout === undefined) { + throw err; + } + const out = String(err.stdout); + assertComplete(out, 'host'); + return parseDeltas(out); + } finally { + run('git', ['checkout', '--', 'test/fixtures']); + } +} + +function measureLinux(browser) { + console.log(`measuring linux ${browser} in docker (emulated; chrome takes a while)...`); + const args = ['run', '--rm', '--platform', 'linux/amd64', + // without this chrome hangs outright rather than running slowly + '--shm-size=2g', + '-v', `${process.cwd()}:/src:ro`, + IMAGE, 'bash', '/src/scripts/measure-tolerances/in-container.sh', browser]; + try { + return parseDeltas(run('docker', args)); + } catch (err) { + // karma exits non-zero because every spec fails at tolerance 0; that is the + // point of the run, so the deltas still have to be read out of its output. + if (err.stdout === undefined) { + throw err; + } + const out = String(err.stdout); + assertComplete(out, `linux ${browser}`); + return parseDeltas(out); + } +} + +function derive(measurements) { + return fixtureFiles().map(file => { + const area = pixelCount(file); + const ratio = env => (measurements[env][file] || 0) / area; + const linux = Math.max(ratio('linuxChrome'), ratio('linuxFirefox')); + const host = ratio('host'); + const current = readTolerance(file); + const proposed = (ZERO_STAYS_ZERO && linux === 0 && host === 0) + ? 0 + : Math.max(ceil(linux * LINUX_MARGIN), ceil(host * HOST_MARGIN)); + return {file, linux, host, current, proposed}; + }); +} + +const args = process.argv.slice(2); +const browsers = (args.find(a => a.startsWith('--host-browsers=')) || '').split('=')[1] || 'chrome,firefox'; + +// Fail fast on a dirty tree before spending an hour in docker. +assertFixturesClean(); +run('docker', ['build', '--platform', 'linux/amd64', '-t', IMAGE, HERE], {stdio: 'inherit'}); +const measurements = { + host: measureHost(browsers), + linuxChrome: measureLinux('chrome'), + linuxFirefox: measureLinux('firefox') +}; + +const rows = derive(measurements); +fs.writeFileSync(REPORT, JSON.stringify(rows, null, 2)); + +const changes = rows.filter(r => r.proposed !== r.current); +const pct = v => (v * 100).toFixed(3) + '%'; +console.log(`\n${changes.length} of ${rows.length} fixtures would change:\n`); +for (const r of changes.sort((a, b) => b.proposed - a.proposed)) { + const dir = r.proposed < r.current ? 'tighter' : 'looser '; + console.log(` ${dir} ${r.file.replace('test/fixtures/', '').padEnd(38)} ` + + `${pct(r.current).padStart(8)} -> ${pct(r.proposed).padStart(8)} ` + + `(linux ${pct(r.linux)}, host ${pct(r.host)})`); +} +const budget = rows.reduce((sum, r) => ({ + before: sum.before + r.current * pixelCount(r.file), + after: sum.after + r.proposed * pixelCount(r.file) +}), {before: 0, after: 0}); +console.log(`\ntotal allowed differing pixels: ${Math.round(budget.before).toLocaleString('en')} -> ` + + `${Math.round(budget.after).toLocaleString('en')}`); +console.log(`report written to ${REPORT}`); + +if (args.includes('--apply')) { + changes.forEach(r => writeTolerance(r.file, r.proposed)); + console.log(`\napplied to ${changes.length} fixtures; run the suite to confirm`); +} else if (changes.length) { + console.log('\nre-run with --apply to write these values'); +} diff --git a/scripts/measure-tolerances/prepare.js b/scripts/measure-tolerances/prepare.js new file mode 100644 index 000000000..d3f809249 --- /dev/null +++ b/scripts/measure-tolerances/prepare.js @@ -0,0 +1,51 @@ +// Forces every fixture to `tolerance: 0` so each one reports its real pixel +// delta instead of just pass/fail, and relaxes karma's timeouts because an +// emulated Chrome is far slower than the 2 minute default allows. +// +// Destructive: only run directly inside the container, which works on a throwaway +// copy. measure.js imports the helpers and handles restoring the host checkout. +import fs from 'fs'; +import path from 'path'; +import {pathToFileURL} from 'url'; + +export function fixtureFiles(dir = 'test/fixtures') { + return fs.readdirSync(dir, {withFileTypes: true}).flatMap(e => { + const p = path.join(dir, e.name); + return e.isDirectory() ? fixtureFiles(p) : (e.name.endsWith('.js') ? [p] : []); + }); +} + +export function readTolerance(file) { + const m = fs.readFileSync(file, 'utf-8').match(/^ {2}tolerance: ([\d.]+),$/m); + return m ? parseFloat(m[1]) : 0.001; // chartjs-test-utils default +} + +export function writeTolerance(file, value) { + const src = fs.readFileSync(file, 'utf-8'); + const line = ` tolerance: ${String(value)},`; + fs.writeFileSync(file, /^ {2}tolerance: [\d.]+,$/m.test(src) + ? src.replace(/^ {2}tolerance: [\d.]+,$/m, line) + : src.replace(/^module\.exports = \{$/m, `module.exports = {\n${line}`)); +} + +export function forceZeroTolerance() { + fixtureFiles().forEach(file => writeTolerance(file, 0)); +} + +export function relaxKarmaTimeouts() { + const conf = 'karma.conf.cjs'; + fs.writeFileSync(conf, fs.readFileSync(conf, 'utf-8') + .replace(/browserNoActivityTimeout: \d+/, 'browserNoActivityTimeout: 3600000') + .replace(/browserDisconnectTimeout: \d+/, 'browserDisconnectTimeout: 3600000') + .replace(/captureTimeout: \d+/, 'captureTimeout: 3600000') + // Chrome's sandbox needs user namespaces the container does not grant. + // Verified not to affect rendering: with it, doughnutLabel/contentMultiline + // still measures the 919px observed in CI. + .replace("'--disable-accelerated-2d-canvas'", "'--disable-accelerated-2d-canvas', '--no-sandbox'")); +} + +// Self-executes only when run as a script, never when imported. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + forceZeroTolerance(); + relaxKarmaTimeouts(); +}