diff --git a/docs/about-vertical-geometry.md b/docs/about-vertical-geometry.md index 34d664c..39dc48d 100644 --- a/docs/about-vertical-geometry.md +++ b/docs/about-vertical-geometry.md @@ -136,6 +136,14 @@ why no threshold moved. ### The harness does not reproduce across sessions +**2026-09-13 correction:** [The harness identity investigation](harness-determinism.md) +reproduced Session A exactly from saved pre-integer-advance modules and themes. +Both theme hashes identify the old percentage conversion, not different +node-scope catalogs from the current binary. The earlier harness did not verify +generated input identity; this subsection's cross-session nondeterminism +interpretation is superseded. The before/after table above retains its stated +`757dc4a` version context. + The supplied `about` baseline of 11.24 / 6.92 / 4.37 was **not** reproduced by an unmodified build of the same commit. Popup's supplied `f26ad027d6de` theme was likewise not reproduced: isolated acquisition repeatedly produced diff --git a/docs/harness-determinism-evidence.json b/docs/harness-determinism-evidence.json new file mode 100644 index 0000000..bf7a8c3 --- /dev/null +++ b/docs/harness-determinism-evidence.json @@ -0,0 +1,542 @@ +{ + "baseCommit": "d039623b0d1a", + "date": "2026-09-13", + "repeatedExact": true, + "screenCount": 15, + "inputSetHashEncoding": "SHA-256 of UTF-8 json.dumps(pathToSha256, sort_keys=True, separators=(comma, colon)); default ensure_ascii=True", + "oldAboutThemeSha256": "d3137cabf5a5692a22406f361a00a85122aecf5d1e24929caaebaae92200c434", + "aboutThemeDifferences": [ + { + "path": "/theme/typography/bodyLg/0/lineHeight", + "old": 1.6, + "current": "26px" + }, + { + "path": "/theme/typography/bodyLg/3/lineHeight", + "old": 1.6, + "current": "29px" + }, + { + "path": "/theme/typography/bodyLgBold/0/lineHeight", + "old": 1.6, + "current": "26px" + }, + { + "path": "/theme/typography/bodyLgBold/3/lineHeight", + "old": 1.6, + "current": "29px" + }, + { + "path": "/theme/typography/bodyXlg/0/lineHeight", + "old": 1.7, + "current": "29px" + }, + { + "path": "/theme/typography/bodyXlg/3/lineHeight", + "old": 1.8, + "current": "36px" + }, + { + "path": "/theme/typography/bodyXlgBold/0/lineHeight", + "old": 1.7, + "current": "29px" + }, + { + "path": "/theme/typography/bodyXlgBold/3/lineHeight", + "old": 1.8, + "current": "36px" + }, + { + "path": "/theme/typography/bodyXs/0/lineHeight", + "old": 1.6, + "current": "19px" + }, + { + "path": "/theme/typography/bodyXs/3/lineHeight", + "old": 1.6, + "current": "21px" + }, + { + "path": "/theme/typography/bodyXsMed/0/lineHeight", + "old": 1.6, + "current": "19px" + }, + { + "path": "/theme/typography/bodyXsMed/3/lineHeight", + "old": 1.6, + "current": "21px" + }, + { + "path": "/theme/typography/button/lineHeight", + "old": 1.4, + "current": "22px" + }, + { + "path": "/theme/typography/caption/0/lineHeight", + "old": 1.6, + "current": "21px" + }, + { + "path": "/theme/typography/caption/3/lineHeight", + "old": 1.6, + "current": "22px" + }, + { + "path": "/theme/typography/captionSemiBold/0/lineHeight", + "old": 1.6, + "current": "21px" + }, + { + "path": "/theme/typography/captionSemiBold/3/lineHeight", + "old": 1.6, + "current": "22px" + }, + { + "path": "/theme/typography/h3/0/lineHeight", + "old": 1.2, + "current": "38px" + }, + { + "path": "/theme/typography/h3/3/lineHeight", + "old": 1.2, + "current": "53px" + }, + { + "path": "/theme/typography/h4/0/lineHeight", + "old": 1.2, + "current": "29px" + }, + { + "path": "/theme/typography/h4/3/lineHeight", + "old": 1.2, + "current": "43px" + }, + { + "path": "/theme/typography/header/lineHeight", + "old": 1.4, + "current": "22px" + }, + { + "path": "/theme/typography/title/0/lineHeight", + "old": 1.4, + "current": "27px" + }, + { + "path": "/theme/typography/title/3/lineHeight", + "old": 1.4, + "current": "34px" + }, + { + "path": "/theme/typography/titleSm/lineHeight", + "old": 1.4, + "current": "24px" + } + ], + "oldPopupReconstruction": { + "lineHeightRatios": { + "buttonSm": 1.2, + "modalBtn": 1.2, + "modalText": 1.8, + "textboxTitle": 1.5 + }, + "sha256": "f26ad027d6dea5dfa4ed10159225888ea2bc6a34325c5612970a8a192d42a9ab", + "encoding": "json.dumps(..., ensure_ascii=False, indent=2) plus LF" + }, + "staleAboutModuleHashes": { + "about-422-2987.tsx": "39121a74184c74e6a82d935ef72726f05920d68eead40666f81cc52d97c96a89", + "about-422-3180.tsx": "34f40844160359c59a9b821f81c7f12e1fd1aef685ec6c2d1b0886e3109a61e8", + "about-422-3376.tsx": "d7d7c28e5830d8dc56c170da296dc407ceb985d77afa4a12e58eb296948c6d67" + }, + "staleAboutMeasuredRatios": { + "about-422-3376": 0.1123611111111111, + "about-422-3180": 0.06920733370075033, + "about-422-2987": 0.043702591794548384 + }, + "runs": [ + { + "round": 1, + "group": "about", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "about-422-3376", + "themeHash": "67d6de70e67967432dcafb51a491b2105f11f5f5024cb4f79e1c8aa29b3503b7", + "changedRatio": 0.07457565991405771, + "environmentStatus": "valid", + "inputFileCount": 84, + "inputSetSha256": "455112c55d7ea512fda77af6cb50eac9506b5a9b195395c991f20c639bba7b6e" + }, + { + "name": "about-422-3180", + "themeHash": "67d6de70e67967432dcafb51a491b2105f11f5f5024cb4f79e1c8aa29b3503b7", + "changedRatio": 0.04063815022762631, + "environmentStatus": "valid", + "inputFileCount": 84, + "inputSetSha256": "157d2c3abbedf766536d67010b6373029ac95cc27ded85ad38e577aac3a2b757" + }, + { + "name": "about-422-2987", + "themeHash": "67d6de70e67967432dcafb51a491b2105f11f5f5024cb4f79e1c8aa29b3503b7", + "changedRatio": 0.024149936935043095, + "environmentStatus": "valid", + "inputFileCount": 84, + "inputSetSha256": "c68bd052a3039b789bd8cfac2f6c6a95dbf7487569fe8c41f2bcbfc929ce7404" + } + ], + "serverPid": 128552 + }, + { + "round": 1, + "group": "popup", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 1, + "screens": [ + { + "name": "popup-422-5682", + "themeHash": "d541f2ae904988f388a0dc3a0aeed6922924ad181e5d0fe266964deedf62e9ae", + "changedRatio": 0.03644230769230769, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "c6d368f7e43753e335831f39d2ccfb476ddc1d6a27552d34a46d3caea28379d2" + }, + { + "name": "popup-422-5705", + "themeHash": "d541f2ae904988f388a0dc3a0aeed6922924ad181e5d0fe266964deedf62e9ae", + "changedRatio": 0.020626068115234375, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "213aa60e6d10d2b7f5660eed60cc534022d58ef9c6df110cf6b3c7decf997cdd" + }, + { + "name": "popup-422-5728", + "themeHash": "d541f2ae904988f388a0dc3a0aeed6922924ad181e5d0fe266964deedf62e9ae", + "changedRatio": 0.00853539737654321, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "eb6840ef841088558d8b07d61d8c9a5c66365ac27ed8a93d4eafd6b043887aba" + } + ], + "serverPid": 18020 + }, + { + "round": 1, + "group": "landing", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "landing-833-3640", + "themeHash": "e18d9d7e25b4e288f369d28230c89f2e6a6da90e9275134b667bf79be7844134", + "changedRatio": 0.049872156420379773, + "environmentStatus": "valid", + "inputFileCount": 21, + "inputSetSha256": "0ea4fbc5c41bd7db5b7f9f81754a56ecb817d640cc33469563a517e48492988e" + }, + { + "name": "landing-833-3322", + "themeHash": "87ef9f58fdb853cb6603a7704dc4b685e4af4242eb3e0b2058e3fbbd968c011d", + "changedRatio": 0.024662550063123068, + "environmentStatus": "valid", + "inputFileCount": 21, + "inputSetSha256": "88f11ae5be1dd08b466cd97b7975a48018ede52f6cfc74388d41c8026e35bc76" + }, + { + "name": "landing-832-2975", + "themeHash": "3ae50e9a165f5060e6c977a6c6b610a4238192437492acd7de908a65c2274018", + "changedRatio": 0.015037458117163857, + "environmentStatus": "valid", + "inputFileCount": 25, + "inputSetSha256": "05eb15fe88b44529e5f7beb631ee2537014a5cb20a7020e78eef895eee6cfec8" + } + ], + "serverPid": 56456 + }, + { + "round": 1, + "group": "grid", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "grid-429-1966", + "themeHash": "a1a8437993a02c99489dfdc71ca7e2f5d6447f4956600fd939272816bc0f3f63", + "changedRatio": 0.029639884816046395, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "6d9883f69032813e8a8f59187d78d08ca65ada83ab3f3fdab45d1aa6512fd3d9" + } + ], + "serverPid": 118464 + }, + { + "round": 1, + "group": "keyframes", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "keyframes-458-2021", + "themeHash": "a1a8437993a02c99489dfdc71ca7e2f5d6447f4956600fd939272816bc0f3f63", + "changedRatio": 0.067138671875, + "environmentStatus": "valid", + "inputFileCount": 4, + "inputSetSha256": "b568c9fe380c1a6688ca4bc4e629647eb4194b92643c5c2b8ddc0e07e6493938" + } + ], + "serverPid": 55156 + }, + { + "round": 1, + "group": "report", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "report-446-1971", + "themeHash": "46d713e0a8523f77d004ec78a2997810a6bf58d1e380a13bcda9de96388fd56a", + "changedRatio": 0.012764674159854678, + "environmentStatus": "valid", + "inputFileCount": 8, + "inputSetSha256": "a535185c9db11cc7e749e45e15fa67fc4bff52705fdd78bb027f63b9148fa276" + } + ], + "serverPid": 49184 + }, + { + "round": 1, + "group": "notice", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 1, + "screens": [ + { + "name": "notice-422-6914", + "themeHash": "89fe4447f2addc8a19460a819ce4d81a19e14b96cee73d5692d10b1bfdfd5b2f", + "changedRatio": 0.05541838134430727, + "environmentStatus": "valid", + "inputFileCount": 32, + "inputSetSha256": "360f16c33fcfe2c15427b459c4b270183e74ec8137e0b4163961619aa813d7f3" + }, + { + "name": "notice-422-7088", + "themeHash": "89fe4447f2addc8a19460a819ce4d81a19e14b96cee73d5692d10b1bfdfd5b2f", + "changedRatio": 0.033521234676007004, + "environmentStatus": "valid", + "inputFileCount": 32, + "inputSetSha256": "c113d21fb0456670b96dd758acc81ea601124d5304c9321d856da03ce0793af0" + }, + { + "name": "notice-422-6865", + "themeHash": "89fe4447f2addc8a19460a819ce4d81a19e14b96cee73d5692d10b1bfdfd5b2f", + "changedRatio": 0.022197898423817863, + "environmentStatus": "valid", + "inputFileCount": 32, + "inputSetSha256": "6ec0926e7e5bff96d997d6a4a14ceaaa6ed8349aa99be9b041754c3bde37ae59" + } + ], + "serverPid": 123260 + }, + { + "round": 2, + "group": "about", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "about-422-3376", + "themeHash": "67d6de70e67967432dcafb51a491b2105f11f5f5024cb4f79e1c8aa29b3503b7", + "changedRatio": 0.07457565991405771, + "environmentStatus": "valid", + "inputFileCount": 84, + "inputSetSha256": "455112c55d7ea512fda77af6cb50eac9506b5a9b195395c991f20c639bba7b6e" + }, + { + "name": "about-422-3180", + "themeHash": "67d6de70e67967432dcafb51a491b2105f11f5f5024cb4f79e1c8aa29b3503b7", + "changedRatio": 0.04063815022762631, + "environmentStatus": "valid", + "inputFileCount": 84, + "inputSetSha256": "157d2c3abbedf766536d67010b6373029ac95cc27ded85ad38e577aac3a2b757" + }, + { + "name": "about-422-2987", + "themeHash": "67d6de70e67967432dcafb51a491b2105f11f5f5024cb4f79e1c8aa29b3503b7", + "changedRatio": 0.024149936935043095, + "environmentStatus": "valid", + "inputFileCount": 84, + "inputSetSha256": "c68bd052a3039b789bd8cfac2f6c6a95dbf7487569fe8c41f2bcbfc929ce7404" + } + ], + "serverPid": 67468 + }, + { + "round": 2, + "group": "popup", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 1, + "screens": [ + { + "name": "popup-422-5682", + "themeHash": "d541f2ae904988f388a0dc3a0aeed6922924ad181e5d0fe266964deedf62e9ae", + "changedRatio": 0.03644230769230769, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "c6d368f7e43753e335831f39d2ccfb476ddc1d6a27552d34a46d3caea28379d2" + }, + { + "name": "popup-422-5705", + "themeHash": "d541f2ae904988f388a0dc3a0aeed6922924ad181e5d0fe266964deedf62e9ae", + "changedRatio": 0.020626068115234375, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "213aa60e6d10d2b7f5660eed60cc534022d58ef9c6df110cf6b3c7decf997cdd" + }, + { + "name": "popup-422-5728", + "themeHash": "d541f2ae904988f388a0dc3a0aeed6922924ad181e5d0fe266964deedf62e9ae", + "changedRatio": 0.00853539737654321, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "eb6840ef841088558d8b07d61d8c9a5c66365ac27ed8a93d4eafd6b043887aba" + } + ], + "serverPid": 94264 + }, + { + "round": 2, + "group": "landing", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "landing-833-3640", + "themeHash": "e18d9d7e25b4e288f369d28230c89f2e6a6da90e9275134b667bf79be7844134", + "changedRatio": 0.049872156420379773, + "environmentStatus": "valid", + "inputFileCount": 21, + "inputSetSha256": "0ea4fbc5c41bd7db5b7f9f81754a56ecb817d640cc33469563a517e48492988e" + }, + { + "name": "landing-833-3322", + "themeHash": "87ef9f58fdb853cb6603a7704dc4b685e4af4242eb3e0b2058e3fbbd968c011d", + "changedRatio": 0.024662550063123068, + "environmentStatus": "valid", + "inputFileCount": 21, + "inputSetSha256": "88f11ae5be1dd08b466cd97b7975a48018ede52f6cfc74388d41c8026e35bc76" + }, + { + "name": "landing-832-2975", + "themeHash": "3ae50e9a165f5060e6c977a6c6b610a4238192437492acd7de908a65c2274018", + "changedRatio": 0.015037458117163857, + "environmentStatus": "valid", + "inputFileCount": 25, + "inputSetSha256": "05eb15fe88b44529e5f7beb631ee2537014a5cb20a7020e78eef895eee6cfec8" + } + ], + "serverPid": 78716 + }, + { + "round": 2, + "group": "grid", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "grid-429-1966", + "themeHash": "a1a8437993a02c99489dfdc71ca7e2f5d6447f4956600fd939272816bc0f3f63", + "changedRatio": 0.029639884816046395, + "environmentStatus": "valid", + "inputFileCount": 10, + "inputSetSha256": "6d9883f69032813e8a8f59187d78d08ca65ada83ab3f3fdab45d1aa6512fd3d9" + } + ], + "serverPid": 87388 + }, + { + "round": 2, + "group": "keyframes", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "keyframes-458-2021", + "themeHash": "a1a8437993a02c99489dfdc71ca7e2f5d6447f4956600fd939272816bc0f3f63", + "changedRatio": 0.067138671875, + "environmentStatus": "valid", + "inputFileCount": 4, + "inputSetSha256": "b568c9fe380c1a6688ca4bc4e629647eb4194b92643c5c2b8ddc0e07e6493938" + } + ], + "serverPid": 79944 + }, + { + "round": 2, + "group": "report", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 0, + "screens": [ + { + "name": "report-446-1971", + "themeHash": "46d713e0a8523f77d004ec78a2997810a6bf58d1e380a13bcda9de96388fd56a", + "changedRatio": 0.012764674159854678, + "environmentStatus": "valid", + "inputFileCount": 8, + "inputSetSha256": "a535185c9db11cc7e749e45e15fa67fc4bff52705fdd78bb027f63b9148fa276" + } + ], + "serverPid": 20548 + }, + { + "round": 2, + "group": "notice", + "binarySha256": "1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133", + "acquireExit": 0, + "renderExit": 1, + "screens": [ + { + "name": "notice-422-6914", + "themeHash": "89fe4447f2addc8a19460a819ce4d81a19e14b96cee73d5692d10b1bfdfd5b2f", + "changedRatio": 0.05541838134430727, + "environmentStatus": "valid", + "inputFileCount": 32, + "inputSetSha256": "360f16c33fcfe2c15427b459c4b270183e74ec8137e0b4163961619aa813d7f3" + }, + { + "name": "notice-422-7088", + "themeHash": "89fe4447f2addc8a19460a819ce4d81a19e14b96cee73d5692d10b1bfdfd5b2f", + "changedRatio": 0.033521234676007004, + "environmentStatus": "valid", + "inputFileCount": 32, + "inputSetSha256": "c113d21fb0456670b96dd758acc81ea601124d5304c9321d856da03ce0793af0" + }, + { + "name": "notice-422-6865", + "themeHash": "89fe4447f2addc8a19460a819ce4d81a19e14b96cee73d5692d10b1bfdfd5b2f", + "changedRatio": 0.022197898423817863, + "environmentStatus": "valid", + "inputFileCount": 32, + "inputSetSha256": "6ec0926e7e5bff96d997d6a4a14ceaaa6ed8349aa99be9b041754c3bde37ae59" + } + ], + "serverPid": 92056 + } + ], + "combinedRender": { + "exit": 0, + "screens": 15, + "matchesRepeatedMetrics": true + } +} diff --git a/docs/harness-determinism.md b/docs/harness-determinism.md new file mode 100644 index 0000000..a512d9e --- /dev/null +++ b/docs/harness-determinism.md @@ -0,0 +1,200 @@ +# Render input identity and reproducibility + +Base: `d039623b0d1a`; investigation on 2026-09-13. No generator, theme generator, +provenance, plugin corpus, or golden files were changed. + +## Root cause and causal reproduction + +The two reported theme hashes identify **different generations of saved inputs**. +They do not demonstrate nondeterministic theme generation by the current binary. +The harness previously trusted files already present under `themes/`, `src/screens/` +and `out/`, without recording which binary produced them or checking their bytes. +`render.mjs` does not invoke the generator: pointing `DEVUP_MCP_BIN` at a current +binary did not establish that the files being rendered came from that binary. + +The exact `d3137cabf5a5` about theme survives in the W11 worktree. Comparing it +with `67d6de70e679` gives precisely 25 changes, all typography `lineHeight` values: +unitless ratios such as `1.6` become rounded pixel strings such as `"26px"`. +There are no changed token names, selected brands, colours, or resource ordering. +Commit `757dc4a` changed that exact percentage conversion in the theme generator. + +For popup, reversing only the same conversion in the current theme reproduces +`f26ad027d6de` exactly, including the serializer's final newline. The twelve +slots use `1.2` for `buttonSm`/`modalBtn`, `1.8` for `modalText`, and `1.5` for +`textboxTitle`; current output is `d541f2ae9049` with pixel line heights. + +The asymmetry is explained by percentage-based typography in about and popup. +Grid and keyframes have no typography entries; landing already uses pixel line +heights. The historical conversion therefore changes only the former hashes. + +Controlled reproduction in the main harness, with no binary or generator edits: + +| About inputs | Theme hash | Mobile / tablet / desktop divergence | +| --- | --- | --- | +| Current modules and current theme | `67d6de70e679` | 7.46 / 4.06 / 2.41 | +| Current modules and saved W11 theme | `d3137cabf5a5` | 7.46 / 4.06 / 2.41 | +| Saved W11 modules and saved W11 theme | `d3137cabf5a5` | **11.24 / 6.92 / 4.37** | + +The final row reproduces all three Session A figures exactly. Theme replacement +alone does not recreate them because current TSX explicitly writes pixel line +heights. This also eliminates a stale Vite theme cache as the explanation for +those absolute values. Saved module bytes matter as well as theme bytes. + +The historical filesystem operation that put old bytes into the main harness +was not logged, so this report does not invent one. What is proven is the exact +old input identity, its ability to recreate Session A, and the missing harness +validation that allowed those inputs to be attributed to a different generator. + +## Requested leads + +1. **Call bank:** `CallCache::path_for` computes a single filename from the tool + name and sorted argument keys. It directly reads that path; it neither scans + candidates nor elects among multiple responses. All 605 initial entries were + inside their 48-hour TTL. Bank growth does not select another matching entry. + The bank remains a live capture cache, not an immutable design fixture; a + future expired or changed upstream response can legitimately change inputs. +2. **Node scope/history:** acquisition still requests node scope. The old/new + theme diff changes no used token set. The exact hashes are explained by the + historical percentage conversion, without a history-dependent resource set. +3. **Group leader:** targets are traversed in manifest order and grouped by theme + content hash. All three about themes are byte-identical; popup explicitly + shares the first frame's responsive module/theme. No leader race was found. +4. **Accumulated artifacts:** this is the demonstrated unsafe boundary. An old + manifest could render old generated files without any acquisition identity. + A separate failing test proves that a refused acquisition would even append + a target when an old snapshot/module/theme/reference happened to exist. + +## Harness changes + +Acquisition removes only the explicit outputs it is about to request, saves the +full server response, checks completion status and required files, and records +failed frames as skipped while continuing later frames. Missing asset files are +also skipped. The existing shared asset-path cache is preserved; responsive +followers require a successfully acquired leader. + +Each target records the actual server identity, executable SHA-256, and SHA-256 +of its module, theme, reference PNG, raw snapshot and required assets. Rendering +validates those inputs before building, compares the binary hash when +`DEVUP_MCP_BIN` is supplied, and refuses a missing explicit theme instead of +falling back to another file. Old manifests require reacquisition. Hand-authored +`*-answer` comparison screens retain their existing separate workflow. + +The render report includes acquisition evidence and the full theme hash, and +reports skipped acquisitions with a failing overall exit status. No thresholds +are used to hide an invalid environment or missing input. + +Regression tests were run red first: missing outputs raised `FileNotFoundError`, +stale outputs were incorrectly appended to the manifest, and old/replaced input +validation lacked the expected rejection. Those tests pass after the fix. +The four pre-existing pyright errors were fixed in touched code: subprocess +stream assertions and guarded stdout reconfiguration; acquire.py is now clean. + +## Repeated measurements and gates + +| Run | Group | Server PID | Theme hash | Divergence percentages | +| --- | --- | ---: | --- | --- | +| 1 | about | 128552 | `67d6de70e679` | 7.46 / 4.06 / 2.41 | +| 2 | about | 67468 | `67d6de70e679` | 7.46 / 4.06 / 2.41 | +| 1 | popup | 18020 | `d541f2ae9049` | 3.64 / 2.06 / 0.85 | +| 2 | popup | 94264 | `d541f2ae9049` | 3.64 / 2.06 / 0.85 | + +The control and previously failing screens also repeat exactly: + +| Screen | Theme hash | Round 1 | Round 2 | +| --- | --- | ---: | ---: | +| landing-833-3640 | `e18d9d7e25b4` | 4.99 | 4.99 | +| landing-833-3322 | `87ef9f58fdb8` | 2.47 | 2.47 | +| landing-832-2975 | `3ae50e9a165f` | 1.50 | 1.50 | +| grid-429-1966 | `a1a8437993a0` | 2.96 | 2.96 | +| keyframes-458-2021 | `a1a8437993a0` | 6.71 | 6.71 | +| report-446-1971 | `46d713e0a852` | 1.28 | 1.28 | +| notice-422-6914 | `89fe4447f2ad` | 5.54 | 5.54 | +| notice-422-7088 | `89fe4447f2ad` | 3.35 | 3.35 | +| notice-422-6865 | `89fe4447f2ad` | 2.22 | 2.22 | + +All 15 screens were acquired and measured in both rounds, with zero skipped +screens or missing assets. Report and all three notice frames returned +`status=partial`, `quality.acquisition=complete`, and lossy projection; there is +no acquisition blocker in this tested environment. Their original refusal was +not reproduced, so no unsupported historical refusal cause is asserted. + +`thresholds.json` was updated **only after both rounds matched**; its generated- +screen numbers are now backed by repeated measurements. This includes increases +where the old unsupported number was lower. These are corrected measurement +baselines, not claimed generator improvements. The plugin-answer thresholds +remain unchanged and are explicitly not claimed as remeasured. + +A final combined render of all 15 screens passed the updated thresholds with +exit zero and exactly the same metrics. Temporary copies of the changed harness +scripts and thresholds in the main checkout were restored afterward; generated +inputs and ignored evidence remain there. No task-owned server process remained +after the runs. + +Full hashes, PIDs, exact ratios and the old-theme diff are retained in +[harness-determinism-evidence.json](harness-determinism-evidence.json). Raw run +logs and reports are in main `harness/render/out/w15-round{1,2}--*`. + +Every group in each round uses a new `acquire.py` process, immediately followed +by a new `render.mjs` process for that group, in the main checkout. The executable +is held constant and hashed before every acquisition and after both rounds: +`1c511f2d863eb092ec9301425cc1e1228bdf8d718786a4d0ad64316e9180f133`. +The second round checks equality of every recorded input hash, the full theme +hash and the unrounded divergence value, not just the displayed percentages. + +All Cargo gates ran against this worktree's own target, with +`CARGO_PROFILE_DEV_DEBUG=0`, `CARGO_PROFILE_TEST_DEBUG=0`, `CARGO_INCREMENTAL=0`, +and no `CARGO_TARGET_DIR` override. + +| Gate | Result | +| --- | --- | +| `cargo fmt --all -- --check` | Pass | +| `cargo clippy --locked --workspace --all-targets --all-features -j 2 -- -D warnings` | Pass, zero warnings | +| `cargo test --workspace -j 2 --no-fail-fast` | 1,059 passed, zero failed, two ignored | +| `cargo insta test --workspace --all-features --check` | Same counts; no snapshots to review | +| `cargo test --locked -p devup-mcp --test stdio_smoke` | Two passed | +| `node --test crates/devup-mcp-figma/tests/explore_script_behavior.mjs` | 13 passed | +| Python acquisition regression tests | Two passed | +| Node input identity regression tests | Two passed | +| `npx --yes pyright harness/render/scripts/acquire.py` | Zero errors or warnings | + +An initial workspace test build collided with the still-running probe's Windows +executable lock; it was rerun after that probe exited. The successful test and +snapshot builds emitted Windows linker informational-output warnings; the +required clippy command emitted zero warnings. No lint suppression was added. + +To repeat a group after installing these harness changes: + +```powershell +cd C:\Users\owjs3\Desktop\projects\devup-mcp\harness\render +$env:DEVUP_MCP_BIN = "\target\debug\devup-mcp.exe" +python scripts\acquire.py about +node scripts\render.mjs about +``` + +Repeat those two commands in another process, and use `popup` for the other +required group. For migration from an old manifest, acquire all groups once; +unproven saved inputs intentionally require reacquisition. A changed upstream +design or expired call bank is a changed input set, not a guarantee of matching +an older design's pixels. + +## Which earlier claims this supersedes + +The Session A about values are invalid **as a measurement of `d039623`**. They +remain reproducible historical values for the saved older generated inputs. +The Session A popup theme is likewise the older percentage representation. + +In `docs/about-vertical-geometry.md`, the section "The harness does not reproduce +across sessions" incorrectly treats those older hashes as evidence that the same +current generator produced different themes. This report supersedes that +interpretation and the attribution of Session A to the current baseline. +Its `757dc4a` before column (7.44 / 6.90 / 4.19) is a historical pre-weight-fix +measurement, not a baseline for `d039623`; its shipped after column is +7.46 / 4.06 / 2.41. Neither historical about baseline is the current absolute +level. Popup's current values match Session B. + +No claim is made that historical before/after experiments in +`docs/line-box-displacement.md`, `docs/integer-line-advance-verification.md`, or +`docs/line-box-fidelity-investigation.md` are invalid merely because their base +commits have different absolute figures. Their numbers must retain their stated +version context. The former report's final landing baseline is checked again +here. Old unverified current thresholds are replaced only after repetition. diff --git a/harness/render/scripts/acquire.py b/harness/render/scripts/acquire.py index 90f214b..ec81064 100644 --- a/harness/render/scripts/acquire.py +++ b/harness/render/scripts/acquire.py @@ -31,11 +31,13 @@ """ import json +import hashlib import os import subprocess import sys import threading import time +from pathlib import Path HERE = os.path.dirname(os.path.abspath(__file__)) HARNESS = os.path.dirname(HERE) @@ -105,23 +107,31 @@ class Server: def __init__(self): environment = dict(os.environ) environment["DEVUP_FIGMA_CALL_CACHE"] = BANK + executable = resolve_exe() + self.binary_sha256 = hashlib.sha256(Path(executable).read_bytes()).hexdigest() self.proc = subprocess.Popen( - [resolve_exe()], cwd=HARNESS, env=environment, + [executable], cwd=HARNESS, env=environment, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding="utf-8", bufsize=1, ) self.lines = [] self.next_id = 1 threading.Thread(target=self._read, daemon=True).start() - self.call_raw("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, - "clientInfo": {"name": "render-harness", "version": "0"}}) - self._send({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}) + try: + self.call_raw("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "render-harness", "version": "0"}}) + self._send({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}) + except BaseException: + self.close() + raise def _read(self): + assert self.proc.stdout is not None for line in self.proc.stdout: self.lines.append(line) def _send(self, message): + assert self.proc.stdin is not None self.proc.stdin.write(json.dumps(message) + "\n") self.proc.stdin.flush() @@ -144,9 +154,9 @@ def call_raw(self, method, params, limit=1800): if message.get("id") == request_id: return message if self.proc.poll() is not None: - raise SystemExit("devup-mcp exited") + raise RuntimeError("devup-mcp exited") time.sleep(0.2) - raise SystemExit(f"no response to {method} within {limit}s") + raise RuntimeError(f"no response to {method} within {limit}s") def export(self, arguments, allow_error=False): """One export, polled to completion. @@ -175,11 +185,20 @@ def _export_call(self, arguments, allow_error): if "error" in response: message = response["error"].get("message") if allow_error: - return {"error": message} - raise SystemExit(f"export refused: {message}") + return {"error": response["error"]} + raise RuntimeError(f"export refused: {message}") + if response["result"].get("isError"): + if allow_error: + return {"error": response["result"]} + raise RuntimeError(f"export refused: {json.dumps(response['result'], ensure_ascii=False)}") text = "".join(part.get("text", "") for part in response["result"].get("content", []) if part.get("type") == "text") - return json.loads(text) + try: + return json.loads(text) + except json.JSONDecodeError: + if allow_error: + return {"error": {"message": "export returned non-JSON content", "response": response["result"]}} + raise RuntimeError(f"export returned non-JSON content: {text}") def _settle(self, body, allow_error, limit=3600): """Poll `jobId` until the top-level status is no longer `in_progress`. @@ -195,12 +214,12 @@ def _settle(self, body, allow_error, limit=3600): job = body.get("exportJob") or body.get("assetJob") or {} job_id = job.get("jobId") if not job_id: - raise SystemExit( + raise RuntimeError( "export reported in_progress without a jobId: " + json.dumps(body)[:400] ) if time.time() > deadline: - raise SystemExit(f"export job {job_id} did not settle within {limit}s") + raise RuntimeError(f"export job {job_id} did not settle within {limit}s") poll = {"jobId": job_id} next_action = (job.get("nextAction") or {}).get("arguments") or {} if next_action.get("jobAction") == "resume": @@ -262,117 +281,166 @@ def acquire_theme(server, frame): def acquire(server, name, target, manifest): + for index, frame in enumerate(target["frames"]): + try: + _acquire_frame(server, name, target, manifest, index, frame) + except (RuntimeError, ValueError, OSError, KeyError, TypeError, AttributeError) as error: + module_name = name if target["output"] == "responsiveTsx" else f"{name}-{frame.replace(':', '-')}" + manifest.setdefault("skipped", []).append({"name": module_name, "screen": module_name, + "frame": frame, "reason": str(error), "responsePath": None}) + print(f" {frame}: skipped: {type(error).__name__}: {error}", flush=True) + + +def _acquire_frame(server, name, target, manifest, index, frame): output = target["output"] - frames = target["frames"] - for index, frame in enumerate(frames): - module_name = name if output == "responsiveTsx" else f"{name}-{frame.replace(':', '-')}" - module_path = f"src/screens/{module_name}.tsx" - reference_path = f"out/{module_name}-{frame.replace(':', '-')}.reference.png" if output == "responsiveTsx" else f"out/{module_name}.reference.png" - wants_module = output == "tsx" or index == 0 - scratch = f"out/{module_name}-{frame.replace(':', '-')}" - # Every output goes to a file: a file has no size limit, where an - # inline answer is capped at 1 MiB and a larger one is delivered as - # resources this does not read. - outputs = ["referencePng", "rawSnapshot"] - paths = {"referencePng": reference_path, "rawSnapshot": f"{scratch}.snapshot.json"} - theme_path = f"themes/{module_name}.json" - if wants_module: - outputs.append(output) - paths[output] = module_path - # The screen's own theme, at node scope. A file-scope theme holds - # every collection in the file, and this file holds several - # brands: more than one defines `primary`, so one of them wins and - # the rest render in the wrong brand's colour - the notice screen - # came out violet where Figma draws it blue. Scoped to the node, - # the same token resolves to that screen's own value and nothing - # conflicts. - outputs.append("devupJson") - paths["devupJson"] = theme_path - # rawSnapshot describes the design rather than the screen, so it needs - # debug: true. This harness is the case that flag is for - it compares - # what the browser drew against what the design says. + module_name = name if output == "responsiveTsx" else f"{name}-{frame.replace(':', '-')}" + module_path = f"src/screens/{module_name}.tsx" + reference_path = f"out/{module_name}-{frame.replace(':', '-')}.reference.png" if output == "responsiveTsx" else f"out/{module_name}.reference.png" + wants_module = output == "tsx" or index == 0 + scratch = f"out/{module_name}-{frame.replace(':', '-')}" + # Every output goes to a file: a file has no size limit, where an + # inline answer is capped at 1 MiB and a larger one is delivered as + # resources this does not read. + outputs = ["referencePng", "rawSnapshot"] + paths = {"referencePng": reference_path, "rawSnapshot": f"{scratch}.snapshot.json"} + theme_path = f"themes/{module_name}.json" + if wants_module: + outputs.append(output) + paths[output] = module_path + # The screen's own theme, at node scope. A file-scope theme holds + # every collection in the file, and this file holds several + # brands: more than one defines `primary`, so one of them wins and + # the rest render in the wrong brand's colour - the notice screen + # came out violet where Figma draws it blue. Scoped to the node, + # the same token resolves to that screen's own value and nothing + # conflicts. + outputs.append("devupJson") + paths["devupJson"] = theme_path + # rawSnapshot describes the design rather than the screen, so it needs + # debug: true. This harness is the case that flag is for - it compares + # what the browser drew against what the design says. + # A failed write must not turn last session's files into this session's + # measurement. Remove only the explicit outputs about to be requested. + for path in paths.values(): + Path(HARNESS, path).unlink(missing_ok=True) + try: body = server.export({"url": url_for(target, frame), "outputs": outputs, "scope": "node", - "outputPaths": paths, "includeDiagnostics": True, "debug": True}) - print(f" {frame}: status={body.get('status')} quality={body.get('quality')}", flush=True) - # The module refers to assets by layer name; the manifest by node id. - with open(os.path.join(HARNESS, paths["rawSnapshot"]), encoding="utf-8") as handle: - snapshot = json.load(handle) - # The manifest is the one output that cannot be written to a file; - # alone it is small enough to arrive inline. - asset_manifest = server.export({"url": url_for(target, frame), "outputs": ["assetManifest"], "scope": "node", - "delivery": "inline"}).get("assetManifest") or {} - nodes = snapshot.get("nodes") or {} - requests = [] - for asset in asset_manifest.get("assets", []): - if asset.get("status") != "available": - continue - # The manifest says where the code refers to the asset; the - # fallback re-derives it from the layer name for a manifest that - # does not. - if asset.get("path"): - path = "public" + asset["path"] - fmt = "svg" if path.endswith(".svg") else "png" - else: - node = nodes.get(asset["nodeId"]) or {} - layer = (node.get("fields") or {}).get("name") or "Asset" - path, fmt = asset_path(layer, asset) - # Done when the file is there; a record without the file is stale. - if path in manifest["assets"] and os.path.exists(os.path.join(HARNESS, path)): - continue - manifest["assets"][path] = asset["assetId"] - requests.append({"assetId": asset["assetId"], "format": fmt, "scale": 1, "outputPath": path}) - # The server caps a call at 6 asset requests and recommends 3, so a - # batch of 16 was refused every single time: each batch paid one - # guaranteed-failing round trip before the one-at-a-time fallback below - # did the actual work. The fallback meant nothing was lost, which is why - # this went unnoticed - the cost was a wasted call per batch, not a - # missing asset. Batching at the recommended size makes the happy path - # actually succeed. - for start in range(0, len(requests), ASSET_BATCH): - batch = requests[start:start + ASSET_BATCH] - # An artifact holds only the assets its own collection exported, - # so the bytes are asked for by URL; the node reads are replayed - # from the bank and only the export itself is new. - # The bytes go to their files whatever the delivery; the answer - # itself may be too large to inline, and is not read. - # `refresh`: the collection the process cached for this URL holds - # no exports, and the server asks for one that does. The node - # reads replay from the bank; only the export itself is new. - body = server.export({"url": url_for(target, frame), "outputs": ["assetManifest"], "scope": "node", - "assetRequests": batch, "refresh": True}, allow_error=True) - if body.get("error"): - # One node the server will not export refuses the whole call, - # and the other fifteen are lost with it. Asked for one at a - # time, the refusal is confined to the node that caused it and - # says which one that is. - print(f" assets {start + 1}-{start + len(batch)}: {body['error']}", flush=True) - refused = [] - for entry in batch: - one = server.export({"url": url_for(target, frame), "outputs": ["assetManifest"], "scope": "node", - "assetRequests": [entry], "refresh": True}, allow_error=True) - if one.get("error"): - refused.append(entry["assetId"]) - manifest["assets"].pop(entry["outputPath"], None) - if refused: - print(f" refused: {', '.join(refused)}", flush=True) - body = {} - missing = [entry["outputPath"] for entry in batch if not os.path.exists(os.path.join(HARNESS, entry["outputPath"]))] - print(f" assets {start + 1}-{start + len(batch)}: quality={body.get('quality', {}).get('assets')} missing={missing}", flush=True) - manifest["targets"].append({ - # One entry per frame; a responsive module is rendered once per - # frame, at that frame's size, so the entries share a screen. - "name": f"{module_name}-{frame.replace(':', '-')}" if output == "responsiveTsx" else module_name, - "screen": module_name, - "module": module_path, - "frame": frame, - "reference": reference_path, - "responsive": output == "responsiveTsx", - "theme": theme_path, - }) + "outputPaths": paths, "includeDiagnostics": True, "debug": True}, allow_error=True) + except (RuntimeError, ValueError, OSError) as error: + body = {"error": str(error)} + response_path = f"{scratch}.response.json" + Path(HARNESS, response_path).parent.mkdir(parents=True, exist_ok=True) + Path(HARNESS, response_path).write_text(json.dumps(body, ensure_ascii=False, indent=2), encoding="utf-8") + print(f" {frame}: status={body.get('status')} quality={body.get('quality')}", flush=True) + missing = [path for path in paths.values() if not Path(HARNESS, path).is_file()] + # Responsive reference frames also require their successfully acquired + # leader's module and theme; an old leader cannot rescue a failed one. + missing.extend(path for path in (module_path, theme_path) + if path not in paths.values() and not Path(HARNESS, path).is_file()) + if body.get("status") not in ("complete", "partial") or missing: + skipped = {"name": module_name, "screen": module_name, "frame": frame, + "reason": "export did not produce the required inputs", "missing": missing, + "response": body, "responsePath": response_path} + manifest.setdefault("skipped", []).append(skipped) + print(f" skipped: missing={missing}; server response={json.dumps(body, ensure_ascii=False)[:2000]}; full response: {response_path}", flush=True) + return + export_identity = body.get("server") + # The module refers to assets by layer name; the manifest by node id. + with open(os.path.join(HARNESS, paths["rawSnapshot"]), encoding="utf-8") as handle: + snapshot = json.load(handle) + # The manifest is the one output that cannot be written to a file; + # alone it is small enough to arrive inline. + asset_manifest = server.export({"url": url_for(target, frame), "outputs": ["assetManifest"], "scope": "node", + "delivery": "inline"}).get("assetManifest") or {} + nodes = snapshot.get("nodes") or {} + requests = [] + asset_paths = [] + for asset in asset_manifest.get("assets", []): + if asset.get("status") != "available": + continue + # The manifest says where the code refers to the asset; the + # fallback re-derives it from the layer name for a manifest that + # does not. + if asset.get("path"): + path = "public" + asset["path"] + fmt = "svg" if path.endswith(".svg") else "png" + else: + node = nodes.get(asset["nodeId"]) or {} + layer = (node.get("fields") or {}).get("name") or "Asset" + path, fmt = asset_path(layer, asset) + # Done when the file is there; a record without the file is stale. + asset_paths.append(path) + if path in manifest["assets"] and os.path.exists(os.path.join(HARNESS, path)): + continue + manifest["assets"][path] = asset["assetId"] + requests.append({"assetId": asset["assetId"], "format": fmt, "scale": 1, "outputPath": path}) + # The server caps a call at 6 asset requests and recommends 3, so a + # batch of 16 was refused every single time: each batch paid one + # guaranteed-failing round trip before the one-at-a-time fallback below + # did the actual work. The fallback meant nothing was lost, which is why + # this went unnoticed - the cost was a wasted call per batch, not a + # missing asset. Batching at the recommended size makes the happy path + # actually succeed. + for start in range(0, len(requests), ASSET_BATCH): + batch = requests[start:start + ASSET_BATCH] + # An artifact holds only the assets its own collection exported, + # so the bytes are asked for by URL; the node reads are replayed + # from the bank and only the export itself is new. + # The bytes go to their files whatever the delivery; the answer + # itself may be too large to inline, and is not read. + # `refresh`: the collection the process cached for this URL holds + # no exports, and the server asks for one that does. The node + # reads replay from the bank; only the export itself is new. + body = server.export({"url": url_for(target, frame), "outputs": ["assetManifest"], "scope": "node", + "assetRequests": batch, "refresh": True}, allow_error=True) + if body.get("error"): + # One node the server will not export refuses the whole call, + # and the other fifteen are lost with it. Asked for one at a + # time, the refusal is confined to the node that caused it and + # says which one that is. + print(f" assets {start + 1}-{start + len(batch)}: {body['error']}", flush=True) + refused = [] + for entry in batch: + one = server.export({"url": url_for(target, frame), "outputs": ["assetManifest"], "scope": "node", + "assetRequests": [entry], "refresh": True}, allow_error=True) + if one.get("error"): + refused.append(entry["assetId"]) + manifest["assets"].pop(entry["outputPath"], None) + if refused: + print(f" refused: {', '.join(refused)}", flush=True) + body = {} + missing = [entry["outputPath"] for entry in batch if not os.path.exists(os.path.join(HARNESS, entry["outputPath"]))] + print(f" assets {start + 1}-{start + len(batch)}: quality={(body.get('quality') or {}).get('assets')} missing={missing}", flush=True) + missing_assets = [path for path in asset_paths if not Path(HARNESS, path).is_file()] + if missing_assets: + manifest.setdefault("skipped", []).append({"name": module_name, "screen": module_name, + "frame": frame, "reason": "asset export left missing inputs", "missing": missing_assets, + "responsePath": response_path}) + print(f" skipped: missing assets={missing_assets}", flush=True) + return + manifest["targets"].append({ + # One entry per frame; a responsive module is rendered once per + # frame, at that frame's size, so the entries share a screen. + "name": f"{module_name}-{frame.replace(':', '-')}" if output == "responsiveTsx" else module_name, + "screen": module_name, + "module": module_path, + "frame": frame, + "reference": reference_path, + "responsive": output == "responsiveTsx", + "theme": theme_path, + "acquisition": { + "binarySha256": getattr(server, "binary_sha256", None), + "server": export_identity, + "responsePath": response_path, + "files": {path: hashlib.sha256(Path(HARNESS, path).read_bytes()).hexdigest() + for path in dict.fromkeys([module_path, theme_path, reference_path, paths["rawSnapshot"], *asset_paths])}, + }, + }) def main(): - sys.stdout.reconfigure(encoding="utf-8", errors="replace") + if hasattr(sys.stdout, "reconfigure"): + getattr(sys.stdout, "reconfigure")(encoding="utf-8", errors="replace") wanted = sys.argv[1:] or list(TARGETS) os.makedirs(os.path.join(HARNESS, "src", "screens"), exist_ok=True) os.makedirs(os.path.join(HARNESS, "public", "icons"), exist_ok=True) @@ -384,7 +452,9 @@ def main(): with open(manifest_path, encoding="utf-8") as handle: manifest = json.load(handle) manifest["targets"] = [t for t in manifest["targets"] if family_of(t.get("screen", t["name"])) not in wanted] + manifest["skipped"] = [t for t in manifest.get("skipped", []) if family_of(t["screen"]) not in wanted] server = Server() + print(f"server: pid={server.proc.pid} binarySha256={server.binary_sha256}", flush=True) try: if not os.path.exists(os.path.join(HARNESS, "devup.json")) or "theme" in wanted: wanted = [name for name in wanted if name != "theme"] @@ -397,6 +467,8 @@ def main(): with open(manifest_path, "w", encoding="utf-8") as handle: json.dump(manifest, handle, ensure_ascii=False, indent=2) print(f"targets: {len(manifest['targets'])}, assets: {len(manifest['assets'])}") + if any(family_of(t["screen"]) in wanted for t in manifest.get("skipped", [])): + raise SystemExit(1) if __name__ == "__main__": diff --git a/harness/render/scripts/inputs.mjs b/harness/render/scripts/inputs.mjs new file mode 100644 index 0000000..cc0823a --- /dev/null +++ b/harness/render/scripts/inputs.mjs @@ -0,0 +1,19 @@ +import { createHash } from "node:crypto"; +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +export function validateInputs(directory, target, binarySha256) { + const evidence = target.acquisition; + if (!evidence?.binarySha256 || !evidence.files || !evidence.files[target.theme]) { + throw new Error(`${target.name}: no acquisition evidence; reacquire this screen`); + } + if (binarySha256 && evidence.binarySha256 !== binarySha256) { + throw new Error(`${target.name}: binary differs from acquisition; reacquire this screen`); + } + for (const [path, expected] of Object.entries(evidence.files)) { + const absolute = join(directory, path); + if (!existsSync(absolute)) throw new Error(`${target.name}: ${path} missing; reacquire this screen`); + const actual = createHash("sha256").update(readFileSync(absolute)).digest("hex"); + if (actual !== expected) throw new Error(`${target.name}: ${path} changed since acquisition; reacquire this screen`); + } +} diff --git a/harness/render/scripts/render.mjs b/harness/render/scripts/render.mjs index c6511ba..6c4b971 100644 --- a/harness/render/scripts/render.mjs +++ b/harness/render/scripts/render.mjs @@ -16,6 +16,7 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { chromium } from "playwright"; import { PNG } from "pngjs"; +import { validateInputs } from "./inputs.mjs"; const HARNESS = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const REPO = resolve(HARNESS, "..", ".."); @@ -108,6 +109,10 @@ async function waitForServer(url, attempts = 50) { // at node scope. An answer screen borrows the theme of the screen it answers // for, so the two are compared under the same colours. function themeFor(target) { + if (target.theme) { + if (!existsSync(join(HARNESS, target.theme))) throw new Error(`${target.name}: missing explicit theme ${target.theme}; reacquire`); + return target.theme; + } const screen = target.screen ?? target.name; const candidates = [target.theme, `themes/${screen}.json`, `themes/${screen.replace(/-answer$/, "")}.json`]; return candidates.find((candidate) => candidate && existsSync(join(HARNESS, candidate))) ?? null; @@ -117,7 +122,15 @@ async function main() { const wanted = process.argv.slice(2); const manifest = JSON.parse(readFileSync(join(HARNESS, "targets.json"), "utf8")); const targets = manifest.targets.filter((target) => wanted.length === 0 || wanted.some((name) => target.name === name || target.name.startsWith(`${name}-`))); + const skipped = (manifest.skipped ?? []).filter((target) => wanted.length === 0 || wanted.some((name) => target.name === name || target.name.startsWith(`${name}-`))); + for (const target of skipped) console.error(`${target.name} (${target.frame}): acquisition-skipped ${target.reason}; ${target.responsePath}`); + if (skipped.length) process.exitCode = 1; if (targets.length === 0) throw new Error("no targets; run scripts/acquire.py first"); + const binarySha256 = process.env.DEVUP_MCP_BIN ? createHash("sha256").update(readFileSync(process.env.DEVUP_MCP_BIN)).digest("hex") : null; + for (const target of targets) { + // Hand-authored plugin comparison screens do not come from acquire.py. + if (!(target.screen ?? target.name).endsWith("-answer")) validateInputs(HARNESS, target, binarySha256); + } if (!existsSync(VISUAL)) { const cargo = run("cargo", ["build", "-p", "devup-mcp-visual", "--release"], { cwd: REPO }); @@ -138,7 +151,8 @@ async function main() { } const browser = await chromium.launch(); - const report = []; + const report = skipped.map((target) => ({ name: target.name, frame: target.frame, + environmentStatus: "acquisition-skipped", reason: target.reason, responsePath: target.responsePath })); try { for (const [index, [key, group]] of [...groups].entries()) { if (group.theme) writeFileSync(join(HARNESS, "devup.json"), readFileSync(join(HARNESS, group.theme))); @@ -208,7 +222,7 @@ async function renderGroup(browser, targets, report, port) { await page.goto(`http://localhost:${port}/?screen=${encodeURIComponent(screen)}`, { waitUntil: "networkidle" }); await page.waitForSelector("body[data-ready]", { timeout: 30000 }); const ready = await page.evaluate(() => document.body.dataset.ready); - const entry = { name: target.name, frame: target.frame, viewport: size, reference: target.reference, actual: `out/${target.name}.actual.png` }; + const entry = { name: target.name, frame: target.frame, viewport: size, reference: target.reference, actual: `out/${target.name}.actual.png`, acquisition: target.acquisition, themeHash: target.theme ? createHash("sha256").update(readFileSync(join(HARNESS, target.theme))).digest("hex") : null }; if (ready !== "1" || errors.length > 0) { entry.environmentStatus = "environment-invalid"; entry.errors = errors.concat(ready !== "1" ? [await page.evaluate(() => document.body.dataset.error ?? "not ready")] : []); diff --git a/harness/render/scripts/test_acquire.py b/harness/render/scripts/test_acquire.py new file mode 100644 index 0000000..6cda50c --- /dev/null +++ b/harness/render/scripts/test_acquire.py @@ -0,0 +1,51 @@ +import contextlib +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import acquire + + +class AcquisitionFailures(unittest.TestCase): + def test_refused_frame_is_recorded_and_later_frame_is_acquired(self): + class FakeServer: + def export(self, arguments, allow_error=False): + if "node-id=1-1" in arguments["url"]: + return {"code": "DEVUP_FIXTURE_REFUSED", "message": "capture refused"} + paths = arguments.get("outputPaths", {}) + for kind, path in paths.items(): + destination = Path(acquire.HARNESS, path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text("{}" if kind == "rawSnapshot" else "new", encoding="utf-8") + return {"status": "complete", "quality": {}, "outputPaths": paths} + + with tempfile.TemporaryDirectory() as directory, patch.object(acquire, "HARNESS", directory): + manifest = {"targets": [], "assets": {}} + with contextlib.redirect_stdout(io.StringIO()): + acquire.acquire(FakeServer(), "example", {"output": "tsx", "frames": ["1:1", "1:2"]}, manifest) + self.assertEqual([t["frame"] for t in manifest["targets"]], ["1:2"]) + self.assertEqual(manifest["skipped"][0]["frame"], "1:1") + self.assertIn("capture refused", json.dumps(manifest["skipped"][0])) + + def test_failed_export_cannot_reuse_old_files(self): + class FakeServer: + def export(self, arguments, allow_error=False): + return {"status": "failed", "message": "no output this time"} + + with tempfile.TemporaryDirectory() as directory, patch.object(acquire, "HARNESS", directory): + for relative in ["out/example-1-1-1-1.snapshot.json", "out/example-1-1.reference.png", "themes/example-1-1.json", "src/screens/example-1-1.tsx"]: + path = Path(directory, relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}", encoding="utf-8") + manifest = {"targets": [], "assets": {}} + with contextlib.redirect_stdout(io.StringIO()): + acquire.acquire(FakeServer(), "example", {"output": "tsx", "frames": ["1:1"]}, manifest) + self.assertEqual(manifest["targets"], []) + self.assertEqual(len(manifest["skipped"]), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/render/scripts/test_inputs.mjs b/harness/render/scripts/test_inputs.mjs new file mode 100644 index 0000000..b9d4b15 --- /dev/null +++ b/harness/render/scripts/test_inputs.mjs @@ -0,0 +1,27 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createHash } from "node:crypto"; +import { validateInputs } from "./inputs.mjs"; + +test("a replaced old theme is rejected before rendering", () => { + const directory = mkdtempSync(join(tmpdir(), "harness-inputs-")); + try { + writeFileSync(join(directory, "theme.json"), "old theme"); + const target = { name: "about", theme: "theme.json", acquisition: { + binarySha256: "binary", files: { "theme.json": createHash("sha256").update("new theme").digest("hex") }, + } }; + assert.throws(() => validateInputs(directory, target), /theme.json.*changed/); + writeFileSync(join(directory, "theme.json"), "new theme"); + assert.doesNotThrow(() => validateInputs(directory, target)); + assert.throws(() => validateInputs(directory, target, "different binary"), /binary/); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +test("an old manifest without acquisition evidence requires reacquisition", () => { + assert.throws(() => validateInputs(".", { name: "popup", theme: "themes/popup.json" }), /reacquire/); +}); diff --git a/harness/render/thresholds.json b/harness/render/thresholds.json index 8867733..8069601 100644 --- a/harness/render/thresholds.json +++ b/harness/render/thresholds.json @@ -1,22 +1,22 @@ { - "note": "The most each screen may differ from Figma's own PNG, as a percentage. Written from a measured run; `render.mjs` fails when a screen exceeds its own figure, so a change that makes a screen worse cannot pass unnoticed. Tighten a figure whenever a run comes in under it. The `popup-answer-*` entries are the plugin's own answer rendered the same way - a baseline to compare against, not something to improve.", + "note": "Current generated-screen baselines are backed by two fresh-process acquire/render measurements on d039623, with identical input hashes and exact divergence values; see docs/harness-determinism.md and its evidence JSON. Values are percentages; render.mjs fails above each value plus tolerance. The unchanged popup-answer entries are historical plugin-answer comparisons and were not remeasured in this task.", "tolerance": 0.05, "screens": { - "popup-422-5682": 3.59, - "popup-422-5705": 2.19, - "popup-422-5728": 0.83, + "popup-422-5682": 3.64, + "popup-422-5705": 2.06, + "popup-422-5728": 0.85, "popup-answer-422-5682": 12.02, "popup-answer-422-5705": 7.14, "popup-answer-422-5728": 21.06, "keyframes-458-2021": 6.71, "grid-429-1966": 2.96, - "report-446-1971": 1.87, - "notice-422-6914": 7.35, - "notice-422-7088": 3.24, - "notice-422-6865": 2.15, - "about-422-3376": 11.25, - "about-422-3180": 6.93, - "about-422-2987": 4.42, + "report-446-1971": 1.28, + "notice-422-6914": 5.54, + "notice-422-7088": 3.35, + "notice-422-6865": 2.22, + "about-422-3376": 7.46, + "about-422-3180": 4.06, + "about-422-2987": 2.41, "landing-833-3640": 4.99, "landing-833-3322": 2.47, "landing-832-2975": 1.5