diff --git a/docs/tessar-10x-performance.png b/docs/tessar-10x-performance.png new file mode 100644 index 00000000000..64d767046a2 Binary files /dev/null and b/docs/tessar-10x-performance.png differ diff --git a/docs/tessar-10x-performance.svg b/docs/tessar-10x-performance.svg new file mode 100644 index 00000000000..a855859dd0d --- /dev/null +++ b/docs/tessar-10x-performance.svg @@ -0,0 +1,2947 @@ + + + + + + + + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tessar-benchmark-results.json b/docs/tessar-benchmark-results.json new file mode 100644 index 00000000000..bca3fc518f6 --- /dev/null +++ b/docs/tessar-benchmark-results.json @@ -0,0 +1,4412 @@ +{ + "version": 2, + "status": "implementation-checkpoint-full-poc-incomplete", + "connected10x": { + "version": 2, + "date": "2026-09-11", + "setupMode": "committed-json-index-only", + "readiness": "realm subscription connected; server markup removed; complete DOM oracle", + "machine": { + "browser": "149.0.7827.55", + "node": "v24.17.0", + "platform": "darwin", + "arch": "arm64", + "cpu": "Apple M5", + "physicalCores": 10, + "memoryGiB": 24, + "database": "PostgreSQL 16.3, Docker disk volume, localhost", + "cpuThrottling": "none", + "networkThrottling": "none" + }, + "workers": { + "allPriority": 1, + "userIndex": 1 + }, + "provenance": { + "commit": "6ff4ca411c992241505ceed69354a73793389bf6", + "runtimeHash": "31039889bd6a93d064c3ff3adc6acc09070c2e76230320746f829cf7cf718d9e", + "hostHash": "1460100b6accd40007ff99eee07098f1230a043e5e4573a13955736683e1fc1b" + }, + "finalGuardNote": "The no-watch first-registration generation guard was added after this 10\u00d7 series; the new real-Postgres regression and fresh private-application build validate it. The measured established-owner read path is unchanged.", + "manifest": { + "version": 1, + "synthetic": true, + "seed": 1729, + "preset": "10x", + "shape": "distributed", + "instanceCount": 12550, + "counts": { + "Classroom": 20, + "Student": 130, + "Staff": 260, + "Slot": 1730, + "Observation": 1640, + "Report": 440, + "Activity": 630, + "Reference": 7660, + "DaySummary": 40 + }, + "recordsSha256": "91d0e279fa214fcc58b5d9f6bb673021865c6a4eda5c08246010da060a0d4ef5" + }, + "candidate": { + "reads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "firstContentMs": 483.83341599999994, + "displayMs": 763.498875, + "interactiveMs": 763.498875, + "fetchRequests": 99, + "metrics": { + "Nodes": 1850, + "ScriptDuration": 0.067208, + "TaskDuration": 0.35960400000000003, + "JSHeapUsedSize": 42796192 + }, + "observationMs": 1018.22975, + "transferredBytes": 8068192, + "inputRequestCount": 0 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 461.3497500000001, + "displayMs": 655.5739999999998, + "interactiveMs": 655.5739999999998, + "fetchRequests": 100, + "metrics": { + "Nodes": 4384, + "ScriptDuration": 0.03468199999999999, + "TaskDuration": 0.262593, + "JSHeapUsedSize": 102422560 + }, + "observationMs": 908.7734170000001, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 503.86474999999973, + "displayMs": 715.5450409999999, + "interactiveMs": 715.5450409999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.037744000000000014, + "TaskDuration": 0.33000399999999996, + "JSHeapUsedSize": 62263460 + }, + "observationMs": 967.6945409999998, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 464.565333, + "displayMs": 645.7421250000002, + "interactiveMs": 645.7421250000002, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.03344, + "TaskDuration": 0.2741379999999999, + "JSHeapUsedSize": 122107004 + }, + "observationMs": 899.8228330000006, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 500.724209, + "displayMs": 699.5431670000007, + "interactiveMs": 699.5431670000007, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.03730800000000001, + "TaskDuration": 0.334025, + "JSHeapUsedSize": 62093028 + }, + "observationMs": 951.8135840000004, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 5, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 486.29812500000025, + "displayMs": 684.9240410000002, + "interactiveMs": 684.9240410000002, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.03565299999999999, + "TaskDuration": 0.28508, + "JSHeapUsedSize": 121771468 + }, + "observationMs": 938.0564160000004, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 6, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 512.3542079999997, + "displayMs": 714.6979579999997, + "interactiveMs": 714.6979579999997, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.03639999999999999, + "TaskDuration": 0.34062900000000007, + "JSHeapUsedSize": 62066408 + }, + "observationMs": 967.8192499999996, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 7, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 465.9053329999997, + "displayMs": 658.0373749999999, + "interactiveMs": 658.0373749999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.038244, + "TaskDuration": 0.2860610000000001, + "JSHeapUsedSize": 120615056 + }, + "observationMs": 911.5980419999996, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 8, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 510.7063330000001, + "displayMs": 696.0204169999997, + "interactiveMs": 696.0204169999997, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.039109000000000005, + "TaskDuration": 0.33923999999999976, + "JSHeapUsedSize": 62080476 + }, + "observationMs": 947.574458000001, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 9, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 472.32179200000064, + "displayMs": 655.4452920000003, + "interactiveMs": 655.4452920000003, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.03553299999999998, + "TaskDuration": 0.2757329999999998, + "JSHeapUsedSize": 121810648 + }, + "observationMs": 908.2158749999999, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 10, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 489.19174999999996, + "displayMs": 673.3225419999999, + "interactiveMs": 673.3225419999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.038609000000000004, + "TaskDuration": 0.330619, + "JSHeapUsedSize": 61585632 + }, + "observationMs": 924.4773339999992, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 11, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 464.01479200000176, + "displayMs": 655.3517500000016, + "interactiveMs": 655.3517500000016, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.03378700000000001, + "TaskDuration": 0.28120699999999976, + "JSHeapUsedSize": 122130776 + }, + "observationMs": 908.3952500000014, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 12, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 514.2720000000008, + "displayMs": 703.1392500000002, + "interactiveMs": 703.1392500000002, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.03735099999999997, + "TaskDuration": 0.34889400000000004, + "JSHeapUsedSize": 61607128 + }, + "observationMs": 955.9302080000016, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 13, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 460.76179199999933, + "displayMs": 646.9172500000004, + "interactiveMs": 646.9172500000004, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.03553700000000004, + "TaskDuration": 0.2805199999999992, + "JSHeapUsedSize": 121365180 + }, + "observationMs": 898.4933339999989, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 14, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 489.627625000001, + "displayMs": 673.7828750000008, + "interactiveMs": 673.7828750000008, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.03547199999999995, + "TaskDuration": 0.33109500000000036, + "JSHeapUsedSize": 61627728 + }, + "observationMs": 926.838667, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 15, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 458.52229100000113, + "displayMs": 643.048291000001, + "interactiveMs": 643.048291000001, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.03435700000000008, + "TaskDuration": 0.2823570000000002, + "JSHeapUsedSize": 121077244 + }, + "observationMs": 894.4465, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 16, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 518.8892500000002, + "displayMs": 715.1108750000003, + "interactiveMs": 715.1108750000003, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.03769499999999992, + "TaskDuration": 0.34101099999999995, + "JSHeapUsedSize": 61614476 + }, + "observationMs": 967.6047500000004, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 17, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 462.2157499999994, + "displayMs": 650.1183329999985, + "interactiveMs": 650.1183329999985, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.034162000000000026, + "TaskDuration": 0.2777769999999995, + "JSHeapUsedSize": 121265264 + }, + "observationMs": 903.182166999999, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 18, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 512.1080829999992, + "displayMs": 703.5772909999978, + "interactiveMs": 703.5772909999978, + "fetchRequests": 100, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.03708200000000006, + "TaskDuration": 0.3431569999999997, + "JSHeapUsedSize": 61593324 + }, + "observationMs": 955.8107079999972, + "transferredBytes": 7693455, + "inputRequestCount": 0 + }, + { + "iteration": 19, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 461.1697079999976, + "displayMs": 646.9675409999982, + "interactiveMs": 646.9675409999982, + "fetchRequests": 100, + "metrics": { + "Nodes": 6234, + "ScriptDuration": 0.03547899999999993, + "TaskDuration": 0.2819390000000004, + "JSHeapUsedSize": 119701528 + }, + "observationMs": 899.3042499999974, + "transferredBytes": 7693455, + "inputRequestCount": 0 + } + ], + "writes": [ + { + "name": "matching content edit", + "startedAt": 1789108971192, + "valid": true, + "ackAt": 1789108974093, + "ackMs": 2901, + "status": 200, + "ownerRevision": 83, + "observedAt": 1789108974210, + "ackToDisplayMs": 117, + "transitions": [ + { + "at": 1789108971380, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108974206, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "transitive row label edit", + "startedAt": 1789108974224, + "valid": true, + "ackAt": 1789108976785, + "ackMs": 2561, + "status": 200, + "ownerRevision": 85, + "observedAt": 1789108976923, + "ackToDisplayMs": 138, + "transitions": [ + { + "at": 1789108974384, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108976915, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "query exit", + "startedAt": 1789108976942, + "valid": true, + "ackAt": 1789108980345, + "ackMs": 3403, + "status": 200, + "ownerRevision": 87, + "observedAt": 1789108980473, + "ackToDisplayMs": 128, + "transitions": [ + { + "at": 1789108977093, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108980459, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "query entry", + "startedAt": 1789108980485, + "valid": true, + "ackAt": 1789108983823, + "ackMs": 3338, + "status": 200, + "ownerRevision": 89, + "observedAt": 1789108983939, + "ackToDisplayMs": 116, + "transitions": [ + { + "at": 1789108980632, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108983935, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "asynchronous source write", + "startedAt": 1789108983953, + "valid": true, + "ackAt": 1789108983973, + "ackMs": 20, + "status": 204, + "ownerRevision": 91, + "observedAt": 1789108986561, + "ackToDisplayMs": 2588, + "transitions": [ + { + "at": 1789108984104, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108986553, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "unrelated write", + "startedAt": 1789108986580, + "valid": true, + "ackAt": 1789108987370, + "ackMs": 790, + "status": 200, + "ownerRevision": 91, + "observedAt": 1789108987504, + "ackToDisplayMs": 134, + "transitions": [ + { + "at": 1789108986752, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108987496, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "matching insertion", + "startedAt": 1789108987518, + "valid": true, + "ackAt": 1789108987533, + "ackMs": 15, + "status": 204, + "ownerRevision": 94, + "observedAt": 1789108989219, + "ackToDisplayMs": 1686, + "transitions": [ + { + "at": 1789108987677, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108989212, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "matching deletion", + "startedAt": 1789108989236, + "valid": true, + "ackAt": 1789108991459, + "ackMs": 2223, + "status": 204, + "ownerRevision": 96, + "observedAt": 1789108991604, + "ackToDisplayMs": 145, + "transitions": [ + { + "at": 1789108989384, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108991589, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "reconnect after missed notification", + "startedAt": 1789108991615, + "valid": true, + "ackAt": 1789108991650, + "ackMs": 35, + "status": 204, + "ownerRevision": 98, + "observedAt": 1789108996827, + "ackToDisplayMs": 5177, + "transitions": [ + { + "at": 1789108991617, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789108991666, + "visibleStats": 0 + }, + { + "at": 1789108996819, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + } + ], + "pageErrorCount": 0 + }, + "reference": { + "reads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "firstContentMs": 12634.655291000001, + "displayMs": 12641.938708, + "interactiveMs": 12641.938708, + "fetchRequests": 198, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.101013, + "TaskDuration": 1.704106, + "JSHeapUsedSize": 94432712 + }, + "observationMs": 12897.162625, + "transferredBytes": 25850737, + "inputRequestCount": 86 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 12914.107959, + "displayMs": 12920.800375000003, + "interactiveMs": 12920.800375000003, + "fetchRequests": 199, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07506799999999998, + "TaskDuration": 1.681603, + "JSHeapUsedSize": 96126580 + }, + "observationMs": 13174.425459000002, + "transferredBytes": 25448282, + "inputRequestCount": 86 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13014.159457999998, + "displayMs": 13019.407499999998, + "interactiveMs": 13019.407499999998, + "fetchRequests": 199, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07326500000000002, + "TaskDuration": 1.6920939999999995, + "JSHeapUsedSize": 95939800 + }, + "observationMs": 13272.783207999997, + "transferredBytes": 25448282, + "inputRequestCount": 86 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13324.988250000002, + "displayMs": 13330.481249999997, + "interactiveMs": 13330.481249999997, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07849800000000001, + "TaskDuration": 1.7762220000000006, + "JSHeapUsedSize": 95916720 + }, + "observationMs": 13585.068832999998, + "transferredBytes": 25448282, + "inputRequestCount": 86 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13338.013375000002, + "displayMs": 13343.310707999997, + "interactiveMs": 13343.310707999997, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07839299999999999, + "TaskDuration": 1.7766760000000001, + "JSHeapUsedSize": 95921284 + }, + "observationMs": 13597.439332999995, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 5, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13344.153791999997, + "displayMs": 13349.563999999998, + "interactiveMs": 13349.563999999998, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07909, + "TaskDuration": 1.7800919999999998, + "JSHeapUsedSize": 95381980 + }, + "observationMs": 13603.015, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 6, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13317.323917000002, + "displayMs": 13322.57424999999, + "interactiveMs": 13322.57424999999, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07676999999999995, + "TaskDuration": 1.7694299999999998, + "JSHeapUsedSize": 95629576 + }, + "observationMs": 13574.823541999998, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 7, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13550.532666999992, + "displayMs": 13556.256083, + "interactiveMs": 13556.256083, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07760300000000009, + "TaskDuration": 1.7975720000000006, + "JSHeapUsedSize": 95699988 + }, + "observationMs": 13810.652875, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 8, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13510.149166999996, + "displayMs": 13516.071916999994, + "interactiveMs": 13516.071916999994, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07808799999999994, + "TaskDuration": 1.7915779999999994, + "JSHeapUsedSize": 95541860 + }, + "observationMs": 13770.264958, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 9, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13545.285374999992, + "displayMs": 13550.968875000006, + "interactiveMs": 13550.968875000006, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07851700000000006, + "TaskDuration": 1.787879000000002, + "JSHeapUsedSize": 95743040 + }, + "observationMs": 13803.628916000001, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 10, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13645.552249999979, + "displayMs": 13651.478624999989, + "interactiveMs": 13651.478624999989, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.08153899999999992, + "TaskDuration": 1.8461679999999987, + "JSHeapUsedSize": 95888556 + }, + "observationMs": 13906.774042000005, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 11, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13660.211958, + "displayMs": 13665.48116699999, + "interactiveMs": 13665.48116699999, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.08108599999999999, + "TaskDuration": 1.8378179999999986, + "JSHeapUsedSize": 95454892 + }, + "observationMs": 13919.850791999983, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 12, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13945.583374999987, + "displayMs": 13954.609582999983, + "interactiveMs": 13954.609582999983, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.08133000000000001, + "TaskDuration": 1.8155409999999996, + "JSHeapUsedSize": 95798364 + }, + "observationMs": 14209.712624999986, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 13, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13590.374958, + "displayMs": 13595.982583000005, + "interactiveMs": 13595.982583000005, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.08106900000000006, + "TaskDuration": 1.8097600000000007, + "JSHeapUsedSize": 95905420 + }, + "observationMs": 13849.050707999995, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 14, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13718.593666, + "displayMs": 13723.76991599999, + "interactiveMs": 13723.76991599999, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07948899999999992, + "TaskDuration": 1.8050420000000003, + "JSHeapUsedSize": 95526624 + }, + "observationMs": 13977.724083000008, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 15, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13592.903416999994, + "displayMs": 13599.157500000001, + "interactiveMs": 13599.157500000001, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.07929399999999998, + "TaskDuration": 1.8256650000000008, + "JSHeapUsedSize": 95415244 + }, + "observationMs": 13853.369417000009, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 16, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13655.53237500001, + "displayMs": 13661.064165999996, + "interactiveMs": 13661.064165999996, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.08062400000000003, + "TaskDuration": 1.8349579999999968, + "JSHeapUsedSize": 95926172 + }, + "observationMs": 13914.648875000014, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 17, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 13796.059374999983, + "displayMs": 13801.233708999993, + "interactiveMs": 13801.233708999993, + "fetchRequests": 200, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.079264, + "TaskDuration": 1.8144659999999995, + "JSHeapUsedSize": 95599760 + }, + "observationMs": 14053.019958999997, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 18, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 14282.547709000006, + "displayMs": 14287.496542000008, + "interactiveMs": 14287.496542000008, + "fetchRequests": 201, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.08405300000000016, + "TaskDuration": 1.8588740000000001, + "JSHeapUsedSize": 95547360 + }, + "observationMs": 14540.508334000013, + "transferredBytes": 25448302, + "inputRequestCount": 86 + }, + { + "iteration": 19, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 17110.52016700001, + "displayMs": 17119.28470900003, + "interactiveMs": 17119.28470900003, + "fetchRequests": 203, + "metrics": { + "Nodes": 28703, + "ScriptDuration": 0.09708299999999981, + "TaskDuration": 2.0692359999999965, + "JSHeapUsedSize": 95595020 + }, + "observationMs": 17372.488708999997, + "transferredBytes": 25448302, + "inputRequestCount": 86 + } + ], + "writes": [ + { + "name": "matching content edit", + "startedAt": 1789108364141, + "valid": false, + "ackAt": 1789108364637, + "ackMs": 496, + "status": 200, + "failure": "page.waitForFunction: Timeout 9999ms exceeded.", + "transitions": [ + { + "at": 1789108364344, + "state": "pending", + "visibleStats": 0 + } + ], + "observationMs": 10498, + "inputRequestCount": 35 + } + ], + "pageErrorCount": 0, + "provenance": { + "variant": "get-cards", + "logicalRecordsSha256": "f23fde199e43b0f97ad1d22682838a975518e7394465e270b822f081418a6d49", + "sourceDatasetRecordsSha256": "91d0e279fa214fcc58b5d9f6bb673021865c6a4eda5c08246010da060a0d4ef5", + "note": "Historical in-place switch manifests inherited the original record hash. This corrected hash includes the changed owner definition; inputs are identical. Future switch manifests compute it directly." + } + }, + "concurrentReadCases": [ + { + "readers": 1, + "durationMs": 10000, + "startedAt": 1789107733667, + "readyReads": 392, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "version": 1, + "startedAt": 1789107734668, + "valid": true, + "ackAt": 1789107734683, + "ackMs": 15, + "observedAt": 1789107737160, + "ackToDisplayMs": 2477 + }, + { + "version": 2, + "startedAt": 1789107738163, + "valid": true, + "ackAt": 1789107738179, + "ackMs": 16, + "observedAt": 1789107740657, + "ackToDisplayMs": 2478 + }, + { + "version": 3, + "startedAt": 1789107741657, + "valid": true, + "ackAt": 1789107741672, + "ackMs": 15, + "observedAt": 1789107744079, + "ackToDisplayMs": 2407 + } + ], + "responseBytes": 11660040, + "browserInputRequests": 0, + "elapsedMs": 10413, + "latency": { + "n": 392, + "min": 5.597834000000148, + "median": 8.08087499999965, + "p95": 11.442417000000205, + "max": 2347.966124999999 + }, + "readyReadsPerSecond": 37.64525112839719, + "valid": true + }, + { + "readers": 10, + "durationMs": 10000, + "startedAt": 1789107747391, + "readyReads": 1709, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "version": 1, + "startedAt": 1789107748393, + "valid": true, + "ackAt": 1789107748425, + "ackMs": 32, + "observedAt": 1789107751125, + "ackToDisplayMs": 2700 + }, + { + "version": 2, + "startedAt": 1789107752124, + "valid": true, + "ackAt": 1789107752157, + "ackMs": 33, + "observedAt": 1789107754996, + "ackToDisplayMs": 2839 + }, + { + "version": 3, + "startedAt": 1789107755996, + "valid": true, + "ackAt": 1789107756025, + "ackMs": 29, + "observedAt": 1789107758657, + "ackToDisplayMs": 2632 + } + ], + "responseBytes": 50834205, + "browserInputRequests": 0, + "elapsedMs": 11266, + "latency": { + "n": 1709, + "min": 10.522499999999127, + "median": 18.79962499999965, + "p95": 32.67229199999929, + "max": 2673.623625 + }, + "readyReadsPerSecond": 151.69536658973902, + "valid": true + }, + { + "readers": 50, + "durationMs": 10000, + "startedAt": 1789107762036, + "readyReads": 2292, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "version": 1, + "startedAt": 1789107763045, + "valid": true, + "ackAt": 1789107763136, + "ackMs": 91, + "observedAt": 1789107765957, + "ackToDisplayMs": 2821 + }, + { + "version": 2, + "startedAt": 1789107766957, + "valid": true, + "ackAt": 1789107767077, + "ackMs": 120, + "observedAt": 1789107769910, + "ackToDisplayMs": 2833 + }, + { + "version": 3, + "startedAt": 1789107770910, + "valid": true, + "ackAt": 1789107771050, + "ackMs": 140, + "observedAt": 1789107773741, + "ackToDisplayMs": 2691 + } + ], + "responseBytes": 68175540, + "browserInputRequests": 0, + "elapsedMs": 11706, + "latency": { + "n": 2292, + "min": 50.639334000006784, + "median": 81.72404200000165, + "p95": 2665.7049590000024, + "max": 2731.087625 + }, + "readyReadsPerSecond": 195.79702716555613, + "valid": true + } + ], + "crashCases": [ + { + "name": "worker crash before-publication", + "valid": true, + "ackMs": 16, + "interruptedReservation": { + "job_id": 22, + "reservation_id": 32, + "worker_id": "worker-pid-97095" + }, + "ackToCrashMs": 543, + "ackToDisplayMs": 3730, + "ownerRevision": 21, + "reservations": [ + { + "worker_id": "worker-pid-97095", + "completion_reason": "interrupted", + "completed_at": "2026-09-11T10:18:51.748Z" + }, + { + "worker_id": "worker-pid-99553", + "completion_reason": "completed", + "completed_at": "2026-09-11T10:18:54.771Z" + } + ], + "explicitPending": true, + "transitions": [ + { + "state": "pending", + "visibleStats": 0, + "at": 1789107531318 + }, + { + "state": "ready", + "visibleStats": 6, + "at": 1789107534918 + } + ], + "realmEvents": [ + { + "name": "index", + "at": 1789107531317, + "origin": 1789107531187 + }, + { + "name": "update", + "at": 1789107531317, + "origin": 1789107531188 + }, + { + "name": "index", + "at": 1789107534902, + "origin": 1789107534786 + } + ], + "initialConnection": true, + "pageErrors": [], + "inputRequests": [] + }, + { + "name": "worker crash after-source-publication", + "valid": true, + "ackMs": 20, + "committedDirtyGeneration": 24, + "interruptedReservation": { + "job_id": 26, + "reservation_id": 39, + "worker_id": "worker-pid-98125" + }, + "ackToCrashMs": 1500, + "ackToDisplayMs": 5556, + "ownerRevision": 26, + "reservations": [ + { + "worker_id": "worker-pid-98125", + "completion_reason": "interrupted", + "completed_at": "2026-09-11T10:19:32.134Z" + }, + { + "worker_id": "worker-pid-99826", + "completion_reason": "completed", + "completed_at": "2026-09-11T10:19:36.005Z" + } + ], + "explicitPending": true, + "transitions": [ + { + "state": "pending", + "visibleStats": 0, + "at": 1789107570744 + }, + { + "state": "ready", + "visibleStats": 6, + "at": 1789107576162 + } + ], + "realmEvents": [ + { + "name": "update", + "at": 1789107570743, + "origin": 1789107570605 + }, + { + "name": "index", + "at": 1789107570743, + "origin": 1789107570608 + }, + { + "name": "index", + "at": 1789107576135, + "origin": 1789107576018 + } + ], + "initialConnection": true, + "pageErrors": [], + "inputRequests": [] + } + ], + "allOwners": { + "tested": 40, + "passed": 40 + }, + "interpretation": "Read distributions cover stable data. The reference fails write freshness, so it cannot establish a freshness-preserving speedup. Full 10\u00d7 HTML replay remains failed. No capacity or AI-throughput claim." + }, + "connectedPrivateApplication": { + "fixture": { + "synthetic": true, + "preset": "smoke", + "fixture": "private-definition-adaptation", + "day": "2026-01-12", + "instanceCount": 34, + "sourceHash": "7e8ff3aad5130c6f4661dd22f5955f6c9d662f1384e485792693af3a12a0ca3d" + }, + "provenance": { + "commit": "6ff4ca411c992241505ceed69354a73793389bf6", + "runtimeHash": "5b6ea1650706e54bdb47449dda9034f733a2816a8903f59bc0acdb7009a7595d", + "hostHash": "0422471ddc8b8a8dab252c428ea6b62248af151358ad5d20e0bd9c689f650d5c", + "workers": { + "allPriority": 1, + "userIndex": 1 + } + }, + "setupMs": 151579.859792, + "indexStatus": { + "index": [ + { + "type": "instance", + "total": 34, + "errors": 0 + }, + { + "type": "file", + "total": 187, + "errors": 0 + } + ], + "html": { + "total": 221, + "errors": 0 + } + }, + "reads": [ + { + "id": "DaySummary/00000", + "valid": true, + "firstContentMs": 496.316916, + "interactiveMs": 926.736958, + "inputRequests": [] + }, + { + "id": "TessarSchedule/00000", + "valid": true, + "firstContentMs": 465.1794589999997, + "interactiveMs": 755.6151669999997, + "inputRequests": [] + }, + { + "id": "TessarCoverage/00000", + "valid": true, + "firstContentMs": 557.8626670000003, + "interactiveMs": 877.9712920000002, + "inputRequests": [] + } + ], + "writes": [ + { + "name": "capture status reaches dashboard through coverage feeder", + "valid": true, + "ackMs": 16, + "ackToDisplayMs": 5929, + "transitions": [ + { + "state": "pending", + "rows": 0, + "at": 1789108827293 + }, + { + "state": "ready", + "rows": 3, + "at": 1789108833053 + } + ], + "inputRequests": [], + "newClientValid": true + }, + { + "name": "distinct report coverage changes through coverage feeder", + "valid": true, + "ackMs": 11, + "ackToDisplayMs": 2831, + "transitions": [ + { + "state": "pending", + "rows": 0, + "at": 1789108837141 + }, + { + "state": "ready", + "rows": 3, + "at": 1789108839832 + } + ], + "inputRequests": [], + "newClientValid": true + }, + { + "name": "uncancelled schedule appears through schedule feeder", + "valid": true, + "ackMs": 18, + "ackToDisplayMs": 2783, + "transitions": [ + { + "state": "pending", + "rows": 0, + "at": 1789108843676 + }, + { + "state": "ready", + "rows": 4, + "at": 1789108846297 + } + ], + "inputRequests": [], + "newClientValid": true + } + ], + "drillDown": [ + { + "name": "explicit drill-down", + "valid": true, + "requests": ["http://localhost:53777/tessar/Lesson/00000"] + } + ], + "pageErrorCount": 0, + "scope": "Projected dashboard, schedule feeder and coverage feeder using private definitions and fabricated data. Full capture/report-generation UI and personal overlays remain outside this checkpoint." + }, + "checks": { + "tessarBrowserTests": 7, + "tessarBrowserAssertions": 80, + "tessarPostgresTests": 6, + "generatorAndBrowserOracleTests": 4, + "matrixSlidingSyncRegressionAssertions": 4, + "packageLint": "host, runtime-common, realm-server and realm-test-harness pass; two existing host RSVP warnings", + "privateDefinitions": "153 parsed; zero errors in the three new views; 830 pre-existing errors" + }, + "historical": { + "version": 1, + "status": "in-progress", + "candidate": { + "commit": "fead7a13ab02154a79e743877798166765959fbe", + "runtimeHash": "cacabe30049993e6a84d2f25b33bf5e130d27d3f3752dac760efdb8573de56cd", + "hostHash": "78492c253fcf49c641cd1022de72fa503796d1d23f7ab383116e7c161bfce09f" + }, + "manifest": { + "version": 1, + "synthetic": true, + "seed": 1729, + "preset": "smoke", + "shape": "distributed", + "instanceCount": 34, + "counts": { + "Classroom": 2, + "Student": 4, + "Staff": 4, + "Slot": 4, + "Observation": 4, + "Report": 4, + "Activity": 4, + "Reference": 4, + "DaySummary": 4 + }, + "recordsSha256": "cbbee835e1b22fc96222fc1019fdba48380433ec8cf8c3eec1d44f2a811cc8d2" + }, + "startupMs": 83182.196375, + "httpReads": [ + { + "iteration": 0, + "elapsedMs": 24.365875000003143, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 1, + "elapsedMs": 8.401167000003625, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 2, + "elapsedMs": 10.184207999991486, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 3, + "elapsedMs": 8.670916999995825, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 4, + "elapsedMs": 8.254000000000815, + "responseBytes": 7164, + "status": 200, + "valid": true + } + ], + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 538.87025, + "inputRequests": [], + "fetchRequests": 101, + "metrics": { + "Nodes": 1469, + "ScriptDuration": 0.08405, + "TaskDuration": 0.378386, + "JSHeapUsedSize": 75435092 + } + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 428.66262500000016, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 2752, + "ScriptDuration": 0.119587, + "TaskDuration": 0.629178, + "JSHeapUsedSize": 97491200 + } + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 467.7433329999999, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.157086, + "TaskDuration": 0.926614, + "JSHeapUsedSize": 61492352 + } + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 443.6385829999999, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 4043, + "ScriptDuration": 0.190751, + "TaskDuration": 1.172988, + "JSHeapUsedSize": 118185404 + } + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 494.621083, + "inputRequests": [], + "fetchRequests": 99, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.226318, + "TaskDuration": 1.4741, + "JSHeapUsedSize": 61486640 + } + } + ], + "mutations": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 826, + "status": 200, + "ownerRevision": 4, + "ackToDisplayMs": 16, + "inputRequests": [], + "pendingNotificationMs": -662, + "transitions": [ + { + "sinceWriteMs": 118, + "state": "ready", + "visibleStats": 6 + }, + { + "sinceWriteMs": 164, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 813, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 1247, + "status": 200, + "ownerRevision": 6, + "ackToDisplayMs": 121, + "inputRequests": [], + "pendingNotificationMs": -1107, + "transitions": [ + { + "sinceWriteMs": 140, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1363, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "query exit", + "valid": true, + "ackMs": 1553, + "status": 200, + "ownerRevision": 8, + "ackToDisplayMs": 119, + "inputRequests": [], + "pendingNotificationMs": -1412, + "transitions": [ + { + "sinceWriteMs": 141, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1668, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "query entry", + "valid": true, + "ackMs": 893, + "status": 200, + "ownerRevision": 10, + "ackToDisplayMs": 115, + "inputRequests": [], + "pendingNotificationMs": -747, + "transitions": [ + { + "sinceWriteMs": 146, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1003, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 15, + "status": 204, + "ownerRevision": 12, + "ackToDisplayMs": 887, + "inputRequests": [], + "pendingNotificationMs": 124, + "transitions": [ + { + "sinceWriteMs": 139, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 894, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 950, + "status": 200, + "ownerRevision": 12, + "ackToDisplayMs": 123, + "inputRequests": [], + "pendingNotificationMs": -807, + "transitions": [ + { + "sinceWriteMs": 143, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1066, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 15, + "status": 204, + "ownerRevision": 16, + "ackToDisplayMs": 1289, + "inputRequests": [], + "pendingNotificationMs": 125, + "transitions": [ + { + "sinceWriteMs": 140, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 1301, + "state": "ready", + "visibleStats": 6 + } + ] + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 471, + "status": 204, + "ownerRevision": 18, + "ackToDisplayMs": 137, + "inputRequests": [], + "pendingNotificationMs": -309, + "transitions": [ + { + "sinceWriteMs": 162, + "state": "pending", + "visibleStats": 0 + }, + { + "sinceWriteMs": 595, + "state": "ready", + "visibleStats": 6 + } + ] + } + ], + "browserErrors": [], + "trace": { + "lcpMs": 1878, + "ttfbMs": 31, + "cls": 0, + "cpuThrottling": 1, + "networkThrottling": "none", + "note": "One DevTools trace; separate from headless readiness measurements. No base/candidate speedup established." + }, + "exploratory1x": { + "storage": "PostgreSQL 16.3 on 3.9 GiB tmpfs; excluded from disk-backed comparisons", + "candidate": { + "commit": "fead7a13ab02154a79e743877798166765959fbe", + "runtimeHash": "cacabe30049993e6a84d2f25b33bf5e130d27d3f3752dac760efdb8573de56cd", + "hostHash": "78492c253fcf49c641cd1022de72fa503796d1d23f7ab383116e7c161bfce09f" + }, + "manifest": { + "version": 1, + "synthetic": true, + "seed": 1729, + "preset": "1x", + "shape": "distributed", + "instanceCount": 1255, + "counts": { + "Classroom": 2, + "Student": 13, + "Staff": 26, + "Slot": 173, + "Observation": 164, + "Report": 44, + "Activity": 63, + "Reference": 766, + "DaySummary": 4 + }, + "recordsSha256": "60ecddc1bd7b8203f1cd0ffda583325cd0b33ed8552734bc79bed99f9e671b14" + }, + "startupMs": 334014.01620799996, + "httpReads": [ + { + "iteration": 0, + "elapsedMs": 28.61095900001237, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 1, + "elapsedMs": 11.054750000010245, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 2, + "elapsedMs": 9.332875000021886, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 3, + "elapsedMs": 10.277624999987893, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 4, + "elapsedMs": 9.901791999989655, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 5, + "elapsedMs": 9.857124999980442, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 6, + "elapsedMs": 11.62100000004284, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 7, + "elapsedMs": 9.131791999971028, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 8, + "elapsedMs": 8.908166000037454, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 9, + "elapsedMs": 11.87562499998603, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 10, + "elapsedMs": 8.843791999970563, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 11, + "elapsedMs": 8.613874999980908, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 12, + "elapsedMs": 8.231292000040412, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 13, + "elapsedMs": 10.982457999954931, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 14, + "elapsedMs": 8.227375000016764, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 15, + "elapsedMs": 6.772209000017028, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 16, + "elapsedMs": 8.223916999995708, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 17, + "elapsedMs": 8.872124999994412, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 18, + "elapsedMs": 9.011999999987893, + "responseBytes": 29698, + "status": 200, + "valid": true + }, + { + "iteration": 19, + "elapsedMs": 9.746916999982204, + "responseBytes": 29698, + "status": 200, + "valid": true + } + ], + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 582.6893750000002, + "fetchRequests": 98, + "metrics": { + "Nodes": 1471, + "ScriptDuration": 0.087699, + "TaskDuration": 0.384828, + "JSHeapUsedSize": 74252456 + }, + "inputRequestCount": 0 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 464.07220900000016, + "fetchRequests": 103, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.127607, + "TaskDuration": 0.699661, + "JSHeapUsedSize": 60268616 + }, + "inputRequestCount": 0 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 434.560833, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.16159, + "TaskDuration": 0.951681, + "JSHeapUsedSize": 118875236 + }, + "inputRequestCount": 0 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 538.8636660000002, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.198748, + "TaskDuration": 1.274711, + "JSHeapUsedSize": 62192904 + }, + "inputRequestCount": 0 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 450.24112499999956, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.232656, + "TaskDuration": 1.526551, + "JSHeapUsedSize": 120423820 + }, + "inputRequestCount": 0 + }, + { + "iteration": 5, + "cache": "warm-reload", + "valid": true, + "displayMs": 493.1064580000002, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.26902, + "TaskDuration": 1.842421, + "JSHeapUsedSize": 62418356 + }, + "inputRequestCount": 0 + }, + { + "iteration": 6, + "cache": "warm-reload", + "valid": true, + "displayMs": 453.88287500000024, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.305986, + "TaskDuration": 2.096136, + "JSHeapUsedSize": 120947520 + }, + "inputRequestCount": 0 + }, + { + "iteration": 7, + "cache": "warm-reload", + "valid": true, + "displayMs": 490.76133299999947, + "fetchRequests": 99, + "metrics": { + "Nodes": 8772, + "ScriptDuration": 0.341787, + "TaskDuration": 2.381155, + "JSHeapUsedSize": 158195248 + }, + "inputRequestCount": 0 + }, + { + "iteration": 8, + "cache": "warm-reload", + "valid": true, + "displayMs": 531.8857500000004, + "fetchRequests": 99, + "metrics": { + "Nodes": 11308, + "ScriptDuration": 0.380472, + "TaskDuration": 2.704089, + "JSHeapUsedSize": 196422944 + }, + "inputRequestCount": 0 + }, + { + "iteration": 9, + "cache": "warm-reload", + "valid": true, + "displayMs": 537.3765000000003, + "fetchRequests": 99, + "metrics": { + "Nodes": 7464, + "ScriptDuration": 0.417063, + "TaskDuration": 3.08325, + "JSHeapUsedSize": 96883632 + }, + "inputRequestCount": 0 + }, + { + "iteration": 10, + "cache": "warm-reload", + "valid": true, + "displayMs": 482.1635839999999, + "fetchRequests": 99, + "metrics": { + "Nodes": 1850, + "ScriptDuration": 0.452137, + "TaskDuration": 3.392069, + "JSHeapUsedSize": 44846444 + }, + "inputRequestCount": 0 + }, + { + "iteration": 11, + "cache": "warm-reload", + "valid": true, + "displayMs": 442.59000000000015, + "fetchRequests": 99, + "metrics": { + "Nodes": 4386, + "ScriptDuration": 0.489469, + "TaskDuration": 3.639918, + "JSHeapUsedSize": 101884296 + }, + "inputRequestCount": 0 + }, + { + "iteration": 12, + "cache": "warm-reload", + "valid": true, + "displayMs": 482.96099999999933, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.526201, + "TaskDuration": 3.938568, + "JSHeapUsedSize": 61734792 + }, + "inputRequestCount": 0 + }, + { + "iteration": 13, + "cache": "warm-reload", + "valid": true, + "displayMs": 464.44058399999994, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.560771, + "TaskDuration": 4.203367, + "JSHeapUsedSize": 120747992 + }, + "inputRequestCount": 0 + }, + { + "iteration": 14, + "cache": "warm-reload", + "valid": true, + "displayMs": 493.4795830000003, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.600257, + "TaskDuration": 4.524254, + "JSHeapUsedSize": 61717764 + }, + "inputRequestCount": 0 + }, + { + "iteration": 15, + "cache": "warm-reload", + "valid": true, + "displayMs": 462.02329199999986, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.635034, + "TaskDuration": 4.788153, + "JSHeapUsedSize": 120212712 + }, + "inputRequestCount": 0 + }, + { + "iteration": 16, + "cache": "warm-reload", + "valid": true, + "displayMs": 502.05570799999987, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.672395, + "TaskDuration": 5.111127, + "JSHeapUsedSize": 61703224 + }, + "inputRequestCount": 0 + }, + { + "iteration": 17, + "cache": "warm-reload", + "valid": true, + "displayMs": 467.5910410000015, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.707947, + "TaskDuration": 5.372618, + "JSHeapUsedSize": 120490356 + }, + "inputRequestCount": 0 + }, + { + "iteration": 18, + "cache": "warm-reload", + "valid": true, + "displayMs": 514.6342499999992, + "fetchRequests": 99, + "metrics": { + "Nodes": 3700, + "ScriptDuration": 0.746014, + "TaskDuration": 5.698184, + "JSHeapUsedSize": 61710756 + }, + "inputRequestCount": 0 + }, + { + "iteration": 19, + "cache": "warm-reload", + "valid": true, + "displayMs": 476.0184999999983, + "fetchRequests": 99, + "metrics": { + "Nodes": 6236, + "ScriptDuration": 0.782276, + "TaskDuration": 5.970163, + "JSHeapUsedSize": 120202736 + }, + "inputRequestCount": 0 + } + ], + "writes": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 5401, + "status": 200, + "ownerRevision": 4, + "ackToDisplayMs": 38, + "inputRequestCount": 0 + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 2771, + "status": 200, + "ownerRevision": 6, + "ackToDisplayMs": 150, + "inputRequestCount": 0 + }, + { + "name": "query exit", + "valid": true, + "ackMs": 3016, + "status": 200, + "ownerRevision": 8, + "ackToDisplayMs": 120, + "inputRequestCount": 0 + }, + { + "name": "query entry", + "valid": true, + "ackMs": 2423, + "status": 200, + "ownerRevision": 10, + "ackToDisplayMs": 119, + "inputRequestCount": 0 + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 16, + "status": 204, + "ownerRevision": 12, + "ackToDisplayMs": 2002, + "inputRequestCount": 0 + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 907, + "status": 200, + "ownerRevision": 12, + "ackToDisplayMs": 131, + "inputRequestCount": 0 + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 15, + "status": 204, + "ownerRevision": 15, + "ackToDisplayMs": 1092, + "inputRequestCount": 0 + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 1521, + "status": 204, + "ownerRevision": 17, + "ackToDisplayMs": 147, + "inputRequestCount": 0 + } + ], + "pageErrorCount": 0 + }, + "failedSetupRuns": [ + { + "preset": "10x", + "instanceCount": 12550, + "storage": "3.9 GiB tmpfs", + "valid": false, + "reason": "PostgreSQL SQLSTATE 53100: no space left on device", + "sourceFilesVisitedApproximately": 10037 + }, + { + "preset": "10x", + "instanceCount": 12550, + "storage": "disk-backed Docker volume", + "valid": false, + "reason": "Harness 600000 ms full-index startup timeout", + "sourceFilesVisitedApproximately": 11278 + } + ], + "smokeV5": { + "candidate": { + "commit": "fead7a13ab02154a79e743877798166765959fbe", + "runtimeHash": "23dd3be3f52df4414a9e29cc71fad5d2a5ce249800fd8eb83614cbce37404895", + "hostHash": "da5c453fe08e2bdfdf11cd201677aac8eeb2b8bf09f20cf5a8d273e0f2bfc26e" + }, + "storage": "disk-backed PostgreSQL 16.3 Docker volume", + "manifest": { + "version": 1, + "synthetic": true, + "seed": 1729, + "preset": "smoke", + "shape": "distributed", + "variant": "materialized", + "instanceCount": 34, + "counts": { + "Classroom": 2, + "Student": 4, + "Staff": 4, + "Slot": 4, + "Observation": 4, + "Report": 4, + "Activity": 4, + "Reference": 4, + "DaySummary": 4 + }, + "recordsSha256": "cbbee835e1b22fc96222fc1019fdba48380433ec8cf8c3eec1d44f2a811cc8d2" + }, + "httpReads": [ + { + "iteration": 0, + "elapsedMs": 23.093459000010625, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 1, + "elapsedMs": 10.064541999992798, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 2, + "elapsedMs": 11.75020900000527, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 3, + "elapsedMs": 8.958333000002312, + "responseBytes": 7164, + "status": 200, + "valid": true + }, + { + "iteration": 4, + "elapsedMs": 8.91874999999709, + "responseBytes": 7164, + "status": 200, + "valid": true + } + ], + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 550.5600419999998, + "fetchRequests": 98, + "metrics": { + "Nodes": 1204, + "ScriptDuration": 0.08798, + "TaskDuration": 0.38672700000000004, + "JSHeapUsedSize": 73473880 + }, + "inputRequestCount": 0 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 446.2209999999998, + "fetchRequests": 103, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.03707700000000001, + "TaskDuration": 0.28576199999999996, + "JSHeapUsedSize": 59505292 + }, + "inputRequestCount": 0 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 445.364834, + "fetchRequests": 99, + "metrics": { + "Nodes": 4043, + "ScriptDuration": 0.033658999999999994, + "TaskDuration": 0.2414559999999999, + "JSHeapUsedSize": 116162848 + }, + "inputRequestCount": 0 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 476.3268330000001, + "fetchRequests": 99, + "metrics": { + "Nodes": 2582, + "ScriptDuration": 0.03739200000000001, + "TaskDuration": 0.31247899999999995, + "JSHeapUsedSize": 61460216 + }, + "inputRequestCount": 0 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 440.77620799999977, + "fetchRequests": 99, + "metrics": { + "Nodes": 4043, + "ScriptDuration": 0.033944, + "TaskDuration": 0.247892, + "JSHeapUsedSize": 118292476 + }, + "inputRequestCount": 0 + } + ], + "writes": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 851, + "status": 200, + "ownerRevision": 4, + "ackToDisplayMs": 14, + "inputRequestCount": 0 + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 1262, + "status": 200, + "ownerRevision": 6, + "ackToDisplayMs": 128, + "inputRequestCount": 0 + }, + { + "name": "query exit", + "valid": true, + "ackMs": 1566, + "status": 200, + "ownerRevision": 8, + "ackToDisplayMs": 121, + "inputRequestCount": 0 + }, + { + "name": "query entry", + "valid": true, + "ackMs": 887, + "status": 200, + "ownerRevision": 10, + "ackToDisplayMs": 121, + "inputRequestCount": 0 + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 21, + "status": 204, + "ownerRevision": 12, + "ackToDisplayMs": 852, + "inputRequestCount": 0 + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 952, + "status": 200, + "ownerRevision": 12, + "ackToDisplayMs": 120, + "inputRequestCount": 0 + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 13, + "status": 204, + "ownerRevision": 16, + "ackToDisplayMs": 1309, + "inputRequestCount": 0 + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 483, + "status": 204, + "ownerRevision": 18, + "ackToDisplayMs": 142, + "inputRequestCount": 0 + }, + { + "name": "reconnect after missed notification", + "valid": true, + "ackMs": 32, + "status": 204, + "ownerRevision": 20, + "ackToDisplayMs": 5159, + "inputRequestCount": 0 + } + ], + "correctedReference": { + "note": "Single read and nine mutation validity checks; not a controlled performance comparison", + "reads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 1333.212375, + "fetchRequests": 105, + "metrics": { + "Nodes": 1357, + "ScriptDuration": 0.078716, + "TaskDuration": 0.392484, + "JSHeapUsedSize": 46309572 + }, + "inputRequestCount": 5 + } + ], + "writes": [ + { + "name": "matching content edit", + "valid": true, + "ackMs": 341, + "status": 200, + "ackToDisplayMs": 254, + "inputRequestCount": 9 + }, + { + "name": "transitive row label edit", + "valid": true, + "ackMs": 492, + "status": 200, + "ackToDisplayMs": 264, + "inputRequestCount": 12 + }, + { + "name": "query exit", + "valid": true, + "ackMs": 878, + "status": 200, + "ackToDisplayMs": 258, + "inputRequestCount": 9 + }, + { + "name": "query entry", + "valid": true, + "ackMs": 243, + "status": 200, + "ackToDisplayMs": 296, + "inputRequestCount": 9 + }, + { + "name": "asynchronous source write", + "valid": true, + "ackMs": 10, + "status": 204, + "ackToDisplayMs": 496, + "inputRequestCount": 9 + }, + { + "name": "unrelated write", + "valid": true, + "ackMs": 428, + "status": 200, + "ackToDisplayMs": 1, + "inputRequestCount": 4 + }, + { + "name": "matching insertion", + "valid": true, + "ackMs": 13, + "status": 204, + "ackToDisplayMs": 1262, + "inputRequestCount": 12 + }, + { + "name": "matching deletion", + "valid": true, + "ackMs": 77, + "status": 204, + "ackToDisplayMs": 282, + "inputRequestCount": 6 + }, + { + "name": "reconnect after missed notification", + "valid": true, + "ackMs": 12, + "status": 204, + "ackToDisplayMs": 5309, + "inputRequestCount": 10 + } + ], + "pageErrorCount": 0 + }, + "feederChecks": [ + { + "name": "unrelated module epoch", + "valid": true, + "ackToReadyMs": 868 + }, + { + "name": "five downstream feeder stages", + "valid": true, + "ackMs": 14, + "ackToReadyMs": 3908, + "snapshots": [ + { + "id": "DaySummary/00000", + "revision": 70, + "scoreTotal": 88 + }, + { + "id": "TessarStage0/00000", + "revision": 71, + "scoreTotal": 88 + }, + { + "id": "TessarStage1/00000", + "revision": 72, + "scoreTotal": 88 + }, + { + "id": "TessarStage2/00000", + "revision": 73, + "scoreTotal": 88 + }, + { + "id": "TessarStage3/00000", + "revision": 74, + "scoreTotal": 88 + }, + { + "id": "TessarStage4/00000", + "revision": 75, + "scoreTotal": 88 + } + ] + } + ], + "concurrencyChecks": { + "note": "Correctness smoke only. Other regression checks overlapped part of the run. HTTP readers plus one browser, not 50 browser CPUs.", + "cases": [ + { + "readers": 1, + "valid": true, + "readyReads": 874, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "valid": true, + "ackMs": 22, + "ackToDisplayMs": 1145, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 12, + "ackToDisplayMs": 638, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 20, + "ackToDisplayMs": 664, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 13, + "ackToDisplayMs": 654, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 12, + "ackToDisplayMs": 652, + "inputRequestCount": 0 + } + ] + }, + { + "readers": 10, + "valid": true, + "readyReads": 3886, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "valid": true, + "ackMs": 33, + "ackToDisplayMs": 714, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 22, + "ackToDisplayMs": 712, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 20, + "ackToDisplayMs": 732, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 27, + "ackToDisplayMs": 788, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 23, + "ackToDisplayMs": 761, + "inputRequestCount": 0 + } + ] + }, + { + "readers": 50, + "valid": true, + "readyReads": 4192, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "valid": true, + "ackMs": 87, + "ackToDisplayMs": 863, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 119, + "ackToDisplayMs": 864, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 103, + "ackToDisplayMs": 816, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 113, + "ackToDisplayMs": 819, + "inputRequestCount": 0 + }, + { + "valid": true, + "ackMs": 100, + "ackToDisplayMs": 782, + "inputRequestCount": 0 + } + ] + } + ] + } + }, + "exploratory10x": { + "status": "source-correct; HTML recovery and write-freshness validation incomplete", + "synthetic": true, + "instanceCount": 12550, + "storage": "disk-backed PostgreSQL 16.3", + "sourceJob": { + "id": 4, + "type": "from-scratch-index", + "result": { + "stats": { + "fileErrors": 0, + "filesIndexed": 12591, + "instanceErrors": 0, + "instancesIndexed": 12590, + "totalIndexEntries": 25101 + }, + "generation": 2, + "ignoreData": {}, + "phaseTimings": { + "swapMs": 25231, + "orderMs": 145, + "setupMs": 2, + "totalMs": 668837, + "writeMs": 85054, + "mtimesMs": 0, + "discoverMs": 80, + "tessarWaves": 1, + "visitLoopMs": 606013, + "tessarSwapMs": 20224, + "tessarWriteMs": 700, + "tessarOwnersRendered": 40, + "tessarMaterializationMs": 37329 + } + }, + "status": "resolved" + }, + "failedHtmlJob": { + "id": 5, + "type": "prerender_html", + "result": { + "status": 500, + "message": "Worker manager detected fatal error in worker worker-pid-57868 for job 5 with job_reservation id 5: FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory\n----- Native stack trace -----\n\n" + }, + "status": "rejected" + }, + "ownerOracle": { + "checked": 40, + "passed": 40 + }, + "browserReadContext": "HTML rendering active; diagnostic, not controlled idle benchmark", + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 1770.2680830000002, + "inputRequests": 0, + "fetchRequests": 102, + "metrics": { + "Nodes": 1844, + "ScriptDuration": 0.122523, + "TaskDuration": 0.556575, + "JSHeapUsedSize": 43262540 + } + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 1485.4174160000002, + "inputRequests": 0, + "fetchRequests": 99, + "metrics": { + "Nodes": 3779, + "ScriptDuration": 0.04695100000000001, + "TaskDuration": 0.32675300000000007, + "JSHeapUsedSize": 102357452 + } + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 1009.2649160000001, + "inputRequests": 0, + "fetchRequests": 98, + "metrics": { + "Nodes": 5551, + "ScriptDuration": 0.045156999999999975, + "TaskDuration": 0.3794700000000001, + "JSHeapUsedSize": 78072344 + } + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 1108.7790829999994, + "inputRequests": 0, + "fetchRequests": 99, + "metrics": { + "Nodes": 7486, + "ScriptDuration": 0.04425399999999999, + "TaskDuration": 0.35226999999999986, + "JSHeapUsedSize": 137075004 + } + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 1034.143750000001, + "inputRequests": 0, + "fetchRequests": 99, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.04583200000000004, + "TaskDuration": 0.42023299999999986, + "JSHeapUsedSize": 62867388 + } + } + ], + "browserErrors": [], + "prewarmProbe": { + "scope": "read-only dependency collection with inert cache sink; base-realm mappings installed", + "maxOldSpaceMiB": 512, + "before": "SIGABRT: heap exhaustion", + "boundedValid": true, + "boundedMaxRSSKiB": 498752, + "boundedReads": 51 + }, + "workers": { + "allPriority": 1, + "userIndex": 0 + } + }, + "realDefinitionApplication": { + "setup": "Isolated local runtime, fabricated 34-instance fixture with private definition-only fork. Diagnostic source clone; separate cold build remains pending.", + "reads": [ + { + "view": "DaySummary", + "valid": true, + "displayMs": 1691.9508749999998, + "inputRequests": 0 + }, + { + "view": "TessarSchedule", + "valid": true, + "displayMs": 3314.530333, + "inputRequests": 0 + }, + { + "view": "TessarCoverage", + "valid": true, + "displayMs": 2126.03675, + "inputRequests": 0 + } + ], + "writes": [ + { + "name": "capture status reaches dashboard through coverage feeder", + "valid": true, + "ackMs": 27, + "ackToDisplayMs": 4361, + "newClientValid": true, + "pendingHidesRows": true, + "inputRequests": 0 + }, + { + "name": "distinct report coverage changes through coverage feeder", + "valid": true, + "ackMs": 26, + "ackToDisplayMs": 6652, + "newClientValid": true, + "pendingHidesRows": true, + "inputRequests": 0 + }, + { + "name": "uncancelled schedule appears through schedule feeder", + "valid": true, + "ackMs": 23, + "ackToDisplayMs": 7285, + "newClientValid": true, + "pendingHidesRows": true, + "inputRequests": 0 + } + ], + "explicitDrillDown": true, + "browserPageErrors": 0, + "interpretation": "Correctness/freshness diagnostic with background 10x HTML work; timings are not a controlled performance comparison." + }, + "tenfoldJsonPath": { + "setupMode": "committed-json-index-only", + "workers": { + "allPriority": 1, + "userIndex": 1 + }, + "instanceCount": 12550, + "coldSourceIndex": "Previously committed successful source index; see exploratory10x. HTML replay later failed on a report render timeout and is not part of this successful JSON-path measurement.", + "httpReads": [ + { + "iteration": 0, + "elapsedMs": 35.867874999996275, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 1, + "elapsedMs": 19.922708999998576, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 2, + "elapsedMs": 24.35479199999827, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 3, + "elapsedMs": 22.84441599999991, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 4, + "elapsedMs": 14.711416999998619, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 5, + "elapsedMs": 15.243792000001122, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 6, + "elapsedMs": 13.681084000003466, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 7, + "elapsedMs": 10.34824999999546, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 8, + "elapsedMs": 9.940875000000233, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 9, + "elapsedMs": 12.795791000004101, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 10, + "elapsedMs": 11.427833999994618, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 11, + "elapsedMs": 11.646542000002228, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 12, + "elapsedMs": 14.281999999999243, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 13, + "elapsedMs": 15.72899999999936, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 14, + "elapsedMs": 12.942208000000392, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 15, + "elapsedMs": 13.120208000000275, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 16, + "elapsedMs": 12.717791999995825, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 17, + "elapsedMs": 13.848666999998386, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 18, + "elapsedMs": 11.179082999995444, + "responseBytes": 29742, + "status": 200, + "valid": true + }, + { + "iteration": 19, + "elapsedMs": 12.093124999999418, + "responseBytes": 29742, + "status": 200, + "valid": true + } + ], + "browserReads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "displayMs": 1057.781083, + "fetchRequests": 102, + "metrics": { + "Nodes": 1844, + "ScriptDuration": 0.089399, + "TaskDuration": 0.439633, + "JSHeapUsedSize": 42941348 + }, + "inputRequestCount": 0 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "displayMs": 734.3342499999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 3779, + "ScriptDuration": 0.036499000000000004, + "TaskDuration": 0.26299399999999995, + "JSHeapUsedSize": 102326384 + }, + "inputRequestCount": 0 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "displayMs": 754.5465829999994, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03770599999999999, + "TaskDuration": 0.322369, + "JSHeapUsedSize": 62322640 + }, + "inputRequestCount": 0 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "displayMs": 752.0025829999995, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.035246, + "TaskDuration": 0.2752729999999999, + "JSHeapUsedSize": 121375344 + }, + "inputRequestCount": 0 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "displayMs": 876.8787090000005, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.039610000000000006, + "TaskDuration": 0.33228199999999997, + "JSHeapUsedSize": 62689612 + }, + "inputRequestCount": 0 + }, + { + "iteration": 5, + "cache": "warm-reload", + "valid": true, + "displayMs": 831.9686250000004, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.035584000000000005, + "TaskDuration": 0.2900280000000002, + "JSHeapUsedSize": 122461340 + }, + "inputRequestCount": 0 + }, + { + "iteration": 6, + "cache": "warm-reload", + "valid": true, + "displayMs": 735.4579999999996, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03858499999999998, + "TaskDuration": 0.3313840000000001, + "JSHeapUsedSize": 62360708 + }, + "inputRequestCount": 0 + }, + { + "iteration": 7, + "cache": "warm-reload", + "valid": true, + "displayMs": 728.6370829999996, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.03698299999999999, + "TaskDuration": 0.28565200000000024, + "JSHeapUsedSize": 122020840 + }, + "inputRequestCount": 0 + }, + { + "iteration": 8, + "cache": "warm-reload", + "valid": true, + "displayMs": 764.531167000001, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03685000000000005, + "TaskDuration": 0.3302559999999999, + "JSHeapUsedSize": 62350624 + }, + "inputRequestCount": 0 + }, + { + "iteration": 9, + "cache": "warm-reload", + "valid": true, + "displayMs": 745.8575419999997, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.03464999999999996, + "TaskDuration": 0.275687, + "JSHeapUsedSize": 122521852 + }, + "inputRequestCount": 0 + }, + { + "iteration": 10, + "cache": "warm-reload", + "valid": true, + "displayMs": 763.9299589999991, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03697400000000001, + "TaskDuration": 0.33788600000000013, + "JSHeapUsedSize": 61831432 + }, + "inputRequestCount": 0 + }, + { + "iteration": 11, + "cache": "warm-reload", + "valid": true, + "displayMs": 744.9626660000013, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.04094000000000003, + "TaskDuration": 0.28154400000000024, + "JSHeapUsedSize": 118205712 + }, + "inputRequestCount": 0 + }, + { + "iteration": 12, + "cache": "warm-reload", + "valid": true, + "displayMs": 770.1560000000009, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03733700000000001, + "TaskDuration": 0.334584, + "JSHeapUsedSize": 61821972 + }, + "inputRequestCount": 0 + }, + { + "iteration": 13, + "cache": "warm-reload", + "valid": true, + "displayMs": 747.7020409999986, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.040767, + "TaskDuration": 0.29415800000000036, + "JSHeapUsedSize": 121130200 + }, + "inputRequestCount": 0 + }, + { + "iteration": 14, + "cache": "warm-reload", + "valid": true, + "displayMs": 799.1140420000011, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03773599999999999, + "TaskDuration": 0.350333, + "JSHeapUsedSize": 61855212 + }, + "inputRequestCount": 0 + }, + { + "iteration": 15, + "cache": "warm-reload", + "valid": true, + "displayMs": 767.5240420000009, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.04225699999999999, + "TaskDuration": 0.28315999999999963, + "JSHeapUsedSize": 119146712 + }, + "inputRequestCount": 0 + }, + { + "iteration": 16, + "cache": "warm-reload", + "valid": true, + "displayMs": 754.5986249999987, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03715199999999996, + "TaskDuration": 0.32651699999999995, + "JSHeapUsedSize": 61865548 + }, + "inputRequestCount": 0 + }, + { + "iteration": 17, + "cache": "warm-reload", + "valid": true, + "displayMs": 726.5414170000004, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.03569600000000006, + "TaskDuration": 0.26559000000000044, + "JSHeapUsedSize": 121200588 + }, + "inputRequestCount": 0 + }, + { + "iteration": 18, + "cache": "warm-reload", + "valid": true, + "displayMs": 757.1587080000027, + "fetchRequests": 100, + "metrics": { + "Nodes": 3688, + "ScriptDuration": 0.03806299999999996, + "TaskDuration": 0.3226610000000001, + "JSHeapUsedSize": 61842340 + }, + "inputRequestCount": 0 + }, + { + "iteration": 19, + "cache": "warm-reload", + "valid": true, + "displayMs": 756.5308750000004, + "fetchRequests": 100, + "metrics": { + "Nodes": 5623, + "ScriptDuration": 0.03877300000000006, + "TaskDuration": 0.2907250000000001, + "JSHeapUsedSize": 119182480 + }, + "inputRequestCount": 0 + } + ], + "writes": [ + { + "name": "matching content edit", + "startedAt": 1789105584339, + "valid": true, + "ackAt": 1789105587591, + "ackMs": 3252, + "status": 200, + "ownerRevision": 4, + "observedAt": 1789105587616, + "ackToDisplayMs": 25, + "inputRequestCount": 0 + }, + { + "name": "transitive row label edit", + "startedAt": 1789105587631, + "valid": true, + "ackAt": 1789105590477, + "ackMs": 2846, + "status": 200, + "ownerRevision": 6, + "observedAt": 1789105590623, + "ackToDisplayMs": 146, + "inputRequestCount": 0 + }, + { + "name": "query exit", + "startedAt": 1789105590638, + "valid": true, + "ackAt": 1789105594571, + "ackMs": 3933, + "status": 200, + "ownerRevision": 8, + "observedAt": 1789105594691, + "ackToDisplayMs": 120, + "inputRequestCount": 0 + }, + { + "name": "query entry", + "startedAt": 1789105594707, + "valid": true, + "ackAt": 1789105598216, + "ackMs": 3509, + "status": 200, + "ownerRevision": 10, + "observedAt": 1789105598334, + "ackToDisplayMs": 118, + "inputRequestCount": 0 + }, + { + "name": "asynchronous source write", + "startedAt": 1789105598348, + "valid": true, + "ackAt": 1789105598373, + "ackMs": 25, + "status": 204, + "ownerRevision": 12, + "observedAt": 1789105601034, + "ackToDisplayMs": 2661, + "inputRequestCount": 0 + }, + { + "name": "unrelated write", + "startedAt": 1789105601047, + "valid": true, + "ackAt": 1789105601804, + "ackMs": 757, + "status": 200, + "ownerRevision": 12, + "observedAt": 1789105601935, + "ackToDisplayMs": 131, + "inputRequestCount": 0 + }, + { + "name": "matching insertion", + "startedAt": 1789105601949, + "valid": true, + "ackAt": 1789105601972, + "ackMs": 23, + "status": 204, + "ownerRevision": 15, + "observedAt": 1789105603743, + "ackToDisplayMs": 1771, + "inputRequestCount": 0 + }, + { + "name": "matching deletion", + "startedAt": 1789105603759, + "valid": true, + "ackAt": 1789105606416, + "ackMs": 2657, + "status": 204, + "ownerRevision": 17, + "observedAt": 1789105606572, + "ackToDisplayMs": 156, + "inputRequestCount": 0 + }, + { + "name": "reconnect after missed notification", + "startedAt": 1789105606586, + "valid": true, + "ackAt": 1789105606611, + "ackMs": 25, + "status": 204, + "ownerRevision": 19, + "observedAt": 1789105611774, + "ackToDisplayMs": 5163, + "inputRequestCount": 0 + } + ], + "pageErrors": [], + "concurrentReadCases": [ + { + "readers": 1, + "durationMs": 10000, + "startedAt": 1789105645093, + "readyReads": 376, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "version": 1, + "startedAt": 1789105646095, + "valid": true, + "ackAt": 1789105646119, + "ackMs": 24, + "observedAt": 1789105648783, + "ackToDisplayMs": 2664 + }, + { + "version": 2, + "startedAt": 1789105649786, + "valid": true, + "ackAt": 1789105649813, + "ackMs": 27, + "observedAt": 1789105652431, + "ackToDisplayMs": 2618 + }, + { + "version": 3, + "startedAt": 1789105653431, + "valid": true, + "ackAt": 1789105653444, + "ackMs": 13, + "observedAt": 1789105655999, + "ackToDisplayMs": 2555 + } + ], + "responseBytes": 11184120, + "browserInputRequests": 0, + "elapsedMs": 10906, + "latency": { + "n": 376, + "min": 5.575999999999112, + "median": 8.12945800000125, + "p95": 13.577250000000276, + "max": 2533.916833 + }, + "readyReadsPerSecond": 34.476434989913805, + "valid": true + }, + { + "readers": 10, + "durationMs": 10000, + "startedAt": 1789105659535, + "readyReads": 1515, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "version": 1, + "startedAt": 1789105660537, + "valid": true, + "ackAt": 1789105660582, + "ackMs": 45, + "observedAt": 1789105663549, + "ackToDisplayMs": 2967 + }, + { + "version": 2, + "startedAt": 1789105664548, + "valid": true, + "ackAt": 1789105664579, + "ackMs": 31, + "observedAt": 1789105667387, + "ackToDisplayMs": 2808 + }, + { + "version": 3, + "startedAt": 1789105668387, + "valid": true, + "ackAt": 1789105668444, + "ackMs": 57, + "observedAt": 1789105671437, + "ackToDisplayMs": 2993 + } + ], + "responseBytes": 45063675, + "browserInputRequests": 0, + "elapsedMs": 11902, + "latency": { + "n": 1515, + "min": 11.078999999997905, + "median": 21.944707999999082, + "p95": 35.031582999999955, + "max": 2874.6979580000007 + }, + "readyReadsPerSecond": 127.28953117123173, + "valid": true + }, + { + "readers": 50, + "durationMs": 10000, + "startedAt": 1789105675175, + "readyReads": 2249, + "pendingReads": 0, + "failures": [], + "writes": [ + { + "version": 1, + "startedAt": 1789105676181, + "valid": true, + "ackAt": 1789105676325, + "ackMs": 144, + "observedAt": 1789105679450, + "ackToDisplayMs": 3125 + }, + { + "version": 2, + "startedAt": 1789105680450, + "valid": true, + "ackAt": 1789105680567, + "ackMs": 117, + "observedAt": 1789105683716, + "ackToDisplayMs": 3149 + } + ], + "responseBytes": 66896505, + "browserInputRequests": 0, + "elapsedMs": 10043, + "latency": { + "n": 2249, + "min": 45.386749999997846, + "median": 90.09966700000223, + "p95": 137.59887499999604, + "max": 3032.98575 + }, + "readyReadsPerSecond": 223.93707059643535, + "valid": true + } + ] + }, + "htmlReplayAfterBoundedPrewarm": { + "valid": false, + "elapsedMs": 2662163.1567920004, + "stats": { + "fileErrors": 1, + "filesIndexed": 12550, + "instanceErrors": 1, + "instancesIndexed": 12549, + "totalIndexEntries": 12551 + }, + "failure": "Report/00144 embedded HTML timed out; Chrome main thread unresponsive. Full replay failed despite completing without worker heap exhaustion.", + "interpretation": "Background diagnostic; overlapping work prevents a controlled setup-time claim." + }, + "browserQualification": "Historical browser samples used a DOM-only oracle that could accept server HTML before client boot. Timings and graph-fetch assertions are provisional until rerun with subscribed client and server markup removed.", + "historicalSamplesNote": "Per-request load latency arrays remain in local raw artifacts; this file retains distributions and counts." + }, + "finalHTMLGuardValidation": { + "provenance": { + "commit": "6ff4ca411c992241505ceed69354a73793389bf6", + "runtimeHash": "95745d923222c62ca5f915a07c33ca6bc8c8e0beeaf88872c882c815e1f8eefa", + "hostHash": "1460100b6accd40007ff99eee07098f1230a043e5e4573a13955736683e1fc1b", + "workers": { + "allPriority": 1, + "userIndex": 1 + }, + "setupMode": "committed-json-index-only" + }, + "reads": [ + { + "iteration": 0, + "cache": "new-browser-context", + "valid": true, + "firstContentMs": 1023.1257500000002, + "displayMs": 1041.326417, + "interactiveMs": 1041.326417, + "fetchRequests": 99, + "metrics": { + "Nodes": 1845, + "ScriptDuration": 0.098782, + "TaskDuration": 0.461451, + "JSHeapUsedSize": 42882072 + }, + "observationMs": 1294.0034580000001, + "transferredBytes": 8026818, + "inputRequestCount": 0 + }, + { + "iteration": 1, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 775.773291, + "displayMs": 782.031, + "interactiveMs": 782.031, + "fetchRequests": 100, + "metrics": { + "Nodes": 3781, + "ScriptDuration": 0.03651700000000001, + "TaskDuration": 0.27297, + "JSHeapUsedSize": 102641812 + }, + "observationMs": 1033.732333, + "transferredBytes": 7670631, + "inputRequestCount": 0 + }, + { + "iteration": 2, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 789.4728339999997, + "displayMs": 799.6539589999998, + "interactiveMs": 799.6539589999998, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.03619900000000001, + "TaskDuration": 0.32264400000000004, + "JSHeapUsedSize": 62003236 + }, + "observationMs": 1051.665834, + "transferredBytes": 7670631, + "inputRequestCount": 0 + }, + { + "iteration": 3, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 794.8917080000001, + "displayMs": 800.1467499999999, + "interactiveMs": 800.1467499999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.03640199999999999, + "TaskDuration": 0.29677200000000004, + "JSHeapUsedSize": 122726176 + }, + "observationMs": 1053.15575, + "transferredBytes": 7670631, + "inputRequestCount": 0 + }, + { + "iteration": 4, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 805.4794579999998, + "displayMs": 809.9929999999995, + "interactiveMs": 809.9929999999995, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.037859000000000004, + "TaskDuration": 0.34494, + "JSHeapUsedSize": 62045356 + }, + "observationMs": 1061.243375, + "transferredBytes": 7670631, + "inputRequestCount": 0 + }, + { + "iteration": 5, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 751.8595839999998, + "displayMs": 757.6037919999999, + "interactiveMs": 757.6037919999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.037275, + "TaskDuration": 0.28639499999999996, + "JSHeapUsedSize": 120110804 + }, + "observationMs": 1008.6266249999999, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 6, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 825.4370420000005, + "displayMs": 830.5347920000004, + "interactiveMs": 830.5347920000004, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.042468000000000006, + "TaskDuration": 0.35923400000000005, + "JSHeapUsedSize": 62023628 + }, + "observationMs": 1081.9652920000008, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 7, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 777.9810840000009, + "displayMs": 783.5997920000009, + "interactiveMs": 783.5997920000009, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.036534999999999984, + "TaskDuration": 0.2859289999999999, + "JSHeapUsedSize": 122023276 + }, + "observationMs": 1035.0848750000005, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 8, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 807.9947919999995, + "displayMs": 813.525834, + "interactiveMs": 813.525834, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.03806999999999999, + "TaskDuration": 0.3457720000000002, + "JSHeapUsedSize": 62016732 + }, + "observationMs": 1064.9824590000007, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 9, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 768.0817079999997, + "displayMs": 772.6224579999998, + "interactiveMs": 772.6224579999998, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.03500600000000004, + "TaskDuration": 0.2854510000000001, + "JSHeapUsedSize": 121922000 + }, + "observationMs": 1024.4283749999995, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 10, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 803.0086250000004, + "displayMs": 808.3292500000007, + "interactiveMs": 808.3292500000007, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.040933, + "TaskDuration": 0.3579319999999999, + "JSHeapUsedSize": 61522268 + }, + "observationMs": 1059.7241250000006, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 11, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 763.514709000001, + "displayMs": 768.8409169999995, + "interactiveMs": 768.8409169999995, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.03777999999999998, + "TaskDuration": 0.2944490000000002, + "JSHeapUsedSize": 119986748 + }, + "observationMs": 1020.4826670000002, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 12, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 803.8143749999999, + "displayMs": 812.1637499999997, + "interactiveMs": 812.1637499999997, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.037734000000000045, + "TaskDuration": 0.35007699999999975, + "JSHeapUsedSize": 61530056 + }, + "observationMs": 1063.6568749999988, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 13, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 756.8191659999993, + "displayMs": 761.9914159999989, + "interactiveMs": 761.9914159999989, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.03665499999999999, + "TaskDuration": 0.2889609999999996, + "JSHeapUsedSize": 121698136 + }, + "observationMs": 1013.4649579999987, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 14, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 816.3234170000014, + "displayMs": 821.5709999999981, + "interactiveMs": 821.5709999999981, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.039700999999999986, + "TaskDuration": 0.3529970000000002, + "JSHeapUsedSize": 61529800 + }, + "observationMs": 1073.2470830000002, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 15, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 783.5977500000008, + "displayMs": 794.192332999999, + "interactiveMs": 794.192332999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.040788999999999964, + "TaskDuration": 0.30196100000000037, + "JSHeapUsedSize": 121527380 + }, + "observationMs": 1045.1912080000002, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 16, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 803.7130830000024, + "displayMs": 809.1827499999999, + "interactiveMs": 809.1827499999999, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.03836600000000001, + "TaskDuration": 0.3539519999999996, + "JSHeapUsedSize": 61524292 + }, + "observationMs": 1061.3474160000005, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 17, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 756.4148749999986, + "displayMs": 763.1697500000009, + "interactiveMs": 763.1697500000009, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.036270000000000024, + "TaskDuration": 0.28883499999999973, + "JSHeapUsedSize": 121248736 + }, + "observationMs": 1015.0797500000008, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 18, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 808.3443329999973, + "displayMs": 813.653457999997, + "interactiveMs": 813.653457999997, + "fetchRequests": 100, + "metrics": { + "Nodes": 3690, + "ScriptDuration": 0.03790099999999996, + "TaskDuration": 0.36127800000000043, + "JSHeapUsedSize": 61540320 + }, + "observationMs": 1065.9607499999984, + "transferredBytes": 7670632, + "inputRequestCount": 0 + }, + { + "iteration": 19, + "cache": "warm-reload", + "valid": true, + "firstContentMs": 788.3051250000026, + "displayMs": 794.0604170000006, + "interactiveMs": 794.0604170000006, + "fetchRequests": 100, + "metrics": { + "Nodes": 5626, + "ScriptDuration": 0.03966500000000006, + "TaskDuration": 0.30046000000000017, + "JSHeapUsedSize": 119678736 + }, + "observationMs": 1045.3259580000013, + "transferredBytes": 7670632, + "inputRequestCount": 0 + } + ], + "writes": [ + { + "name": "matching content edit", + "startedAt": 1789110564256, + "valid": true, + "ackAt": 1789110567380, + "ackMs": 3124, + "status": 200, + "ownerRevision": 17, + "observedAt": 1789110567400, + "ackToDisplayMs": 20, + "transitions": [ + { + "at": 1789110564449, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110567266, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "transitive row label edit", + "startedAt": 1789110567418, + "valid": true, + "ackAt": 1789110570605, + "ackMs": 3187, + "status": 200, + "ownerRevision": 19, + "observedAt": 1789110570770, + "ackToDisplayMs": 165, + "transitions": [ + { + "at": 1789110567578, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110570757, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "query exit", + "startedAt": 1789110570831, + "valid": true, + "ackAt": 1789110574925, + "ackMs": 4094, + "status": 200, + "ownerRevision": 21, + "observedAt": 1789110575051, + "ackToDisplayMs": 126, + "transitions": [ + { + "at": 1789110571042, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110575038, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "query entry", + "startedAt": 1789110575065, + "valid": true, + "ackAt": 1789110579119, + "ackMs": 4054, + "status": 200, + "ownerRevision": 23, + "observedAt": 1789110579250, + "ackToDisplayMs": 131, + "transitions": [ + { + "at": 1789110575258, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110579238, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "asynchronous source write", + "startedAt": 1789110579272, + "valid": true, + "ackAt": 1789110579292, + "ackMs": 20, + "status": 204, + "ownerRevision": 25, + "observedAt": 1789110582016, + "ackToDisplayMs": 2724, + "transitions": [ + { + "at": 1789110579428, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110582007, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "unrelated write", + "startedAt": 1789110582031, + "valid": true, + "ackAt": 1789110582839, + "ackMs": 808, + "status": 200, + "ownerRevision": 25, + "observedAt": 1789110582970, + "ackToDisplayMs": 131, + "transitions": [ + { + "at": 1789110582206, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110582966, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "matching insertion", + "startedAt": 1789110582983, + "valid": true, + "ackAt": 1789110583003, + "ackMs": 20, + "status": 204, + "ownerRevision": 28, + "observedAt": 1789110584862, + "ackToDisplayMs": 1859, + "transitions": [ + { + "at": 1789110583149, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110584855, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "matching deletion", + "startedAt": 1789110584876, + "valid": true, + "ackAt": 1789110587420, + "ackMs": 2544, + "status": 204, + "ownerRevision": 30, + "observedAt": 1789110587567, + "ackToDisplayMs": 147, + "transitions": [ + { + "at": 1789110585026, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110587561, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + }, + { + "name": "reconnect after missed notification", + "startedAt": 1789110587584, + "valid": true, + "ackAt": 1789110587604, + "ackMs": 20, + "status": 204, + "ownerRevision": 32, + "observedAt": 1789110592826, + "ackToDisplayMs": 5222, + "transitions": [ + { + "at": 1789110587586, + "state": "pending", + "visibleStats": 0 + }, + { + "at": 1789110587631, + "visibleStats": 0 + }, + { + "at": 1789110592816, + "state": "ready", + "visibleStats": 6 + } + ], + "inputRequestCount": 0 + } + ], + "crashCases": [ + { + "name": "worker crash before-publication", + "valid": true, + "ackMs": 29, + "initialHTMLHasOldStats": false, + "initialHTMLCacheControl": "no-store", + "interruptedReservation": { + "job_id": 10, + "reservation_id": 11, + "worker_id": "worker-pid-11912" + }, + "ackToCrashMs": 1352, + "ackToDisplayMs": 6231, + "newClientAckToDisplayMs": 6247, + "ownerRevision": 8, + "reservations": [ + { + "worker_id": "worker-pid-11912", + "completion_reason": "interrupted", + "completed_at": "2026-09-11T11:04:35.212Z" + }, + { + "worker_id": "worker-pid-12580", + "completion_reason": "completed", + "completed_at": "2026-09-11T11:04:39.884Z" + } + ], + "explicitPending": true, + "transitions": [ + { + "state": "pending", + "visibleStats": 0, + "at": 1789110273985 + }, + { + "state": "ready", + "visibleStats": 6, + "at": 1789110280056 + } + ], + "realmEvents": [ + { + "name": "index", + "at": 1789110273983, + "origin": 1789110273821 + }, + { + "name": "update", + "at": 1789110273983, + "origin": 1789110273821 + }, + { + "name": "index", + "at": 1789110280036, + "origin": 1789110279904 + } + ], + "initialConnection": true, + "pageErrors": [], + "inputRequests": [] + }, + { + "name": "worker crash after-source-publication", + "valid": true, + "ackMs": 33, + "committedDirtyGeneration": 11, + "interruptedReservation": { + "job_id": 14, + "reservation_id": 18, + "worker_id": "worker-pid-12580" + }, + "ackToCrashMs": 2240, + "ackToDisplayMs": 6121, + "ownerRevision": 13, + "reservations": [ + { + "worker_id": "worker-pid-12580", + "completion_reason": "interrupted", + "completed_at": "2026-09-11T11:06:42.161Z" + }, + { + "worker_id": "worker-pid-14606", + "completion_reason": "completed", + "completed_at": "2026-09-11T11:06:45.815Z" + } + ], + "explicitPending": true, + "transitions": [ + { + "state": "pending", + "visibleStats": 0, + "at": 1789110400042 + }, + { + "state": "ready", + "visibleStats": 6, + "at": 1789110406011 + } + ], + "realmEvents": [ + { + "name": "update", + "at": 1789110400041, + "origin": 1789110399903 + }, + { + "name": "index", + "at": 1789110400041, + "origin": 1789110399905 + }, + { + "name": "index", + "at": 1789110405972, + "origin": 1789110405841 + } + ], + "initialConnection": true, + "pageErrors": [], + "inputRequests": [] + } + ], + "note": "Final initial-HTML and cache guards added after the matched series. Zero page errors and input/search requests. Before-publication crash includes a separately authenticated client opened after acknowledgement." + } +} diff --git a/docs/tessar-performance-report.md b/docs/tessar-performance-report.md new file mode 100644 index 00000000000..8a029c0a131 --- /dev/null +++ b/docs/tessar-performance-report.md @@ -0,0 +1,262 @@ +# Tessar performance checkpoint — DO NOT MERGE + +Tessar's materialized JSON path passes the tested 10× read, mutation, reconnect, +concurrent-reader and worker-recovery cases. It displays a complete dashboard +without input-card or query-search requests. The equivalent getCards reference +produces correct initial output but fails the fixed write-freshness deadline. +The full 10× HTML pipeline also has a failed render. This is an implementation +checkpoint with measured results, **not a completed, production-ready POC or a +qualified freshness-preserving speedup claim**. + +Remaining work includes a controlled 1× reference, increased matching cardinality, +write bursts and multiple writing clients, broader replica/lifecycle coverage, +completion of the application workflows, and the failed 10× HTML path. No +additional AI-generation capacity has been established. + +## Contract and measurement correction + +Every qualifying display must match independently derived statistics and every +row's membership, order and values. An acknowledged relevant write must reach +the complete view within **10,000 ms**. Pending output hides old results; failed +work must not appear current. Write acknowledgement latency is separate from +acknowledgement-to-display delay. Setup allowances do not extend this deadline. + +The original browser harness could accept server HTML before Ember booted. +Historical version 1 browser timings and short graph-request observations are +therefore provisional. The version 2 harness requires a connected realm-event +subscription and removal of server markup, then validates the live DOM. It records +first content separately from interactive readiness. A Chromium regression +reproduces the old false positive and verifies its rejection. Results below use +the corrected gate; raw failures remain retained. + +Final review also found that the HTML injection path could serve an older ready +dashboard while JSON was pending. It now checks both materialization readiness +and the HTML revision, suppressing old output for queued/failed writes or delayed +HTML. Realm pages with active materializations use `no-store`; a deployment ETag +cannot bypass those checks. A failing-before/passing-after Postgres regression +covers head and isolated HTML. This guard was added after the matched read series +below and is measured separately as a final validation. + +The final guarded build passes another **20/20 connected reads and 9/9 writes**. +Interactive readiness is 800 ms median and 831 ms p95; the longest +acknowledgement-to-display delay is 5,222 ms, for reconnect. It makes zero +input-card/search requests and has zero page errors. This is a separate +regression run, not a new matched comparison with the reference. + +## 10× stable-data reads + +The fixture contains **12,550 fabricated instances**, seed 1729, distributed over +20 classroom keys with 40 saved views: 130 students, 260 staff, 1,730 slots, +1,640 observations, 440 reports, 630 activities and 7,660 references. The selected +view has 44 displayed rows. All 40 materialized owners pass the complete HTTP +oracle. No production records are inputs. The other presets are smoke (34) and +1× (1,255); 100× is unsupported. + +Both variants ran sequentially on the same local runtime and disk-backed +PostgreSQL 16.3 container, with one all-priority worker and one index-only worker. +The machine is an Apple M5, 10 physical cores, 24 GiB RAM. Headless Chromium +149.0.7827.55 used pinned Node 24.17.0, without CPU or network throttling. Other +builds and tests were excluded from these read series. The private application +runtime remained idle during the final candidate series. + +Each series has one new-context navigation and 19 reloads in that context. +Reloads recreate the client card store; this is not warm in-memory application +navigation. The database was warm. Variants were switched in place, so process +caches and allocator history were not reset between them. These are local +observations, not isolated-machine capacity estimates. + +| Observation | Tessar materialized view | Corrected getCards reference | +| -------------------------------------- | -----------------------: | ---------------------------------------: | +| Complete stable-data displays | 20/20 | 20/20 | +| Interactive readiness, median | 674 ms | 13,576 ms | +| Interactive readiness, p95 | 716 ms | 14,287 ms | +| Input-card/search requests per display | 0 | 86 | +| Total transferred bytes, median | 7,693,455 | 25,448,302 | +| Browser task duration, median | 308 ms | 1,801 ms | +| Browser DOM nodes, median | 4,042 | 28,703 | +| Browser JS heap, median | 78.5 MiB | 91.2 MiB | +| Page errors | 0 | 0 | +| First content-edit freshness gate | Pass | **Fail: still pending after 10 seconds** | + +Transfer counts include application modules, styles and authentication traffic; +they are not just business-data JSON. The candidate's first-content median is +485 ms, distinct from its 674 ms interactive median. CDP task duration includes +the 250 ms post-read observation window. Heap samples depend on garbage +collection: the candidate's maximum sample was 116.5 MiB, above the reference's +91.7 MiB maximum. These samples do not establish a general memory reduction. + +Earlier candidate HTTP samples returned approximately 29.7 kB each without +repeating query membership or input assembly. A bare getCards owner HTTP response +does not contain equivalent computed output and cannot be the baseline. The +browser reference pages all matching types in batches of 100 without a fixed +page-count cutoff. + +The reference includes explicit pending handling and complete re-querying on +publication/reconnect. It is a **corrected reference**, not an untouched incumbent +page. Both its initial trial and repeated-read series failed the first mutation's +freshness gate. After acknowledgement the browser hid its statistics and issued +35 input/search requests, but had not finished within 10 seconds. The failure +disqualifies an overall speedup claim; it is not removed from a successful +throughput denominator. + +![Tessar 10× observations and freshness limits](tessar-10x-performance.svg) + +## 10× writes and recovery + +All nine candidate mutations passed in the already-open connected browser with +zero input-card/search requests. The independent oracle checks full output. + +| Synthetic change | Write acknowledgement | Acknowledgement → complete display | +| --------------------------- | --------------------: | ---------------------------------: | +| Matching content edit | 2,901 ms | 117 ms | +| Transitive reference label | 2,561 ms | 138 ms | +| Query exit | 3,403 ms | 128 ms | +| Query entry | 3,338 ms | 116 ms | +| Asynchronous source edit | 20 ms | 2,588 ms | +| Unrelated reference edit | 790 ms | 134 ms | +| Matching insertion | 15 ms | 1,686 ms | +| Matching deletion | 2,223 ms | 145 ms | +| Offline write and reconnect | 35 ms | 5,177 ms | + +PATCH and DELETE wait for indexing before acknowledgement, so their small +subsequent display delays do not mean cheap writes. Source POSTs acknowledge +queued capture first. The unrelated edit leaves the selected owner's revision +unchanged. Before indexing classifies a change, the client conservatively marks +loaded realm snapshots pending; unrelated writes can still cause cheap owner +revalidation. + +Two deterministic worker interruptions passed the full connected-client gate: +before publication recovered in **3,730 ms**, and after durable source publication +in **5,556 ms**, from acknowledgement. Both showed pending with no old statistics, +used a replacement worker to finish the interrupted reservation, and returned +correct HTTP/browser output without graph fetching or page errors. The existing +queue restart mechanism is reused. + +Earlier crash trials sometimes missed pending while accepting prerendered HTML. +Those failures remain recorded. The corrected harness and first-sync +reconciliation pass the two tested crash points; this does not prove arbitrary +replica failures or silently lost notifications. + +With the final HTML guard, a second test opened a separately authenticated browser +after the source acknowledgement and before the worker crash. Its initial response +had no old statistics and used `no-store`. The existing client recovered in +6,231 ms and the new client in **6,247 ms**, with zero graph requests and page +errors. The post-source-publication crash case also passed again in 6,121 ms. + +## Readers during capture + +Each row is one approximately 10-second interval with sequential source writes, +simultaneous HTTP readers and one already-open connected browser. Every complete +response passes its full oracle and acknowledged revision floor. All nine writes +across these intervals met the unchanged deadline. + +| HTTP readers + one browser | Ready reads | Observed ready reads/s | Read p95 | Maximum acknowledgement → display | +| -------------------------: | ----------: | ---------------------: | ---------: | --------------------------------: | +| 1 | 392 | 37.6 | 11.4 ms | 2,478 ms | +| 10 | 1,709 | 151.7 | 32.7 ms | 2,839 ms | +| 50 | 2,292 | 195.8 | 2,665.7 ms | 2,833 ms | + +There were zero invalid responses and no browser input/search requests. GETs +waiting for indexing account for the long read tail at 50 readers; they remain +in the distribution. Throughput uses the observed elapsed interval, including +draining in-flight work. This is neither a 50-browser simulation nor a sustained +maximum-capacity test. Reference throughput is unqualified because its +single-browser freshness case already fails. + +## Cost moved to indexing, and the remaining HTML failure + +The 10× source pass indexed every record and materialized all 40 owners with +zero errors. It reported **668,837 ms total**, including **37,329 ms** for one +Tessar materialization wave. Its Tessar publication-swap counter was 20,224 ms; +these counters overlap and must not be added as independent totals. + +The following HTML pass exhausted the Node worker heap before its first file. +Prewarming chunked SQL parameters but retained dependency rows for the entire +realm, including millions of repeated dependency strings. The new implementation +processes 250 URLs at a time while preserving complete module coverage. + +A read-only probe on the actual synthetic index, with a 512 MiB Node old-space +limit, aborts on the old collector and completes on the bounded collector +(51 reads, approximately 487 MiB maximum RSS). Its module-cache sink is inert: +this isolates collection memory and is not an indexing-throughput comparison. + +The subsequent complete HTML replay avoided that heap failure but **still failed**: +one report's embedded render timed out with an unresponsive Chrome main thread. +It reported 12,550 files / 12,549 instances completed and one file / instance +error, after about 44 minutes. Other diagnostics overlapped that replay, so its +elapsed time is not controlled. The read/write benchmarks above cloned the +successful committed JSON index and explicitly skipped replay, recording +`setupMode: committed-json-index-only`. They do not turn this into a successful +full 10× setup. + +Earlier retained setup failures include a full 3.9 GiB PostgreSQL tmpfs and a +600-second harness startup timeout. Moving the disposable database to disk and +extending setup allowance did not alter the freshness gate. The existing +dedicated indexing lane prevents a long HTML job occupying the only worker +available for foreground indexing; the harness changes no production defaults. + +## Private application proof + +Three private-fork definitions project the existing schedule, coverage and +classroom-day models into compact rows and computed statistics. A fresh build +of 153 definitions and 34 fabricated instances completed with **zero source-index +and HTML errors**. HTML-only visits now consume published snapshots; index +discovery and materialization still compute from their inputs. + +All three connected browser displays pass exact row/statistic checks with no +input/search requests. Dashboard, schedule and coverage observations were 927, +756 and 878 ms, respectively. These three samples are correctness evidence, +not latency distributions. + +| Source change through a feeder | Acknowledgement | Acknowledgement → complete dashboard | +| ------------------------------ | --------------: | -----------------------------------: | +| Capture status | 16 ms | 5,929 ms | +| Distinct report coverage | 11 ms | 2,831 ms | +| Uncancelled schedule | 18 ms | 2,783 ms | + +Every case passed in the open browser and after a reload, hid old rows while +pending, and restored the fabricated input afterward. Explicit drill-down loaded +the selected source record. Portable generator/read/write checks are saved with +the private fork. No deployment realm or shared staging runtime was modified. +Full capture/report-generation UI flows and personal overlays remain to integrate. + +## Regression evidence and limits + +- Seven focused Tessar browser tests / 80 assertions pass, including empty values, + no getter/query/input work, nested computeds, edits, monotonic refresh, HTML + snapshot consumption and first-sync reconciliation. +- Six focused Postgres cases pass: predicate parity, atomic watch/output + publication, rollback, durable dirty state, generation guards, queued/failed + writes, module epochs, bounded prewarm and the first-watch registration race. + The new race test failed before the guard moved ahead of the no-watch fast + path, then passed. Initial-HTML tests also reject queued/failed or older output. + These are transaction interleavings, not multi-replica load. +- The existing sliding-sync regression passes four assertions. Earlier unchanged + computed and index-writer paths passed 15/41 and 56/153 tests/assertions. +- Four generator/browser-oracle tests pass. Host, runtime-common, realm-server + and harness lint/type checks pass, with two existing host RSVP warnings. + The three new private views have no parse/type errors; 830 pre-existing errors + remain among the other private definition files. + +The 10× measurements record exact dirty-source and built-host hashes. The +first-watch generation and initial-HTML guards were added after the matched +series and validated by the new Postgres regressions and final 10× browser run. +The private application build also includes the first-watch guard. Historical smoke and 1× results +remain in the sanitized JSON, subject to the old DOM-only qualification and, +for 1×, a different tmpfs storage configuration. + +The supported contract is opt-in saved views with same-realm Boxel query ASTs, +`computeVia`, and compact contained rows. Arbitrary iteration of query-backed +card instances can still hydrate their graph. Reverse watches supplement concrete +transitive dependencies and select owners for recomputation; they do not perform +arithmetic delta updates of arbitrary JavaScript. Large matching sets can still +cost heavily at index time. Cross-realm watches and cyclic feeders are outside +this POC. + +Browser lifecycle checks use authenticated Matrix sessions. Anonymous published +pages are not validated by this checkpoint and need their own subscription and +freshness contract before enabling materialized views there. + +Commands are in `scripts/tessar/README.md`; sanitized observations are in +`tessar-benchmark-results.json`. Large raw logs, traces, generated datasets and +failure diagnostics stay outside Git. The branch and PR remain DO NOT MERGE. diff --git a/docs/tessar-query-materialization-plan.md b/docs/tessar-query-materialization-plan.md new file mode 100644 index 00000000000..1a892b35ba4 --- /dev/null +++ b/docs/tessar-query-materialization-plan.md @@ -0,0 +1,477 @@ +# Tessar query materialization: DO NOT MERGE POC plan + +Status: implementation resumed September 10, 2026. Run focused correctness and +freshness tests and browser/network performance checks at each working increment. + +This is a draft experiment. Do not merge it, enable auto-merge or deploy its +runtime changes to production. Use **Tessar** as the codename in every new +monorepo-facing document, PR description, fixture family and benchmark artifact. +Keep real deployment names, account identifiers and realm URLs outside the +monorepo. Production records must never become fixtures or benchmark inputs. + +## Goal + +Use Boxel indexing to maintain shared dashboard lists and statistics. Readers +consume the materialized results instead of reconstructing the business-data +graph and repeating the computation. Source changes identify which saved views +need to be recomputed, including queries that previously returned no results. + +Reuse Boxel's query language, `computeVia`, indexed JSON, dependency tracking and +indexing workers. The initial implementation maintains results through +incremental invalidation followed by recomputation. It does not implement +arithmetic delta maintenance of arbitrary JavaScript aggregates. + +Success has two independent parts: + +1. A clean indexed view can display supplied computed results without calling + their getters or fetching their input graph again. +2. A relevant source change reliably refreshes that view, while unrelated writes + and additional readers do not repeatedly execute its query and computation. + +Correctness and freshness are non-negotiable validity gates. Performance is the +question the POC must measure and report once those gates pass. Lower latency, +less memory or higher throughput cannot compensate for wrong, incomplete or +stale results. A negative performance result is still useful evidence; a +correctness or freshness violation is a failed case, not an accepted tradeoff. + +## Current checkpoint + +September 11: the opt-in snapshot path, Postgres watch registry and index lifecycle +are implemented. The client and HTML renderer consume published computed values +and compact rows. Watches, outputs and durable dirty state publish together; +worker recovery and first-sync reconciliation reuse the existing runtime. +Initial HTML now shares the pending/failed and revision gates, and materialized +realm pages bypass deployment-ETag caching. A newly authenticated browser opened +during a pending write/crash reaches the complete result within 6.25 seconds. +The final guarded build passes another 20 connected reads and nine mutations: +800 ms median / 831 ms p95 interactive readiness, zero input/search requests, +and at most 5,222 ms acknowledgement-to-display delay across the nine mutations. + +The corrected browser harness requires live realm subscriptions and removal of +server HTML. Earlier DOM-only browser samples are provisional. On the synthetic +12,550-instance JSON path, 20/20 connected reads and nine mutations pass with +zero input/search requests. All 40 owner oracles pass. Two worker crash points +recover within the fixed 10-second deadline, and 1/10/50 HTTP-reader checks with +one connected browser pass their full output/revision gates. The corrected +getCards reference passes 20 stable-data reads but fails write freshness; it +cannot establish a qualified overall speedup. + +The complete 10× HTML pipeline remains failed. Bounded prewarming fixes the +initial dependency-collection heap exhaustion, but a subsequent full replay has +one unresponsive-Chrome render timeout. JSON-path diagnostic results are labeled +explicitly and do not count as a successful full build. The report records both +costs and failures rather than relaxing correctness or freshness. + +Three private application definitions use the existing schedule and coverage +models. A fresh build of 153 definitions and 34 fabricated records has zero JSON +or HTML errors. All three connected displays pass complete row/statistic checks; +three feeder-to-dashboard mutations, reloads and explicit drill-down pass. +Private generator and browser tests are saved with the staging fork mirror. +Full capture/report-generation UI flows and personal overlays remain to integrate. + +Seven focused browser tests / 80 assertions, six Postgres cases, four generator/ +browser-oracle tests and the four-assertion sliding-sync regression pass. The +first-watch registration race has a failing-before/passing-after transaction +regression. Modified package lint/type checks pass; private definitions retain +830 pre-existing errors, with zero in the three new views. + +M3 has tested lifecycle coverage, M4 proves the projected application display, +and M5 has a qualified checkpoint report and exportable chart. None is declared +fully complete: controlled 1× and increased matching-cardinality measurements, +write bursts and multiple writing clients, broader replica/lifecycle coverage, +the full application workflows and the failed 10× HTML path remain. See +[tessar-performance-report.md](tessar-performance-report.md) for evidence and limits. + +## Scope and constraints + +The first slice supports saved materialized summary cards, same-realm queries, +stable classroom/day parameters, and the typed equality, `in`, range and boolean +predicates needed by Tessar. Computations must derive from indexed inputs and +stable parameters. Keep draft state, current identity, date navigation, generation +progress and clock-based presentation local to each user. Store stable deadlines +as data rather than reindexing every minute. + +Use a small acyclic feeder graph. Keep concrete dependencies on consumed input +cards and their transitive inputs. Query watches supplement those dependencies. +Do not globally enable query-only dependencies for all existing cards. + +Defer cross-realm watches, full-text/ranking dependencies, arbitrary cyclic views, +field-level JavaScript read-set inference, a new aggregation engine, and a new +projected-query authoring API. Unsupported query shapes must be rejected for +materialization with a clear explanation or use a proven conservative fallback; +they must not silently lose invalidations. + +## Revalidate current main before implementing + +The earlier investigation used `730081f8b4`. The updated base has client behavior +that the implementation must preserve: + +| Area | Confirmed behavior and consequence | +| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `packages/base/field-support.ts` | Computed getters still invoke `computeVia` even when deserialization supplied a value. A pass-scoped compute memo is not a persisted snapshot. | +| `packages/host/app/routes/render/meta.ts` | Index serialization still uses `omitQueryFields: true` and excludes query-only runtime dependencies. | +| `packages/base/query-field-support.ts` | Eager query resolution, `eager: false`, newer-document seed handover and generation ordering already exist. Integrate with these paths. | +| `packages/host/app/resources/search.ts` | Seed ordering uses realm-specific generation floors. Ordinary live resources still refresh on realm events. Existing floors are not an exact materialization revision contract. | +| `packages/host/app/services/store.ts` | Selective search-entry inflation and scoped card searches exist. `addResourceFromSearchData` still adds a single-resource document; inspect compound-resource reuse before adding a cache. | +| `packages/runtime-common/realm-index-query-engine.ts` | GET/search assembly can populate query fields and expand links. Client snapshot support alone does not remove this server work. | +| `packages/runtime-common/index-writer.ts` | Working-index buffers, batch promotion and generation guards already exist. Integrate watch publication and invalidation with these mechanisms. | + +Record the exact base commit, existing behavior and relevant feature flags in +each benchmark run. Recheck these paths if main is updated again. + +## Synthetic data and baseline design + +Generate new records from a fixed seed. Use fabricated names, narratives, IDs, +scores, statuses and dates. Do not anonymize production records or use them as +templates. Keep credentials and runtime deployment addresses out of output. + +| Preset | Instance count, excluding realm/index configuration | Purpose | +| ------ | --------------------------------------------------: | ------------------------------------------------------------ | +| Smoke | Small explicit fixture | Human-auditable counts, membership and graph behavior | +| 1x | 1,255 | Reference workload approximating the current production size | +| 10x | 12,550 | Primary performance benchmark target | + +The benchmark target is 10x the recorded current instance count. Retain the 1x +reference and smoke fixtures for comparison and validation; do not generate or +run a 100x dataset. + +Retain a representative mix of reference, schedule, roster, observation, report +and summary records. Match the intended topology and approximate payload sizes, +not merely the total record count. The exact production-shaped distribution can +remain local; committed manifests describe only synthetic data. + +Vary these axes separately: + +- **Realm size:** add unrelated classrooms/dates while holding one watched view's + matching inputs and displayed rows constant. +- **Query cardinality:** increase matching inputs for one classroom/day while + holding the displayed summary size constant where possible. +- **Graph cost:** vary link depth, fan-out, shared targets and record body size. +- **Readers and writes:** use 1, 10 and 50 readers, then relevant and unrelated + writes, bursts, moves, deletes and changes to transitive dependencies. +- **Watch selectivity:** compare tightly scoped queries with broad/fallback + predicates. Broad dependencies are expected to cause broader invalidation. + +Implement a deterministic generator and run manifests, not a checked-in tree of +12,550 JSON files. Materialize datasets on demand, with bounded generator +memory and resumable seeding. Keep generated data and raw benchmark output out of +Git. Confirm referential integrity and expected summary values before measuring. + +Proposed monorepo locations: `scripts/tessar/` for generation/benchmark tooling, +small `tessar-*` GTS fixtures in the existing test fixture directories, and +`docs/tessar-*` for the plan and sanitized results. Use only locations needed by +the first experiment; avoid introducing a new package. + +Record the current implementation against the same synthetic fixture and the +same runtime/database resources used for the candidate. Separate cold indexing, +cold client load, warm reads, and post-write refresh. A staging baseline on a +different runtime or machine is contextual evidence, not a controlled speedup. + +## Contract to establish first + +Use one explicit opt-in on a materialized summary definition. Keep query authors +on existing Boxel query definitions and ordinary `computeVia`. Source card types +should not need new annotations simply because a summary consumes them. Settle +the smallest declaration syntax in the contract spike before changing APIs. + +An indexed response must identify its materialized fields and carry enough +provenance to establish: + +- Which computed values and query memberships are complete and successful. +- The definition/query revision and source realm that produced them. +- The input/index revision observed by the computation and the published owner + revision. Do not equate today's generation floor with an exact result revision. +- Whether a value is missing, unresolved, partial, errored or successfully empty. + +Use existing JSON attributes and relationship IDs for the values themselves. +Add the smallest versioned metadata necessary; do not invent a second card wire +format. Successful `0`, `false`, `null` and empty arrays must survive round trips. + +The mode applies to clean indexed views. Missing metadata follows existing +behavior. Unsaved cards, indexing and explicit live editing compute from their +inputs. Editing a snapshot-backed owner must explicitly leave snapshot mode or +use a local overlay; stale snapshot values must not masquerade as edited results. + +## Step 1: read the indexed results without rebuilding their inputs + +1. Capture supplied computed values in a snapshot structure separate from the + authored/default data bucket. Mark only values with valid server provenance. +2. In indexed-view mode, have the computed getter read that snapshot. Propagate + the mode through contained fields. Preserve ordinary computation elsewhere. +3. Keep complete query membership separately from hydrated member instances. + Bypass eager query resolution and realm-wide live searches for an authoritative + materialized field. Reuse existing query signatures and seed ordering. +4. Preserve already-returned raw resources in a bounded, revision-aware map where + current main loses them. Hydrate on demand through the existing identity map. + Passing the entire compound document into today's eager deserializer is not + sufficient: it can still instantiate the graph. +5. Do not bulk-fetch seed URLs simply to recover already-known membership. The + first UI uses serialized statistics and compact contained rows. Zero-hydration + arbitrary iteration of a `linksToMany` array is not a prerequisite. +6. Refresh snapshot values when the owner is invalidated, with generation and + definition checks. A local source write must refresh dependent summaries in + that same client; self-originated write suppression protects the actual edit. +7. Add the corresponding server read path: return stored membership and computed + values together, without rerunning query membership or recursively assembling + its input graph. Preserve ordinary explicit source-card reads. + +Initial tests may use supplied serialized fixtures to isolate client behavior. +This step is not ready for continuously refreshed views until step 2 works. + +Primary files: `packages/base/card-api.gts`, `packages/base/field-support.ts`, +`packages/base/query-field-support.ts`, `packages/host/app/services/store.ts`, +`packages/host/app/resources/search.ts`, +`packages/runtime-common/realm-index-query-engine.ts`, and the relevant handlers +in `packages/runtime-common/realm.ts`. + +Gate: rendering the supplied output executes zero materialized computed getters, +makes zero requests for its input dependencies, and does not run a query-field +search. Nested computeds, empty results, owner refresh, module replacement, +editing and legacy fallbacks work. Server GET does not repeat membership work. +Opening a source record is allowed to hydrate that record. + +## Step 2: durable reverse-query invalidation + +### Predicate semantics and candidate lookup + +Keep the resolved Boxel query AST as the authority. Resolve `$this` parameters +during indexing and include the source scope/type plus query/definition revision. +For a predicate `Pq` and changed indexed document `d`: + +```text +matches(d) = { q in saved queries | Pq(d) } +affected(old,new) = matches(old) union matches(new) +matches(d) is a subset of candidates(d) +``` + +An absent old/new version matches nothing. A content edit that matches both +versions still invalidates the summary: membership can stay the same while a +score, status or nested dependency changes a computed result. + +Build a registry keyed by owner and field, with source scope, resolved query and +revision. Derive a companion routing-term table; use ordinary indexed columns +such as realm, source type, field path, typed value and watch ID. The first +extractor can select one mandatory equality anchor, expand `in` anchors, and +fall back to a broader bucket when no safe anchor exists. Include type ancestry. + +Every boolean branch must be covered. Do not naively intersect term lists for +queries with different constrained fields, `OR`, or `NOT`. Candidate selection +may return extras but must never miss a matching query. Verify candidates against +the old/new documents using Boxel's existing semantics. Reuse or factor the +existing compiler rather than introduce an independently interpreted filter DSL. +Test nulls, plural paths, type inheritance and reference normalization: `eq` and +`in` currently have different rules. + +For sorted/paginated queries, watch the unpaginated membership predicate. A row +outside the current page can enter it after an insert or sort-key edit. Initially +invalidate conservatively; result fingerprints can be a later optimization. + +The algorithmic reference is [Elasticsearch Percolator](https://www.elastic.co/docs/reference/query-languages/query-dsl/query-dsl-percolate-query). +Its implementation can inform extraction/fallback review, but no Elasticsearch +dependency is required. PostgreSQL indexes route candidates; they do not +automatically reverse-evaluate a JSON query AST. + +### Registration, dependencies and publication + +1. Register watches during owner indexing, independent of open clients. Capture + successful empty queries and replace obsolete watches when parameters or + definitions change. Delete watches when their owner is deleted. +2. Retain concrete dependencies on consumed result cards and their transitive + inputs for opted-in computations. Keep ordinary cards' dependency behavior. +3. Capture old/new effective indexed rows before replacement or tombstoning. + Include dependency-driven reindexes and indexing-error transitions, not just + raw file writes. Avoid invalidating solely because unrelated HTML changed. +4. Select and verify candidate watches, then mark owners dirty durably. Deduplicate + owners across a batch and coalesce bursts through the existing indexing queue. + Process owners after the changed source rows are visible to their queries. +5. Publish computed values, membership and dependency/watch registrations as one + consistent owner revision. Ordinary GETs must not mix that snapshot with a + newly executed membership query. +6. Establish an input-watermark/retry protocol compatible with current working + index buffers and batch promotion. A source change racing registration cannot + disappear, and a worker finishing older work cannot clear newer dirty work. + Do not hold a database transaction open across long asynchronous card renders. +7. Propagate changed feeder outputs to their parents, with cycle detection and + an actionable failure mode. Full rebuilds need source-first scheduling or a + convergence pass. Cover retries, restarts and definition/module replacement. +8. Keep pending work in durable storage. Notifications only wake workers. Maintain + explicit error/freshness state when recomputation fails; do not publish partial + results as a successful complete snapshot. + +Primary files: `packages/runtime-common/index-writer.ts`, +`packages/runtime-common/index-query-engine.ts`, +`packages/runtime-common/index-runner.ts`, +`packages/runtime-common/index-runner/card-indexer.ts`, +`packages/runtime-common/index-runner/relationship-dependency-extractor.ts`, +`packages/runtime-common/dependency-tracker.ts`, +`packages/host/app/routes/render/meta.ts`, `packages/base/searchable.ts`, and the +query normalization/signature helpers. Add one focused runtime module for watch +registration/matching if that keeps responsibilities clear. Reuse existing queue +integration before adding a separate worker or scheduler. + +Create migrations with `pnpm create ` in `packages/postgres`, +then regenerate the SQLite schema with `pnpm make-schema`. Cover both adapters. + +Gate: insert into an empty result, deletion, moves between groups/dates, +content-only changes, transitive changes, pagination, watch replacement and +definition changes refresh the correct owner. Unrelated scoped writes do not. +Race/restart tests show no lost invalidation. Feeder chains converge. One client's +write refreshes its own displayed dependent summary as well as other clients'. + +## Tessar realm adaptation + +Create a persisted day summary and only the feeder views that have independent +consumers. Query existing source card types and compute compact contained rows +and statistics. Reuse the existing schedule composition, coverage, report-version +and pipeline rules; move their shared deterministic parts out of component getters. + +Display rows contain the labels, IDs, statuses, ordering values and counts the UI +actually needs. Preserve source IDs for drill-down/editing without embedding every +source card object. Keep user selection, unsaved overlays, timers and AI-generation +state local. Do not infer historical attendance from a current location field. + +Provision summary instances through synthetic seeding/day setup so indexing +can precompute them before a user visits. Decide explicitly how an uncached day +is provisioned; do not write shared navigation state onto a singleton dashboard. + +Retain truly ad hoc and AI-input queries where needed. Remove the broad read-time +loads serving the shared displayed lists/statistics. Verify no hidden component +getter or default eager query path reconstructs the same input graph. + +Do this work in the isolated development realm. Commit only synthetic Tessar +fixtures and generic runtime changes to the monorepo. + +## Milestones and reviewable increments + +| Milestone | Deliverable | Exit evidence | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------- | +| M0: baseline and contract | Revalidate main; small deterministic fixture/generator; define opt-in, metadata, modes, revisions and mandatory freshness guarantees | Baseline request/compute trace, independent expected results and fixed validity gates | +| M1: snapshot consumption | Base/host/server read path from step 1 | Zero recomputation/input requests for supplied outputs, with legacy/editing regression checks | +| M2: reverse matcher | Registry schema and conservative routing/verification primitives | Predicate parity and no-missed-candidate cases, including empty, boolean and pagination cases | +| M3: index lifecycle | Register watches, retain dependencies, coalesce dirty owners and publish consistently | End-to-end change propagation plus race, restart and feeder convergence tests | +| M4: Tessar display views | Synthetic realm summaries, compact rows and dashboard consumption | Matching UI output; successful source edits/drill-down and local overlays | +| M5: performance report | Controlled base/candidate runs across sizes and concurrency, enforcing correctness/freshness gates throughout | Reproducible Tessar performance report with measured gains, regressions, resource costs and limits | + +Keep these as distinct commits or small reviewable groups on the draft POC. M1 +alone does not establish freshness. M2 alone does not establish durable view +maintenance. Do not present a primitive-only proof as the completed use case. + +## Validation and measurements + +### Non-negotiable validity gates + +Establish and freeze the freshness contract in M0 before comparing performance. +Specify when a write is acknowledged, which source revision a view incorporates, +when that revision must be visible to existing and newly connected clients, and +the maximum permitted propagation delay. These requirements must not be relaxed +to make larger datasets or higher concurrency appear to pass. + +- Compare membership, order, displayed rows and statistics against independently + derived expected results from the synthetic inputs, including after changes. +- Check correctness and freshness during concurrent reads/writes and write bursts, + not only after the queue is empty. Include the writer's own client, another + client, newly connected clients, and a reconnect after missed notifications. +- Track every acknowledged relevant write through its materialized owner revision + to client visibility. Require no lost changes, backward revisions or mismatched + membership/statistics. Verify convergence after retries and worker restarts. +- If refresh is asynchronous, represent pending/failed freshness honestly; a + previous successful value cannot be labeled current while required writes are + missing. A source-write acknowledgement and a completed materialization are + distinct events unless the implemented contract explicitly makes them one. +- Count every wrong result, stale response, deadline breach, dropped operation + and error. A violated gate disqualifies the case from successful throughput or + speedup claims; retain the failed case in the report. + +Apply equivalent result semantics and freshness requirements to the baseline and +candidate. If an existing baseline truncates results or is stale, mark that case +invalid and use a correct reference path for the comparison. Never obtain an +apparent speedup by returning fewer results or by excluding failed requests. + +Extend the nearest existing tests, with small synthetic GTS fixtures and explicit +expected results rather than tests that repeat the implementation: + +- Host query-field acceptance, seed/refresh, membership-status and barrier tests. +- Host index-query-engine, index-writer and runtime-dependency-tracker tests. +- Realm-server skip-query-backed-expansion, live-search-cache, indexing and queue + tests where the corresponding integration behavior is changed. +- Focused generator integrity checks: repeatable output, valid links and known + summary values. Never commit a large generated dataset as test fixtures. + +Use the pinned mise/Node/pnpm toolchain. Use Glint, never direct `tsc` or +`glint --declaration`. Run package lint before commits that change package code. +Run focused host/realm-server tests and capture complete host output to files. +Do not run the entire host suite locally. Run realm lint and render validation +for changed GTS, recording any pre-existing baseline separately. + +For each benchmark, record commit, runtime/database configuration, seed, dataset +manifest, query shape/cardinality, graph shape, reader count and cache state. +Collect: + +- Cold/warm dashboard latency and response bytes. +- Client requests, instantiated cards and computed-getter invocations. +- Server query counts, assembly work and relevant database query plans/timing. +- Initial indexing time, worker CPU/memory and queue backlog. +- Reverse candidates examined, watches matched and owners actually reindexed. +- Source-write-to-summary freshness and work performed during write bursts. + +Use enough repeated observations to report distributions meaningfully; do not +label a single run p95. Hold displayed output size fixed in the unrelated-growth +case. Measure membership-ID serialization separately from source-graph expansion; +do not claim constant payload size when the returned list itself grows. + +Structural performance criteria are zero redundant computation and input fetching +on a snapshot read, and no reindex of an unrelated scoped summary. Numerical read +latency, memory and throughput targets will be set from M0 on the chosen runtime +rather than invented now. The correctness/freshness gates above apply regardless +of those performance targets. Large matching sets and broad watches can still +require substantial indexing work; quantify it. + +### Required performance benchmark report + +Deliver `docs/tessar-performance-report.md` and a compact, sanitized results file +such as `docs/tessar-benchmark-results.json`. Keep large raw traces and generated +datasets outside Git, with reproducible commands, seeds and manifest hashes. +The report is required to complete the POC; passing correctness tests alone does +not complete the work. + +Include: + +1. **Validity:** the fixed correctness/freshness contract, evidence that each + qualifying case passed, and all excluded/failed cases with their reasons. +2. **Controlled comparison:** base and candidate commits, runtime/database + configuration, warmup, repetitions, dataset/query shapes, cache conditions, + reader counts and write rates. Report latency distributions and error rates. +3. **Scaling:** the 10x synthetic benchmark with a production-sized reference, + separating unrelated realm growth from query cardinality and graph cost. Plot read + latency, resource use and refresh delay against these axes and concurrency. +4. **Cost placement:** initial materialization cost, incremental indexing and + reverse-matching work, write amplification, queue depth, client work, server + CPU/memory, database time and network bytes. Quantify the additional work on + writes alongside the work removed from reads. +5. **Capacity:** sustained readers and writes supported while the validity gates + hold. Report source-record capture latency and backlog under load, rather than + measuring fast reads in isolation from the ingestion/indexing workload. +6. **Conclusion:** measured improvements or regressions, the workload where the + approach pays off, limiting resources, and the next smallest justified change. + Do not promise that performance will improve before the measurements exist. + +Present base/candidate tables and standalone exportable charts. Compute totals +over declared read/write workloads so shifting work into indexing is visible in +the report. Do not infer spare AI-generation capacity from read latency alone; +support any such claim with measured resource headroom or a separate mixed-load +experiment. + +## Scope estimate and completion + +Working estimate, subject to the M0 recheck: 1,600–2,700 monorepo implementation +lines plus 1,400–2,300 focused test lines. Realm adaptation is about 600–1,000 +changed GTS lines, including moving existing logic. Synthetic generation and +benchmark tooling are a separate, not-yet-sized work item. These are estimates, +not a promised diff size or elapsed-time commitment. + +Finish with the required Tessar performance report, the tested contract, and a +list of remaining limitations. Only cases satisfying the non-negotiable +correctness/freshness gates qualify as successful benchmark results. The PR +remains draft and DO NOT MERGE even after its +tests pass. Production promotion and any shared-runtime deployment require a +separate decision. Implementation resumed on explicit user instruction. diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 0d06ea943cd..f22977dbc1d 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -2,6 +2,11 @@ import type Modifier from 'ember-modifier'; import GlimmerComponent from '@glimmer/component'; import { isEqual } from 'lodash-es'; import { WatchedArray, rawArrayValues } from './watched-array'; +import { + currentTessarInputSnapshot, + tessarSnapshotFields, + type TessarMaterialization, +} from '@cardstack/runtime-common'; import { BoxelInput, BrokenLinkTemplate, @@ -119,6 +124,7 @@ import { peekQueryFieldSearchResource, queryFieldHasUnreachableRealms, resolveQueryFieldEagerly, + tessarQueryWatch, validateRelationshipQuery, } from './query-field-support'; import { isSavedInstance } from './-private'; @@ -188,6 +194,11 @@ import { getFields, getRelationshipMembershipState, getter, + hasTessarQueryMembership, + hasTessarSnapshot, + leaveTessarSnapshot, + setTessarSnapshot, + tessarSnapshotState, registerRelationshipProbe, relationshipStateForEntry, readFieldLoadingSignal, @@ -236,6 +247,7 @@ import type { } from '@cardstack/runtime-common'; export const BULK_GENERATED_ITEM_COUNT = 3; +export { hasTessarSnapshot, markTessarPending } from './field-support'; interface CardOrFieldTypeIconSignature { Element: SVGSVGElement; @@ -1400,7 +1412,10 @@ class LinksTo implements Field { let deserialized = getDataBucket(instance); entangleWithCardTracking(instance); - if (this.queryDefinition) { + if ( + this.queryDefinition && + !hasTessarQueryMembership(instance, this.name) + ) { let dependencyTrackingContext = runtimeQueryDependencyContext({ queryField: this.name, consumer: instance.id, @@ -1876,7 +1891,10 @@ class LinksToMany implements Field< let deserialized = getDataBucket(instance); - if (this.queryDefinition) { + if ( + this.queryDefinition && + !hasTessarQueryMembership(instance, this.name) + ) { let dependencyTrackingContext = runtimeQueryDependencyContext({ queryField: this.name, consumer: instance.id, @@ -3915,6 +3933,12 @@ export class CardInfoField extends FieldDef { } export class CardDef extends BaseDef { + // Tessar POC: explicit, inherited opt-in for an indexed summary definition. + // Normal views stay on their existing live-computation path. + static tessarMaterialized = false; + get tessarState() { + return tessarSnapshotState(this); + } readonly [localId]: string = uuidv4(); [isSavedInstance] = false; get [fieldsUntracked](): Record | undefined { @@ -4612,6 +4636,24 @@ registerRelationshipProbe((instance, field) => { // bump (load start / settle, or query resource creation) re-evaluates this. readFieldLoadingSignal(instance, field.name); if (field.queryDefinition) { + if (hasTessarQueryMembership(instance, field.name)) { + let value = getDataBucket(instance).get(field.name); + let entries = Array.isArray(value) + ? rawArrayValues(value) + : value == null + ? [] + : [value]; + return { + isLoading: false, + isQueryField: true, + queryMembership: entries.map((entry) => + relationshipStateForEntry(entry), + ), + queryTotalMatchCount: entries.length, + queryIsPartial: false, + queryHasUnreachableRealms: false, + }; + } let resource = peekQueryFieldSearchResource(instance, field.name); let isLoading = resource?.isLoading ?? false; let bucketEntry = getDataBucket(instance).get(field.name); @@ -4918,6 +4960,29 @@ async function getDeserializedValue({ if (!field) { throw new Error(`could not find field ${fieldName} in card ${card.name}`); } + if (opts?.tessarSnapshot) { + let snapshot = opts.tessarSnapshot; + if (field.fieldType === 'contains' || field.fieldType === 'containsMany') { + let prefix = `${fieldName}.${field.fieldType === 'containsMany' ? '*.' : ''}`; + opts = { + ...opts, + tessarSnapshot: { + ...snapshot, + computedFields: snapshot.computedFields + .filter((path) => path.startsWith(prefix)) + .map((path) => path.slice(prefix.length)), + queryFields: snapshot.queryFields + .filter((path) => path.startsWith(prefix)) + .map((path) => path.slice(prefix.length)), + }, + }; + } else { + // Carry membership IDs without inflating the included input graph. An + // explicit later link read uses the ordinary lazy/identity-map path. + if (snapshot.queryFields.includes(fieldName)) doc = { data: resource }; + opts = { ...opts, tessarSnapshot: undefined }; + } + } let result = await field.deserialize( value, doc, @@ -5089,7 +5154,47 @@ async function _createFromSerialized( }); } -async function _updateFromSerialized({ +const tessarDeserializations = initSharedState( + 'tessarDeserializations', + () => new WeakMap>(), +); + +async function _updateFromSerialized(args: { + instance: BaseInstanceType; + resource: LooseCardResource; + doc: LooseSingleCardDocument | CardDocument; + store: CardStore; + opts?: DeserializeOpts; +}): Promise> { + let { instance, resource } = args; + if (!resource.meta.tessar) return _applySerialized(args); + // Serialize a snapshot's contained-field assembly too: rejecting an old + // root only after await is too late if it already mutated contained values. + let previous = tessarDeserializations.get(instance); + let update = (previous ?? Promise.resolve()) + .catch(() => undefined) + .then(async () => { + let accepted = (instance as any)[meta]?.tessar?.publishedGeneration; + let incoming = resource.meta.tessar?.publishedGeneration; + if ( + hasTessarSnapshot(instance) && + Number.isSafeInteger(accepted) && + Number.isSafeInteger(incoming) && + incoming! < accepted + ) + return instance; + return _applySerialized(args); + }); + tessarDeserializations.set(instance, update); + try { + return await update; + } finally { + if (tessarDeserializations.get(instance) === update) + tessarDeserializations.delete(instance); + } +} + +async function _applySerialized({ instance, resource, doc, @@ -5102,6 +5207,57 @@ async function _updateFromSerialized({ store: CardStore; opts?: DeserializeOpts; }): Promise> { + let inPrerender = (globalThis as any).__boxelRenderContext === true; + if ( + !opts?.tessarSnapshot && + resource.meta.tessar && + (!inPrerender || currentTessarInputSnapshot()) + ) { + opts = { ...opts, tessarSnapshot: tessarSnapshotFields(resource) }; + } + if (opts?.tessarSnapshot) { + for (let path of opts.tessarSnapshot.computedFields) { + let name = path.split('.')[0]; + if ( + !Object.prototype.hasOwnProperty.call(resource.attributes ?? {}, name) + ) { + throw new Error(`Tessar snapshot is missing computed output '${path}'`); + } + } + for (let name of opts.tessarSnapshot.queryFields) { + if (name.includes('.')) continue; // checked by contained deserialization + let relationship = resource.relationships?.[name]; + if (Array.isArray(relationship)) { + throw new Error( + `Tessar snapshot query '${name}' needs complete umbrella membership`, + ); + } + let membership = relationship?.data; + let count = Array.isArray(membership) + ? membership.length + : membership === null + ? 0 + : membership + ? 1 + : undefined; + if ( + count === undefined || + relationship?.meta?.total !== count || + relationship?.meta?.errors?.length + ) { + throw new Error( + `Tessar snapshot query '${name}' is missing, partial or errored`, + ); + } + } + opts = { + ...opts, + tessarSnapshot: { + ...opts.tessarSnapshot, + scope: opts.tessarSnapshot.scope ?? { active: true }, + }, + }; + } // because our store uses a tracked map for its identity map all the assembly // work that we are doing to deserialize the instance below is "live". so we // add the actual instance silently in a non-tracked way and only track it at @@ -5380,6 +5536,9 @@ async function _updateFromSerialized({ } let deserialized = getDataBucket(instance); + let snapshotOptions = opts?.tessarSnapshot; + let snapshotValues = new Map(); + for (let [field, value] of values) { if (!field) { continue; @@ -5404,9 +5563,26 @@ async function _updateFromSerialized({ applySubscribersToInstanceValue(instance, field, existingValue, value); } deserialized.set(field.name as string, value); + if ( + snapshotOptions?.computedFields.includes(field.name) && + field.computeVia + ) { + snapshotValues.set(field.name, value); + } field.captureQueryFieldSeedData?.(instance, value, resource); } + setTessarSnapshot( + instance, + snapshotOptions + ? { + scope: snapshotOptions.scope!, + values: snapshotValues, + queryFields: new Set(snapshotOptions.queryFields), + } + : undefined, + ); + // assign the realm meta before we compute as computeds may be relying on this if (!isFieldInstance(instance) && resource.id != null) { (instance as any)[meta] = resource.meta; @@ -5439,7 +5615,10 @@ async function _updateFromSerialized({ for (let field of Object.values( getFields(instance, { includeComputeds: true }), )) { - if (field?.queryDefinition) { + if ( + field?.queryDefinition && + !hasTessarQueryMembership(instance, field.name) + ) { resolveQueryFieldEagerly(store, instance, field); } } @@ -5515,6 +5694,7 @@ function setField(instance: BaseDef, field: Field, value: any) { propagateRealmContext(value, instance); // TODO: refactor validate to not have a return value and accomplish this normalization another way value = field.validate(instance, value); + leaveTessarSnapshot(instance); let deserialized = getDataBucket(instance); deserialized.set(field.name, value); notifySubscribers(instance, field.name, value); @@ -5527,6 +5707,7 @@ function notifySubscribers( value: any, visited = new WeakSet(), ) { + leaveTessarSnapshot(instance); if (visited.has(instance)) { return; } @@ -5803,6 +5984,122 @@ declare module 'ember-provide-consume-context/context-registry' { } } +// Pull a materialized owner's declared query inputs before collecting its +// computed output. Kept explicit so ordinary cards retain their lazy policy. +export async function prepareTessarQueries(instance: CardDef): Promise { + if (!(instance.constructor as typeof CardDef).tessarMaterialized) return; + for (let field of Object.values( + getFields(instance, { includeComputeds: true }), + )) { + if (field.queryDefinition) { + if (field.fieldType !== 'linksToMany') { + throw new Error( + 'Tessar currently materializes complete linksToMany queries', + ); + } + tessarQueryWatch(getStore(instance), instance, field); + ensureQueryFieldSearchResource(getStore(instance), instance, field); + } + } + await getStore(instance).loaded(); +} + +// serializeCard continues to omit query fields, so it never recursively +// serializes the input graph. Add complete membership umbrellas directly. +export function tessarIndexManifest( + instance: CardDef, + doc: LooseSingleCardDocument, + inputGeneration?: number, +): TessarMaterialization | undefined { + if (!(instance.constructor as typeof CardDef).tessarMaterialized) return; + let computedFields: string[] = []; + let watches: TessarMaterialization['watches'] = []; + let collect = ( + value: BaseDef, + prefix = '', + ancestors = new Set(), + ) => { + if (ancestors.has(value)) + throw new Error('Tessar does not support cyclic contained outputs'); + let nextAncestors = new Set(ancestors).add(value); + for (let [name, field] of Object.entries( + getFields(value, { includeComputeds: true }), + )) { + if (name === 'id') continue; + let path = `${prefix}${name}`; + if (field.queryDefinition) { + if (prefix || field.fieldType !== 'linksToMany') { + throw new Error( + 'Tessar currently requires top-level linksToMany queries', + ); + } + let watch = tessarQueryWatch(getStore(instance), instance, field); + watches.push({ fieldPath: path, query: watch.query }); + if (inputGeneration === undefined) continue; + let resource = peekQueryFieldSearchResource(instance, name); + let membership = getRelationshipMembershipState(instance, name); + if ( + !resource || + resource.isLoading || + resource.errors?.length || + !membership.isLoaded || + membership.isPartial || + membership.totalMatchCount !== membership.membership?.length || + membership.membership?.some((member) => member.kind !== 'present') + ) { + throw new Error( + `Tessar query '${name}' is unresolved, partial or errored`, + ); + } + doc.data.relationships ??= {}; + doc.data.relationships[name] = { + links: { self: null, search: watch.searchURL }, + data: membership.membership!.map((member) => ({ + type: 'card', + id: member.reference!, + })), + meta: { total: membership.totalMatchCount }, + }; + continue; + } + if (field.computeVia) { + if ( + field.fieldType !== 'contains' && + field.fieldType !== 'containsMany' + ) { + // Computed links (including CardDef.cardTheme) retain their ordinary + // lazy behavior. Only contained outputs are covered by this stamp. + continue; + } + computedFields.push(path); + } + if ( + primitive in field.card || + (field.fieldType !== 'contains' && field.fieldType !== 'containsMany') + ) + continue; + let child = (value as any)[name]; + if (field.fieldType === 'containsMany') { + for (let item of child ?? []) { + if (item) collect(item, `${path}.*.`, nextAncestors); + } + } else if (child) collect(child, `${path}.`, nextAncestors); + } + }; + collect(instance); + return { + version: 1, + state: 'pending', + computedFields: [...new Set(computedFields)], + queryFields: watches.map((watch) => watch.fieldPath), + watches, + // A discovery pass registers watches but cannot publish an authoritative + // result. Only the writer, after checking the pinned input revision, may + // change this marker to ready. + inputGeneration: inputGeneration ?? 0, + }; +} + function getStore(instance: BaseDef): CardStore { return stores.get(instance as BaseDef) ?? new FallbackCardStore(); } diff --git a/packages/base/card-serialization.ts b/packages/base/card-serialization.ts index a4803f9ee12..0c119783f46 100644 --- a/packages/base/card-serialization.ts +++ b/packages/base/card-serialization.ts @@ -82,6 +82,14 @@ export interface SerializeOpts { export interface DeserializeOpts { ignoreBrokenLinks?: true; dependencyTrackingContext?: RuntimeDependencyTrackingContext; + // Internal Tessar read mode, supplied only after the caller verifies server + // provenance/completeness. Never inferred from authored attributes. Contained + // paths use dots and '*' for containsMany entries; links remain lazy. + tessarSnapshot?: { + computedFields: string[]; + queryFields: string[]; + scope?: { active: boolean }; + }; } // --- Serialization Symbols --- diff --git a/packages/base/field-support.ts b/packages/base/field-support.ts index ceae18b86ad..4af70ad2e32 100644 --- a/packages/base/field-support.ts +++ b/packages/base/field-support.ts @@ -57,6 +57,74 @@ const deserializedData = initSharedState( 'deserializedData', () => new WeakMap>(), ); + +// Tessar's reader overlay is separate from authored/default values. The caller +// must validate the indexed revision before requesting snapshot deserialization. +// A contained edit leaves snapshot mode for the entire owner graph. +export interface TessarSnapshotScope { + active: boolean; + pending?: boolean; +} +const tessarSnapshots = initSharedState( + 'tessarSnapshots', + () => + new WeakMap< + BaseDef, + { + scope: TessarSnapshotScope; + values: Map; + queryFields: Set; + } + >(), +); + +export function setTessarSnapshot( + instance: BaseDef, + snapshot?: { + scope: TessarSnapshotScope; + values: Map; + queryFields: Set; + }, +): void { + let previous = tessarSnapshots.get(instance); + if (previous && previous.scope !== snapshot?.scope) + previous.scope.active = false; + if (snapshot) tessarSnapshots.set(instance, snapshot); + else tessarSnapshots.delete(instance); +} + +export function leaveTessarSnapshot(instance: BaseDef): void { + let snapshot = tessarSnapshots.get(instance); + if (snapshot) snapshot.scope.active = false; +} + +export function hasTessarSnapshot(instance: BaseDef): boolean { + return Boolean(tessarSnapshots.get(instance)?.scope.active); +} + +export function tessarSnapshotState( + instance: BaseDef, +): 'live' | 'ready' | 'pending' { + entangleWithCardTracking(instance); + let scope = tessarSnapshots.get(instance)?.scope; + return scope?.active ? (scope.pending ? 'pending' : 'ready') : 'live'; +} + +export function markTessarPending(instance: BaseDef): void { + let scope = tessarSnapshots.get(instance)?.scope; + if (scope?.active && !scope.pending) { + scope.pending = true; + notifyCardTracking(instance); + } +} + +export function hasTessarQueryMembership( + instance: BaseDef, + fieldName: string, +): boolean { + let snapshot = tessarSnapshots.get(instance); + return Boolean(snapshot?.scope.active && snapshot.queryFields.has(fieldName)); +} // Cache for resolved field configurations per instance/field const fieldConfigurationCache = initSharedState( 'fieldConfigurationCache', @@ -162,6 +230,10 @@ export function getter( cardTracking.get(instance); if (field.computeVia) { + let snapshot = tessarSnapshots.get(instance); + if (snapshot?.scope.active && snapshot.values.has(field.name)) { + return snapshot.values.get(field.name) as BaseInstanceType; + } // Fast path when no pass is open: skip the counter + memo entirely // so production reads pay only one branch on the module-local null // check. JIT branch-predicts this and the original behaviour is diff --git a/packages/base/query-field-support.ts b/packages/base/query-field-support.ts index d0edfe5037d..c68cd5cdf70 100644 --- a/packages/base/query-field-support.ts +++ b/packages/base/query-field-support.ts @@ -902,6 +902,36 @@ function resolveInstancePathValue(instance: BaseDef, path: string): any { return current; } +export function tessarQueryWatch( + store: CardStore, + instance: BaseDef, + field: Field, +) { + let definition = buildFieldDefinition(field); + if (!definition) + throw new Error(`Tessar cannot resolve query field '${field.name}'`); + let normalized = resolveQueryAndRealm(store, instance, field, definition); + if (!normalized) + throw new Error( + `Tessar cannot resolve query parameters for '${field.name}'`, + ); + let realm = (instance as any)[realmURLSymbol] as URL | undefined; + if ( + !realm || + normalized.realmHrefs.length !== 1 || + normalized.realmHrefs[0] !== realm.href + ) { + throw new Error( + 'Tessar materialization currently requires same-realm queries', + ); + } + return { + fieldPath: field.name, + query: normalized.query, + searchURL: normalized.searchURL, + }; +} + function buildFieldDefinition(field: Field): FieldDefinition | undefined { let ref = identifyCard(field.card); if (!ref) { diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index 2b98795270f..bc7c71f895a 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -263,6 +263,10 @@ export default class CardPrerender extends Component { ): RenderRouteOptions => { let out: RenderRouteOptions = { ...baseOptions, + tessarUseSnapshot: + visitType === 'prerender-html' && pass === 'cardRender' + ? true + : undefined, // The fused index pass carries both flags — the render route // serves it from the card branch and folds the file extract into // the render.meta payload. diff --git a/packages/host/app/lib/gc-card-store.ts b/packages/host/app/lib/gc-card-store.ts index 300f63fd76c..5d456e6c04f 100644 --- a/packages/host/app/lib/gc-card-store.ts +++ b/packages/host/app/lib/gc-card-store.ts @@ -25,6 +25,7 @@ import { type SingleFileMetaDocument, type VirtualNetwork, } from '@cardstack/runtime-common'; +import { currentTessarInputSnapshot } from '@cardstack/runtime-common/tessar-materialization'; import type { BaseDef, @@ -131,9 +132,14 @@ function currentRenderScope(): string | undefined { if (g.__boxelRenderContext !== true || typeof g.__boxelJobId !== 'string') { return undefined; } - return typeof g.__boxelRenderScope === 'string' - ? g.__boxelRenderScope - : g.__boxelJobId; + let scope = + typeof g.__boxelRenderScope === 'string' + ? g.__boxelRenderScope + : g.__boxelJobId; + let tessar = currentTessarInputSnapshot(); + return tessar + ? `${scope}:tessar:${tessar.realmURL}:${tessar.generation}` + : scope; } // we use this 2 way mapping between local ID and remote ID because if we end up @@ -570,7 +576,12 @@ export default class CardStoreWithGarbageCollection implements CardStore { } return await promise; } - promise = loadCardDocument(this.#fetch, url, this.#virtualNetwork); + promise = loadCardDocument( + this.#fetch, + url, + this.#virtualNetwork, + currentTessarInputSnapshot(), + ); // Held locally as well as in the map: a scope boundary clears the map // mid-flight, so reading it back in the `finally` would time this load // against a newer load's start — or find nothing and drop the entry. diff --git a/packages/host/app/lib/prerender-fetch-headers.ts b/packages/host/app/lib/prerender-fetch-headers.ts index 8e5ddc9b873..e874c8a5806 100644 --- a/packages/host/app/lib/prerender-fetch-headers.ts +++ b/packages/host/app/lib/prerender-fetch-headers.ts @@ -4,6 +4,10 @@ import { X_BOXEL_JOB_ID_HEADER, X_BOXEL_LOGGING_CORRELATION_ID_HEADER, } from '@cardstack/runtime-common'; +import { + currentTessarInputSnapshot, + TESSAR_INPUT_GENERATION_HEADER, +} from '@cardstack/runtime-common/tessar-materialization'; // Set by the prerender server's `evaluateOnNewDocument` before the // SPA boots, and also by the host's prerender-shaped routes @@ -16,7 +20,15 @@ import { export function duringPrerenderHeaders(): Record { let flag = (globalThis as unknown as { __boxelRenderContext?: boolean }) .__boxelRenderContext; - return flag === true ? { [DURING_PRERENDER_HEADER]: '1' } : {}; + let tessar = currentTessarInputSnapshot(); + return flag === true + ? { + [DURING_PRERENDER_HEADER]: '1', + ...(tessar + ? { [TESSAR_INPUT_GENERATION_HEADER]: String(tessar.generation) } + : {}), + } + : {}; } // The same marker, for a card write rather than a search. Gated on the diff --git a/packages/host/app/routes/render.ts b/packages/host/app/routes/render.ts index 69cf6a55e9f..7bec2ac8975 100644 --- a/packages/host/app/routes/render.ts +++ b/packages/host/app/routes/render.ts @@ -1,4 +1,5 @@ import type Controller from '@ember/controller'; + import { registerDestructor } from '@ember/destroyable'; import { action } from '@ember/object'; import Route from '@ember/routing/route'; @@ -38,6 +39,10 @@ import { coerceErrorMessage, serializableError, } from '@cardstack/runtime-common/error'; +import { + currentTessarInputSnapshot, + TESSAR_INPUT_GENERATION_HEADER, +} from '@cardstack/runtime-common/tessar-materialization'; import { windowErrorHandler, @@ -559,13 +564,23 @@ export default class RenderRoute extends Route { (globalThis as any).__renderModel = undefined; (globalThis as any).__boxelSetRenderStage?.('buildModel:fetching-source'); + let tessarInput = currentTessarInputSnapshot(); let response: Response; try { response = await this.#authGuard.race(() => - this.network.authedFetch(id, { + this.network.authedFetch(tessarInput ? id.replace(/\.json$/, '') : id, { method: 'GET', headers: { - Accept: SupportedMimeType.CardSource, + Accept: tessarInput + ? SupportedMimeType.CardJson + : SupportedMimeType.CardSource, + ...(tessarInput + ? { + [TESSAR_INPUT_GENERATION_HEADER]: String( + tessarInput.generation, + ), + } + : {}), }, }), ); @@ -581,6 +596,17 @@ export default class RenderRoute extends Route { let lastModified = new Date(response.headers.get('last-modified')!); let doc: LooseSingleCardDocument | CardErrorsJSONAPI = await response.json(); + if (tessarInput && 'data' in doc && doc.data.meta.tessar) { + // This is the owner being recomputed. Its previous membership is an + // output, never an input seed for the new generation's query. + for (let field of doc.data.meta.tessar.queryFields) { + for (let key of Object.keys(doc.data.relationships ?? {})) { + if (key === field || key.startsWith(`${field}.`)) + delete doc.data.relationships![key]; + } + } + delete doc.data.meta.tessar; + } let canonicalId = id.replace(/\.json$/, ''); let state = new TrackedMap(); @@ -623,29 +649,58 @@ export default class RenderRoute extends Route { throw new Error(JSON.stringify(doc.errors[0], null, 2)); } (globalThis as any).__boxelSetRenderStage?.('buildModel:deriving-type'); + let sourceDoc = doc; let { derivedCardType, hydratedInstance } = await this.#authGuard.race( async () => { + let renderDoc: LooseSingleCardDocument = sourceDoc; + let tessarUseSnapshot: true | undefined; let derivedCardType = await deriveCardTypeFromDoc( - doc, + renderDoc, id, this.loaderService.loader, ); + if (parsedOptions.tessarUseSnapshot && !tessarInput) { + let Klass = await loadCardDef(renderDoc.data.meta.adoptsFrom, { + loader: this.loaderService.loader, + relativeTo: this.network.virtualNetwork.toURL(canonicalId), + }); + if ((Klass as typeof CardDef).tessarMaterialized) { + let indexedResponse = await this.network.authedFetch( + canonicalId, + { + headers: { Accept: SupportedMimeType.CardJson }, + }, + ); + let indexed = await indexedResponse.json(); + if ( + !indexedResponse.ok || + indexed.data?.meta?.tessar?.state !== 'ready' + ) { + throw new Error( + 'Tessar HTML requires a ready published materialization', + ); + } + renderDoc = indexed as LooseSingleCardDocument; + tessarUseSnapshot = true; + } + } + await this.realm.ensureRealmMeta(realmURL); let screenshotsMeta = await this.declarationScreenshotsMeta( - doc, + renderDoc, canonicalId, realmURL, ); let enhancedDoc: LooseSingleCardDocument = { - ...doc, + ...renderDoc, data: { - ...doc.data, + ...renderDoc.data, id: canonicalId, type: 'card', meta: { - ...doc.data.meta, + ...renderDoc.data.meta, lastModified: lastModified.getTime(), realmURL: realmURL as RealmIdentifier, realmInfo: { ...this.realm.info(id) }, @@ -659,6 +714,7 @@ export default class RenderRoute extends Route { relativeTo: rri(id), realm: realmURL, doNotPersist: true, + tessarUseSnapshot, }); (globalThis as any).__boxelSetRenderStage?.( 'buildModel:store-settle', diff --git a/packages/host/app/routes/render/meta.ts b/packages/host/app/routes/render/meta.ts index 9090bb70936..f660f5641af 100644 --- a/packages/host/app/routes/render/meta.ts +++ b/packages/host/app/routes/render/meta.ts @@ -24,6 +24,7 @@ import { type PrerenderMetaDiagnostics, type RenderError, } from '@cardstack/runtime-common'; +import { currentTessarInputSnapshot } from '@cardstack/runtime-common/tessar-materialization'; import type CardService from '@cardstack/host/services/card-service'; import type EnvironmentService from '@cardstack/host/services/environment-service'; @@ -168,6 +169,9 @@ export default class RenderMetaRoute extends Route { // The search doc comes from the searchable-driven generator in its own base // module. It derives link depth from the explicit `searchable` annotations // rather than from what the render happened to load. + let tessar = (instance.constructor as typeof CardDef).tessarMaterialized; + let tessarInput = currentTessarInputSnapshot(); + if (tessar && tessarInput) await api.prepareTessarQueries(instance); let searchable = await this.cardService.getSearchable(); // Produce the search doc by walking until the store's load state is @@ -238,7 +242,7 @@ export default class RenderMetaRoute extends Route { ...new Set([ SEARCHABLE_MODULE_URL, ...(renderModel?.capturedDeps ?? []), - ...snapshotRuntimeDependencies({ excludeQueryOnly: true }).deps, + ...snapshotRuntimeDependencies({ excludeQueryOnly: !tessar }).deps, ...searchableDeps, ]), ]; @@ -301,6 +305,11 @@ export default class RenderMetaRoute extends Route { delete relationship.data; } delete serialized.included; + let tessarManifest = tessar + ? api.tessarIndexManifest(instance, serialized, tessarInput?.generation) + : undefined; + delete serialized.data.meta.tessar; + if (tessarManifest) serialized.data.meta.tessar = tessarManifest; } finally { if (passOpen && typeof api.endComputePass === 'function') { passSnapshot = api.endComputePass(); @@ -361,6 +370,11 @@ export default class RenderMetaRoute extends Route { if (typeof api.getBrokenLinks === 'function') { let brokenLinks = api.getBrokenLinks(instance); if (brokenLinks.length > 0) { + if (tessar && tessarInput) { + throw new Error( + 'Tessar cannot publish with broken input dependencies', + ); + } diagnostics.brokenLinks = brokenLinks.map( ({ fieldName, reference, kind }) => ({ fieldName, reference, kind }), ); @@ -512,6 +526,11 @@ export default class RenderMetaRoute extends Route { continue; } if (!stable) { + if ((instance.constructor as typeof CardDef).tessarMaterialized) { + throw new Error( + 'Tessar materialization did not reach a stable input graph', + ); + } computePerfLog.warn( `render.meta searchable walk for ${instance.id} did not reach a stable load generation within ${SEARCHABLE_SETTLE_MAX_PASSES} passes; using the current store state`, ); diff --git a/packages/host/app/services/matrix-service.ts b/packages/host/app/services/matrix-service.ts index e5cf963d493..259c634394e 100644 --- a/packages/host/app/services/matrix-service.ts +++ b/packages/host/app/services/matrix-service.ts @@ -1287,6 +1287,8 @@ export default class MatrixService extends Service { let wasAuthenticated = this.session.isAuthenticated; this.setPostLoginCompleted(true, 'start-success'); + if (this.initialSyncCompleted) + this.messageService.tessarConnectionChanged(true); loginCompletedThisRun = true; if (isTesting()) console.warn('[start-phase] postLoginCompleted=true'); // Symmetric to logout()'s notifySessionEnded(): tell every session @@ -1411,6 +1413,8 @@ export default class MatrixService extends Service { let roomIds: string[] = list?.ops?.[0]?.room_ids ?? []; switch (state) { case SlidingSyncState.Complete: + if (this.session.isAuthenticated && this.initialSyncCompleted) + this.messageService.tessarConnectionChanged(true); if (!this.initialSyncCompleted) { Promise.allSettled([ this.drainRoomState(), @@ -1418,6 +1422,8 @@ export default class MatrixService extends Service { this.drainTimeline(), ]).then(() => { this.initialSyncCompleted = true; + if (this.session.isAuthenticated) + this.messageService.tessarConnectionChanged(true); this.initialSyncCompletedDeferred.fulfill(); this.slidingSync?.setList( SLIDING_SYNC_AI_ROOM_LIST_NAME, @@ -1428,6 +1434,7 @@ export default class MatrixService extends Service { roomIds.forEach((id) => this.roomsWaitingForSync.get(id)?.fulfill()); break; case SlidingSyncState.RequestFinished: + if (!resp) this.messageService.tessarConnectionChanged(false); roomIds.forEach((id) => this.aiRoomIds.add(id)); break; } diff --git a/packages/host/app/services/message-service.ts b/packages/host/app/services/message-service.ts index 8ac7efbb8da..d849bd35c8f 100644 --- a/packages/host/app/services/message-service.ts +++ b/packages/host/app/services/message-service.ts @@ -1,3 +1,4 @@ +import { registerDestructor } from '@ember/destroyable'; import Service, { service } from '@ember/service'; import { tracked } from '@glimmer/tracking'; @@ -11,10 +12,46 @@ export default class MessageService extends Service { new Map(); @service declare private network: NetworkService; @service declare private session: SessionService; + // The first completed sync must reconcile views fetched before realm-event + // subscriptions were ready. Treating startup as connected drops that edge. + private tessarConnected = false; + private tessarConnectionListeners = new Set<(connected: boolean) => void>(); constructor(...args: ConstructorParameters) { super(...args); this.session.register(this); + let offline = () => this.tessarConnectionChanged(false); + let online = () => this.tessarConnectionChanged(true); + let visible = () => { + if (document.visibilityState === 'visible') { + this.tessarConnectionChanged(false); + this.tessarConnectionChanged(navigator.onLine); + } + }; + window.addEventListener('offline', offline); + window.addEventListener('online', online); + document.addEventListener('visibilitychange', visible); + registerDestructor(this, () => { + window.removeEventListener('offline', offline); + window.removeEventListener('online', online); + document.removeEventListener('visibilitychange', visible); + this.tessarConnectionListeners.clear(); + }); + } + + get isTessarConnected() { + return this.tessarConnected; + } + + subscribeTessarConnection(callback: (connected: boolean) => void) { + this.tessarConnectionListeners.add(callback); + return () => this.tessarConnectionListeners.delete(callback); + } + + tessarConnectionChanged(connected: boolean) { + if (this.tessarConnected === connected) return; + this.tessarConnected = connected; + for (let callback of this.tessarConnectionListeners) callback(connected); } register() { @@ -31,6 +68,7 @@ export default class MessageService extends Service { // Re-login re-subscribes fresh (RealmResource.subscribe(), etc.), so a clear // here can't strand a live session's wiring. this.listenerCallbacks = new Map(); + this.tessarConnected = false; if ((globalThis as any)._CARDSTACK_REALM_SUBSCRIBE === this) { delete (globalThis as any)._CARDSTACK_REALM_SUBSCRIBE; } diff --git a/packages/host/app/services/store.ts b/packages/host/app/services/store.ts index c47ee0a26a3..0ed7812c7bb 100644 --- a/packages/host/app/services/store.ts +++ b/packages/host/app/services/store.ts @@ -96,6 +96,11 @@ import { type VirtualNetwork, } from '@cardstack/runtime-common'; +import { + currentTessarInputSnapshot, + tessarSnapshotFields, +} from '@cardstack/runtime-common/tessar-materialization'; + import CardStore, { getDeps, type ReferenceCount } from '../lib/gc-card-store'; import { @@ -223,6 +228,7 @@ const queryFieldSeedFromSearchSymbol = Symbol.for( type PersistOptions = CreateOptions & { clientRequestId?: string }; type DependencyTrackingOptions = { + tessarUseSnapshot?: true; dependencyTrackingContext?: RuntimeDependencyTrackingContext; }; type TrackedCreateOptions = CreateOptions & DependencyTrackingOptions; @@ -339,8 +345,21 @@ export default class StoreService extends Service implements StoreInterface { this.store = this.createCardStore(); this.session.register(this); this.ready = this.setup(); + let unsubscribeTessar = this.messageService.subscribeTessarConnection( + (connected) => { + if (this.isRenderStore || (globalThis as any).__boxelPrerenderApp) + return; + for (let instance of this.store.allCardInstances()) { + if (instance.id && this.cardApiCache?.hasTessarSnapshot?.(instance)) { + this.cardApiCache.markTessarPending(instance); + if (connected) this.reloadTask.perform(instance); + } + } + }, + ); registerDestructor(this, () => { clearInterval(this.gcInterval); + unsubscribeTessar(); }); } @@ -825,6 +844,7 @@ export default class StoreService extends Service implements StoreInterface { instanceOrDoc, opts?.relativeTo, opts?.dependencyTrackingContext, + opts?.tessarUseSnapshot, ); } else { instance = instanceOrDoc; @@ -1518,6 +1538,11 @@ export default class StoreService extends Service implements StoreInterface { // the undefined wire default — and so must share one key; keying on the raw // `scope` would split them and defeat the dedup. let wireScope = this.resolveWireScope(query, scope); + // Tessar reads are pinned to a committed generation. Never reuse a raw + // source pass's job cache or an in-flight request from an earlier wave. + if (currentTessarInputSnapshot()) { + return this.fetchSearchDocUncoalesced(query, realms, wireScope); + } // Resolved-doc cache eligibility: prerender + jobId + same-realm. // Cross-realm reads bypass — see field comment. @@ -2022,14 +2047,23 @@ export default class StoreService extends Service implements StoreInterface { doc: LooseSingleCardDocument | CardDocument, relativeTo?: RealmResourceIdentifier | URL | undefined, dependencyTrackingContext?: RuntimeDependencyTrackingContext, + tessarUseSnapshot?: true, ): Promise { let api = await this.cardService.getAPI(); + let needsTessarConnection = + !!resource.meta.tessar && + !this.isRenderStore && + !(globalThis as any).__boxelPrerenderApp && + !this.messageService.isTessarConnected; let shouldStubTimers = this.renderContextBlocksPersistence() && !isTesting(); let performCreate = async () => (await api.createFromSerialized(resource, doc, relativeTo, { store: this.store, dependencyTrackingContext, + ...(tessarUseSnapshot + ? { tessarSnapshot: tessarSnapshotFields(resource) } + : {}), })) as T; // Time the deserialize and report it (no-op when telemetry is disabled). let telemetry = this.#clientTelemetry(); @@ -2037,6 +2071,12 @@ export default class StoreService extends Service implements StoreInterface { let card = shouldStubTimers ? await withStubbedRenderTimers(performCreate) : await performCreate(); + if (needsTessarConnection && api.hasTessarSnapshot(card)) { + // A snapshot fetched before subscriptions are ready has a read/sync + // gap. Keep it explicitly pending until one connected revalidation. + api.markTessarPending(card); + if (this.messageService.isTessarConnected) this.reloadTask.perform(card); + } if (telemetry?.isEnabled && deserializeStart !== undefined) { telemetry.recordDeserialize({ durationMs: performance.now() - deserializeStart, @@ -2148,6 +2188,20 @@ export default class StoreService extends Service implements StoreInterface { } private handleInvalidations = (event: RealmEventContent) => { + if (event.eventName === 'update') { + // Before indexing has classified a source change, conservatively mark + // this realm's loaded snapshots pending. Keep their inputs unhydrated. + for (let instance of this.store.allCardInstances()) { + if ( + instance.id && + asURL(instance.id, this.network.virtualNetwork).startsWith( + event.realmURL, + ) + ) + this.cardApiCache?.markTessarPending?.(instance); + } + return; + } if (event.eventName !== 'index') { return; } @@ -2172,6 +2226,18 @@ export default class StoreService extends Service implements StoreInterface { // can be the only word a card being held as awaiting-index ever gets // that the row it is waiting for now exists. let reloadsTriggered = this.reloadAwaitingIndexInstances(event.realmURL); + for (let instance of this.store.allCardInstances()) { + if ( + instance.id && + asURL(instance.id, this.network.virtualNetwork).startsWith( + event.realmURL, + ) && + this.cardApiCache?.hasTessarSnapshot?.(instance) + ) { + this.reloadTask.perform(instance); + reloadsTriggered++; + } + } // Report the pass as a thin realm-event so the dashboard still sees it. telemetry?.recordEvent({ event_type: 'realm-event', @@ -2190,7 +2256,21 @@ export default class StoreService extends Service implements StoreInterface { if (event.indexType !== 'incremental') { return; } - let invalidations = event.invalidations as string[]; + let invalidations = [...(event.invalidations as string[])]; + // An unrelated write can leave the owner revision unchanged. Revalidate + // the pending snapshot after the lane settles to clear its pending state. + for (let instance of this.store.allCardInstances()) { + if ( + instance.id && + asURL(instance.id, this.network.virtualNetwork).startsWith( + event.realmURL, + ) && + instance.tessarState === 'pending' + ) { + let url = asURL(instance.id, this.network.virtualNetwork); + if (!invalidations.includes(url)) invalidations.push(url); + } + } let ownWrite = event.clientRequestId ? this.cardService.clientRequestIds.has(event.clientRequestId) : false; @@ -2340,6 +2420,11 @@ export default class StoreService extends Service implements StoreInterface { ); } + // A clean materialized owner may depend on the card this client + // edited. Local edits leave snapshot mode, so this exception cannot + // overwrite a draft owner while suppressing self-originated events. + if (this.cardApiCache?.hasTessarSnapshot?.(instance)) + reloadFile = true; if (reloadFile) { this.reloadTask.perform(instance); reloadsTriggered++; diff --git a/packages/host/config/schema/1788216332849_schema.sql b/packages/host/config/schema/1789091944988_schema.sql similarity index 88% rename from packages/host/config/schema/1788216332849_schema.sql rename to packages/host/config/schema/1789091944988_schema.sql index fbf74a8e90d..3f7b3c69751 100644 --- a/packages/host/config/schema/1788216332849_schema.sql +++ b/packages/host/config/schema/1789091944988_schema.sql @@ -222,6 +222,34 @@ PRIMARY KEY ( realm_url, username ) ); + CREATE TABLE IF NOT EXISTS tessar_owners ( + realm_url TEXT NOT NULL, + owner_url TEXT NOT NULL, + published_generation NOT NULL, + input_generation NOT NULL, + dirty_generation, + definition_revision TEXT NOT NULL, + retired BOOLEAN DEFAULT false NOT NULL, + PRIMARY KEY ( realm_url, owner_url ) +); + + CREATE TABLE IF NOT EXISTS tessar_query_terms ( + realm_url TEXT NOT NULL, + owner_url TEXT NOT NULL, + field_path TEXT NOT NULL, + path TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY ( realm_url, owner_url, field_path, path, value ) +); + + CREATE TABLE IF NOT EXISTS tessar_query_watches ( + realm_url TEXT NOT NULL, + owner_url TEXT NOT NULL, + field_path TEXT NOT NULL, + query BLOB NOT NULL, + PRIMARY KEY ( realm_url, owner_url, field_path ) +); + CREATE TABLE IF NOT EXISTS unlisted_realm_paths ( id DEFAULT (hex(randomblob(16))) NOT NULL, source_realm_url TEXT NOT NULL, diff --git a/packages/host/tests/integration/components/tessar-snapshot-test.gts b/packages/host/tests/integration/components/tessar-snapshot-test.gts new file mode 100644 index 00000000000..c3336ba5b70 --- /dev/null +++ b/packages/host/tests/integration/components/tessar-snapshot-test.gts @@ -0,0 +1,523 @@ +import { getService } from '@universal-ember/test-support'; +import { module, test } from 'qunit'; + +import { + rri, + meta, + isSingleCardDocument, + parseRenderRouteOptions, + serializeRenderRouteOptions, +} from '@cardstack/runtime-common'; + +import { + setupIntegrationTestRealm, + setupLocalIndexing, + testRealmURL, + testRRI, +} from '../../helpers'; +import { + setupBaseRealm, + CardDef, + FieldDef, + contains, + containsMany, + linksToMany, + field, + StringField, + NumberField, +} from '../../helpers/base-realm'; +import { setupMockMatrix } from '../../helpers/mock-matrix'; +import { setupRenderingTest } from '../../helpers/setup'; + +module('Integration | Tessar snapshot', function (hooks) { + setupRenderingTest(hooks); + setupBaseRealm(hooks); + setupLocalIndexing(hooks); + let mockMatrixUtils = setupMockMatrix(hooks); + + test('first sync after session reset reconciles Tessar readers exactly once', function (assert) { + let messages = getService('message-service'); + messages.resetState(); + let transitions: boolean[] = []; + let unsubscribe = messages.subscribeTessarConnection((connected) => + transitions.push(connected), + ); + try { + messages.tessarConnectionChanged(true); + assert.deepEqual( + transitions, + [true], + 'initial sync closes the subscription gap', + ); + messages.tessarConnectionChanged(true); + assert.deepEqual( + transitions, + [true], + 'routine sync does not repeatedly reload clean views', + ); + messages.tessarConnectionChanged(false); + messages.tessarConnectionChanged(true); + assert.deepEqual( + transitions, + [true, false, true], + 'a later reconnect reconciles again', + ); + } finally { + unsubscribe(); + } + }); + + test('supplied outputs and membership do not compute, search or inflate included inputs', async function (assert) { + let calls = 0; + class TessarInput extends CardDef { + @field name = contains(StringField); + } + class TessarSummary extends CardDef { + @field inputs = linksToMany(TessarInput, { + query: { filter: { eq: { name: 'Tessar' } } }, + }); + @field count = contains(NumberField, { + computeVia: function () { + calls++; + return 999; + }, + }); + @field empty = containsMany(StringField, { + computeVia: function () { + calls++; + return ['live']; + }, + }); + } + await setupIntegrationTestRealm({ + skipBootIndex: true, + mockMatrixUtils, + contents: { 'tessar.gts': { TessarInput, TessarSummary } }, + }); + let api: typeof import('@cardstack/base/card-api') = await getService( + 'loader-service', + ).loader.import('@cardstack/base/card-api'); + let summary = new TessarSummary(); + let store = api.getStore(summary); + let searches = 0; + store.resolvesQueryFieldsEagerly = true; + store.getSearchResource = () => { + searches++; + throw new Error('Tessar snapshot must not search'); + }; + let inputId = rri(`${testRealmURL}TessarInput/1`); + let doc = { + data: { + id: `${testRealmURL}TessarSummary/1`, + type: 'card' as const, + attributes: { count: 0, empty: [] }, + relationships: { + inputs: { data: [{ type: 'card', id: inputId }], meta: { total: 1 } }, + }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + }, + }, + included: [ + { + id: inputId, + type: 'card' as const, + attributes: { name: 'Tessar' }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarInput', + }, + }, + }, + ], + }; + let opts = { + tessarSnapshot: { + computedFields: ['count', 'empty'], + queryFields: ['inputs'], + }, + }; + await api.updateFromSerialized(summary, doc, store, opts); + assert.strictEqual(summary.count, 0); + assert.deepEqual([...summary.empty], []); + assert.strictEqual(calls, 0, 'no computed getter was invoked'); + assert.strictEqual(searches, 0, 'no query search was started'); + assert.strictEqual( + store.getCard(inputId), + undefined, + 'included input stayed uninflated', + ); + let membership = api.getRelationshipMembershipState(summary, 'inputs'); + assert.strictEqual(membership.totalMatchCount, 1); + assert.strictEqual(membership.membership?.length, 1); + assert.false(membership.isPartial); + doc.data.attributes.count = 7; + await api.updateFromSerialized(summary, doc, store, opts); + assert.strictEqual( + summary.count, + 7, + 'new owner snapshot replaces old output', + ); + Object.assign(doc.data.meta, { + generation: 2, + tessar: { + version: 1, + state: 'ready', + inputGeneration: 1, + publishedGeneration: 2, + definitionRevision: 'tessar-test', + computedFields: ['count', 'empty'], + queryFields: ['inputs'], + watches: [], + }, + }); + await api.updateFromSerialized(summary, doc, store); + assert.strictEqual( + summary.count, + 7, + 'an indexed response activates snapshot consumption', + ); + assert.strictEqual(calls, 0); + assert.strictEqual(searches, 0); + assert.strictEqual(summary.tessarState, 'ready'); + api.markTessarPending(summary); + assert.strictEqual( + summary.tessarState, + 'pending', + 'source changes expose pending state without expanding inputs', + ); + assert.strictEqual(searches, 0); + await api.updateFromSerialized(summary, doc, store); + assert.strictEqual( + summary.tessarState, + 'ready', + 'revalidation clears pending state', + ); + let older = structuredClone(doc); + let newer = structuredClone(doc); + newer.data.attributes.count = 11; + Object.assign(newer.data.meta, { + generation: 3, + tessar: { + ...(newer.data.meta as any).tessar, + inputGeneration: 2, + publishedGeneration: 3, + }, + }); + await Promise.all([ + api.updateFromSerialized(summary, newer, store), + api.updateFromSerialized(summary, older, store), + ]); + assert.strictEqual( + summary.count, + 11, + 'overlapping refreshes cannot move output backwards', + ); + assert.strictEqual((summary as any)[meta].tessar.publishedGeneration, 3); + await api.updateFromSerialized(summary, older, store); + assert.strictEqual( + summary.count, + 11, + 'a delayed older response is ignored', + ); + Object.assign(doc.data.meta, structuredClone(newer.data.meta)); + (doc.data.meta as any).tessar.state = 'pending'; + await assert.rejects( + api.updateFromSerialized(summary, doc, store), + /pending or has invalid provenance/, + ); + delete (doc.data.meta as any).tessar; + delete (doc.data.meta as any).generation; + assert.strictEqual(calls, 0); + doc.data.relationships.inputs.meta.total = 2; + await assert.rejects( + api.updateFromSerialized(summary, doc, store, opts), + /missing, partial or errored/, + 'partial membership cannot be accepted as a complete materialization', + ); + }); + + test('publication registers empty queries and refuses incomplete membership', async function (assert) { + class TessarInput extends CardDef { + @field name = contains(StringField); + } + class TessarSummary extends CardDef { + static tessarMaterialized = true; + @field name = contains(StringField); + @field inputs = linksToMany(TessarInput, { + query: { filter: { eq: { name: '$this.name' } } }, + }); + @field count = contains(NumberField, { + computeVia: function (this: TessarSummary) { + return this.inputs.length; + }, + }); + } + await setupIntegrationTestRealm({ + skipBootIndex: true, + mockMatrixUtils, + contents: { 'tessar.gts': { TessarInput, TessarSummary } }, + }); + let api: typeof import('@cardstack/base/card-api') = await getService( + 'loader-service', + ).loader.import('@cardstack/base/card-api'); + let summary = new TessarSummary({ name: 'Tessar' }); + summary.id = rri(`${testRealmURL}summary`); + (summary as any)[meta] = { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + realmURL: testRealmURL, + }; + let store = api.getStore(summary); + let result = { + instances: [], + instancesByRealm: [], + isLoading: false, + meta: { page: { total: 0 } }, + totalMatchCount: 0, + isPartial: false, + }; + store.getSearchResource = () => result; + await api.updateFromSerialized( + summary, + { + data: { + id: summary.id, + type: 'card', + attributes: { name: 'Tessar' }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + realmURL: testRealmURL, + }, + }, + }, + store, + ); + await api.prepareTessarQueries(summary); + let doc = api.serializeCard(summary, { + includeComputeds: true, + omitQueryFields: true, + }); + let discovery = api.tessarIndexManifest(summary, doc); + assert.strictEqual( + discovery?.state, + 'pending', + 'a discovery render cannot claim readiness', + ); + assert.strictEqual( + discovery?.watches.length, + 1, + 'the empty query is registered', + ); + assert.false( + JSON.stringify(discovery?.watches).includes('$this'), + 'watch parameters are resolved', + ); + assert.true(JSON.stringify(discovery?.watches).includes('Tessar')); + let snapshot = api.tessarIndexManifest(summary, doc, 7); + assert.true( + isSingleCardDocument(doc), + 'the published wire shape passes the same validator used by the host store', + ); + let membership = doc.data.relationships?.inputs; + if (Array.isArray(membership)) + throw new Error('Expected a Tessar membership umbrella'); + assert.deepEqual( + membership?.data, + [], + 'known empty membership is serialized explicitly', + ); + assert.strictEqual(membership?.meta?.total, 0); + assert.strictEqual(snapshot?.inputGeneration, 7); + assert.true(snapshot?.computedFields.includes('count')); + assert.notOk(doc.included, 'the input graph is never serialized'); + result.totalMatchCount = 2; + result.isPartial = true; + assert.throws( + () => api.tessarIndexManifest(summary, doc, 7), + /unresolved, partial or errored/, + ); + }); + + test('Tessar HTML explicitly consumes a published snapshot while index discovery computes', async function (assert) { + let calls = 0; + class TessarSummary extends CardDef { + static tessarMaterialized = true; + @field total = contains(NumberField, { + computeVia: function () { + calls++; + return 999; + }, + }); + } + await setupIntegrationTestRealm({ + skipBootIndex: true, + mockMatrixUtils, + contents: { 'tessar.gts': { TessarSummary } }, + }); + let doc = { + data: { + id: `${testRealmURL}TessarSummary/html`, + type: 'card' as const, + attributes: { total: 3 }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + tessar: { + version: 1 as const, + state: 'ready' as const, + inputGeneration: 1, + publishedGeneration: 2, + definitionRevision: 'tessar-html-test', + computedFields: ['total'], + queryFields: [], + watches: [], + }, + }, + }, + }; + let globals = globalThis as any; + let previous = globals.__boxelRenderContext; + globals.__boxelRenderContext = true; + try { + let options = parseRenderRouteOptions( + serializeRenderRouteOptions({ + cardRender: true, + tessarUseSnapshot: true, + }), + ); + assert.true(options.tessarUseSnapshot); + let rendered = await getService('render-store').add(doc, { + doNotPersist: true, + relativeTo: testRRI(''), + tessarUseSnapshot: options.tessarUseSnapshot, + }); + assert.strictEqual(rendered.total, 3, 'HTML uses the supplied output'); + assert.strictEqual(calls, 0, 'HTML never ran the computed getter'); + doc.data.id = `${testRealmURL}TessarSummary/index`; + let indexed = await getService('render-store').add(doc, { + doNotPersist: true, + relativeTo: testRRI(''), + }); + assert.strictEqual( + indexed.total, + 999, + 'discovery keeps its computation path', + ); + assert.ok(calls > 0); + doc.data.id = `${testRealmURL}TessarSummary/pending`; + await assert.rejects( + getService('render-store').add( + { + data: { + ...doc.data, + meta: { + ...doc.data.meta, + tessar: { ...doc.data.meta.tessar, state: 'pending' }, + }, + }, + }, + { + doNotPersist: true, + relativeTo: testRRI(''), + tessarUseSnapshot: true, + }, + ), + /pending or has invalid provenance/, + 'HTML cannot accept pending output', + ); + } finally { + globals.__boxelRenderContext = previous; + } + }); + + test('nested values share edit invalidation and legacy documents keep computing', async function (assert) { + let calls = 0; + class TessarDetails extends FieldDef { + @field label = contains(StringField); + @field total = contains(NumberField, { + computeVia: function () { + calls++; + return 12; + }, + }); + } + class TessarSummary extends CardDef { + @field label = contains(StringField); + @field details = contains(TessarDetails); + @field rows = containsMany(TessarDetails); + @field total = contains(NumberField, { + computeVia: function () { + calls++; + return 42; + }, + }); + } + await setupIntegrationTestRealm({ + skipBootIndex: true, + mockMatrixUtils, + contents: { 'tessar.gts': { TessarSummary, TessarDetails } }, + }); + let api: typeof import('@cardstack/base/card-api') = await getService( + 'loader-service', + ).loader.import('@cardstack/base/card-api'); + let summary = new TessarSummary(); + let doc = { + data: { + type: 'card' as const, + attributes: { + label: 'Tessar', + total: 0, + details: { label: 'Nested', total: 3 }, + rows: [{ label: 'Row', total: 5 }], + }, + meta: { + adoptsFrom: { + module: rri(`${testRealmURL}tessar`), + name: 'TessarSummary', + }, + }, + }, + }; + let opts = { + tessarSnapshot: { + computedFields: ['total', 'details.total', 'rows.*.total'], + queryFields: [], + }, + }; + await api.updateFromSerialized(summary, doc, api.getStore(summary), opts); + assert.strictEqual(summary.total, 0); + assert.strictEqual(summary.details.total, 3); + assert.strictEqual(summary.rows[0].total, 5); + assert.strictEqual(calls, 0); + summary.details.label = 'Edited'; + assert.strictEqual( + summary.total, + 42, + 'contained edit clears the owner overlay', + ); + assert.strictEqual( + summary.rows[0].total, + 12, + 'sibling overlay also leaves snapshot mode', + ); + await api.updateFromSerialized(summary, doc); + assert.strictEqual( + summary.total, + 42, + 'ordinary deserialization never trusts supplied computeds', + ); + assert.strictEqual(summary.details.total, 12); + }); +}); diff --git a/packages/host/tests/unit/index-query-engine-test.ts b/packages/host/tests/unit/index-query-engine-test.ts index 478a7ea1043..f8ac6046370 100644 --- a/packages/host/tests/unit/index-query-engine-test.ts +++ b/packages/host/tests/unit/index-query-engine-test.ts @@ -14,6 +14,8 @@ import { identifyCard, getFieldDefinitions, rri, + coerceTypes, + type Filter, type RealmResourceIdentifier, type ResolvedCodeRef, type Definition, @@ -22,7 +24,11 @@ import { isFilterRefersToNonsearchableFieldError, } from '@cardstack/runtime-common'; +import { query } from '@cardstack/runtime-common/expression'; +import { TessarQueryRegistry } from '@cardstack/runtime-common/tessar-query-registry'; + import ENV from '@cardstack/host/config/environment'; + import { shimExternals } from '@cardstack/host/lib/externals'; import type SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter'; @@ -296,6 +302,212 @@ module('Unit | query', function (hooks) { ); }); + test('Tessar reverse verification agrees with forward predicates for indexed documents', async function (assert) { + let { mango, vangogh, ringo, paper } = testCards; + await setupIndex(dbAdapter, [ + { + card: mango, + data: { + search_doc: { + name: 'Mango', + age: 5, + isHairy: true, + friends: [{ name: 'Ringo' }], + }, + }, + }, + { + card: vangogh, + data: { + search_doc: { name: 'Van Gogh', age: 8, isHairy: false, friends: [] }, + }, + }, + { + card: ringo, + data: { + search_doc: { + name: 'Ringo', + age: null, + isHairy: true, + friends: [{ name: 'Mango' }], + }, + }, + }, + { card: paper, data: { search_doc: { name: 'Mango' } } }, + ]); + let documents = (await dbAdapter.execute( + 'SELECT url, search_doc, types FROM boxel_index WHERE type = $1', + { bind: ['instance'], coerceTypes }, + )) as unknown as Array< + Parameters[1] + >; + let on = { module: rri(`${testRealmURL}person`), name: 'Person' }; + let predicates: Filter[] = [ + { on, eq: { name: 'Mango' } }, + { on, in: { name: ['Mango', 'Ringo'] } }, + { on, eq: { age: null } }, + { on, eq: { 'friends.name': 'Ringo' } }, + { on, any: [{ eq: { name: 'Mango' } }, { eq: { name: 'Van Gogh' } }] }, + { on, not: { eq: { name: 'Mango' } } }, + { on, range: { age: { gte: 5, lt: 8 } } }, + { on, eq: { isHairy: true } }, + ]; + let expectedCounts = [1, 2, 1, 1, 2, 2, 1, 2]; + for (let [index, filter] of predicates.entries()) { + let forward = await indexQueryEngine.searchCards(new URL(testRealmURL), { + filter, + }); + let matched: string[] = []; + assert.strictEqual( + forward.cards.length, + expectedCounts[index], + 'forward fixture has the expected real matches', + ); + for (let document of documents) { + if (await indexQueryEngine.tessarMatchesDocument(filter, document)) + matched.push(document.url.replace(/\.json$/, '')); + } + assert.deepEqual( + matched.sort(), + forward.cards.map((card) => String(card.id)).sort(), + stringify(filter), + ); + } + assert.deepEqual( + await indexQueryEngine.tessarRoutingTerms({ on, eq: { name: 'Mango' } }), + [{ path: 'name', value: 'Mango' }], + ); + assert.strictEqual( + await indexQueryEngine.tessarRoutingTerms({ + on, + any: [{ eq: { name: 'Mango' } }, { not: { eq: { name: 'Ringo' } } }], + }), + undefined, + 'an unanchored OR branch requires the broad bucket', + ); + assert.strictEqual( + await indexQueryEngine.tessarRoutingTerms({ on, eq: { age: null } }), + undefined, + 'null routes broadly, including absent values', + ); + }); + + test('Tessar watches route conservatively and preserve durable invalidation', async function (assert) { + let registry = new TessarQueryRegistry(dbAdapter, indexQueryEngine); + let tx = (expression: Parameters[1]) => + query(dbAdapter, expression); + let on = { module: rri(`${testRealmURL}person`), name: 'Person' }; + let owner = (name: string, generation = 1) => ({ + realmURL: testRealmURL, + ownerURL: `${testRealmURL}${name}`, + generation, + inputGeneration: generation, + definitionRevision: 'tessar-v1', + }); + let publish = async (name: string, filter: Filter) => + registry.publish(tx, { + ...owner(name), + watches: await registry.prepare([ + { + fieldPath: 'people', + query: { filter, page: { size: 1, number: 0 } }, + }, + ]), + }); + await publish('mango-summary', { on, eq: { name: 'Mango' } }); + await publish('ringo-summary', { on, eq: { name: 'Ringo' } }); + await publish('empty-summary', { on, eq: { name: 'Future' } }); + await publish('broad-summary', { on, not: { eq: { name: 'Excluded' } } }); + await setupIndex(dbAdapter, [ + { card: testCards.mango, data: { search_doc: { name: 'Mango' } } }, + { card: testCards.ringo, data: { search_doc: { name: 'Ringo' } } }, + ]); + let rows = await dbAdapter.execute( + "SELECT url, search_doc, types FROM boxel_index WHERE type = 'instance'", + { coerceTypes }, + ); + let mango = rows.find( + (row) => (row.search_doc as any).name === 'Mango', + ) as unknown as Parameters[1]; + let moved = { + ...mango, + search_doc: { ...mango.search_doc, name: 'Ringo' }, + }; + let affected = await registry.affected(testRealmURL, mango, moved); + assert.deepEqual( + affected, + ['broad-summary', 'mango-summary', 'ringo-summary'] + .map((name) => `${testRealmURL}${name}`) + .sort(), + 'old and new scopes both invalidate', + ); + assert.deepEqual( + await registry.affected(testRealmURL, undefined, { + ...mango, + search_doc: { ...mango.search_doc, name: 'Future' }, + }), + ['broad-summary', 'empty-summary'] + .map((name) => `${testRealmURL}${name}`) + .sort(), + 'an initially empty watch sees an insertion', + ); + assert.deepEqual( + await registry.affected(testRealmURL, mango, undefined), + ['broad-summary', 'mango-summary'] + .map((name) => `${testRealmURL}${name}`) + .sort(), + 'deletions use the old document', + ); + assert.deepEqual( + await registry.affected('http://elsewhere/', mango, moved), + [], + 'realm isolation', + ); + await registry.markDirty(tx, testRealmURL, affected, 3); + await registry.markDirty(tx, testRealmURL, affected, 2); + assert.true( + ( + await new TessarQueryRegistry(dbAdapter, indexQueryEngine).pending( + testRealmURL, + ) + ).every((row) => row.generation === 3), + 'dirty watermark survives reconstruction and never regresses', + ); + let watches = await registry.prepare([ + { fieldPath: 'people', query: { filter: { on, eq: { name: 'Mango' } } } }, + ]); + assert.false( + await registry.publish(tx, { ...owner('mango-summary', 2), watches }), + 'a result that missed a newer change cannot publish', + ); + assert.true( + await registry.publish(tx, { ...owner('mango-summary', 3), watches }), + 'observing the required revision permits publication', + ); + assert.false( + (await registry.pending(testRealmURL)).some( + (row) => row.ownerURL === owner('mango-summary').ownerURL, + ), + 'successful publication clears dirty state', + ); + assert.true( + await registry.publish(tx, { + ...owner('mango-summary', 4), + watches: [], + retired: true, + }), + ); + assert.false( + await registry.publish(tx, { ...owner('mango-summary', 3), watches }), + 'a delayed worker cannot resurrect retired watches', + ); + assert.false( + (await registry.affected(testRealmURL, mango, undefined)).includes( + owner('mango-summary').ownerURL, + ), + ); + }); + test('can get all cards with empty filter', async function (assert) { let { mango, vangogh, paper } = testCards; await setupIndex(dbAdapter, [mango, vangogh, paper]); diff --git a/packages/postgres/migrations/1789091944988_tessar-query-watches.js b/packages/postgres/migrations/1789091944988_tessar-query-watches.js new file mode 100644 index 00000000000..e4120ddb890 --- /dev/null +++ b/packages/postgres/migrations/1789091944988_tessar-query-watches.js @@ -0,0 +1,39 @@ +exports.shorthands = undefined; + +exports.up = (pgm) => { + // Tessar is opt-in. The owner row survives retirement to prevent an older + // worker from resurrecting watches after a definition change or deletion. + pgm.createTable('tessar_owners', { + realm_url: { type: 'varchar', notNull: true, primaryKey: true }, + owner_url: { type: 'varchar', notNull: true, primaryKey: true }, + published_generation: { type: 'bigint', notNull: true }, + input_generation: { type: 'bigint', notNull: true }, + dirty_generation: 'bigint', + definition_revision: { type: 'varchar', notNull: true }, + retired: { type: 'boolean', notNull: true, default: false }, + }); + pgm.createTable('tessar_query_watches', { + realm_url: { type: 'varchar', notNull: true, primaryKey: true }, + owner_url: { type: 'varchar', notNull: true, primaryKey: true }, + field_path: { type: 'varchar', notNull: true, primaryKey: true }, + query: { type: 'jsonb', notNull: true }, + }); + // One safe disjunctive set of equality anchors per watch. The empty path is + // a broad route. These rows are candidates only; the ordinary SQL compiler + // verifies the saved predicate against both old and new indexed documents. + pgm.createTable('tessar_query_terms', { + realm_url: { type: 'varchar', notNull: true, primaryKey: true }, + owner_url: { type: 'varchar', notNull: true, primaryKey: true }, + field_path: { type: 'varchar', notNull: true, primaryKey: true }, + path: { type: 'varchar', notNull: true, primaryKey: true }, + value: { type: 'varchar', notNull: true, primaryKey: true }, + }); + pgm.createIndex('tessar_query_terms', ['realm_url', 'path', 'value']); + pgm.createIndex('tessar_owners', ['realm_url', 'dirty_generation']); +}; + +exports.down = (pgm) => { + pgm.dropTable('tessar_query_terms'); + pgm.dropTable('tessar_query_watches'); + pgm.dropTable('tessar_owners'); +}; diff --git a/packages/realm-server/handlers/handle-search.ts b/packages/realm-server/handlers/handle-search.ts index fcbd75f9fc3..71d941d86ac 100644 --- a/packages/realm-server/handlers/handle-search.ts +++ b/packages/realm-server/handlers/handle-search.ts @@ -1,6 +1,12 @@ import type Koa from 'koa'; +import { + assertTessarGeneration, + tessarRequestedGeneration, + tessarHasMaterializations, +} from '@cardstack/runtime-common/tessar-materialization'; import { applyServerSearchPageBound, + CardError, buildSearchErrorResponse, DURING_PRERENDER_HEADER, ifNoneMatchMatches, @@ -185,6 +191,7 @@ export default function handleSearch(opts: { // ride the run-time opts instead. let runSearchOpts = { ...searchOpts, + tessarInput: request.headers.has('x-boxel-tessar-input-generation'), ...(loggingCorrelationId !== null ? { loggingCorrelationId } : {}), ...(timings ? { timings } : {}), }; @@ -236,6 +243,43 @@ export default function handleSearch(opts: { let jobId = searchCache ? prerenderJobId : null; try { + let tessarGeneration = tessarRequestedGeneration(request); + if (tessarGeneration !== undefined) { + if (realmList.length !== 1) { + throw new CardError('Tessar inputs require exactly one realm', { + status: 400, + }); + } + await assertTessarGeneration(dbAdapter, realmList[0], tessarGeneration); + let body = await runSearch(); + await assertTessarGeneration(dbAdapter, realmList[0], tessarGeneration); + await setContextResponse( + ctxt, + new Response(body, { + headers: { + 'content-type': SupportedMimeType.CardJson, + 'cache-control': 'no-store', + }, + }), + ); + emitTimeline(); + return; + } + // Generation-keyed caches cannot see a queued source write until its + // first index swap. Tessar responses must revalidate pending state. + if (await tessarHasMaterializations(dbAdapter, realmList)) { + setContextResponse( + ctxt, + new Response(await runSearch(), { + headers: { + 'content-type': SupportedMimeType.CardJson, + 'cache-control': 'no-store', + }, + }), + ); + emitTimeline(); + return; + } await respondWithJobScopedSearchCache(ctxt, { searchCache, jobId, @@ -251,7 +295,7 @@ export default function handleSearch(opts: { // The per-request time budget fired inside `runSearch`. A bounded search // is never cacheable (cacheable ⟹ during-prerender ⟹ not bounded), so // this only surfaces on the fresh-compute path and leaves no cache entry. - if (e instanceof SearchBoundError) { + if (e instanceof SearchBoundError || e instanceof CardError) { await setContextResponse( ctxt, buildSearchErrorResponse(e.message, e.status), diff --git a/packages/realm-server/handlers/serve-index.ts b/packages/realm-server/handlers/serve-index.ts index ffe2af8ba83..6ebe54d71f8 100644 --- a/packages/realm-server/handlers/serve-index.ts +++ b/packages/realm-server/handlers/serve-index.ts @@ -18,6 +18,7 @@ import { sanitizeHeadHTMLToString, } from '@cardstack/runtime-common'; import type { MatrixClient } from '@cardstack/runtime-common/matrix-client'; +import { tessarHasMaterializations } from '@cardstack/runtime-common/tessar-materialization'; import { ensureSingleTitle, injectHeadHTML, @@ -395,8 +396,14 @@ export function createServeIndex(deps: ServeIndexDeps): ServeIndexHandlers { publishedRealmInfo = await getPublishedRealmInfo(requestURL, routingDeps); } let lastPublishedAt = publishedRealmInfo?.lastPublishedAt; + // A deployment timestamp cannot validate mutable materialized output. + // Keep a cached ready page from bypassing the queued-write/HTML checks. + let tessarRealm = + routedRealm && + (await tessarHasMaterializations(dbAdapter, [routedRealm.url])); + if (tessarRealm) ctxt.set('Cache-Control', 'no-store'); let etag = - lastPublishedAt && indexHTMLHash + !tessarRealm && lastPublishedAt && indexHTMLHash ? `"${lastPublishedAt}-${indexHTMLHash}"` : null; diff --git a/packages/realm-server/lib/index-html-injection.ts b/packages/realm-server/lib/index-html-injection.ts index 66d75e556f2..02158ea9e52 100644 --- a/packages/realm-server/lib/index-html-injection.ts +++ b/packages/realm-server/lib/index-html-injection.ts @@ -1,10 +1,41 @@ -import type { DBAdapter } from '@cardstack/runtime-common'; -import { query } from '@cardstack/runtime-common'; +import type { + DBAdapter, + TessarMaterialization, +} from '@cardstack/runtime-common'; +import { CardError, query } from '@cardstack/runtime-common'; +import { tessarReadState } from '@cardstack/runtime-common/tessar-materialization'; import { indexURLCandidates, indexCandidateExpressions, } from './index-url-utils.ts'; +type TessarHTMLRow = { + url: string; + realm_url: string; + generation: string | number; + html_generation: string | number; + html_error: unknown; + tessar: TessarMaterialization | string | null; +}; + +async function tessarHTMLIsCurrent(db: DBAdapter, row?: TessarHTMLRow) { + if (!row?.tessar) return true; + if (row.html_error || Number(row.html_generation) !== Number(row.generation)) + return false; + let stamp = + typeof row.tessar === 'string' ? JSON.parse(row.tessar) : row.tessar; + try { + return ( + (await tessarReadState(db, row.realm_url, row.url, stamp)) === 'ready' + ); + } catch (error) { + // Leave the shell to show the explicit JSON failure after boot, rather + // than injecting a previous successful dashboard into a failed page. + if (error instanceof CardError && error.status === 503) return false; + throw error; + } +} + export async function retrieveHeadHTML({ cardURL, dbAdapter, @@ -31,7 +62,9 @@ export async function retrieveHeadHTML({ // lookup to a live instance row and supplies the generation for logging. let rows = await query(dbAdapter, [ ` - SELECT ph.head_html AS head_html, i.generation + SELECT ph.head_html AS head_html, i.generation, i.url, i.realm_url, + ph.generation AS html_generation, ph.error_doc AS html_error, + i.pristine_doc->'meta'->'tessar' AS tessar FROM boxel_index AS i JOIN prerendered_html AS ph ON ph.url = i.url AND ph.realm_url = i.realm_url AND ph.type = i.type @@ -49,6 +82,14 @@ export async function retrieveHeadHTML({ log?.debug('Head query result for %s', cardURL.href, rows); + if ( + !(await tessarHTMLIsCurrent( + dbAdapter, + rows[0] as TessarHTMLRow | undefined, + )) + ) + return null; + let headRow = rows[0] as | { head_html?: string | null; generation?: string | number } | undefined; @@ -89,7 +130,9 @@ export async function retrieveIsolatedHTML({ // the lookup to a live instance row and supplies the generation for logging. let rows = await query(dbAdapter, [ ` - SELECT ph.isolated_html AS isolated_html, i.generation + SELECT ph.isolated_html AS isolated_html, i.generation, i.url, i.realm_url, + ph.generation AS html_generation, ph.error_doc AS html_error, + i.pristine_doc->'meta'->'tessar' AS tessar FROM boxel_index AS i JOIN prerendered_html AS ph ON ph.url = i.url AND ph.realm_url = i.realm_url AND ph.type = i.type @@ -107,6 +150,14 @@ export async function retrieveIsolatedHTML({ log?.debug('Isolated query result for %s', cardURL.href, rows); + if ( + !(await tessarHTMLIsCurrent( + dbAdapter, + rows[0] as TessarHTMLRow | undefined, + )) + ) + return null; + let isolatedRow = rows[0] as | { isolated_html?: string | null; generation?: string | number } | undefined; diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 2bc829cafbb..13ea97f2ed0 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -1182,6 +1182,24 @@ export function buildPrerenderApp(options: { ? rawRenderScope : undefined; + let tessarInputSnapshot = attrs.tessarInputSnapshot; + if ( + tessarInputSnapshot !== undefined && + (!tessarInputSnapshot || + typeof tessarInputSnapshot !== 'object' || + tessarInputSnapshot.realmURL !== realm || + !Number.isSafeInteger(tessarInputSnapshot.generation) || + tessarInputSnapshot.generation < 0) + ) { + ctxt.status = 400; + ctxt.body = { + errors: [ + { status: 400, message: 'Invalid Tessar indexed input snapshot' }, + ], + }; + return; + } + let start = Date.now(); // Hoisted so a re-render after a host-shell change replays the same // visit rather than an approximation of it. @@ -1201,6 +1219,7 @@ export function buildPrerenderApp(options: { ...(jobId ? { jobId } : {}), ...(screenshots ? { screenshots } : {}), ...(renderScope ? { renderScope } : {}), + ...(tessarInputSnapshot ? { tessarInputSnapshot } : {}), signal: ac.signal, }; let shellAtStart = options.getHostShellHash?.(); diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 0c26853b595..534ecb3a218 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -732,6 +732,7 @@ export class Prerenderer { jobId, screenshots, renderScope, + tessarInputSnapshot, } = this.#gateClearCache(rawArgs); let signal = (rawArgs as { signal?: AbortSignal }).signal; let testOnTabAcquired = ( @@ -794,6 +795,7 @@ export class Prerenderer { jobId, screenshots, renderScope, + tessarInputSnapshot, signal, onTabAcquired, }); @@ -831,6 +833,7 @@ export class Prerenderer { jobId, screenshots, renderScope, + tessarInputSnapshot, signal, onTabAcquired, }); diff --git a/packages/realm-server/prerender/remote-prerenderer.ts b/packages/realm-server/prerender/remote-prerenderer.ts index 8d32290c0c3..6c40ab60267 100644 --- a/packages/realm-server/prerender/remote-prerenderer.ts +++ b/packages/realm-server/prerender/remote-prerenderer.ts @@ -286,6 +286,7 @@ export function createRemotePrerenderer( jobId, screenshots, renderScope, + tessarInputSnapshot, }: PrerenderVisitArgs): Promise { return await requestWithRetry( 'prerender-visit', @@ -306,6 +307,7 @@ export function createRemotePrerenderer( ...(jobId ? { jobId } : {}), ...(screenshots ? { screenshots } : {}), ...(renderScope ? { renderScope } : {}), + ...(tessarInputSnapshot ? { tessarInputSnapshot } : {}), }, ); }, diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 522dccc03e3..fdfda62cac1 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -887,6 +887,7 @@ export class RenderRunner { jobId, screenshots, renderScope, + tessarInputSnapshot, signal, onTabAcquired, }: PrerenderVisitArgs & { @@ -1030,6 +1031,7 @@ export class RenderRunner { id: string | undefined, jobPriority: number | undefined, scope: string | undefined, + tessar: PrerenderVisitArgs['tessarInputSnapshot'], ) => { localStorage.setItem('boxel-session', sessionAuth); (globalThis as unknown as { __boxelJobId?: string }).__boxelJobId = @@ -1044,6 +1046,11 @@ export class RenderRunner { ( globalThis as unknown as { __boxelRenderScope?: string } ).__boxelRenderScope = scope; + ( + globalThis as unknown as { + __tessarInputSnapshot?: PrerenderVisitArgs['tessarInputSnapshot']; + } + ).__tessarInputSnapshot = tessar; return ( ( globalThis as unknown as { @@ -1056,6 +1063,7 @@ export class RenderRunner { jobId, priority, renderScope, + tessarInputSnapshot, ), ); // A card-instance index visit fuses the file extract into the @@ -1099,6 +1107,10 @@ export class RenderRunner { ) => { let optionsForThisPass: RenderRouteOptions = { ...baseOptions, + tessarUseSnapshot: + visitType === 'prerender-html' && pass === 'cardRender' + ? true + : undefined, // Set only the flag(s) for the current pass so the host route // picks the right branch in #buildModel regardless of what other // passes are part of this visit. The fused index pass carries both diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 195dd5f9f1b..bce4156a2b1 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -847,7 +847,7 @@ export class RealmServer { // this list the preflight fails and the player errors before any // bytes flow. allowHeaders: - 'Authorization, Content-Type, If-Match, If-None-Match, If-Range, Range, X-Requested-With, X-Boxel-Client-Request-Id, X-Boxel-Assume-User, X-HTTP-Method-Override, X-Boxel-Disable-Module-Cache, X-Filename, X-Boxel-During-Prerender, X-Boxel-Consuming-Realm, X-Boxel-Job-Id, X-Boxel-Job-Priority, X-Boxel-Logging-Correlation-Id, X-Grafana-Device-Id, X-Grafana-Action', + 'Authorization, Content-Type, If-Match, If-None-Match, If-Range, Range, X-Requested-With, X-Boxel-Client-Request-Id, X-Boxel-Assume-User, X-HTTP-Method-Override, X-Boxel-Disable-Module-Cache, X-Filename, X-Boxel-During-Prerender, X-Boxel-Tessar-Input-Generation, X-Boxel-Consuming-Realm, X-Boxel-Job-Id, X-Boxel-Job-Priority, X-Boxel-Logging-Correlation-Id, X-Grafana-Device-Id, X-Grafana-Action', // Without an explicit expose list, @koa/cors only emits the // CORS-safelisted response headers (cache-control, content-*, // expires, last-modified, pragma). ETag is not on that list, diff --git a/packages/realm-server/tests/tessar-query-registry-test.ts b/packages/realm-server/tests/tessar-query-registry-test.ts new file mode 100644 index 00000000000..663f4f60680 --- /dev/null +++ b/packages/realm-server/tests/tessar-query-registry-test.ts @@ -0,0 +1,627 @@ +import QUnit from 'qunit'; +import type { PgAdapter } from '@cardstack/postgres'; +import { + IndexQueryEngine, + IndexWriter, + VirtualNetwork, + baseRealmRRI, + rri, + logger, + param, + type Definition, + type DefinitionLookup, + type Filter, + type InstanceEntry, + type Reader, +} from '@cardstack/runtime-common'; +import { preWarmModulesTable } from '@cardstack/runtime-common/index-runner/prewarm-modules'; +import { + TessarQueryRegistry, + type TessarDocument, +} from '@cardstack/runtime-common/tessar-query-registry'; +import { setupDB } from './helpers/index.ts'; +import { + retrieveHeadHTML, + retrieveIsolatedHTML, +} from '../lib/index-html-injection.ts'; +import { + assertTessarGeneration, + tessarRequestedGeneration, + TESSAR_INPUT_GENERATION_HEADER, + tessarReadState, +} from '@cardstack/runtime-common/tessar-materialization'; + +const { module, test } = QUnit; +const realmURL = 'https://tessar.example/'; +module('Tessar | bounded module prewarm', function () { + test('a large invalidation set keeps complete module coverage with bounded dependency reads', async function (assert) { + let invalidations = Array.from( + { length: 751 }, + (_, i) => new URL(`${realmURL}${i}.json`), + ); + invalidations.push(new URL(`${realmURL}helper.ts`)); + let reads: string[][] = []; + let warmed = new Set(); + let sourceReads: string[] = []; + let log = logger('tessar-prewarm-test'); + let count = await preWarmModulesTable({ + realmURL: new URL(realmURL), + invalidations, + allRealmCardModules: [`${realmURL}sibling.gts`], + virtualNetwork: new VirtualNetwork(), + getDependencyRows: async (urls) => { + reads.push(urls); + return urls.flatMap((url) => + url.endsWith('/750.json') + ? [] + : [ + { + url, + type: 'file' as const, + deps: null, + hasError: false, + isDeleted: false, + errorDoc: null, + }, + { + url, + type: 'instance' as const, + deps: [ + `${realmURL}record`, + `${realmURL}input.json`, + `${realmURL}styles.glimmer-scoped.css`, + 'https://other.example/external', + ], + hasError: false, + isDeleted: false, + errorDoc: null, + }, + ], + ); + }, + reader: { + readFile: async (url: URL) => { + sourceReads.push(url.href); + return { + content: JSON.stringify({ + data: { meta: { adoptsFrom: { module: './novel' } } }, + }), + }; + }, + } as Reader, + definitionLookup: { + populateDefinitionCacheEntry: async ({ + moduleURL, + }: { + moduleURL: string; + }) => { + warmed.add(moduleURL); + }, + } as unknown as DefinitionLookup, + getModuleCacheContext: async () => ({ + resolvedRealmURL: realmURL, + cacheScope: 'public', + authUserId: '', + }), + prerenderUserId: 'tessar', + jobPriority: 0, + jobInfo: { jobId: 1, reservationId: 1, priority: 0, queueWaitMs: null }, + log, + perfLog: log, + }); + assert.true( + reads.every((urls) => urls.length <= 250), + 'dependency payloads are bounded instead of retained for the whole realm', + ); + assert.deepEqual( + reads.flat(), + invalidations.map((url) => url.href), + 'every invalidated URL is inspected', + ); + assert.deepEqual( + sourceReads, + [`${realmURL}750.json`], + 'only the missing dependency row reads source', + ); + assert.deepEqual( + [...warmed].sort(), + ['helper.ts', 'novel', 'record', 'sibling.gts'] + .map((path) => realmURL + path) + .sort(), + 'module coverage, best-row selection and cross-realm exclusion are preserved', + ); + assert.strictEqual(count, 4); + }); +}); +const on = { module: rri(`${realmURL}record`), name: 'TessarRecord' }; +const definition: Definition = { + type: 'card-def', + codeRef: on, + displayName: 'Tessar record', + fields: { + name: 'string', + score: 'number', + active: 'boolean', + tags: 'strings', + }, + fieldDefs: { + string: { + type: 'contains', + isPrimitive: true, + isComputed: false, + fieldOrCard: { module: rri(`${baseRealmRRI}string`), name: 'default' }, + }, + strings: { + type: 'containsMany', + isPrimitive: true, + isComputed: false, + fieldOrCard: { module: rri(`${baseRealmRRI}string`), name: 'default' }, + }, + number: { + type: 'contains', + isPrimitive: true, + isComputed: false, + serializerName: 'number', + fieldOrCard: { module: rri(`${baseRealmRRI}number`), name: 'default' }, + }, + boolean: { + type: 'contains', + isPrimitive: true, + isComputed: false, + serializerName: 'boolean', + fieldOrCard: { module: rri(`${baseRealmRRI}boolean`), name: 'default' }, + }, + }, +}; + +module('Tessar | Postgres query registry', function (hooks) { + let db: PgAdapter; + let engine: IndexQueryEngine; + let registry: TessarQueryRegistry; + let writer: IndexWriter; + let publication: ReturnType; + let network: VirtualNetwork; + setupDB(hooks, { + templateDatabase: process.env.TESSAR_TEST_TEMPLATE_DB, + beforeEach: async (adapter) => { + db = adapter; + // Persisted definition fixture: the SQL compiler receives the same + // loaderless field schema that a real module-index entry supplies. + let lookup = { + async lookupDefinition() { + return definition; + }, + } as unknown as DefinitionLookup; + network = new VirtualNetwork(); + network.addRealmMapping(baseRealmRRI, 'https://cardstack.com/base/'); + engine = new IndexQueryEngine(db, lookup, network); + registry = new TessarQueryRegistry(db, engine); + writer = new IndexWriter(db); + publication = writer.tessarPublication(lookup, network); + }, + }); + let record = (name: string): TessarDocument => ({ + url: `${realmURL}record-1.json`, + types: [`${on.module}/${on.name}`], + search_doc: { name, score: 7, active: true, tags: ['red', 'blue'] }, + }); + + test('Tessar source swap, watch registration, dirty marking and guarded owner publication share the index transaction', async function (assert) { + let ownerURL = `${realmURL}summary.json`; + let inputURL = `${realmURL}input.json`; + let entry = ( + name: string, + count?: number, + inputGeneration = 0, + ): InstanceEntry => ({ + type: 'instance', + lastModified: 1, + resourceCreatedAt: 1, + resource: { + id: rri( + name === 'summary' + ? ownerURL.replace('.json', '') + : inputURL.replace('.json', ''), + ), + type: 'card', + attributes: { name, ...(count !== undefined ? { count } : {}) }, + meta: { + adoptsFrom: on, + ...(count !== undefined + ? { + tessar: { + version: 1 as const, + state: 'pending' as const, + computedFields: ['count'], + queryFields: ['inputs'], + watches: [ + { + fieldPath: 'inputs', + query: { filter: { on, eq: { name: 'A' } } }, + }, + ], + inputGeneration, + }, + } + : {}), + }, + }, + searchData: { name, ...(count !== undefined ? { count } : {}) }, + types: [`${on.module}/${on.name}`], + displayNames: ['TessarRecord'], + deps: new Set(), + }); + let source = await writer.createBatch(new URL(realmURL), network); + await source.updateEntry(new URL(ownerURL), entry('summary', 0)); + await source.updateEntry(new URL(inputURL), entry('A')); + // The HTTP PATCH/POST path holds this source-write lock until indexing + // completes. Publication must use a different advisory-lock namespace. + await db.withWriteLock(realmURL, async () => { + let deadline: ReturnType | undefined; + try { + await Promise.race([ + source.done({ tessar: publication }), + new Promise((_, reject) => { + deadline = setTimeout( + () => + reject( + new Error('Tessar publication deadlocked a source write'), + ), + 5000, + ); + }), + ]); + } finally { + clearTimeout(deadline); + } + }); + assert.deepEqual(await registry.pending(realmURL), [ + { ownerURL, generation: 1 }, + ]); + let materialize = await writer.createBatch(new URL(realmURL), network); + await materialize.updateEntry(new URL(ownerURL), entry('summary', 1, 1)); + await materialize.done({ tessar: publication, tessarInputGeneration: 1 }); + let [published] = await db.execute( + 'SELECT pristine_doc FROM boxel_index WHERE url = $1 AND type = $2', + { bind: [ownerURL, 'instance'] }, + ); + let stamp = (published.pristine_doc as any).meta.tessar; + assert.strictEqual(stamp.state, 'ready'); + assert.strictEqual(stamp.publishedGeneration, 2); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp), + 'ready', + ); + await db.execute( + 'INSERT INTO prerendered_html (url, file_alias, realm_url, type, generation, isolated_html, head_html) VALUES ($1, $1, $2, $3, 2, $4, $5)', + { + bind: [ownerURL, realmURL, 'instance', '

1

', '1'], + }, + ); + let initialHTML = () => + Promise.all([ + retrieveIsolatedHTML({ cardURL: new URL(ownerURL), dbAdapter: db }), + retrieveHeadHTML({ cardURL: new URL(ownerURL), dbAdapter: db }), + ]); + assert.deepEqual( + await initialHTML(), + ['

1

', '1'], + 'current HTML may be injected', + ); + await db.execute( + 'UPDATE prerendered_html SET generation = 1 WHERE url = $1', + { bind: [ownerURL] }, + ); + assert.deepEqual( + await initialHTML(), + [null, null], + 'old HTML cannot stand in for a newer materialization', + ); + await db.execute( + 'UPDATE prerendered_html SET generation = 2 WHERE url = $1', + { bind: [ownerURL] }, + ); + let [queued] = await db.execute( + "INSERT INTO jobs (job_type, concurrency_group, timeout, priority, args) VALUES ('incremental-index', $1, 60, 10, '{}') RETURNING id", + { bind: [`indexing:${realmURL}`] }, + ); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp), + 'pending', + 'an acknowledged queued write is pending before the first source swap', + ); + assert.deepEqual( + await initialHTML(), + [null, null], + 'a new client cannot receive ready-looking HTML while its source write is queued', + ); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp, { + tessarInput: true, + }), + 'ready', + 'a revision-pinned worker can consume its ready feeder', + ); + await db.execute("UPDATE jobs SET status = 'resolved' WHERE id = $1", { + bind: [queued.id], + }); + await db.execute("UPDATE jobs SET status = 'rejected' WHERE id = $1", { + bind: [queued.id], + }); + await assert.rejects( + tessarReadState(db, realmURL, ownerURL, stamp), + /indexing failed/, + 'a failed source pass cannot expose the previous snapshot as current', + ); + assert.deepEqual( + await initialHTML(), + [null, null], + 'a failed source pass cannot expose old HTML as current either', + ); + await db.execute("UPDATE jobs SET status = 'resolved' WHERE id = $1", { + bind: [queued.id], + }); + let change = await writer.createBatch(new URL(realmURL), network); + await change.updateEntry(new URL(inputURL), entry('B')); + await change.done({ tessar: publication }); + assert.deepEqual( + await registry.pending(realmURL), + [{ ownerURL, generation: 3 }], + 'a membership exit invalidates without an existing concrete dependency', + ); + assert.strictEqual( + await tessarReadState(db, realmURL, ownerURL, stamp), + 'pending', + 'the old snapshot cannot be served as current', + ); + let obsolete = await writer.createBatch(new URL(realmURL), network); + await obsolete.updateEntry(new URL(ownerURL), entry('summary', 99, 1)); + await assert.rejects( + obsolete.done({ tessar: publication, tessarInputGeneration: 1 }), + /input revision/, + ); + let [unchanged] = await db.execute( + 'SELECT generation, pristine_doc FROM boxel_index WHERE url = $1 AND type = $2', + { bind: [ownerURL, 'instance'] }, + ); + assert.strictEqual( + Number(unchanged.generation), + 2, + 'failed transaction never promoted the obsolete owner', + ); + assert.strictEqual((unchanged.pristine_doc as any).attributes.count, 1); + await assertTessarGeneration(db, realmURL, 3); + let fresh = await writer.createBatch(new URL(realmURL), network); + await fresh.updateEntry(new URL(ownerURL), entry('summary', 0, 3)); + await fresh.done({ tessar: publication, tessarInputGeneration: 3 }); + assert.deepEqual( + await registry.pending(realmURL), + [], + 'a correct recomputation clears durable dirty state', + ); + assert.strictEqual(await registry.activeOwnerCount(realmURL), 1); + // Module writes mint the realm-wide loader epoch before indexing starts. + // An owner outside that module's dependency closure still needs a new + // stamp, or read-time epoch validation would leave it pending forever. + await db.execute( + 'UPDATE realm_generations SET loader_epoch = $1 WHERE realm_url = $2', + { bind: ['tessar-new-module-epoch', realmURL] }, + ); + let moduleChange = await writer.createBatch(new URL(realmURL), network); + await moduleChange.done({ tessar: publication }); + assert.deepEqual( + await registry.pending(realmURL), + [{ ownerURL, generation: 5 }], + 'an epoch change schedules owners outside the changed module closure', + ); + let rematerialize = await writer.createBatch(new URL(realmURL), network); + await rematerialize.updateEntry(new URL(ownerURL), entry('summary', 0, 5)); + await rematerialize.done({ + tessar: publication, + tessarInputGeneration: 5, + }); + let [restamped] = await db.execute( + 'SELECT pristine_doc FROM boxel_index WHERE url = $1 AND type = $2', + { bind: [ownerURL, 'instance'] }, + ); + assert.strictEqual( + await tessarReadState( + db, + realmURL, + ownerURL, + (restamped.pristine_doc as any).meta.tessar, + ), + 'ready', + 'the restamped owner is readable under the new definition epoch', + ); + }); + + test('a source prepared before the first watch cannot publish after that registration', async function (assert) { + // Deterministic interleaving across separate pinned transactions: a source + // worker observed no watches, then another publisher registered the first + // owner. The stale no-watch fast path must still reject its old revision. + await writer.createBatch(new URL(realmURL), network); + let prepared = await publication.prepare(realmURL, 1); + assert.deepEqual(prepared.rows, []); + assert.false(prepared.hadOwners); + let ownerURL = `${realmURL}first-summary.json`; + let watches = await registry.prepare([ + { fieldPath: 'inputs', query: { filter: { on, eq: { name: 'A' } } } }, + ]); + await db.withWriteLock(`tessar:index:${realmURL}`, async (tx) => { + if (!tx) + throw new Error( + 'This regression requires a pinned Postgres transaction', + ); + await registry.publish(tx, { + realmURL, + ownerURL, + generation: 1, + inputGeneration: 0, + definitionRevision: '0', + watches, + pending: true, + }); + await tx([ + 'UPDATE realm_generations SET current_generation = 1 WHERE realm_url =', + param(realmURL), + ]); + }); + await assert.rejects( + db.withWriteLock(`tessar:index:${realmURL}`, async (tx) => { + if (!tx) + throw new Error( + 'This regression requires a pinned Postgres transaction', + ); + await publication.commit(tx, { + realmURL, + generation: 1, + definitionRevision: '0', + prepared, + }); + }), + /input revision changed/, + 'the source must retry against the newly registered watch', + ); + assert.deepEqual(await registry.pending(realmURL), [ + { ownerURL, generation: 1 }, + ]); + await assertTessarGeneration(db, realmURL, 1); + }); + + test('Tessar input reads reject changed generations and malformed revision headers', async function (assert) { + await db.execute( + 'INSERT INTO realm_generations (realm_url, current_generation) VALUES ($1, 7)', + { bind: [realmURL] }, + ); + await assertTessarGeneration(db, realmURL, 7); + assert.strictEqual( + tessarRequestedGeneration( + new Request(realmURL, { + headers: { [TESSAR_INPUT_GENERATION_HEADER]: '7' }, + }), + ), + 7, + ); + for (let value of ['', '-1', '1.5', 'NaN', '9007199254740992']) { + assert.throws( + () => + tessarRequestedGeneration( + new Request(realmURL, { + headers: { [TESSAR_INPUT_GENERATION_HEADER]: value }, + }), + ), + /nonnegative input generation/, + ); + } + await db.execute( + 'UPDATE realm_generations SET current_generation = 8 WHERE realm_url = $1', + { bind: [realmURL] }, + ); + await assert.rejects( + assertTessarGeneration(db, realmURL, 7), + /input revision changed/, + ); + await db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + await assertTessarGeneration(db, realmURL, 8, tx); + assert.ok(true, 'publication can validate its pinned transaction'); + }); + }); + + test('real SQL verifies typed values, null, plural paths and boolean composition', async function (assert) { + let predicates: Array<[Filter, boolean]> = [ + [{ on, eq: { name: 'A' } }, true], + [{ on, eq: { name: 'B' } }, false], + [{ on, in: { name: ['A', 'B'] } }, true], + [{ on, range: { score: { gte: 7, lt: 8 } } }, true], + [{ on, range: { score: { gt: 7 } } }, false], + [{ on, eq: { active: true } }, true], + [{ on, eq: { tags: 'blue' } }, true], + [{ on, eq: { score: null } }, false], + [ + { on, any: [{ eq: { name: 'B' } }, { not: { eq: { score: 8 } } }] }, + true, + ], + ]; + for (let [filter, expected] of predicates) { + assert.strictEqual( + await engine.tessarMatchesDocument(filter, record('A')), + expected, + JSON.stringify(filter), + ); + } + assert.true( + await engine.tessarMatchesDocument( + { on, eq: { score: null } }, + { ...record('A'), search_doc: { name: 'A' } }, + ), + 'missing field follows SQL null semantics', + ); + }); + + test('watch publication rolls back atomically and dirty state survives a new connection', async function (assert) { + let owner = { + realmURL, + ownerURL: `${realmURL}summary`, + generation: 1, + inputGeneration: 1, + definitionRevision: 'tessar-v1', + watches: await registry.prepare([ + { fieldPath: 'records', query: { filter: { on, eq: { name: 'A' } } } }, + ]), + }; + await assert.rejects( + db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + await registry.publish(tx!, owner); + throw new Error('Tessar injected rollback'); + }), + /injected rollback/, + ); + assert.deepEqual( + await registry.affected(realmURL, undefined, record('A')), + [], + 'rollback leaves no partial watch or terms', + ); + await db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + assert.true(await registry.publish(tx!, owner)); + }); + assert.deepEqual( + await registry.affected(realmURL, record('A'), record('B')), + [owner.ownerURL], + 'membership exit invalidates', + ); + await db.withWriteLock(`tessar:${realmURL}`, async (tx) => { + await registry.markDirty(tx!, realmURL, [owner.ownerURL], 3); + }); + let { PgAdapter } = await import('@cardstack/postgres'); + let reopened = new PgAdapter(); + try { + let afterRestart = new TessarQueryRegistry(reopened, engine); + assert.deepEqual( + await afterRestart.pending(realmURL), + [{ ownerURL: owner.ownerURL, generation: 3 }], + 'dirty work is durable', + ); + await reopened.withWriteLock(`tessar:${realmURL}`, async (tx) => { + assert.false( + await afterRestart.publish(tx!, { + ...owner, + generation: 2, + inputGeneration: 2, + }), + 'racing stale publication is rejected', + ); + assert.true( + await afterRestart.publish(tx!, { + ...owner, + generation: 3, + inputGeneration: 3, + }), + ); + }); + assert.deepEqual(await afterRestart.pending(realmURL), []); + } finally { + await reopened.close(); + } + }); +}); diff --git a/packages/realm-test-harness/src/isolated-realm-stack.ts b/packages/realm-test-harness/src/isolated-realm-stack.ts index a756ca88ec3..af8386d5b95 100644 --- a/packages/realm-test-harness/src/isolated-realm-stack.ts +++ b/packages/realm-test-harness/src/isolated-realm-stack.ts @@ -632,6 +632,21 @@ export async function startIsolatedRealmStack({ if (realms.length === 0) { throw new Error('startIsolatedRealmStack requires at least one realm'); } + // Tessar benchmarks exercise foreground writes during long HTML sweeps. + // Opt into the existing worker-manager index lane; preserve the harness + // default and record this resource choice in the benchmark manifest. + let userIndexWorkers = Number( + process.env.TEST_HARNESS_USER_INDEX_WORKERS ?? 0, + ); + if ( + !Number.isSafeInteger(userIndexWorkers) || + userIndexWorkers < 0 || + userIndexWorkers > 4 + ) { + throw new Error( + 'TEST_HARNESS_USER_INDEX_WORKERS must be an integer from 0 to 4', + ); + } let rootDir = mkdtempSync(join(tmpdir(), 'software-factory-realms-')); let workerManagerMetadataFile = join(rootDir, 'worker-manager.runtime.json'); let realmServerMetadataFile = join(rootDir, 'realm-server.runtime.json'); @@ -841,6 +856,7 @@ export async function startIsolatedRealmStack({ let workerArgs = [ 'worker-manager.ts', + `--userIndexCount=${userIndexWorkers}`, `--port=${actualWorkerManagerPort}`, `--matrixURL=${context.matrixURL}`, `--prerendererUrl=${prerenderURL}`, diff --git a/packages/runtime-common/definition-lookup.ts b/packages/runtime-common/definition-lookup.ts index 229b2466424..f32e537cb0b 100644 --- a/packages/runtime-common/definition-lookup.ts +++ b/packages/runtime-common/definition-lookup.ts @@ -1246,7 +1246,11 @@ export class CachingDefinitionLookup implements DefinitionLookup { } registerRealm(realm: LocalRealm): void { - this.#realms.push(realm); + // Tessar workers scope lookups for each indexing job. Replace a previous + // registration for that URL so jobs do not retain retired realm contexts. + let existing = this.#realms.findIndex((item) => item.url === realm.url); + if (existing === -1) this.#realms.push(realm); + else this.#realms[existing] = realm; } forRealm(realm: LocalRealm): DefinitionLookup { diff --git a/packages/runtime-common/document.ts b/packages/runtime-common/document.ts index df5f298a748..29ae4ee4111 100644 --- a/packages/runtime-common/document.ts +++ b/packages/runtime-common/document.ts @@ -8,12 +8,17 @@ import { } from './index.ts'; import type { RealmResourceIdentifier } from './realm-identifiers.ts'; import type { VirtualNetwork } from './virtual-network.ts'; +import { + TESSAR_INPUT_GENERATION_HEADER, + type TessarInputSnapshot, +} from './tessar-materialization.ts'; async function loadDocumentWithRequest( fetch: typeof globalThis.fetch, url: string, requestURL: URL, accept: SupportedMimeType, + extraHeaders?: Record, ) { let response: Response; requestURL.searchParams.set('noCache', 'true'); @@ -28,6 +33,7 @@ async function loadDocumentWithRequest( // documents being indexed and not finding the document yet in the index. headers: { Accept: accept, + ...extraHeaders, }, }); } catch (err: any) { @@ -74,14 +80,27 @@ export async function loadCardDocument( fetch: typeof globalThis.fetch, url: string, virtualNetwork: VirtualNetwork, + tessar?: TessarInputSnapshot, ) { - let target = !url.endsWith('.json') ? `${url}.json` : url; + let target = tessar + ? url.replace(/\.json$/, '') + : !url.endsWith('.json') + ? `${url}.json` + : url; let requestURL = virtualNetwork.toURL(target); + if (tessar && !requestURL.href.startsWith(tessar.realmURL)) { + return new CardError('Tessar indexed inputs must be in the owner realm', { + status: 400, + }); + } let json = await loadDocumentWithRequest( fetch, url, requestURL, - SupportedMimeType.CardSource, + tessar ? SupportedMimeType.CardJson : SupportedMimeType.CardSource, + tessar + ? { [TESSAR_INPUT_GENERATION_HEADER]: String(tessar.generation) } + : undefined, ); if (isCardError(json)) { return json; @@ -95,6 +114,16 @@ export async function loadCardDocument( )}`, ); } + if ( + tessar && + json.data.meta.tessar && + json.data.meta.tessar.state !== 'ready' + ) { + return new CardError( + 'Tessar feeder is pending; defer its dependent owner', + { status: 409 }, + ); + } if (!json.data.id) { // card source format is not serialized with the ID, so we add that back in. json.data.id = url as RealmResourceIdentifier; diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index 07fb330b442..66bcc85b212 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -373,6 +373,122 @@ export class IndexQueryEngine { : this.#query(expression); } + // Tessar reverse-query verification uses the forward query compiler against + // one effective indexed document. No second JavaScript predicate evaluator: + // type ancestry, nulls, plural paths and reference normalization are shared. + // Sort/page never restrict invalidation membership. + async tessarMatchesDocument( + filter: Filter | undefined, + document: Pick & + Partial< + Pick< + BoxelIndexTable, + 'file_alias' | 'type' | 'last_modified' | 'resource_created_at' + > + >, + ): Promise { + assertTessarFilter(filter); + let json = (value: unknown): Expression => [ + dbExpression({ + pg: ['CAST(', param(value as any), 'AS JSONB)'], + sqlite: [param(value as any)], + }), + ]; + let expression: CardExpression = [ + 'SELECT 1 AS matched FROM (SELECT', + ...json(document.search_doc), + 'AS search_doc,', + ...json(document.types), + 'AS types,', + param(document.url), + 'AS url,', + param(document.file_alias ?? null), + 'AS file_alias,', + param(document.type ?? 'instance'), + 'AS type,', + param(document.last_modified ?? null), + 'AS last_modified,', + param(document.resource_created_at ?? null), + 'AS resource_created_at', + ') AS i', + tableValuedFunctionsPlaceholder, + 'WHERE', + ...(filter + ? this.filterCondition(filter, baseCardRef, 'positive') + : ['TRUE']), + 'LIMIT 1', + ]; + return (await this.#queryCards(expression)).length > 0; + } + + // A deliberately narrow routing optimization: direct string fields have an + // identical query/index encoding. Everything else remains in the broad + // bucket until an extractor can prove it will not miss a forward match. + async tessarRoutingTerms( + filter: Filter | undefined, + ): Promise | undefined> { + assertTessarFilter(filter); + let extract = async ( + part: Filter | undefined, + inheritedOn: CodeRef, + ): Promise | undefined> => { + if (!part) return undefined; + let on = 'on' in part && part.on ? part.on : inheritedOn; + if ('every' in part) { + for (let child of part.every) { + let terms = await extract(child, on); + if (terms) return terms; + } + } else if ('any' in part) { + let branches = await Promise.all( + part.any.map((child) => extract(child, on)), + ); + if (branches.every((branch) => branch !== undefined)) + return branches.flat() as Array<{ path: string; value: string }>; + } else if ('eq' in part || 'in' in part) { + let entries = + 'eq' in part + ? Object.entries(part.eq).map( + ([path, value]) => [path, [value]] as const, + ) + : Object.entries(part.in); + for (let [path, values] of entries) { + if ( + path.includes('.') || + isReferenceFilterField(path) || + !Array.isArray(values) || + values.length === 0 || + values.length > 64 || + path.length > 128 || + !values.every( + (value) => typeof value === 'string' && value.length <= 256, + ) + ) + continue; + let definition = await this.getDefinition(on); + let field = await getField(definition, path, this.#definitionLookup); + let key = internalKeyFor( + field.fieldOrCard, + undefined, + this.#virtualNetwork, + ); + if ( + field.type === 'contains' && + field.isPrimitive && + !field.serializerName && + [ + `${baseRealmRRI}string/default`, + `${baseRealmRRI}card-api/StringField`, + ].includes(key) + ) + return values.map((value) => ({ path, value: value as string })); + } + } + return undefined; + }; + return extract(filter, baseCardRef); + } + async getInstance( url: URL, opts?: GetEntryOptions, @@ -2468,6 +2584,17 @@ function assertNever(value: never) { return new Error(`should never happen ${value}`); } +function assertTessarFilter(filter: Filter | undefined): void { + if (!filter) return; + if ('matches' in filter) + throw new InvalidQueryError( + 'Tessar materialization does not support full-text predicates', + ); + if ('every' in filter) filter.every.forEach(assertTessarFilter); + if ('any' in filter) filter.any.forEach(assertTessarFilter); + if ('not' in filter) assertTessarFilter(filter.not); +} + type FilterFieldHandler = ( definition: Definition, expression: T, diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 5810a359ecd..5aae5a45239 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -1,4 +1,6 @@ import { ignore, type Ignore } from './ignore.ts'; +import type { TessarIndexPublication } from './tessar-index-publication.ts'; +import type { TessarInputSnapshot } from './tessar-materialization.ts'; // Isomorphic UUID — works in both Node and the browser (host tests // instantiate IndexRunner inside a Chrome tab, so Node's built-in // `crypto.randomUUID` is not available). @@ -64,6 +66,16 @@ type VisitRenderOutcome = | { status: 'error'; error: unknown }; export class IndexRunner { + #tessar: TessarIndexPublication; + #tessarInputSnapshot: TessarInputSnapshot | undefined; + #tessarTimings = { + tessarMaterializationMs: 0, + tessarWriteMs: 0, + tessarSwapMs: 0, + tessarWaves: 0, + tessarOwnersRendered: 0, + }; + #tessarSourceWriteMs: number | undefined; #indexingInstances = new Map>(); #reader: Reader; #indexWriter: IndexWriter; @@ -169,6 +181,17 @@ export class IndexRunner { }): void; }) { this.#indexWriter = indexWriter; + this.#tessar = indexWriter.tessarPublication( + definitionLookup.forRealm({ + url: realmURL.href, + getRealmOwnerUserId: async () => realmOwnerUserId, + visibility: async () => + (await this.getModuleCacheContext()).cacheScope === 'public' + ? 'public' + : 'private', + }), + virtualNetwork, + ); this.#realmPaths = new RealmPaths(realmURL, virtualNetwork); this.#reader = reader; this.#realmURL = realmURL; @@ -342,8 +365,12 @@ export class IndexRunner { `${jobIdentity(current.#jobInfo)} completed index visit in ${Date.now() - visitStart} ms`, ); let finalizeStart = Date.now(); - let { totalIndexEntries } = await current.batch.done(); + let { totalIndexEntries } = await current.batch.done({ + tessar: current.#tessar, + }); swapMs = Date.now() - finalizeStart; + current.#tessarSourceWriteMs = current.batch.writeMs; + invalidations.push(...(await current.#drainTessar())); current.#perfLog.debug( `${jobIdentity(current.#jobInfo)} completed index finalization in ${swapMs} ms`, ); @@ -390,7 +417,8 @@ export class IndexRunner { discoverMs, orderMs, ...(visitLoopMs !== undefined ? { visitLoopMs } : {}), - writeMs: current.batch.writeMs, + writeMs: current.#tessarSourceWriteMs ?? current.batch.writeMs, + ...(current.#tessarTimings.tessarWaves ? current.#tessarTimings : {}), ...(swapMs !== undefined ? { swapMs } : {}), }, }; @@ -563,8 +591,12 @@ export class IndexRunner { visitLoopMs = Date.now() - loopStart; let finalizeStart = Date.now(); - let { totalIndexEntries } = await current.batch.done(); + let { totalIndexEntries } = await current.batch.done({ + tessar: current.#tessar, + }); swapMs = Date.now() - finalizeStart; + current.#tessarSourceWriteMs = current.batch.writeMs; + invalidations.push(...(await current.#drainTessar())); current.stats.totalIndexEntries = totalIndexEntries; } finally { current.#onProgress?.({ @@ -605,12 +637,95 @@ export class IndexRunner { discoverMs, orderMs, ...(visitLoopMs !== undefined ? { visitLoopMs } : {}), - writeMs: current.batch.writeMs, + writeMs: current.#tessarSourceWriteMs ?? current.batch.writeMs, + ...(current.#tessarTimings.tessarWaves ? current.#tessarTimings : {}), ...(swapMs !== undefined ? { swapMs } : {}), }, }; } + // Drain the durable registry inside the existing per-realm indexing job. + // A crash retries that queue job and discovers pending work again. Failed + // waves remain pending and fail the job; they never publish partial output. + async #drainTessar(): Promise { + let startedAt = Date.now(); + let published = new Set(); + let pending = await this.#tessar.registry.pending(this.realmURL.href); + // A leaf write can activate one new downstream owner per wave. Bound an + // acyclic chain by all owners, not only the initially dirty subset. + let maxWaves = pending.length + ? (await this.#tessar.registry.activeOwnerCount(this.realmURL.href)) + 1 + : 0; + try { + for (let wave = 0; pending.length && wave < maxWaves; wave++) { + this.#tessarTimings.tessarWaves++; + // No job resume seed: previous source-pass working rows have the same + // job id, but must never be re-promoted by a materialization wave. + this.#batch = await this.#indexWriter.createBatch( + this.realmURL, + this.#virtualNetwork, + ); + this.#tessarInputSnapshot = { + realmURL: this.realmURL.href, + generation: this.batch.currentGeneration - 1, + }; + this.#dependencyResolver.reset(); + this.#indexingInstances.clear(); + let succeeded = 0; + let failures: string[] = []; + for (let { ownerURL } of pending) { + let outcome = await this.#renderVisit(new URL(ownerURL)); + if ( + outcome.status !== 'rendered' || + outcome.result.card?.error || + !outcome.result.card?.serialized?.data.meta.tessar + ) { + let error = + outcome.status === 'error' + ? outcome.error + : outcome.status === 'rendered' + ? (outcome.result.card?.error ?? + outcome.result.pageUnusableError) + : undefined; + failures.push( + `${ownerURL}: ${coerceErrorMessage( + error, + outcome.status === 'rendered' + ? `incomplete Tessar materialization (card fields: ${Object.keys(outcome.result.card ?? {}).join(', ')}; metadata: ${Object.keys(outcome.result.card?.serialized?.data.meta ?? {}).join(', ')})` + : `Tessar visit ${outcome.status}`, + )}`, + ); + continue; + } + await this.#finishVisit(outcome.result); + published.add(ownerURL); + succeeded++; + this.#tessarTimings.tessarOwnersRendered++; + } + if (!succeeded) + throw new Error( + `Tessar could not materialize pending owners: ${failures.join(', ')}`, + ); + let swapStartedAt = Date.now(); + await this.batch.done({ + tessar: this.#tessar, + tessarInputGeneration: this.#tessarInputSnapshot.generation, + }); + this.#tessarTimings.tessarSwapMs += Date.now() - swapStartedAt; + this.#tessarTimings.tessarWriteMs += this.batch.writeMs; + pending = await this.#tessar.registry.pending(this.realmURL.href); + } + if (pending.length) + throw new Error( + 'Tessar feeder graph did not converge; owners remain pending', + ); + return [...published].map((url) => new URL(url)); + } finally { + this.#tessarInputSnapshot = undefined; + this.#tessarTimings.tessarMaterializationMs += Date.now() - startedAt; + } + } + // Announce this pass's now-fixed invalidation set, tagged per URL: // genuine deletions (the URLs in `deletes`) as 'delete', everything else — // fan-out dependents are always re-renders — as 'update'. Only fires in @@ -659,6 +774,7 @@ export class IndexRunner { async #renderVisit(url: URL): Promise { try { let result = await renderFileForIndexing({ + tessarInputSnapshot: this.#tessarInputSnapshot, url, realmURL: this.#realmURL, ignoreMap: this.ignoreMap, diff --git a/packages/runtime-common/index-runner/prewarm-modules.ts b/packages/runtime-common/index-runner/prewarm-modules.ts index 85d6c82a7ad..6b7700f8090 100644 --- a/packages/runtime-common/index-runner/prewarm-modules.ts +++ b/packages/runtime-common/index-runner/prewarm-modules.ts @@ -25,6 +25,11 @@ import { canonicalURL } from './dependency-url.ts'; // the requests just queue at the server's per-affinity admission. const DEFAULT_PREWARM_CONCURRENCY = 1; +// Tessar's 10x fixture has millions of repeated dependency strings. Consume +// bounded slices here: SQL parameter chunking alone still retains every row +// in getDependencyRows's result until the entire realm has been read. +const TESSAR_PREWARM_DEPENDENCY_BATCH_SIZE = 250; + // Resolve the pre-warm fan-out width from `INDEXER_PREWARM_CONCURRENCY`, // falling back to the default. Reads `process.env` defensively — pre-warm // runs only in the worker, but the bundle is shared, so guard against a @@ -219,41 +224,49 @@ export async function preWarmModulesTable({ // Base layer: the realm-wide `.gts` / `.gjs` sweep. let toWarm = new Set(allRealmCardModules); - let hrefs = invalidations.map((u) => u.href); - let existingRows = await getDependencyRows(hrefs); - let bestByUrl = new Map(); - for (let row of existingRows) { - // Prefer rows that actually carry deps so the lookup below returns the - // strongest signal available for each URL. - let existing = bestByUrl.get(row.url); - if (!existing || (!existing.deps?.length && row.deps?.length)) { - bestByUrl.set(row.url, { url: row.url, deps: row.deps ?? null }); - } - } - let novelJsonUrls: URL[] = []; - for (let url of invalidations) { - // Module files in the invalidation set are deps that instances in the - // same pass will consume — pre-warm them directly. This includes - // `.ts` / `.js` helpers, but only the ones the pass is actually touching, - // so cost is bounded by invalidation size rather than realm size. - if (hasExecutableExtension(url.href)) { - toWarm.add(url.href); + for ( + let offset = 0; + offset < invalidations.length; + offset += TESSAR_PREWARM_DEPENDENCY_BATCH_SIZE + ) { + let urls = invalidations.slice( + offset, + offset + TESSAR_PREWARM_DEPENDENCY_BATCH_SIZE, + ); + let existingRows = await getDependencyRows(urls.map((url) => url.href)); + let bestByUrl = new Map(); + for (let row of existingRows) { + // Prefer the strongest signal for this URL; every version of a URL + // is returned in the same slice, preserving the whole-realm choice. + let existing = bestByUrl.get(row.url); + if (!existing || (!existing.deps?.length && row.deps?.length)) { + bestByUrl.set(row.url, { url: row.url, deps: row.deps ?? null }); + } } - let row = bestByUrl.get(url.href); - if (row?.deps?.length) { - for (let dep of row.deps) { - let resolved = canonicalURL(dep, url.href, virtualNetwork); - // `.json` marks an instance dep and `.glimmer-scoped.css` marks an - // inline-styles artifact; everything else in the deps array is a - // module URL (stored extensionless after normalizeModuleURL / - // normalizeDependency). - if (!resolved.endsWith('.json') && !isScopedCSSRequest(resolved)) { - toWarm.add(resolved); + for (let url of urls) { + // Module files in the invalidation set are deps that instances in the + // same pass will consume — pre-warm them directly. This includes + // `.ts` / `.js` helpers, but only the ones the pass is actually touching, + // so cost is bounded by invalidation size rather than realm size. + if (hasExecutableExtension(url.href)) { + toWarm.add(url.href); + } + let row = bestByUrl.get(url.href); + if (row?.deps?.length) { + for (let dep of row.deps) { + let resolved = canonicalURL(dep, url.href, virtualNetwork); + // `.json` marks an instance dep and `.glimmer-scoped.css` marks an + // inline-styles artifact; everything else in the deps array is a + // module URL (stored extensionless after normalizeModuleURL / + // normalizeDependency). + if (!resolved.endsWith('.json') && !isScopedCSSRequest(resolved)) { + toWarm.add(resolved); + } } + } else if (url.href.endsWith('.json')) { + novelJsonUrls.push(url); } - } else if (url.href.endsWith('.json')) { - novelJsonUrls.push(url); } } for (let url of novelJsonUrls) { diff --git a/packages/runtime-common/index-runner/visit-file.ts b/packages/runtime-common/index-runner/visit-file.ts index 4e594b528eb..e4e4df9ed3e 100644 --- a/packages/runtime-common/index-runner/visit-file.ts +++ b/packages/runtime-common/index-runner/visit-file.ts @@ -27,6 +27,7 @@ import { resolveFileDefCodeRef } from '../file-def-code-ref.ts'; import type { VirtualNetwork } from '../virtual-network.ts'; interface RenderFileForIndexingOptions { + tessarInputSnapshot?: import('../tessar-materialization.ts').TessarInputSnapshot; url: URL; realmURL: URL; ignoreMap: Map; @@ -127,6 +128,7 @@ interface RouteIndexVisitCallbacks { // render before this one's row writes land. Returns `undefined` when the // file is ignored or belongs to a different realm. export async function renderFileForIndexing({ + tessarInputSnapshot, url, realmURL, ignoreMap, @@ -235,6 +237,7 @@ export async function renderFileForIndexing({ } let visitArgs = { + ...(tessarInputSnapshot ? { tessarInputSnapshot } : {}), affinityType: 'realm' as const, affinityValue: realmURL.href, realm: realmURL.href, diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 6470da105e5..9af9784fd4c 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -1,4 +1,9 @@ import { flatten } from 'lodash-es'; +import { TessarIndexPublication } from './tessar-index-publication.ts'; +import { TessarQueryRegistry } from './tessar-query-registry.ts'; +import { IndexQueryEngine } from './index-query-engine.ts'; +import type { DefinitionLookup } from './definition-lookup.ts'; +import type { Querier } from './expression.ts'; import { flattenDeep } from 'lodash-es'; import { type CardResource, @@ -63,6 +68,16 @@ export class IndexWriter { this.#dbAdapter = dbAdapter; } + tessarPublication(definitions: DefinitionLookup, network: VirtualNetwork) { + return new TessarIndexPublication( + this.#dbAdapter, + new TessarQueryRegistry( + this.#dbAdapter, + new IndexQueryEngine(this.#dbAdapter, definitions, network), + ), + ); + } + async createBatch( realmURL: URL, virtualNetwork: VirtualNetwork, @@ -276,6 +291,7 @@ function prerenderedHtmlEntryFrom( const WRITE_BUFFER_FLUSH_THRESHOLD = 100; export class Batch { + #tessarTransaction: Querier | undefined; readonly ready: Promise; #invalidations = new Set(); #nodeResolvedInvalidations: string[] | undefined; @@ -1684,6 +1700,8 @@ export class Batch { } async done(opts?: { + tessar?: TessarIndexPublication; + tessarInputGeneration?: number; // Write the previous generation's realm_meta value at this batch's // generation instead of recomputing it from `boxel_index_working`. The // setup-phase failure recovery finalizes while the working table still @@ -1709,15 +1727,45 @@ export class Batch { await this.#query(['COMMIT']); return { totalIndexEntries: this.#invalidations.size }; } - await this.#query(['BEGIN']); - if (opts?.carryForwardRealmMeta) { - await this.carryForwardRealmMeta(); - } else { - await this.updateRealmMeta(); - } - await this.applyBatchUpdates(); - await this.pruneObsoleteEntries(); - await this.#query(['COMMIT']); + let tessarPrepared = await opts?.tessar?.prepare( + this.realmURL.href, + this.generation, + ); + // Source writes hold the realm's write lock while awaiting indexing. + // Publication needs its own namespace or that wait deadlocks the worker. + await this.#dbAdapter.withWriteLock( + `tessar:index:${this.realmURL.href}`, + async (tx) => { + // Pg pins this connection and owns BEGIN/COMMIT/ROLLBACK. SQLite's + // adapter has one connection, so explicitly bracket that fallback. + this.#tessarTransaction = tx; + if (!tx) await this.#query(['BEGIN']); + try { + if (opts?.tessar && tessarPrepared) { + await opts.tessar.commit(tx ?? ((expr) => this.#query(expr)), { + realmURL: this.realmURL.href, + generation: this.generation, + definitionRevision: this.loaderEpoch, + prepared: tessarPrepared, + inputGeneration: opts.tessarInputGeneration, + }); + } + if (opts?.carryForwardRealmMeta) { + await this.carryForwardRealmMeta(); + } else { + await this.updateRealmMeta(); + } + await this.applyBatchUpdates(); + await this.pruneObsoleteEntries(); + if (!tx) await this.#query(['COMMIT']); + } catch (error) { + if (!tx) await this.#query(['ROLLBACK']); + throw error; + } finally { + this.#tessarTransaction = undefined; + } + }, + ); let totalIndexEntries = await this.numberOfIndexEntries(); return { totalIndexEntries }; @@ -1767,7 +1815,9 @@ export class Batch { } #query(expression: Expression) { - return query(this.#dbAdapter, expression, coerceTypes); + return this.#tessarTransaction + ? this.#tessarTransaction(expression) + : query(this.#dbAdapter, expression, coerceTypes); } private async getProductionVersion( diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index cd604b448de..0c08f533951 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -932,6 +932,7 @@ export type VisitPass = (typeof VISIT_PASS_ORDER)[number]; export type PrerenderVisitType = 'index' | 'prerender-html'; export type PrerenderVisitArgs = { + tessarInputSnapshot?: import('./tessar-materialization.ts').TessarInputSnapshot; affinityType: AffinityType; affinityValue: string; realm: string; @@ -1996,3 +1997,8 @@ export { type BotCommandFilter, type BotCommandMatrixFilter, } from './bot-command.ts'; +export { + currentTessarInputSnapshot, + tessarSnapshotFields, + type TessarMaterialization, +} from './tessar-materialization.ts'; diff --git a/packages/runtime-common/realm-index-query-engine.ts b/packages/runtime-common/realm-index-query-engine.ts index d33acaac341..33f38c8bb84 100644 --- a/packages/runtime-common/realm-index-query-engine.ts +++ b/packages/runtime-common/realm-index-query-engine.ts @@ -1,4 +1,5 @@ import { isScopedCSSRequest } from './scoped-css.ts'; +import { tessarReadState } from './tessar-materialization.ts'; import { cloneDeep } from 'lodash-es'; import { SupportedMimeType, @@ -107,6 +108,7 @@ import { const RECURSING_DEPTH = 3; type Options = { + tessarInput?: boolean; loadLinks?: true; linkFields?: string[]; // When true, populateQueryFields will only use cached definitions from the @@ -257,6 +259,7 @@ export class RealmIndexQueryEngine { realm.virtualNetwork, ); this.#definitionLookup = definitionLookup; + this.#tessarDB = dbAdapter; this.#realm = realm; this.#fetch = fetch; } @@ -264,6 +267,32 @@ export class RealmIndexQueryEngine { private get realmURL() { return (this.#realmURL ??= new URL(this.#realm.url)); } + #tessarDB: DBAdapter; + + private async tessarReadResource( + resource: LooseCardResource, + realmURL: URL, + opts?: Options, + ) { + if (!resource.meta.tessar || !resource.id) return; + let url = + this.#realm.virtualNetwork + .toURL(resource.id) + .href.replace(/\.json$/, '') + '.json'; + resource.meta = { + ...resource.meta, + tessar: { + ...resource.meta.tessar, + state: await tessarReadState( + this.#tessarDB, + realmURL.href, + url, + resource.meta.tessar, + opts, + ), + }, + }; + } // The entry engine. Runs the parsed entry query — the // `item.` membership query against the SQL core, then the htmlQuery @@ -572,6 +601,9 @@ export class RealmIndexQueryEngine { id: cardUrl as RealmResourceIdentifier, links: { self: cardUrl }, }; + // This check also applies when assembly is omitted or fields are + // sparse; neither path may advertise a dirty snapshot as ready. + await this.tessarReadResource(item, this.realmURL, opts); if (fieldset.item.kind === 'sparse') { item = buildSparseItemResource(item, fieldset.item.fields); } else { @@ -768,7 +800,8 @@ export class RealmIndexQueryEngine { `bug: should never get here--search index doc is undefined`, ); } - if (opts?.loadLinks) { + await this.tessarReadResource(doc.data, this.realmURL, opts); + if (opts?.loadLinks && !doc.data.meta.tessar) { let included = await this.loadLinks( { realmURL: this.realmURL, @@ -1685,6 +1718,14 @@ export class RealmIndexQueryEngine { try { await Promise.all( layer.map(async ({ resource, applyLinkFields }) => { + if ((resource as LooseCardResource).meta.tessar) { + await this.tessarReadResource( + resource as LooseCardResource, + realmURL, + opts, + ); + return; + } let popOpts = applyLinkFields ? opts : opts?.linkFields @@ -1796,6 +1837,12 @@ export class RealmIndexQueryEngine { for (let entry of relationshipEntries(resource.relationships)) { let { relationship, key, fieldName } = entry; + if ( + (resource as LooseCardResource).meta.tessar?.queryFields.includes( + fieldName, + ) + ) + continue; if (processed.has(key)) { continue; } diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 9c2b0792721..b946e59621c 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -1,4 +1,9 @@ import { Deferred } from './deferred.ts'; +import { + assertTessarGeneration, + tessarRequestedGeneration, + TESSAR_INPUT_GENERATION_HEADER, +} from './tessar-materialization.ts'; import { resolveRangeHeader } from './http-range.ts'; import { awaitRealmIndexSettled, @@ -3480,11 +3485,43 @@ export class Realm { }); } if (this.#router.handles(request)) { + let tessarGeneration = tessarRequestedGeneration(request); + if (tessarGeneration !== undefined) { + if (!['GET', 'QUERY'].includes(request.method)) { + return badRequest({ + requestContext, + message: 'Tessar input revisions apply only to reads', + }); + } + await assertTessarGeneration( + this.#dbAdapter, + this.url, + tessarGeneration, + ); + let response = await this.#router.handle(request, requestContext); + await assertTessarGeneration( + this.#dbAdapter, + this.url, + tessarGeneration, + ); + response.headers.set('cache-control', 'no-store'); + return response; + } return this.#router.handle(request, requestContext); } else { return this.fallbackHandle(request, requestContext); } } catch (e) { + if ( + request.headers.has(TESSAR_INPUT_GENERATION_HEADER) && + e instanceof CardError + ) { + return systemError({ + requestContext, + status: e.status, + message: e.message, + }); + } if (e instanceof AuthenticationError) { return createResponse({ body: e.message, @@ -6489,7 +6526,10 @@ export class Realm { // under the old schema. Read endpoints serve the realm's canonical // indexed view; the small wait when indexing is genuinely pending is // the right tradeoff vs returning stale state. - let pending = this.incrementalIndexing(); + let tessarInput = request.headers.has(TESSAR_INPUT_GENERATION_HEADER); + // An indexer's revision-pinned input read must not wait on the job that + // issued it. Its before/after generation checks reject concurrent swaps. + let pending = tessarInput ? undefined : this.incrementalIndexing(); if (pending) { await pending; } @@ -6535,7 +6575,7 @@ export class Realm { // peek when the client sent a validator — otherwise we'd be // paying an extra DB round-trip on cache misses since // `cardDocument()` below does its own `instance()` lookup. - if (ifNoneMatch) { + if (ifNoneMatch && !tessarInput) { await this.getRealmInfo(); let realmInfoHash = this.getCachedRealmInfoHash(); let instanceEntry = await this.#realmIndexQueryEngine.instance(url, { @@ -6565,6 +6605,7 @@ export class Realm { if ( !this.hasForeignRealmDeps(instanceEntry.deps) && instanceEntry.type === 'instance' && + !instanceEntry.instance.meta.tessar && instanceEntry.indexedAt != null ) { let etag = buildCardJsonEtag( @@ -6601,8 +6642,10 @@ export class Realm { // read for the response ETag below reflects the post-assembly // realm info. let maybeError = await this.#realmIndexQueryEngine.cardDocument(url, { - loadLinks: true, - skipQueryBackedExpansion: isDuringPrerenderRequest(request), + tessarInput, + ...(tessarInput ? {} : { loadLinks: true as const }), + skipQueryBackedExpansion: + tessarInput || isDuringPrerenderRequest(request), }); if (maybeError === undefined) { if (await this.nonJsonFileExists(localPath)) { @@ -6712,19 +6755,20 @@ export class Realm { // could 304 a follow-up request whose `included[]` should have // been re-fetched from the foreign realm. let foreignDeps = this.hasForeignRealmDeps(maybeError.deps); - let responseEtag = foreignDeps - ? undefined - : buildCardJsonEtag( - maybeError.indexedAt, - this.getCachedRealmInfoHash(), - screenshotsEtagFingerprint(maybeError.screenshots), - ); + let responseEtag = + foreignDeps || card.data.meta.tessar + ? undefined + : buildCardJsonEtag( + maybeError.indexedAt, + this.getCachedRealmInfoHash(), + screenshotsEtagFingerprint(maybeError.screenshots), + ); return createResponse({ body: JSON.stringify(card, null, 2), init: { headers: { 'content-type': SupportedMimeType.CardJson, - 'cache-control': cacheControl, + 'cache-control': card.data.meta.tessar ? 'no-store' : cacheControl, ...(responseEtag ? { etag: responseEtag } : {}), ...etagSuppressedHeader(foreignDeps), ...lastModifiedHeader(card), @@ -7075,6 +7119,7 @@ export class Realm { ): Promise { let engineOpts = { loadLinks: true as const, + ...(opts?.tessarInput ? { tessarInput: true } : {}), ...(opts?.cacheOnlyDefinitions ? { cacheOnlyDefinitions: true } : {}), ...(opts?.omitIncluded ? { omitIncluded: true } : {}), // `!== undefined` so an explicit priority 0 (system-initiated) survives. @@ -7142,6 +7187,7 @@ export class Realm { } let runSearch = (signal?: AbortSignal) => this.searchEntries(searchEntryQuery, { + tessarInput: request.headers.has(TESSAR_INPUT_GENERATION_HEADER), cacheOnlyDefinitions: duringPrerender, // Inside a prerender the search skips the `loadLinks` // relationship-assembly pass entirely: the host re-resolves every diff --git a/packages/runtime-common/render-route-options.ts b/packages/runtime-common/render-route-options.ts index f7ad0ebd30e..44f4d05db65 100644 --- a/packages/runtime-common/render-route-options.ts +++ b/packages/runtime-common/render-route-options.ts @@ -10,6 +10,8 @@ export interface RenderRouteOptions { // for, then records it — one reset per tab per module change. loaderEpoch?: string; cardRender?: true; + // HTML-only Tessar visits consume the published output, never recompute it. + tessarUseSnapshot?: true; fileExtract?: true; fileRender?: true; fileDefCodeRef?: ResolvedCodeRef; @@ -40,6 +42,9 @@ export function parseRenderRouteOptions( if (typeof parsed.loaderEpoch === 'string') { options.loaderEpoch = parsed.loaderEpoch; } + if (parsed.tessarUseSnapshot === true) { + options.tessarUseSnapshot = true; + } if (parsed.cardRender) { options.cardRender = true; } @@ -83,6 +88,9 @@ export function serializeRenderRouteOptions( if (options.loaderEpoch !== undefined) { serialized.loaderEpoch = options.loaderEpoch; } + if (options.tessarUseSnapshot) { + serialized.tessarUseSnapshot = true; + } if (options.cardRender) { serialized.cardRender = true; } diff --git a/packages/runtime-common/resource-types.ts b/packages/runtime-common/resource-types.ts index ef214538ded..c1eedbe3431 100644 --- a/packages/runtime-common/resource-types.ts +++ b/packages/runtime-common/resource-types.ts @@ -10,6 +10,7 @@ import type { import type { VirtualNetwork } from './virtual-network.ts'; import type { Query } from './query.ts'; import type { ScreenshotsMeta } from './capture-spec.ts'; +import type { TessarMaterialization } from './tessar-materialization.ts'; // Metadata for a query-based linksTo/linksToMany field on a FileDef subclass, // extracted during file prerendering so that file-meta responses can populate @@ -113,6 +114,7 @@ export type CardResourceMeta = Meta & { // data from stale. Additive — absent when the serialization did not come off // the index (e.g. a freshly-built resource that has not been persisted). generation?: number; + tessar?: TessarMaterialization; // The instance's declared-screenshot captures, joined at serve time from // the prerendered manifest (`prerendered_html.screenshots`) — never // persisted into the index or the source file, and stripped from incoming diff --git a/packages/runtime-common/search-utils.ts b/packages/runtime-common/search-utils.ts index 308831dd627..0bc7655fa29 100644 --- a/packages/runtime-common/search-utils.ts +++ b/packages/runtime-common/search-utils.ts @@ -106,6 +106,7 @@ export function resolveSearchRequestMethod(request: Request): string { // dropping a field here (e.g. priority) silently breaks the threading from // the realm-server handler down to searchCards. export type SearchOpts = { + tessarInput?: boolean; cacheOnlyDefinitions?: boolean; // Prerender searches set this so `searchCardsUncoalesced` skips the // `loadLinks` relationship-assembly pass entirely (the host re-resolves diff --git a/packages/runtime-common/tessar-index-publication.ts b/packages/runtime-common/tessar-index-publication.ts new file mode 100644 index 00000000000..d594e248aec --- /dev/null +++ b/packages/runtime-common/tessar-index-publication.ts @@ -0,0 +1,230 @@ +import type { DBAdapter } from './db.ts'; +import type { BoxelIndexTable } from './index-structure.ts'; +import { coerceTypes } from './index-structure.ts'; +import { param, query, type Querier } from './expression.ts'; +import { assertTessarGeneration } from './tessar-materialization.ts'; +import type { + TessarQueryRegistry, + TessarDocument, + TessarPreparedWatch, +} from './tessar-query-registry.ts'; + +// IndexRunner prepares these outside the commit lock. The actual publication, +// watch replacement and dirty markers all use Batch.done's pinned transaction. +export class TessarIndexPublication { + private db: DBAdapter; + readonly registry: TessarQueryRegistry; + constructor(db: DBAdapter, registry: TessarQueryRegistry) { + this.db = db; + this.registry = registry; + } + + async prepare(realmURL: string, generation: number) { + let active = await query(this.db, [ + 'SELECT owner_url FROM tessar_owners WHERE realm_url =', + param(realmURL), + 'AND retired = FALSE', + ]); + let rows = (await query( + this.db, + [ + 'SELECT * FROM boxel_index_working WHERE realm_url =', + param(realmURL), + 'AND generation =', + param(generation), + ...(active.length + ? [] + : ["AND pristine_doc->'meta'->'tessar' IS NOT NULL"]), + ], + coerceTypes, + )) as unknown as BoxelIndexTable[]; + let watches = new Map(); + for (let row of rows) { + let manifest = row.pristine_doc?.meta.tessar; + if ( + row.type === 'instance' && + !row.has_error && + !row.is_deleted && + manifest + ) { + watches.set(row.url, await this.registry.prepare(manifest.watches)); + } + } + return { rows, watches, hadOwners: active.length > 0 }; + } + + async commit( + tx: Querier, + args: { + realmURL: string; + generation: number; + definitionRevision: string; + prepared: Awaited>; + inputGeneration?: number; + }, + ): Promise { + let { + realmURL, + generation, + definitionRevision, + prepared, + inputGeneration, + } = args; + // Even the no-watch fast path was prepared outside the lock. Another + // publisher may have registered the realm's first owner since then; a + // stale source must retry so that new watch participates in invalidation. + await assertTessarGeneration(this.db, realmURL, generation - 1, tx); + if (!prepared.rows.length && !prepared.hadOwners) return; + if (inputGeneration !== undefined && inputGeneration !== generation - 1) { + throw new Error('Tessar publication does not follow its input revision'); + } + if (inputGeneration !== undefined) { + let [realm] = await tx([ + 'SELECT loader_epoch FROM realm_generations WHERE realm_url =', + param(realmURL), + ]); + if (realm?.loader_epoch !== definitionRevision) + throw new Error('Tessar definitions changed during computation'); + } + let changed = new Set(); + let dirty = new Set(); + if (inputGeneration === undefined) { + // The loader epoch is realm-wide. Even a module outside an owner's + // recorded dependencies can change it; every older snapshot must get + // work scheduled instead of remaining permanently pending at read time. + let outdated = await tx([ + 'SELECT owner_url FROM tessar_owners WHERE realm_url =', + param(realmURL), + 'AND retired = FALSE AND definition_revision <>', + param(definitionRevision), + ]); + for (let owner of outdated) dirty.add(owner.owner_url as string); + } + for (let row of prepared.rows) { + if (row.type !== 'instance') continue; + let [old] = await tx([ + 'SELECT * FROM boxel_index WHERE realm_url =', + param(realmURL), + 'AND url =', + param(row.url), + "AND type = 'instance'", + ]); + let previous = old as unknown as BoxelIndexTable | undefined; + // Exclude only provenance from equality: membership and output changes + // propagate to feeders; a recomputation with identical data does not. + if (tessarSemanticRow(previous) !== tessarSemanticRow(row)) { + changed.add(row.url); + for (let owner of await this.registry.affected( + realmURL, + tessarDocument(previous), + tessarDocument(row), + tx, + )) + dirty.add(owner); + } + let manifest = row.pristine_doc?.meta.tessar; + if (!row.has_error && !row.is_deleted && manifest) { + if ( + inputGeneration !== undefined && + manifest.inputGeneration !== inputGeneration + ) { + throw new Error( + 'Tessar renderer returned a different input revision', + ); + } + let ready = inputGeneration !== undefined; + if ( + !(await this.registry.publish(tx, { + realmURL, + ownerURL: row.url, + generation, + inputGeneration: inputGeneration ?? 0, + definitionRevision, + watches: prepared.watches.get(row.url)!, + pending: !ready, + })) + ) + throw new Error('Tessar rejected an obsolete owner publication'); + let resource = structuredClone(row.pristine_doc!); + resource.meta.tessar = { + ...manifest, + state: ready ? 'ready' : 'pending', + publishedGeneration: generation, + definitionRevision, + }; + await tx([ + 'UPDATE boxel_index_working SET pristine_doc =', + param(JSON.stringify(resource)), + 'WHERE realm_url =', + param(realmURL), + 'AND url =', + param(row.url), + "AND type = 'instance' AND generation =", + param(generation), + ]); + } else if (previous?.pristine_doc?.meta.tessar) { + if (row.is_deleted || !row.has_error) { + await this.registry.publish(tx, { + realmURL, + ownerURL: row.url, + generation, + inputGeneration: generation, + definitionRevision, + watches: [], + retired: true, + }); + } else dirty.add(row.url); + } + } + // Concrete dependencies supplement watches, including transitive inputs + // whose fields a getter consumed without appearing in a query predicate. + // Existing source passes already expand their dependency closure. This + // handles a newly published feeder's outputs in a Tessar follow-up wave. + if (changed.size) { + let owners = await tx([ + 'SELECT o.owner_url, i.deps FROM tessar_owners o JOIN boxel_index i ON i.realm_url = o.realm_url AND i.url = o.owner_url', + "AND i.type = 'instance' WHERE o.realm_url =", + param(realmURL), + 'AND o.retired = FALSE', + ]); + for (let owner of owners) { + let deps = ( + typeof owner.deps === 'string' ? JSON.parse(owner.deps) : owner.deps + ) as string[] | null; + if (deps?.some((dep) => changed.has(dep))) + dirty.add(owner.owner_url as string); + } + } + // A successful owner in this wave was computed from the previous revision. + // If another output it consumes changed in the wave, it must run again. + await this.registry.markDirty(tx, realmURL, [...dirty], generation); + // A retry after this commit must discover durable dirty owners, not resume + // and re-promote an already committed source pass under its old generation. + await tx([ + 'UPDATE boxel_index_working SET job_id = NULL WHERE realm_url =', + param(realmURL), + 'AND generation =', + param(generation), + ]); + } +} + +function tessarDocument( + row: BoxelIndexTable | undefined, +): TessarDocument | undefined { + return row && !row.is_deleted && row.search_doc + ? { url: row.url, types: row.types ?? [], search_doc: row.search_doc } + : undefined; +} + +function tessarSemanticRow(row: BoxelIndexTable | undefined): string { + if (!row || row.is_deleted) return 'deleted'; + let resource = row.pristine_doc ? structuredClone(row.pristine_doc) : null; + if (resource?.meta) delete resource.meta.tessar; + return JSON.stringify([ + Boolean(row.has_error), + resource, + row.search_doc, + row.types, + ]); +} diff --git a/packages/runtime-common/tessar-materialization.ts b/packages/runtime-common/tessar-materialization.ts new file mode 100644 index 00000000000..775ddc23bd8 --- /dev/null +++ b/packages/runtime-common/tessar-materialization.ts @@ -0,0 +1,159 @@ +import type { Query } from './query.ts'; +import type { DBAdapter } from './db.ts'; +import { query, param, type Querier } from './expression.ts'; +import { CardError } from './error.ts'; +import type { LooseCardResource } from './resource-types.ts'; +import { indexingConcurrencyGroup } from './jobs/indexing.ts'; + +// Versioned provenance for the ordinary attributes/relationship payload. The +// writer supplies revisions; realm source files must never supply this stamp. +export interface TessarMaterialization { + version: 1; + state: 'pending' | 'ready'; + computedFields: string[]; + queryFields: string[]; + watches: Array<{ fieldPath: string; query: Query }>; + inputGeneration: number; + publishedGeneration?: number; + definitionRevision?: string; +} + +export interface TessarInputSnapshot { + realmURL: string; + generation: number; +} + +export const TESSAR_INPUT_GENERATION_HEADER = 'x-boxel-tessar-input-generation'; + +export async function tessarHasMaterializations( + db: DBAdapter, + realms: string[], +): Promise { + for (let realm of realms) { + let rows = await query(db, [ + 'SELECT 1 FROM tessar_owners WHERE realm_url =', + param(realm), + 'AND retired = FALSE LIMIT 1', + ]); + if (rows.length) return true; + } + return false; +} + +export function tessarRequestedGeneration( + request: Request, +): number | undefined { + let value = request.headers.get(TESSAR_INPUT_GENERATION_HEADER); + if (value === null) return undefined; + let generation = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(generation)) { + throw new CardError('Tessar requires a nonnegative input generation', { + status: 400, + }); + } + return generation; +} + +export async function assertTessarGeneration( + db: DBAdapter, + realmURL: string, + expected: number, + tx?: Querier, +): Promise { + let execute: Querier = tx ?? ((expression) => query(db, expression)); + let [row] = await execute([ + 'SELECT current_generation FROM realm_generations WHERE realm_url =', + param(realmURL), + ]); + if (Number(row?.current_generation ?? 0) !== expected) { + throw new CardError( + 'Tessar input revision changed; recompute the complete owner', + { status: 409 }, + ); + } +} + +// A prerender visit sets and clears this explicitly. Ordinary readers and +// source edits never opt into the indexed-input protocol via this helper. +export function currentTessarInputSnapshot(): TessarInputSnapshot | undefined { + let globals = globalThis as unknown as { + __boxelRenderContext?: boolean; + __tessarInputSnapshot?: TessarInputSnapshot; + }; + return globals.__boxelRenderContext === true + ? globals.__tessarInputSnapshot + : undefined; +} + +export function tessarSnapshotFields(resource: LooseCardResource) { + let stamp = resource.meta.tessar; + if (!stamp) return undefined; + if ( + stamp.version !== 1 || + stamp.state !== 'ready' || + !Number.isSafeInteger(stamp.publishedGeneration) || + !Number.isSafeInteger(stamp.inputGeneration) || + stamp.inputGeneration < 0 || + stamp.publishedGeneration! <= stamp.inputGeneration || + !stamp.definitionRevision || + !Array.isArray(stamp.computedFields) || + !Array.isArray(stamp.queryFields) || + (resource.meta.generation !== undefined && + resource.meta.generation !== stamp.publishedGeneration) + ) { + throw new CardError( + 'Tessar materialization is pending or has invalid provenance', + { status: 409 }, + ); + } + return { + computedFields: stamp.computedFields, + queryFields: stamp.queryFields, + }; +} + +export async function tessarReadState( + db: DBAdapter, + realmURL: string, + ownerURL: string, + stamp: TessarMaterialization, + opts?: { tessarInput?: boolean }, +): Promise<'ready' | 'pending'> { + // Source endpoints durably enqueue before acknowledging. Across replicas, + // an unprocessed write is visible here even before its indexed old/new + // documents exist for precise reverse matching. A revision-pinned worker + // must ignore its own queue job while consuming already-ready feeders. + if (db.kind === 'pg' && !opts?.tessarInput) { + let pending = await query(db, [ + "SELECT 1 FROM jobs WHERE status = 'unfulfilled' AND concurrency_group =", + param(indexingConcurrencyGroup(realmURL)), + 'LIMIT 1', + ]); + if (pending.length) return 'pending'; + let [latest] = await query(db, [ + 'SELECT status FROM jobs WHERE concurrency_group =', + param(indexingConcurrencyGroup(realmURL)), + 'ORDER BY id DESC LIMIT 1', + ]); + if (latest?.status === 'rejected') + throw new CardError( + 'Tessar indexing failed; retry indexing before treating this view as current', + { status: 503 }, + ); + } + let [row] = await query(db, [ + 'SELECT o.published_generation, o.dirty_generation, o.retired, o.definition_revision, r.loader_epoch FROM tessar_owners o JOIN realm_generations r ON r.realm_url = o.realm_url WHERE o.realm_url =', + param(realmURL), + 'AND o.owner_url =', + param(ownerURL), + ]); + return row && + !row.retired && + row.dirty_generation == null && + Number(row.published_generation) === stamp.publishedGeneration && + row.definition_revision === stamp.definitionRevision && + row.loader_epoch === stamp.definitionRevision && + stamp.state === 'ready' + ? 'ready' + : 'pending'; +} diff --git a/packages/runtime-common/tessar-query-registry.ts b/packages/runtime-common/tessar-query-registry.ts new file mode 100644 index 00000000000..6ab1699a7f5 --- /dev/null +++ b/packages/runtime-common/tessar-query-registry.ts @@ -0,0 +1,282 @@ +import type { DBAdapter } from './db.ts'; +import type { IndexQueryEngine } from './index-query-engine.ts'; +import type { Query } from './query.ts'; +import { + any, + every, + param, + query, + type Expression, + type Querier, +} from './expression.ts'; + +export interface TessarWatch { + fieldPath: string; + query: Query; +} + +export interface TessarPreparedWatch extends TessarWatch { + terms: Array<{ path: string; value: string }>; +} + +export type TessarDocument = Parameters< + IndexQueryEngine['tessarMatchesDocument'] +>[1]; + +// All mutations take the caller's pinned transaction. Index publication and +// watch replacement must commit together. A notification is never the queue: +// dirty_generation survives disconnects and worker restarts. +export class TessarQueryRegistry { + private db: DBAdapter; + private engine: IndexQueryEngine; + constructor(db: DBAdapter, engine: IndexQueryEngine) { + this.db = db; + this.engine = engine; + } + + async prepare(watches: TessarWatch[]): Promise { + let paths = new Set(); + let prepared: TessarPreparedWatch[] = []; + for (let watch of watches) { + if (!watch.fieldPath || paths.has(watch.fieldPath)) { + throw new Error('Tessar watch paths must be nonempty and unique'); + } + paths.add(watch.fieldPath); + // Page and sort affect the output, but never narrow invalidation. Even a + // non-returned match can change an aggregate or displace a page member. + let terms = await this.engine.tessarRoutingTerms(watch.query.filter); + prepared.push({ + ...watch, + terms: terms?.length ? terms : [{ path: '', value: '' }], + }); + } + return prepared; + } + + async publish( + tx: Querier, + owner: { + realmURL: string; + ownerURL: string; + generation: number; + inputGeneration: number; + definitionRevision: string; + watches: TessarPreparedWatch[]; + retired?: boolean; + pending?: boolean; + }, + ): Promise { + let { realmURL, ownerURL, generation, inputGeneration } = owner; + if ( + !Number.isSafeInteger(generation) || + generation < 1 || + !Number.isSafeInteger(inputGeneration) || + inputGeneration < 0 || + inputGeneration > generation + ) { + throw new Error( + 'Tessar publication needs valid input and owner generations', + ); + } + for (let watch of owner.watches) { + let realms = + watch.query.realms ?? + (watch.query.realm ? [watch.query.realm] : [realmURL]); + if (realms.length !== 1 || realms[0] !== realmURL) { + throw new Error('Tessar watches currently require the owner realm'); + } + } + let [previous] = await tx([ + 'SELECT published_generation, dirty_generation FROM tessar_owners WHERE', + ...this.ownerCondition(realmURL, ownerURL), + ]); + if ( + (previous && Number(previous.published_generation) > generation) || + (!owner.pending && + !owner.retired && + previous?.dirty_generation != null && + Number(previous.dirty_generation) > inputGeneration) + ) { + return false; + } + await tx([ + `INSERT INTO tessar_owners + (realm_url, owner_url, published_generation, input_generation, + dirty_generation, definition_revision, retired) VALUES (`, + param(realmURL), + ',', + param(ownerURL), + ',', + param(generation), + ',', + param(inputGeneration), + ',', + param(owner.pending ? generation : null), + ',', + param(owner.definitionRevision), + ',', + param(owner.retired ?? false), + `) ON CONFLICT (realm_url, owner_url) DO UPDATE SET + published_generation = EXCLUDED.published_generation, + input_generation = EXCLUDED.input_generation, dirty_generation = EXCLUDED.dirty_generation, + definition_revision = EXCLUDED.definition_revision, + retired = EXCLUDED.retired`, + ]); + for (let table of ['tessar_query_terms', 'tessar_query_watches']) { + await tx([ + `DELETE FROM ${table} WHERE`, + ...this.ownerCondition(realmURL, ownerURL), + ]); + } + if (owner.retired) return true; + for (let watch of owner.watches) { + await tx([ + `INSERT INTO tessar_query_watches + (realm_url, owner_url, field_path, query) VALUES (`, + param(realmURL), + ',', + param(ownerURL), + ',', + param(watch.fieldPath), + ',', + param(JSON.stringify(watch.query)), + ')', + ]); + let seen = new Set(); + for (let term of watch.terms) { + let key = JSON.stringify([term.path, term.value]); + if (seen.has(key)) continue; + seen.add(key); + await tx([ + `INSERT INTO tessar_query_terms + (realm_url, owner_url, field_path, path, value) VALUES (`, + param(realmURL), + ',', + param(ownerURL), + ',', + param(watch.fieldPath), + ',', + param(term.path), + ',', + param(term.value), + ')', + ]); + } + } + return true; + } + + async candidates(realmURL: string, document: TessarDocument, tx?: Querier) { + let routes = [ + every([ + ['t.path =', param('')], + ['t.value =', param('')], + ]), + ]; + for (let [path, value] of Object.entries(document.search_doc ?? {})) { + if (typeof value !== 'string') continue; + routes.push( + every([ + ['t.path =', param(path)], + ['t.value =', param(value)], + ]), + ); + } + let expression: Expression = [ + `SELECT DISTINCT w.owner_url, w.field_path, w.query + FROM tessar_query_terms t JOIN tessar_query_watches w + ON w.realm_url = t.realm_url AND w.owner_url = t.owner_url + AND w.field_path = t.field_path + WHERE`, + ...(every([ + ['t.realm_url =', param(realmURL)], + any(routes), + ]) as Expression), + ]; + let rows = tx + ? await tx(expression) + : await query(this.db, expression, { query: 'JSON' }); + return rows.map((row) => ({ + ownerURL: row.owner_url as string, + fieldPath: row.field_path as string, + query: (typeof row.query === 'string' + ? JSON.parse(row.query) + : row.query) as Query, + })); + } + + async affected( + realmURL: string, + oldDocument: TessarDocument | undefined, + newDocument: TessarDocument | undefined, + tx?: Querier, + ): Promise { + let owners = new Set(); + for (let document of [oldDocument, newDocument]) { + if (!document) continue; + for (let watch of await this.candidates(realmURL, document, tx)) { + if (owners.has(watch.ownerURL)) continue; + if ( + await this.engine.tessarMatchesDocument(watch.query.filter, document) + ) { + owners.add(watch.ownerURL); + } + } + } + return [...owners].sort(); + } + + async markDirty( + tx: Querier, + realmURL: string, + owners: string[], + generation: number, + ) { + for (let ownerURL of new Set(owners)) { + await tx([ + `UPDATE tessar_owners SET dirty_generation = CASE + WHEN dirty_generation IS NULL OR dirty_generation <`, + param(generation), + 'THEN', + param(generation), + 'ELSE dirty_generation END WHERE', + ...(every([ + this.ownerCondition(realmURL, ownerURL), + ['retired = FALSE'], + ['input_generation <', param(generation)], + ]) as Expression), + ]); + } + } + + async activeOwnerCount(realmURL: string): Promise { + let [row] = await query(this.db, [ + 'SELECT COUNT(*) AS total FROM tessar_owners WHERE realm_url =', + param(realmURL), + 'AND retired = FALSE', + ]); + return Number(row.total); + } + + async pending( + realmURL: string, + ): Promise> { + let rows = await query(this.db, [ + `SELECT owner_url, dirty_generation FROM tessar_owners + WHERE realm_url =`, + param(realmURL), + 'AND retired = FALSE AND dirty_generation IS NOT NULL ORDER BY owner_url', + ]); + return rows.map((row) => ({ + ownerURL: row.owner_url as string, + generation: Number(row.dirty_generation), + })); + } + + private ownerCondition(realmURL: string, ownerURL: string) { + return every([ + ['realm_url =', param(realmURL)], + ['owner_url =', param(ownerURL)], + ]) as Expression; + } +} diff --git a/packages/runtime-common/worker.ts b/packages/runtime-common/worker.ts index 92c7b7ae6aa..ebf1828d65d 100644 --- a/packages/runtime-common/worker.ts +++ b/packages/runtime-common/worker.ts @@ -76,6 +76,12 @@ export interface IndexPhaseTimings { // The final atomic swap: `batch.done()` (realm-meta update, working → main // promotion, obsolete-row prune) in one transaction. swapMs?: number; + // Tessar follow-up waves are separate from the source visit/swap timings. + tessarMaterializationMs?: number; + tessarWriteMs?: number; + tessarSwapMs?: number; + tessarWaves?: number; + tessarOwnersRendered?: number; } export interface StreamFileRef { diff --git a/scripts/tessar/.eslintrc.cjs b/scripts/tessar/.eslintrc.cjs new file mode 100644 index 00000000000..3039c39e48b --- /dev/null +++ b/scripts/tessar/.eslintrc.cjs @@ -0,0 +1,4 @@ +// Tessar fixture definitions run through the realm's GTS transform pipeline. +module.exports = { + extends: ['../../packages/base/.eslintrc.cjs'], +}; diff --git a/scripts/tessar/README.md b/scripts/tessar/README.md new file mode 100644 index 00000000000..a0ea063cb2c --- /dev/null +++ b/scripts/tessar/README.md @@ -0,0 +1,132 @@ +# Tessar experimental benchmark tools + +DO NOT MERGE. Use an isolated local runtime from this worktree. All generated +records are fabricated; the generator does not read deployment records. + +From the monorepo root, with the pinned mise toolchain installed: + +```sh +mise exec -- pnpm exec node --test scripts/tessar/generate.test.mjs +mise exec -- pnpm exec node scripts/tessar/generate.mjs --preset smoke --output /private/tmp/tessar-smoke +``` + +Presets are `smoke` (34 instances), `1x` (1,255) and `10x` (12,550). The output +directory must not exist. `manifest.json` records the seed, counts and data hash; +`expected.json` is the independent raw-document oracle. Production data is never +an input. A 100x preset is deliberately unsupported. + +`--variant materialized` is the default. `--variant get-cards` changes only the +dashboard definition; source inputs and expected displayed output stay the same. +The reference pages broad type queries in batches of 100 without a fixed number +of pages. It must pass the browser oracle before being used as a baseline. + +Build the host and serve its icons using the package scripts. From +`packages/realm-server`: + +```sh +mise exec -- pnpm exec node ../../scripts/tessar/bench.mjs --dataset /private/tmp/tessar-smoke --output /private/tmp/tessar-smoke-result.json --serve +``` + +The harness starts disposable Postgres/Synapse/realm/host services and prints the +local realm URL. Runtime URLs and owned process IDs are saved beside the output +as `.runtime.json`. The source fingerprint includes dirty and untracked runtime +files so a cached fixture cannot silently stand in for the current implementation. +Stop the harness with SIGINT/SIGTERM when the browser trial finishes. + +Use a fully migrated template database containing the Tessar migration. If the +default harness template predates the migration, set +`TEST_HARNESS_MIGRATED_TEMPLATE_DB` to a separately prepared disposable template; +do not reset a database that an active harness is using. `--realm-server-url` +can pin a localhost URL so the host is built against the same base realm. +Set `TEST_HARNESS_PGPORT=55437` for the separate disk-backed Tessar Postgres +container. The 10× cold build requires +`TEST_HARNESS_FULL_INDEX_REALM_STARTUP_TIMEOUT_MS=3600000`; this is a setup +allowance, not a change to the fixed 10,000 ms write-freshness deadline. + +Once the harness is ready, run the real Chromium check from the monorepo root: + +```sh +mise exec -- pnpm exec node scripts/tessar/browser-check.mjs --dataset /private/tmp/tessar-smoke --runtime /private/tmp/tessar-smoke-result.json.runtime.json --matrix-url http://localhost:/ --output /private/tmp/tessar-smoke-browser.json --iterations 20 --writes +``` + +The version 2 script authenticates only the synthetic harness account. It waits +for the live realm subscription, connected client and removal of server markup +before qualifying a display or starting writes. `firstContentMs` and +`interactiveMs` are separate observations. Earlier version 1 browser samples +could accept server HTML before client boot and are provisional. Run the +regression with `pnpm exec node --test scripts/tessar/browser-oracle.test.mjs`. + +It compares all +statistics and displayed rows with the raw-document oracle, records input-card +and search requests, and exercises content, transitive, entry/exit, asynchronous, +unrelated, insertion, deletion and offline/reconnect mutations in the already-open browser. Each +write series needs a fresh generated realm: it deliberately changes the fixture. +The `get-cards` variant's write/freshness behavior is still being validated; +passing initial reads alone does not qualify a baseline for a freshness claim. + +`lifecycle-check.mjs --dataset ... --runtime ... --output ...` adds a synthetic +five-stage feeder chain after the size benchmark. It verifies an unrelated +module epoch change and leaf-to-dashboard convergence. Its additional records +must not be included in the fixed-size performance samples. + +For host QUnit tests, build with `RESOLVED_BASE_REALM_URL` pointing at the printed +local server's `/base/` and clear ambient `BOXEL_ENVIRONMENT`, `ENV_SLUG`, and +`ENV_MODE` settings. Filter on `Tessar`. Capture the complete console output. +After a base-module edit, invalidate that local realm's module cache or restart +the isolated stack before claiming that a browser test covered the edit. + +The initial ordinary-query GET case failed correctness: live membership and +stored statistics disagreed. Failed cases remain in the report. The current +10× materialized JSON path passes connected browser reads, nine mutations, +both worker crash points and 1/10/50 HTTP-reader checks. The getCards reference +has produced correct initial output but breached the fixed refresh deadline. +The full 10× HTML pipeline has a separate failed render and must not be labeled +a successful setup. See `docs/tessar-performance-report.md` for qualifications. + +### Tessar diagnostic tools + +`verify-owners.mjs --dataset ... --runtime ... --output ...` validates every saved +view against its independent oracle, requiring ready provenance. It uses four +bounded HTTP readers. + +`load-check.mjs --dataset ... --runtime ... --matrix-url ... --output ... --writes` +checks 1, 10 and 50 simultaneous HTTP readers plus one open browser. It records +pending responses separately and validates complete rows/statistics and revision +floors. This is not a test of 50 concurrent browser processes. + +`restart-check.mjs` takes the same dataset/runtime/matrix/output arguments. It +holds the publication transaction lock, acknowledges a fabricated source edit, +then interrupts only the verified worker child owned by that runtime. The +replacement worker must finish the reserved job and refresh the open browser +within the unchanged 10-second deadline. `--phase after-source-publication` +instead interrupts after the dirty owner generation is durably committed. +The before-publication case also opens a separately authenticated browser after +acknowledgement, rejects stale or cacheable initial HTML, and requires both +clients to reach the complete output within the same deadline. + +Set `TEST_HARNESS_USER_INDEX_WORKERS=1` to give the isolated harness one existing +index-only worker in addition to its one all-priority worker. The default remains +zero. Keep these counts identical in performance comparisons. + +`bench.mjs --resume-index sf_bld_` clones a preserved successful +source index and replays its failed HTML job. This is a recovery diagnostic, +not a cold build. `--skip-html-replay` additionally isolates JSON/browser reads +from the failed HTML path; it requires `--resume-index` and records +`setupMode: committed-json-index-only`. An HTML failure remains a failed full +pipeline result even when the separate JSON/browser tests pass. Failure details +are saved to `.failure.json` before the disposable runtime is cleaned up. + +A separate built host can be selected with `TEST_HARNESS_HOST_DIST_PACKAGE_DIR`. +Its index HTML is fingerprinted. Explicitly set `ICONS_URL=http://localhost:4206` +and the local `RESOLVED_BASE_REALM_URL` when building; inherited hosted asset +settings can invalidate a local test setup. + +`prewarm-check.mjs --database --dataset ... --output ...` is a read-only probe +of real indexed dependency collection with an inert module-cache sink. A Node +old-space cap distinguishes bounded collection from retaining every dependency +row; its elapsed time is not end-to-end indexing throughput. + +`plot-results.py --results docs/tessar-benchmark-results.json --output docs` +uses Matplotlib to reproduce the standalone SVG and PNG. It reads only sanitized +observations; it does not contact a realm. Use a disposable Python virtual +environment if Matplotlib is not already available. diff --git a/scripts/tessar/bench.mjs b/scripts/tessar/bench.mjs new file mode 100644 index 00000000000..5ef105d2001 --- /dev/null +++ b/scripts/tessar/bench.mjs @@ -0,0 +1,260 @@ +import { createHash } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { execFileSync } from 'node:child_process'; +import { validateSummary } from './generate.mjs'; + +// Import the harness only after removing ambient hosted-environment routing. +// Tessar always starts its own local servers and disposable database. +for (let name of [ + 'BOXEL_ENVIRONMENT', + 'ENV_SLUG', + 'ENV_MODE', + 'ICONS_URL', + 'REALM_BASE_URL', + 'REALM_TEST_URL', + 'HOST_URL', + 'MATRIX_URL_VAL', + 'PRERENDER_MGR_URL', + 'REALM_SERVER_TLS_CERT_FILE', + 'REALM_SERVER_TLS_KEY_FILE', +]) + delete process.env[name]; +const { startFactoryRealmServer, startFactorySupportServices } = + await import('../../packages/realm-test-harness/src/index.ts'); + +let { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + output: { type: 'string' }, + iterations: { type: 'string', default: '5' }, + serve: { type: 'boolean', default: false }, + 'realm-server-url': { type: 'string' }, + 'resume-index': { type: 'string' }, + 'skip-html-replay': { type: 'boolean', default: false }, + }, +}); +if (values['skip-html-replay'] && !values['resume-index']) + throw new Error( + '--skip-html-replay requires an explicit committed index clone', + ); +if (!values.dataset || !values.output) + throw new Error('--dataset and --output are required'); +let dataset = resolve(values.dataset); +let output = resolve(values.output); +let iterations = Number(values.iterations); +if (!Number.isSafeInteger(iterations) || iterations < 1) + throw new Error('Invalid iterations'); +let manifest = JSON.parse( + await readFile(join(dataset, 'manifest.json'), 'utf8'), +); +if ( + manifest.synthetic !== true || + !['smoke', '1x', '10x'].includes(manifest.preset) +) + throw new Error('A Tessar synthetic manifest is required'); +let expected = JSON.parse( + await readFile(join(dataset, 'expected.json'), 'utf8'), +); +let hostDir = process.env.TEST_HARNESS_HOST_DIST_PACKAGE_DIR + ? resolve(process.env.TEST_HARNESS_HOST_DIST_PACKAGE_DIR) + : resolve(import.meta.dirname, '../../packages/host'); +process.env.TEST_HARNESS_HOST_DIST_PACKAGE_DIR = hostDir; +let hostHash = createHash('sha256') + .update(await readFile(join(hostDir, 'dist/index.html'))) + .digest('hex'); +let commit = execFileSync('git', ['rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +let repository = resolve(import.meta.dirname, '../..'); +let sourcePaths = execFileSync( + 'git', + [ + 'ls-files', + '-z', + '--cached', + '--others', + '--exclude-standard', + '--', + 'packages/base', + 'packages/runtime-common', + 'packages/host', + 'packages/postgres', + 'packages/realm-server', + 'packages/realm-test-harness', + ], + { cwd: repository, encoding: 'utf8' }, +) + .split('\0') + .filter(Boolean); +let sourceHash = createHash('sha256'); +for (let path of [...new Set(sourcePaths)].sort()) { + try { + sourceHash + .update(path) + .update('\0') + .update(await readFile(join(repository, path))); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + sourceHash.update(`deleted:${path}`); + } +} +let runtimeHash = sourceHash.digest('hex'); +// A tooling-only edit or commit must not force a fresh 12,550-record index. +// Runtime source and built-host contents still invalidate the template; the +// harness separately hashes every fixture module and synthetic source record. +process.env.TEST_HARNESS_CACHE_SALT = `tessar:${hostHash}:${runtimeHash}`; +let started = performance.now(); +let realm; +let support; +let resumedHtml; +if ( + values['resume-index'] && + !/^sf_bld_[a-z0-9_]+$/.test(values['resume-index']) +) + throw new Error( + '--resume-index must name an isolated synthetic builder database', + ); +let realmServerURL = values['realm-server-url'] + ? new URL(values['realm-server-url']) + : undefined; +if ( + realmServerURL && + !['localhost', '127.0.0.1'].includes(realmServerURL.hostname) +) + throw new Error('Tessar benchmarks require a local realm server'); +try { + if (values['resume-index']) support = await startFactorySupportServices(); + realm = await startFactoryRealmServer({ + realmServerURL, + realms: [{ dir: join(dataset, 'realm'), path: 'tessar/' }], + ...(support + ? { + context: support.context, + templateDatabaseName: values['resume-index'], + templateRealmServerURL: realmServerURL, + } + : {}), + }); + await writeFile( + `${output}.runtime.json`, + JSON.stringify({ + realmURL: realm.realmURL.href, + realmServerURL: realm.realmServerURL.href, + databaseName: realm.databaseName, + childPids: realm.childPids, + matrixURL: support?.context.matrixURL, + }), + { mode: 0o600 }, + ); + if (support && !values['skip-html-replay']) { + // Preserve the failed builder; repair only this disposable clone. Reuse + // the committed JSON index explicitly, never report this as a cold build. + const { resumeTessarHtml } = await import('./resume-html.mjs'); + resumedHtml = await resumeTessarHtml({ + sourceDatabase: values['resume-index'], + runtimeDatabase: realm.databaseName, + realmURL: realm.realmURL.href, + }); + } + let startupMs = performance.now() - started; + console.log(`Tessar ready: ${realm.realmURL.href}`); + let results = []; + for (let iteration = 0; iteration < iterations; iteration++) { + let start = performance.now(); + let response = await fetch(realm.cardURL('DaySummary/00000'), { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(60_000), + }); + let bytes = await response.text(); + let elapsedMs = performance.now() - start; + let body = JSON.parse(bytes); + let valid = response.ok; + let failure; + try { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + if ( + manifest.variant !== 'get-cards' && + body.data.meta.tessar?.state !== 'ready' + ) + throw new Error('Tessar view is not ready'); + validateSummary(body.data.attributes, expected['DaySummary/00000']); + } catch (error) { + valid = false; + failure = error.message; + } + results.push({ + iteration, + elapsedMs, + responseBytes: Buffer.byteLength(bytes), + status: response.status, + valid, + failure, + }); + if (iteration === 0) + await writeFile(`${output}.response.json`, JSON.stringify(body, null, 2)); + } + await writeFile( + output, + JSON.stringify( + { + version: 1, + commit, + hostHash, + runtimeHash, + manifest, + workers: { + allPriority: 1, + userIndex: Number(process.env.TEST_HARNESS_USER_INDEX_WORKERS ?? 0), + }, + startupMs, + setupMode: values['skip-html-replay'] + ? 'committed-json-index-only' + : support + ? 'resumed-html-from-committed-index' + : 'cold-or-template-cache', + resumedHtml, + results, + }, + null, + 2, + ) + '\n', + ); + console.log( + JSON.stringify({ + output, + startupMs, + validReads: results.filter((r) => r.valid).length, + totalReads: results.length, + browserReady: values.serve, + }), + ); + if (values.serve) + await new Promise((done) => { + process.once('SIGINT', done); + process.once('SIGTERM', done); + }); + else if (results.some((r) => !r.valid)) process.exitCode = 1; +} catch (error) { + await writeFile( + `${output}.failure.json`, + JSON.stringify( + { + commit, + hostHash, + runtimeHash, + manifest, + elapsedMs: performance.now() - started, + error: error.message, + details: error.tessarDetails, + }, + null, + 2, + ) + '\n', + ); + throw error; +} finally { + await realm?.stop(); + await support?.stop(); +} diff --git a/scripts/tessar/browser-check.mjs b/scripts/tessar/browser-check.mjs new file mode 100644 index 00000000000..dabe0dfc560 --- /dev/null +++ b/scripts/tessar/browser-check.mjs @@ -0,0 +1,427 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { chromium } from '@playwright/test'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; +import { + readDisplay, + validateDisplay, + waitForDisplay, + waitForTessarInteractive, +} from './browser-oracle.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + 'matrix-url': { type: 'string' }, + output: { type: 'string' }, + iterations: { type: 'string', default: '5' }, + writes: { type: 'boolean', default: false }, + }, +}); +if ( + !values.dataset || + !values.runtime || + !values.output || + !values['matrix-url'] +) + throw new Error( + '--dataset, --runtime, --matrix-url and --output are required', + ); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const manifest = JSON.parse( + await readFile(join(resolve(values.dataset), 'manifest.json'), 'utf8'), +); +const realm = new URL(runtime.realmURL); +const matrix = new URL(values['matrix-url']); +for (const url of [realm, matrix, new URL(runtime.realmServerURL)]) + if (!['localhost', '127.0.0.1'].includes(url.hostname)) + throw new Error('Tessar browser checks require isolated local services'); +if (manifest.synthetic !== true || realm.pathname !== '/tessar/') + throw new Error('Only a synthetic Tessar fixture realm may be tested'); +const iterations = Number(values.iterations); +if (!Number.isSafeInteger(iterations) || iterations < 1) + throw new Error('Invalid iterations'); +const { records } = generate(manifest); +const ownerId = 'DaySummary/00000'; +const ownerURL = new URL(ownerId, realm).href; +const url = new URL(ownerURL); +url.searchParams.set( + 'operatorModeState', + JSON.stringify({ + aiAssistantOpen: false, + stacks: [[{ format: 'isolated', id: ownerURL }]], + submode: 'interact', + workspaceChooserOpened: false, + }), +); +process.env.TEST_HARNESS_BROWSER_MATRIX_URL = matrix.href; +const { buildBrowserState, installBrowserState } = + await import('../../packages/software-factory/tests/helpers/browser-auth.ts'); +const state = await buildBrowserState(realm.href, runtime.realmServerURL); +const browser = await chromium.launch({ headless: true }); +const context = await browser.newContext(); +await installBrowserState(context, state); +const page = await context.newPage(); +const reads = []; +const writes = []; +const browserErrors = []; +page.on('pageerror', (error) => browserErrors.push(error.message)); +let requests = []; +page.on('request', (request) => { + if (['fetch', 'xhr'].includes(request.resourceType())) + requests.push({ url: request.url(), method: request.method() }); +}); +const cdp = await context.newCDPSession(page); +await cdp.send('Performance.enable'); +await cdp.send('Network.enable'); +let transferredBytes = 0; +cdp.on('Network.loadingFinished', ({ encodedDataLength }) => { + transferredBytes += encodedDataLength; +}); + +function inputRequests() { + return requests.filter( + (request) => + (request.url.startsWith(realm.href) && + /\/(Student|Staff|Slot|Observation|Report|Reference|Activity)\//.test( + request.url, + )) || + (request.url.startsWith(runtime.realmServerURL) && + /\/_(federated-)?search(?:[?/#]|$)/.test(request.url)), + ); +} + +try { + for (let iteration = 0; iteration < iterations; iteration++) { + requests = []; + transferredBytes = 0; + const beforeMetrics = Object.fromEntries( + (await cdp.send('Performance.getMetrics')).metrics.map((m) => [ + m.name, + m.value, + ]), + ); + const started = performance.now(); + let result = { + iteration, + cache: iteration === 0 ? 'new-browser-context' : 'warm-reload', + valid: false, + }; + try { + if (iteration === 0) + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + else await page.reload({ waitUntil: 'domcontentloaded' }); + await waitForDisplay(page, expectedSummary(records, ownerId), 30_000); + result.firstContentMs = performance.now() - started; + await waitForTessarInteractive(page, realm.href); + await waitForDisplay(page, expectedSummary(records, ownerId), 30_000); + validateDisplay( + await readDisplay(page), + expectedSummary(records, ownerId), + ); + result.displayMs = performance.now() - started; + result.interactiveMs = result.displayMs; + // Keep the readiness timestamp separate from the observation window; + // delayed eager work after the first render still counts as input work. + await page.waitForTimeout(250); + result.valid = true; + result.inputRequests = inputRequests(); + if (manifest.variant !== 'get-cards' && result.inputRequests.length) + throw new Error( + 'Tessar materialized display loaded input cards or searched', + ); + result.fetchRequests = requests.length; + result.metrics = Object.fromEntries( + (await cdp.send('Performance.getMetrics')).metrics + .filter((m) => + [ + 'JSHeapUsedSize', + 'Nodes', + 'TaskDuration', + 'ScriptDuration', + ].includes(m.name), + ) + .map((m) => [ + m.name, + ['TaskDuration', 'ScriptDuration'].includes(m.name) + ? m.value - (beforeMetrics[m.name] ?? 0) + : m.value, + ]), + ); + } catch (error) { + result.valid = false; + result.failure = error.message; + result.visibleFailure = (await page.locator('body').innerText()).slice( + 0, + 3000, + ); + } + // Failed/unfinished reads still consumed resources. Retain their work + // rather than reporting only the cheap successful observations. + result.observationMs = performance.now() - started; + result.transferredBytes = transferredBytes; + result.inputRequests = inputRequests(); + result.fetchRequests = requests.length; + if (!result.metrics) + result.metrics = Object.fromEntries( + (await cdp.send('Performance.getMetrics')).metrics + .filter((m) => + [ + 'JSHeapUsedSize', + 'Nodes', + 'TaskDuration', + 'ScriptDuration', + ].includes(m.name), + ) + .map((m) => [ + m.name, + ['TaskDuration', 'ScriptDuration'].includes(m.name) + ? m.value - (beforeMetrics[m.name] ?? 0) + : m.value, + ]), + ); + reads.push(result); + console.log( + JSON.stringify({ + iteration, + valid: result.valid, + displayMs: result.displayMs, + failure: result.failure, + }), + ); + } + if (values.writes && reads.every((read) => read.valid)) { + const token = buildRealmToken(realm, new URL(runtime.realmServerURL)); + const trials = [ + { + id: 'Observation/00000', + patch: { score: 80 }, + name: 'matching content edit', + }, + { + id: 'Reference/00000', + patch: { label: 'Tessar reference updated' }, + name: 'transitive row label edit', + }, + { + id: 'Observation/00000', + patch: { day: '2026-01-13' }, + name: 'query exit', + }, + { + id: 'Observation/00000', + patch: { day: '2026-01-12' }, + name: 'query entry', + }, + { + id: 'Observation/00000', + patch: { score: 81 }, + name: 'asynchronous source write', + source: true, + }, + { + id: `Reference/${String(manifest.counts.Reference - 1).padStart(5, '0')}`, + patch: { label: 'Tessar unrelated reference updated' }, + name: 'unrelated write', + unrelated: true, + }, + { + id: 'Observation/99999', + patch: { + score: 5, + sequence: 99999, + label: 'Tessar inserted observation', + }, + name: 'matching insertion', + source: true, + insert: true, + }, + { id: 'Observation/99999', name: 'matching deletion', delete: true }, + { + id: 'Observation/00000', + patch: { score: 82 }, + name: 'reconnect after missed notification', + source: true, + offline: true, + }, + ]; + for (const trial of trials) { + if (trial.insert) + records.set( + trial.id, + structuredClone(records.get('Observation/00000')), + ); + const source = records.get(trial.id); + if (trial.delete) records.delete(trial.id); + else Object.assign(source.data.attributes, trial.patch); + const expected = expectedSummary(records, ownerId); + const previousOwner = await ( + await fetch(ownerURL, { + headers: { Accept: 'application/vnd.card+json' }, + }) + ).json(); + await page.evaluate(() => { + window.__tessarTransitions = []; + window.__tessarObserver?.disconnect(); + window.__tessarObserver = new MutationObserver(() => { + const state = document + .querySelector('[data-tessar-state]') + ?.getAttribute('data-tessar-state'); + const events = window.__tessarTransitions; + if (!events.length || events.at(-1).state !== state) + events.push({ + at: Date.now(), + state, + visibleStats: + document.querySelectorAll('[data-tessar-stat]').length, + }); + }); + window.__tessarObserver.observe(document.body, { + attributes: true, + childList: true, + characterData: true, + subtree: true, + }); + }); + requests = []; + const startedAt = Date.now(); + let result = { name: trial.name, startedAt, valid: false }; + try { + if (trial.offline) await context.setOffline(true); + const mime = trial.source + ? 'application/vnd.card+source' + : 'application/vnd.card+json'; + const response = await fetch( + new URL(trial.id + (trial.source ? '.json' : ''), realm), + { + method: trial.delete ? 'DELETE' : trial.source ? 'POST' : 'PATCH', + headers: { + Accept: mime, + 'Content-Type': mime, + Authorization: `Bearer ${token}`, + }, + body: trial.delete + ? undefined + : JSON.stringify( + trial.source + ? source + : { + data: { + type: 'card', + attributes: trial.patch, + meta: source.data.meta, + }, + }, + ), + signal: AbortSignal.timeout(30_000), + }, + ); + result.ackAt = Date.now(); + result.ackMs = result.ackAt - startedAt; + result.status = response.status; + if (!response.ok) + throw new Error( + `Write HTTP ${response.status}: ${await response.text()}`, + ); + await response.arrayBuffer(); + if (manifest.variant !== 'get-cards') { + const ownerResponse = await fetch(ownerURL, { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(10_000), + }); + const document = await ownerResponse.json(); + validateSummary(document.data.attributes, expected); + if (document.data.meta.tessar?.state !== 'ready') + throw new Error('Owner is pending'); + result.ownerRevision = document.data.meta.tessar.publishedGeneration; + if ( + trial.unrelated && + result.ownerRevision !== + previousOwner.data.meta.tessar.publishedGeneration + ) + throw new Error('An unrelated write reindexed the selected owner'); + } + if (trial.offline) { + await page.waitForTimeout(250); + await context.setOffline(false); + } + // Observe the already-open client; a reload would hide lost invalidation. + await waitForDisplay( + page, + expected, + Math.max(1, 10_000 - (Date.now() - result.ackAt)), + ); + result.observedAt = Date.now(); + result.ackToDisplayMs = result.observedAt - result.ackAt; + result.inputRequests = inputRequests(); + result.transitions = await page.evaluate( + () => window.__tessarTransitions, + ); + if ( + trial.source && + !result.transitions.some( + (event) => event.state === 'pending' && event.visibleStats === 0, + ) + ) + throw new Error( + 'The asynchronous write did not expose an explicit pending view', + ); + result.valid = + result.ackToDisplayMs <= 10_000 && + (manifest.variant === 'get-cards' || + result.inputRequests.length === 0); + } catch (error) { + result.failure = error.message; + } + result.inputRequests = inputRequests(); + result.transitions = await page.evaluate( + () => window.__tessarTransitions, + ); + if (!result.valid) { + result.observationMs = Date.now() - startedAt; + result.visibleFailure = (await page.locator('body').innerText()).slice( + 0, + 3000, + ); + } + writes.push(result); + if (!result.valid) break; + } + } +} finally { + await writeFile( + values.output, + JSON.stringify( + { + version: 2, + readiness: 'subscribed-client-with-server-markup-removed', + manifest, + runtime, + reads, + writes, + browserErrors, + }, + null, + 2, + ) + '\n', + ); + await browser.close(); +} +console.log( + JSON.stringify({ + output: values.output, + validReads: reads.filter((r) => r.valid).length, + totalReads: reads.length, + validWrites: writes.filter((w) => w.valid).length, + totalWrites: writes.length, + }), +); +if ( + reads.some((r) => !r.valid) || + writes.some((w) => !w.valid) || + browserErrors.length +) + process.exitCode = 1; diff --git a/scripts/tessar/browser-oracle.mjs b/scripts/tessar/browser-oracle.mjs new file mode 100644 index 00000000000..cd3cedfae3a --- /dev/null +++ b/scripts/tessar/browser-oracle.mjs @@ -0,0 +1,96 @@ +import { validateSummary } from './generate.mjs'; + +// The server can supply the entire dashboard as HTML before Ember starts. +// A DOM-only oracle therefore cannot establish client readiness or prove that +// hydration avoids input-graph requests. Require live realm subscriptions and +// removal of the server markup before starting a mutation or timing a read. +export async function waitForTessarInteractive( + page, + realmURL, + timeout = 30000, +) { + await page.waitForFunction( + (realmURL) => { + const messages = window._CARDSTACK_REALM_SUBSCRIBE; + const start = document.getElementById('boxel-isolated-start'); + const end = document.getElementById('boxel-isolated-end'); + return ( + messages?.isTessarConnected === true && + messages.listenerCallbacks.get(realmURL)?.length > 0 && + (!start || !end || start.nextElementSibling === end) + ); + }, + realmURL, + { timeout }, + ); +} + +export async function readDisplay(page) { + return page.evaluate(() => ({ + stats: Object.fromEntries( + [...document.querySelectorAll('[data-tessar-stat]')].map((el) => [ + el.dataset.tessarStat, + Number(el.textContent), + ]), + ), + rows: [...document.querySelectorAll('[data-tessar-source]')].map((el) => ({ + sourceId: el.dataset.tessarSource, + cells: [...el.querySelectorAll('td')].map((td) => td.textContent.trim()), + })), + })); +} + +export function validateDisplay(actual, expected) { + const { rows, ...stats } = expected; + validateSummary(actual.stats, stats); + const expectedRows = rows.map((row) => ({ + sourceId: row.sourceId, + cells: [row.label, row.studentLabel, row.referenceLabel, row.status], + })); + if (JSON.stringify(actual.rows) !== JSON.stringify(expectedRows)) + throw new Error( + 'Tessar displayed row IDs, order or values differ from the oracle', + ); +} + +export async function waitForDisplay(page, expected, timeout = 10000) { + await page.waitForFunction( + (expected) => { + const stats = Object.fromEntries( + [...document.querySelectorAll('[data-tessar-stat]')].map((el) => [ + el.dataset.tessarStat, + Number(el.textContent), + ]), + ); + const { rows, ...expectedStats } = expected; + const actualRows = [ + ...document.querySelectorAll('[data-tessar-source]'), + ].map((el) => ({ + sourceId: el.dataset.tessarSource, + cells: [...el.querySelectorAll('td')].map((td) => + td.textContent.trim(), + ), + })); + return ( + Object.entries(expectedStats).every( + ([key, value]) => stats[key] === value, + ) && + JSON.stringify(actualRows) === + JSON.stringify( + rows.map((row) => ({ + sourceId: row.sourceId, + cells: [ + row.label, + row.studentLabel, + row.referenceLabel, + row.status, + ], + })), + ) + ); + }, + expected, + { timeout }, + ); + validateDisplay(await readDisplay(page), expected); +} diff --git a/scripts/tessar/browser-oracle.test.mjs b/scripts/tessar/browser-oracle.test.mjs new file mode 100644 index 00000000000..c05595afa56 --- /dev/null +++ b/scripts/tessar/browser-oracle.test.mjs @@ -0,0 +1,39 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { chromium } from '@playwright/test'; +import { waitForDisplay, waitForTessarInteractive } from './browser-oracle.mjs'; + +test('Tessar browser readiness rejects correct server HTML until the subscribed client replaces it', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await page.setContent(` + +
0
+ + `); + const realmURL = 'http://localhost/tessar/'; + // This was the old false positive: the independent value oracle passes + // even though there is no client available to receive a subsequent write. + await waitForDisplay(page, { scoreTotal: 0, rows: [] }); + await assert.rejects(waitForTessarInteractive(page, realmURL, 100)); + await page.evaluate((realmURL) => { + window._CARDSTACK_REALM_SUBSCRIBE = { + isTessarConnected: true, + listenerCallbacks: new Map([[realmURL, [() => {}]]]), + }; + }, realmURL); + await assert.rejects(waitForTessarInteractive(page, realmURL, 100)); + await page.evaluate(() => { + document.querySelector('article').remove(); + document.body.insertAdjacentHTML( + 'beforeend', + '
0
', + ); + }); + await waitForTessarInteractive(page, realmURL); + await waitForDisplay(page, { scoreTotal: 0, rows: [] }); + } finally { + await browser.close(); + } +}); diff --git a/scripts/tessar/generate.mjs b/scripts/tessar/generate.mjs new file mode 100644 index 00000000000..257661b4ac8 --- /dev/null +++ b/scripts/tessar/generate.mjs @@ -0,0 +1,222 @@ +import { createHash } from 'node:crypto'; +import { mkdir, writeFile, copyFile } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { parseArgs, isDeepStrictEqual } from 'node:util'; +import { pathToFileURL } from 'node:url'; + +// Synthetic workload proportions, not production records or identifiers. +export const counts = { + Classroom: 2, + Student: 13, + Staff: 26, + Slot: 173, + Observation: 164, + Report: 44, + Activity: 63, + Reference: 766, + DaySummary: 4, +}; +export const presets = { smoke: 0, '1x': 1, '10x': 10 }; +export const day = '2026-01-12'; +export const nextDay = '2026-01-13'; +const moduleRef = '../tessar'; + +export function generate({ + preset = 'smoke', + seed = 1729, + shape = 'distributed', + variant = 'materialized', +} = {}) { + if (!Object.hasOwn(presets, preset)) + throw new Error('Preset must be smoke, 1x or 10x'); + if (!Number.isSafeInteger(seed) || seed < 0) + throw new Error('Seed must be a nonnegative safe integer'); + if (!['distributed', 'focused'].includes(shape)) + throw new Error('Unknown workload shape'); + if (!['materialized', 'get-cards'].includes(variant)) + throw new Error('Variant must be materialized or get-cards'); + let scale = presets[preset]; + let sizes = Object.fromEntries( + Object.entries(counts).map(([type, count]) => [ + type, + scale ? count * scale : Math.min(count, 4), + ]), + ); + sizes.Classroom = scale ? 2 * scale : 2; + let state = seed >>> 0; + let random = () => + (state = (Math.imul(state, 1664525) + 1013904223) >>> 0) / 2 ** 32; + let id = (type, index) => `${type}/${String(index).padStart(5, '0')}`; + let records = new Map(); + for (let [type, count] of Object.entries(sizes)) { + for (let i = 0; i < count; i++) { + let classroom = i % (shape === 'focused' ? 2 : sizes.Classroom); + let attributes = { + label: `Tessar ${type} ${String(i).padStart(5, '0')}`, + classroomKey: `room-${classroom}`, + day: Math.floor(i / sizes.Classroom) % 2 ? nextDay : day, + sequence: i, + score: Math.floor(random() * 101), + status: i % 3 === 0 ? 'ready' : 'pending', + narrative: `Synthetic Tessar record ${i}. `.repeat( + type === 'Reference' ? 12 : 3, + ), + }; + let relationships; + if (['Slot', 'Observation', 'Report', 'Activity'].includes(type)) { + relationships = { + student: { + links: { self: `../${id('Student', i % sizes.Student)}` }, + }, + reference: { + links: { self: `../${id('Reference', i % sizes.Reference)}` }, + }, + staff: { links: { self: `../${id('Staff', i % sizes.Staff)}` } }, + }; + } + records.set(id(type, i), { + data: { + type: 'card', + attributes, + ...(relationships ? { relationships } : {}), + meta: { + adoptsFrom: + variant === 'get-cards' && type === 'DaySummary' + ? { module: '../get-cards', name: 'TessarGetCardsPage' } + : { module: moduleRef, name: type }, + }, + }, + }); + } + } + return { records, sizes, seed, preset, shape, variant }; +} + +// Reference implementation deliberately operates on raw fixture documents, +// independently of CardDef getters, query fields, server results or the UI. +export function expectedSummary(records, ownerId) { + let owner = records.get(ownerId).data.attributes; + let expected = { + studentCount: 0, + slotCount: 0, + observationCount: 0, + reportCount: 0, + readyReportCount: 0, + scoreTotal: 0, + rows: [], + }; + for (let [sourceId, { data }] of records) { + let attrs = data.attributes; + if (attrs.classroomKey !== owner.classroomKey) continue; + let type = data.meta.adoptsFrom.name; + if (type === 'Student') expected.studentCount++; + if (attrs.day !== owner.day) continue; + if (type === 'Observation') { + expected.observationCount++; + expected.scoreTotal += attrs.score; + } + if (type === 'Report') { + expected.reportCount++; + if (attrs.status === 'ready') expected.readyReportCount++; + } + if (type === 'Slot') { + expected.slotCount++; + let studentId = data.relationships.student.links.self.slice(3); + let referenceId = data.relationships.reference.links.self.slice(3); + expected.rows.push({ + sourceId, + label: attrs.label, + sequence: attrs.sequence, + status: attrs.status, + studentLabel: records.get(studentId).data.attributes.label, + referenceLabel: records.get(referenceId).data.attributes.label, + }); + } + } + expected.rows.sort((a, b) => a.sequence - b.sequence); + return expected; +} + +export function validateSummary(actual, expected) { + let keys = Object.keys(expected); + let normalized = Object.fromEntries(keys.map((key) => [key, actual[key]])); + if (!isDeepStrictEqual(normalized, expected)) + throw new Error( + `Tessar result mismatch: ${JSON.stringify({ expected, actual: normalized })}`, + ); +} + +export async function writeDataset(output, options) { + let dataset = generate(options); + // Exclusive root creation avoids overwriting an existing run or real realm. + await mkdir(output); + let realmDir = join(output, 'realm'); + await mkdir(realmDir); + await copyFile( + new URL('./realm/tessar.gts', import.meta.url), + join(realmDir, 'tessar.gts'), + ); + if (dataset.variant === 'get-cards') + await copyFile( + new URL('./realm/get-cards.gts', import.meta.url), + join(realmDir, 'get-cards.gts'), + ); + let hash = createHash('sha256'); + for (let [id, document] of dataset.records) { + let bytes = JSON.stringify(document) + '\n'; + hash.update(id).update('\0').update(bytes); + await mkdir(join(realmDir, id.split('/')[0]), { recursive: true }); + await writeFile(join(realmDir, `${id}.json`), bytes); + } + let expected = Object.fromEntries( + [...dataset.records.keys()] + .filter((id) => id.startsWith('DaySummary/')) + .map((id) => [id, expectedSummary(dataset.records, id)]), + ); + let manifest = { + version: 1, + synthetic: true, + seed: dataset.seed, + preset: dataset.preset, + shape: dataset.shape, + variant: dataset.variant, + instanceCount: dataset.records.size, + counts: dataset.sizes, + recordsSha256: hash.digest('hex'), + }; + await writeFile( + join(output, 'manifest.json'), + JSON.stringify(manifest, null, 2) + '\n', + ); + await writeFile( + join(output, 'expected.json'), + JSON.stringify(expected, null, 2) + '\n', + ); + return manifest; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + let { values } = parseArgs({ + options: { + output: { type: 'string' }, + preset: { type: 'string', default: 'smoke' }, + seed: { type: 'string', default: '1729' }, + shape: { type: 'string', default: 'distributed' }, + variant: { type: 'string', default: 'materialized' }, + }, + }); + if (!values.output) throw new Error('--output must name a new directory'); + console.log( + JSON.stringify( + await writeDataset(resolve(values.output), { + preset: values.preset, + seed: Number(values.seed), + shape: values.shape, + variant: values.variant, + }), + ), + ); +} diff --git a/scripts/tessar/generate.test.mjs b/scripts/tessar/generate.test.mjs new file mode 100644 index 00000000000..ff8f076d5e3 --- /dev/null +++ b/scripts/tessar/generate.test.mjs @@ -0,0 +1,71 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; + +test('Tessar datasets have fixed sizes, deterministic fabricated records and valid links', () => { + for (let [preset, count] of [ + ['1x', 1255], + ['10x', 12550], + ]) { + let { records } = generate({ preset }); + assert.equal(records.size, count); + assert.deepEqual(records, generate({ preset }).records); + for (let { data } of records.values()) { + assert.match(data.attributes.label, /^Tessar /); + for (let relationship of Object.values(data.relationships ?? {})) { + assert.ok(records.has(relationship.links.self.slice(3))); + } + } + } + assert.throws(() => generate({ preset: '100x' }), /Preset/); +}); + +test('Tessar raw-data oracle reflects membership, content, transitive edits and deletions', () => { + let { records } = generate(); + let id = 'DaySummary/00000'; + let before = expectedSummary(records, id); + assert.equal(before.studentCount, 2); + assert.equal(before.slotCount, 1); + assert.equal(before.observationCount, 1); + assert.equal(before.readyReportCount, 1); + assert.deepEqual( + before.rows.map((r) => r.sourceId), + ['Slot/00000'], + ); + records.get('Student/00000').data.attributes.label = 'Tessar revised student'; + assert.equal( + expectedSummary(records, id).rows[0].studentLabel, + 'Tessar revised student', + ); + records.get('Observation/00000').data.attributes.score = 123; + assert.equal(expectedSummary(records, id).scoreTotal, 123); + records.get('Slot/00002').data.attributes.day = '2026-01-12'; + assert.equal(expectedSummary(records, id).slotCount, 2); + records.delete('Slot/00000'); + assert.deepEqual( + expectedSummary(records, id).rows.map((r) => r.sourceId), + ['Slot/00002'], + ); + assert.throws( + () => validateSummary(before, expectedSummary(records, id)), + /mismatch/, + ); +}); + +test('Tessar getCards and materialized variants have identical source inputs and expected outputs', () => { + let candidate = generate({ preset: '10x' }).records; + let baseline = generate({ preset: '10x', variant: 'get-cards' }).records; + assert.equal(baseline.size, 12550); + for (let [id, document] of candidate) { + if (id.startsWith('DaySummary/')) { + assert.deepEqual( + expectedSummary(baseline, id), + expectedSummary(candidate, id), + ); + assert.equal( + baseline.get(id).data.meta.adoptsFrom.name, + 'TessarGetCardsPage', + ); + } else assert.deepEqual(baseline.get(id), document); + } +}); diff --git a/scripts/tessar/lifecycle-check.mjs b/scripts/tessar/lifecycle-check.mjs new file mode 100644 index 00000000000..54e6618fbb9 --- /dev/null +++ b/scripts/tessar/lifecycle-check.mjs @@ -0,0 +1,149 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { generate, expectedSummary } from './generate.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + output: { type: 'string' }, + }, +}); +if (!values.dataset || !values.runtime || !values.output) + throw new Error('--dataset, --runtime and --output are required'); +const manifest = JSON.parse( + await readFile(join(values.dataset, 'manifest.json'), 'utf8'), +); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const realm = new URL(runtime.realmURL), + server = new URL(runtime.realmServerURL); +if ( + manifest.synthetic !== true || + realm.pathname !== '/tessar/' || + ![realm, server].every((url) => + ['localhost', '127.0.0.1'].includes(url.hostname), + ) +) + throw new Error( + 'Tessar lifecycle checks require isolated synthetic services', + ); +const token = buildRealmToken(realm, server); +const { records } = generate(manifest); +const results = []; +async function post(path, bytes) { + const response = await fetch(new URL(path, realm), { + method: 'POST', + headers: { + Accept: 'application/vnd.card+source', + 'Content-Type': 'application/vnd.card+source', + Authorization: `Bearer ${token}`, + }, + body: bytes, + signal: AbortSignal.timeout(60000), + }); + await response.arrayBuffer(); + if (!response.ok) throw new Error(`${path}: HTTP ${response.status}`); +} +async function awaitScore(id, expected, deadline) { + let last; + while (Date.now() < deadline) { + try { + const response = await fetch(new URL(id, realm), { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())), + }); + last = await response.json(); + if ( + response.ok && + last.data?.meta?.tessar?.state === 'ready' && + last.data.attributes.scoreTotal === expected + ) + return { + id, + revision: last.data.meta.tessar.publishedGeneration, + scoreTotal: expected, + }; + } catch (error) { + last = { error: error.message }; + } + await delay(100); + } + throw new Error( + `Tessar lifecycle deadline exceeded for ${id}: ${JSON.stringify(last)}`, + ); +} + +try { + const source = structuredClone(records.get('Observation/00000')); + await post('Observation/00000.json', JSON.stringify(source)); + const expected = expectedSummary(records, 'DaySummary/00000').scoreTotal; + await awaitScore('DaySummary/00000', expected, Date.now() + 60000); + await post( + 'feeder-chain.gts', + await readFile( + new URL('./realm/feeder-chain.gts', import.meta.url), + 'utf8', + ), + ); + // A new unrelated module changes the loader epoch. Existing views must + // return to ready even though none depended on this new module beforehand. + const epochAck = Date.now(); + await awaitScore('DaySummary/00000', expected, epochAck + 10000); + results.push({ + name: 'unrelated module epoch', + valid: true, + ackToReadyMs: Date.now() - epochAck, + }); + const owner = records.get('DaySummary/00000').data.attributes; + for (let stage = 0; stage < 5; stage++) + await post( + `TessarStage${stage}/00000.json`, + JSON.stringify({ + data: { + type: 'card', + attributes: { classroomKey: owner.classroomKey, day: owner.day }, + meta: { + adoptsFrom: { + module: '../feeder-chain', + name: `TessarStage${stage}`, + }, + }, + }, + }), + ); + const setupDeadline = Date.now() + 60000; + await awaitScore('TessarStage4/00000', expected, setupDeadline); + source.data.attributes.score += 9; + const start = Date.now(); + await post('Observation/00000.json', JSON.stringify(source)); + const ack = Date.now(); + const snapshots = []; + for (const id of [ + 'DaySummary/00000', + ...Array.from({ length: 5 }, (_, i) => `TessarStage${i}/00000`), + ]) + snapshots.push(await awaitScore(id, expected + 9, ack + 10000)); + results.push({ + name: 'five downstream feeder stages', + valid: true, + ackMs: ack - start, + ackToReadyMs: Date.now() - ack, + snapshots, + }); +} catch (error) { + results.push({ + name: 'lifecycle failure', + valid: false, + error: error.message, + }); + process.exitCode = 1; +} finally { + await writeFile( + values.output, + JSON.stringify({ version: 1, manifest, results }, null, 2) + '\n', + ); + console.log(JSON.stringify(results)); +} diff --git a/scripts/tessar/load-check.mjs b/scripts/tessar/load-check.mjs new file mode 100644 index 00000000000..544d4545083 --- /dev/null +++ b/scripts/tessar/load-check.mjs @@ -0,0 +1,290 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { chromium } from '@playwright/test'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; +import { waitForDisplay, waitForTessarInteractive } from './browser-oracle.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + output: { type: 'string' }, + 'matrix-url': { type: 'string' }, + readers: { type: 'string', default: '1,10,50' }, + 'duration-ms': { type: 'string', default: '10000' }, + writes: { type: 'boolean', default: false }, + }, +}); +if ( + !values.dataset || + !values.runtime || + !values.output || + !values['matrix-url'] +) + throw new Error( + '--dataset, --runtime, --output and --matrix-url are required', + ); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const manifest = JSON.parse( + await readFile(join(values.dataset, 'manifest.json'), 'utf8'), +); +const realm = new URL(runtime.realmURL), + server = new URL(runtime.realmServerURL), + matrix = new URL(values['matrix-url']); +for (const url of [realm, server, matrix]) + if (!['localhost', '127.0.0.1'].includes(url.hostname)) + throw new Error('Tessar load checks require local services'); +if ( + realm.pathname !== '/tessar/' || + manifest.synthetic !== true || + manifest.variant === 'get-cards' +) + throw new Error('A synthetic materialized Tessar dataset is required'); +const readers = values.readers.split(',').map(Number), + durationMs = Number(values['duration-ms']); +if ( + readers.some((n) => !Number.isSafeInteger(n) || n < 1 || n > 50) || + !Number.isSafeInteger(durationMs) || + durationMs < 1000 || + durationMs > 60000 +) + throw new Error('Readers must be 1–50 and duration 1000–60000 ms'); +const { records } = generate(manifest), + ownerId = 'DaySummary/00000', + ownerURL = new URL(ownerId, realm).href; +const token = buildRealmToken(realm, server); +const cardHeaders = { Accept: 'application/vnd.card+json' }; +const cases = []; + +async function sourceWrite(id, data) { + const response = await fetch(new URL(`${id}.json`, realm), { + method: 'POST', + headers: { + Accept: 'application/vnd.card+source', + 'Content-Type': 'application/vnd.card+source', + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify(data), + signal: AbortSignal.timeout(30000), + }); + if (!response.ok) + throw new Error(`Tessar source write failed: ${response.status}`); + await response.arrayBuffer(); +} + +// Browser-check mutations are known synthetic records. Restore them before +// each load case; setup costs are separate from the measured workload. +async function reset() { + for (const id of [ + 'Observation/00000', + 'Reference/00000', + `Reference/${String(manifest.counts.Reference - 1).padStart(5, '0')}`, + ]) + await sourceWrite(id, records.get(id)); + const response = await fetch(ownerURL, { + headers: cardHeaders, + signal: AbortSignal.timeout(60000), + }); + const doc = await response.json(); + validateSummary(doc.data.attributes, expectedSummary(records, ownerId)); + if (doc.data.meta.tessar?.state !== 'ready') + throw new Error('Tessar setup did not settle'); +} + +process.env.TEST_HARNESS_BROWSER_MATRIX_URL = matrix.href; +const { buildBrowserState, installBrowserState } = + await import('../../packages/software-factory/tests/helpers/browser-auth.ts'); +const browser = await chromium.launch({ headless: true }); +const context = await browser.newContext(); +await installBrowserState( + context, + await buildBrowserState(realm.href, server.href), +); +const page = await context.newPage(); +const url = new URL(ownerURL); +url.searchParams.set( + 'operatorModeState', + JSON.stringify({ + aiAssistantOpen: false, + stacks: [[{ format: 'isolated', id: ownerURL }]], + submode: 'interact', + workspaceChooserOpened: false, + }), +); + +function distribution(values) { + values.sort((a, b) => a - b); + const quantile = (q) => values[Math.max(0, Math.ceil(values.length * q) - 1)]; + return { + n: values.length, + min: values[0], + median: quantile(0.5), + p95: quantile(0.95), + max: values.at(-1), + }; +} + +try { + for (const concurrency of readers) { + await reset(); + const expected = expectedSummary(records, ownerId), + originalScore = records.get('Observation/00000').data.attributes.score; + const otherScores = expected.scoreTotal - originalScore; + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + await waitForTessarInteractive(page, realm.href); + await waitForDisplay(page, expected, 30000); + const startedAt = Date.now(), + until = startedAt + durationMs; + const result = { + readers: concurrency, + durationMs, + startedAt, + readyReads: 0, + pendingReads: 0, + failures: [], + writes: [], + latenciesMs: [], + responseBytes: 0, + browserInputRequests: 0, + }; + const onRequest = (request) => { + if ( + (request.url().startsWith(realm.href) && + /\/(Student|Staff|Slot|Observation|Report|Reference|Activity)\//.test( + request.url(), + )) || + /\/_(federated-)?search(?:[?/#]|$)/.test(request.url()) + ) + result.browserInputRequests++; + }; + page.on('request', onRequest); + let publishedVersion = 0; + const lastAcknowledgedVersion = (at) => + result.writes.filter((w) => w.ackAt !== undefined && w.ackAt <= at).at(-1) + ?.version ?? 0; + async function reader() { + while (Date.now() < until) { + const readAt = Date.now(), + start = performance.now(), + minimumVersion = lastAcknowledgedVersion(readAt); + try { + const response = await fetch(ownerURL, { + headers: cardHeaders, + signal: AbortSignal.timeout(10000), + }); + const text = await response.text(); + result.responseBytes += Buffer.byteLength(text); + if (!response.ok) throw new Error(`Read HTTP ${response.status}`); + const doc = JSON.parse(text); + if (doc.data.meta.tessar?.state === 'pending') { + result.pendingReads++; + continue; + } + if (doc.data.meta.tessar?.state !== 'ready') + throw new Error('Missing materialization'); + const score = doc.data.attributes.scoreTotal - otherScores; + const version = score === originalScore ? 0 : score - 500; + if ( + !Number.isSafeInteger(version) || + version < minimumVersion || + version > publishedVersion + ) + throw new Error( + `Stale or unknown score version ${version}; minimum ${minimumVersion}`, + ); + validateSummary(doc.data.attributes, { + ...expected, + scoreTotal: otherScores + score, + }); + result.readyReads++; + result.latenciesMs.push(performance.now() - start); + } catch (error) { + result.failures.push({ + at: Date.now(), + kind: 'read', + message: error.message, + }); + } + } + } + async function writer() { + if (!values.writes) return; + while (Date.now() + 1500 < until) { + await delay(1000); + const version = ++publishedVersion, + startedAt = Date.now(); + const write = { version, startedAt, valid: false }; + result.writes.push(write); + try { + const source = structuredClone(records.get('Observation/00000')); + source.data.attributes.score = 500 + version; + await sourceWrite('Observation/00000', source); + write.ackAt = Date.now(); + write.ackMs = write.ackAt - startedAt; + await waitForDisplay(page, { + ...expected, + scoreTotal: otherScores + 500 + version, + }); + write.observedAt = Date.now(); + write.ackToDisplayMs = write.observedAt - write.ackAt; + write.valid = write.ackToDisplayMs <= 10000; + } catch (error) { + result.failures.push({ + at: Date.now(), + kind: 'write', + message: error.message, + }); + } + } + } + await Promise.all([ + ...Array.from({ length: concurrency }, reader), + writer(), + ]); + page.off('request', onRequest); + result.elapsedMs = Date.now() - startedAt; + result.latency = distribution([...result.latenciesMs]); + result.readyReadsPerSecond = result.readyReads / (result.elapsedMs / 1000); + result.valid = + result.failures.length === 0 && + result.writes.every((w) => w.valid) && + result.browserInputRequests === 0; + cases.push(result); + console.log( + JSON.stringify({ + readers: concurrency, + valid: result.valid, + readyReads: result.readyReads, + pendingReads: result.pendingReads, + throughput: result.readyReadsPerSecond, + latency: result.latency, + writes: result.writes.map(({ ackMs, ackToDisplayMs, valid }) => ({ + ackMs, + ackToDisplayMs, + valid, + })), + failures: result.failures.slice(0, 3), + }), + ); + } +} finally { + await writeFile( + values.output, + JSON.stringify( + { + version: 2, + readiness: 'subscribed-client-with-server-markup-removed', + manifest, + note: 'Concurrent HTTP readers plus one already-open Chromium dashboard; this does not simulate the client CPU of 50 browsers.', + cases, + }, + null, + 2, + ) + '\n', + ); + await browser.close(); +} +if (cases.some((result) => !result.valid)) process.exitCode = 1; diff --git a/scripts/tessar/plot-results.py b/scripts/tessar/plot-results.py new file mode 100644 index 00000000000..fb41fd8b4ed --- /dev/null +++ b/scripts/tessar/plot-results.py @@ -0,0 +1,61 @@ +"""Render the sanitized Tessar checkpoint with Matplotlib (no realm access).""" + +import argparse +import json +from pathlib import Path + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--results', required=True) +parser.add_argument('--output', required=True) +args = parser.parse_args() +result = json.loads(Path(args.results).read_text())['connected10x'] +target = Path(args.output) +target.mkdir(parents=True, exist_ok=True) + +plt.rcParams.update({ + 'font.family': 'DejaVu Sans', 'font.size': 10, 'axes.spines.top': False, + 'axes.spines.right': False, 'svg.hashsalt': 'tessar', +}) +fig, axes = plt.subplots(1, 2, figsize=(11, 4.5)) +colors = ['#14696b', '#9a6855'] +for index, (name, label) in enumerate([ + ('candidate', 'Tessar'), ('reference', 'getCards reference'), +]): + samples = result[name]['reads'] + seconds = [sample['interactiveMs'] / 1000 for sample in samples] + axes[0].scatter([index + ((i % 5) - 2) * .035 for i in range(len(seconds))], + seconds, color=colors[index], s=25, alpha=.75, label=label) + middle = sorted(seconds)[len(seconds)//2-1:len(seconds)//2+1] + median = sum(middle) / len(middle) + axes[0].plot([index-.2, index+.2], [median, median], color=colors[index], lw=3) +axes[0].set_xticks([0, 1], ['Tessar', 'getCards reference']) +axes[0].set_ylabel('Complete interactive display (seconds)') +axes[0].set_ylim(bottom=0) +axes[0].set_title('20 correct stable-data reads per variant', loc='left', weight='bold') +axes[0].grid(axis='y', alpha=.2) + +writes = result['candidate']['writes'] +labels = ['Content', 'Transitive', 'Exit', 'Entry', 'Async', 'Unrelated', + 'Insert', 'Delete', 'Reconnect'] +seconds = [write['ackToDisplayMs'] / 1000 for write in writes] +axes[1].barh(labels, seconds, color=colors[0]) +axes[1].axvline(10, color='#b44736', linestyle='--', lw=1.5) +axes[1].set_xlim(0, 10.8) +axes[1].invert_yaxis() +axes[1].set_xlabel('Acknowledged write → complete display (seconds)') +axes[1].set_title('Tessar: all nine tested mutations pass', loc='left', weight='bold') +axes[1].grid(axis='x', alpha=.2) +fig.suptitle('Tessar · 12,550 fabricated records · DO NOT MERGE', x=.04, ha='left', weight='bold') +fig.text(.04, .02, + 'Reference refresh exceeded 10 s and is disqualified for a freshness-preserving speedup claim.\n' + 'JSON-path diagnostic; full 10× HTML replay failed. Samples are local observations, not a capacity guarantee.', + fontsize=9, color='#604936') +fig.tight_layout(rect=[.02, .12, .99, .92]) +fig.savefig(target / 'tessar-10x-performance.svg', metadata={'Date': None}) +fig.savefig(target / 'tessar-10x-performance.png', dpi=160) +plt.close(fig) diff --git a/scripts/tessar/prewarm-check.mjs b/scripts/tessar/prewarm-check.mjs new file mode 100644 index 00000000000..be6a165675a --- /dev/null +++ b/scripts/tessar/prewarm-check.mjs @@ -0,0 +1,131 @@ +// Read-only memory probe over the actual synthetic dependency index. The +// definition-cache sink is inert: this isolates candidate collection from +// module rendering and does not change the preserved failed builder. +import { readFile, writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { getHeapStatistics } from 'node:v8'; +import { PgAdapter } from '../../packages/postgres/index.ts'; +import { + IndexWriter, + VirtualNetwork, + logger, +} from '../../packages/runtime-common/index.ts'; +import { query, param } from '../../packages/runtime-common/expression.ts'; +const { values } = parseArgs({ + options: { + database: { type: 'string' }, + dataset: { type: 'string' }, + output: { type: 'string' }, + before: { type: 'string' }, + }, +}); +if ( + !/^sf_bld_[a-z0-9_]+$/.test(values.database ?? '') || + !values.dataset || + !values.output +) + throw new Error( + '--database (synthetic builder), --dataset and --output are required', + ); +const manifest = JSON.parse( + await readFile(`${values.dataset}/manifest.json`, 'utf8'), +); +if (manifest.synthetic !== true) throw new Error('Synthetic manifest required'); +process.env.PGHOST = '127.0.0.1'; +process.env.PGPORT = process.env.TEST_HARNESS_PGPORT ?? '55437'; +process.env.PGUSER = 'postgres'; +process.env.PGDATABASE = values.database; +process.env.PGOPTIONS = '-c default_transaction_read_only=on'; +const { preWarmModulesTable } = await import( + values.before + ? pathToFileURL(values.before).href + : '../../packages/runtime-common/index-runner/prewarm-modules.ts' +); +const db = new PgAdapter(), + network = new VirtualNetwork(); +const realmURL = new URL('http://localhost:53776/tessar/'); +network.addURLMapping( + new URL('https://cardstack.com/base/'), + new URL('http://localhost:53776/base/'), +); +network.addRealmMapping('@cardstack/base/', 'http://localhost:53776/base/'); +const warmed = new Set(), + reads = []; +let result = { + valid: false, + implementation: values.before ? 'before' : 'bounded', + heapLimitBytes: getHeapStatistics().heap_size_limit, +}; +try { + const [job] = await query(db, [ + "SELECT args FROM jobs WHERE job_type='prerender_html' AND args->>'realmURL'=", + param(realmURL.href), + 'ORDER BY id DESC LIMIT 1', + ]); + const batch = await new IndexWriter(db).createBatch(realmURL, network, null, { + prerenderHtmlOnly: true, + generation: 2, + }); + const started = performance.now(); + const log = logger('tessar-prewarm-probe'); + await preWarmModulesTable({ + realmURL, + invalidations: job.args.changes.map((c) => new URL(c.url)), + allRealmCardModules: [new URL('tessar.gts', realmURL).href], + definitionLookup: { + populateDefinitionCacheEntry: async ({ moduleURL }) => { + warmed.add(moduleURL); + }, + }, + virtualNetwork: network, + reader: { + readFile: async (url) => ({ + content: await readFile( + `${values.dataset}/realm/${url.href.slice(realmURL.href.length)}`, + 'utf8', + ), + }), + }, + getDependencyRows: async (urls) => { + const rows = await batch.getDependencyRows(urls); + reads.push({ + urls: urls.length, + rows: rows.length, + heapBytes: process.memoryUsage().heapUsed, + }); + return rows; + }, + getModuleCacheContext: async () => ({ + resolvedRealmURL: realmURL.href, + cacheScope: 'public', + authUserId: '', + }), + prerenderUserId: 'tessar', + jobPriority: 0, + jobInfo: { jobId: 0, reservationId: 0, priority: 0, queueWaitMs: null }, + log, + perfLog: log, + }); + result = { + ...result, + valid: true, + elapsedMs: performance.now() - started, + maxRSSKiB: process.resourceUsage().maxRSS, + warmed: [...warmed].sort(), + reads, + }; +} catch (error) { + result.error = error.message; + process.exitCode = 1; +} finally { + await db.close(); + await writeFile(values.output, JSON.stringify(result, null, 2) + '\n'); + console.log( + JSON.stringify({ + ...result, + warmed: result.warmed?.length, + reads: result.reads?.length, + }), + ); +} diff --git a/scripts/tessar/realm/feeder-chain.gts b/scripts/tessar/realm/feeder-chain.gts new file mode 100644 index 00000000000..d7db13afaad --- /dev/null +++ b/scripts/tessar/realm/feeder-chain.gts @@ -0,0 +1,60 @@ +import { + Component, + field, + contains, + linksToMany, + NumberField, +} from '@cardstack/base/card-api'; +import { TessarRecord, DaySummary } from './tessar'; + +const tessarPending = (state: string) => state === 'pending'; + +// A deliberately long acyclic chain: one initially dirty leaf is insufficient +// to bound the number of waves needed to refresh all downstream consumers. +class TessarFeederOutput extends TessarRecord { + static tessarMaterialized = true; + + @field scoreTotal = contains(NumberField, { + computeVia: function (this: TessarFeederOutput) { + return ((this as any).inputs ?? []).reduce( + (sum: number, input: any) => sum + input.scoreTotal, + 0, + ); + }, + }); + + static isolated = class extends Component { + + }; +} + +const query = { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, +}; + +export class TessarStage0 extends TessarFeederOutput { + @field inputs = linksToMany(DaySummary, { query }); +} +export class TessarStage1 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage0, { query }); +} +export class TessarStage2 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage1, { query }); +} +export class TessarStage3 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage2, { query }); +} +export class TessarStage4 extends TessarFeederOutput { + @field inputs = linksToMany(TessarStage3, { query }); +} diff --git a/scripts/tessar/realm/get-cards.gts b/scripts/tessar/realm/get-cards.gts new file mode 100644 index 00000000000..96f2c7054a8 --- /dev/null +++ b/scripts/tessar/realm/get-cards.gts @@ -0,0 +1,233 @@ +import { Component } from '@cardstack/base/card-api'; +import { tracked } from '@glimmer/tracking'; +import { associateDestroyableChild, destroy } from '@ember/destroyable'; +import { TessarRecord } from './tessar'; + +// Correct, explicitly paged read-time reference for the same synthetic output. +// It intentionally follows the dashboard's broad type-search/getCards pattern. +// There is no fixed page-count cutoff: the entire result must be available. +export class TessarGetCardsPage extends TessarRecord { + static isolated = class extends Component { + @tracked pages: Record = {}; + @tracked loading = true; + @tracked error = ''; + @tracked freshnessPending = false; + private resourceOwner?: object; + private loadRevision = 0; + private unsubscribe?: () => void; + private offline = () => { + this.freshnessPending = true; + }; + private online = () => { + this.load(); + }; + + constructor(owner: unknown, args: any) { + super(owner, args); + if (!(globalThis as any).__boxelRenderContext) { + // The incumbent's resource loading flag does not expose the interval + // between an async source acknowledgement and index notification. + // The corrected reference uses the host event bus for that interval, + // and re-queries complete pages on publication/reconnect. + this.unsubscribe = ( + globalThis as any + )._CARDSTACK_REALM_SUBSCRIBE?.subscribe( + new URL('./', import.meta.url).href, + (event: any) => { + if (event.eventName === 'update') this.freshnessPending = true; + if (event.eventName === 'index') this.load(); + }, + ); + window.addEventListener('offline', this.offline); + window.addEventListener('online', this.online); + this.load(); + } + } + + willDestroy() { + this.loadRevision++; + this.unsubscribe?.(); + window.removeEventListener('offline', this.offline); + window.removeEventListener('online', this.online); + super.willDestroy(); + } + + async load() { + const revision = ++this.loadRevision; + this.loading = true; + this.error = ''; + const resourceOwner = {}; + associateDestroyableChild(this, resourceOwner); + try { + let context = (this.args as any).context; + let getCards = context?.getCards ?? context?.actions?.getCards; + if (!getCards) throw new Error('Tessar getCards context is missing'); + let realm = new URL('./', import.meta.url).href; + let module = new URL('./tessar', import.meta.url).href; + let pages: Record = {}; + for (let name of ['Student', 'Slot', 'Observation', 'Report']) { + pages[name] = []; + for (let number = 0; ; number++) { + let resource = getCards( + resourceOwner, + () => ({ + filter: { type: { module, name } }, + page: { size: 100, number }, + }), + () => [realm], + ); + let deadline = Date.now() + 30_000; + // Resources begin their asynchronous task on the following turn. + await new Promise((resolve) => setTimeout(resolve, 0)); + while (resource.isLoading) { + if (revision !== this.loadRevision) return; + if (Date.now() >= deadline) + throw new Error('Tessar getCards page timed out'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + if (resource.errors?.length || resource.queryErrors?.length) + throw new Error('Tessar getCards page failed'); + if (!Array.isArray(resource.instances)) + throw new Error('Tessar getCards page has no instances'); + pages[name].push(resource); + if (resource.instances.length < 100) break; + } + } + if (revision !== this.loadRevision) return; + const previousOwner = this.resourceOwner; + this.resourceOwner = resourceOwner; + this.pages = pages; + this.freshnessPending = false; + if (previousOwner) destroy(previousOwner); + } catch (error: any) { + if (revision === this.loadRevision) this.error = error.message; + } finally { + if (resourceOwner !== this.resourceOwner) destroy(resourceOwner); + if (revision === this.loadRevision) this.loading = false; + } + } + + get pending() { + return ( + this.loading || + this.freshnessPending || + Object.values(this.pages).some((pages) => + pages.some((resource) => resource.isLoading), + ) + ); + } + + get tessarState() { + return this.error ? 'error' : this.pending ? 'pending' : 'ready'; + } + + matching(type: string, day = true): any[] { + let owner = this.args.model; + return (this.pages[type] ?? []) + .flatMap((resource) => resource.instances) + .filter( + (item) => + item.classroomKey === owner.classroomKey && + (!day || item.day === owner.day), + ); + } + + get stats() { + let observations = this.matching('Observation'); + let reports = this.matching('Report'); + return [ + { + key: 'studentCount', + label: 'Students', + value: this.matching('Student', false).length, + }, + { + key: 'slotCount', + label: 'Sessions', + value: this.matching('Slot').length, + }, + { + key: 'observationCount', + label: 'Observations', + value: observations.length, + }, + { key: 'reportCount', label: 'Reports', value: reports.length }, + { + key: 'readyReportCount', + label: 'Ready reports', + value: reports.filter((item) => item.status === 'ready').length, + }, + { + key: 'scoreTotal', + label: 'Score total', + value: observations.reduce((sum, item) => sum + item.score, 0), + }, + ]; + } + + get rows() { + return this.matching('Slot') + .sort((a, b) => a.sequence - b.sequence) + .map((slot) => ({ + sourceId: `Slot/${String(slot.sequence).padStart(5, '0')}`, + label: slot.label, + studentLabel: slot.student?.label, + referenceLabel: slot.reference?.label, + status: slot.status, + })); + } + + + }; +} diff --git a/scripts/tessar/realm/tessar.gts b/scripts/tessar/realm/tessar.gts new file mode 100644 index 00000000000..1409f811747 --- /dev/null +++ b/scripts/tessar/realm/tessar.gts @@ -0,0 +1,192 @@ +import { + CardDef, + FieldDef, + Component, + field, + contains, + containsMany, + linksTo, + linksToMany, + StringField, + NumberField, +} from '@cardstack/base/card-api'; + +const tessarPending = (state: string) => state === 'pending'; + +export class TessarRecord extends CardDef { + @field label = contains(StringField); + @field classroomKey = contains(StringField); + @field day = contains(StringField); + @field sequence = contains(NumberField); + @field score = contains(NumberField); + @field status = contains(StringField); + @field narrative = contains(StringField); +} +export class Classroom extends TessarRecord {} +export class Student extends TessarRecord {} +export class Staff extends TessarRecord {} +export class Reference extends TessarRecord {} +export class Activity extends TessarRecord { + @field student = linksTo(Student); + @field staff = linksTo(Staff); + @field reference = linksTo(Reference); +} +export class Slot extends Activity {} +export class Observation extends Activity {} +export class Report extends Activity {} + +export class TessarRow extends FieldDef { + @field sourceId = contains(StringField); + @field label = contains(StringField); + @field sequence = contains(NumberField); + @field status = contains(StringField); + @field studentLabel = contains(StringField); + @field referenceLabel = contains(StringField); +} + +export class DaySummary extends TessarRecord { + static tessarMaterialized = true; + @field students = linksToMany(Student, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey' } }, + page: { size: 2000 }, + }, + }); + @field slots = linksToMany(Slot, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, + }, + }); + @field observations = linksToMany(Observation, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, + }, + }); + @field reports = linksToMany(Report, { + query: { + filter: { eq: { classroomKey: '$this.classroomKey', day: '$this.day' } }, + page: { size: 2000 }, + }, + }); + @field studentCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.students.length; + }, + }); + @field slotCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.slots.length; + }, + }); + @field observationCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.observations.length; + }, + }); + @field reportCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.reports.length; + }, + }); + @field readyReportCount = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.reports.filter((report) => report.status === 'ready').length; + }, + }); + @field scoreTotal = contains(NumberField, { + computeVia: function (this: DaySummary) { + return this.observations.reduce( + (sum, item) => sum + (item.score ?? 0), + 0, + ); + }, + }); + @field rows = containsMany(TessarRow, { + computeVia: function (this: DaySummary) { + return [...this.slots] + .sort((a, b) => (a.sequence ?? 0) - (b.sequence ?? 0)) + .map( + (slot) => + new TessarRow({ + sourceId: `Slot/${String(slot.sequence).padStart(5, '0')}`, + label: slot.label, + sequence: slot.sequence, + status: slot.status, + studentLabel: slot.student?.label, + referenceLabel: slot.reference?.label, + }), + ); + }, + }); + + static isolated = class extends Component { + + }; +} diff --git a/scripts/tessar/restart-check.mjs b/scripts/tessar/restart-check.mjs new file mode 100644 index 00000000000..1dd11376c08 --- /dev/null +++ b/scripts/tessar/restart-check.mjs @@ -0,0 +1,381 @@ +// Interrupt only a verified worker belonging to this synthetic local harness. +// The publication lock makes the crash point deterministic without adding +// production fault-injection hooks or changing the freshness deadline. +import { readFile, writeFile } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { chromium } from '@playwright/test'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { hashRealmUrlForAdvisoryLock } from '../../packages/postgres/pg-adapter.ts'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; +import { waitForDisplay, waitForTessarInteractive } from './browser-oracle.mjs'; + +const require = createRequire( + new URL('../../packages/realm-server/package.json', import.meta.url), +); +const { Client } = require('pg'); +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + output: { type: 'string' }, + 'matrix-url': { type: 'string' }, + phase: { type: 'string', default: 'before-publication' }, + }, +}); +for (const key of ['dataset', 'runtime', 'output', 'matrix-url']) + if (!values[key]) throw new Error(`--${key} is required`); +if (!['before-publication', 'after-source-publication'].includes(values.phase)) + throw new Error('Unknown Tessar crash phase'); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const manifest = JSON.parse( + await readFile(join(values.dataset, 'manifest.json'), 'utf8'), +); +const realm = new URL(runtime.realmURL), + server = new URL(runtime.realmServerURL), + matrix = new URL(values['matrix-url']); +const pgHost = process.env.TEST_HARNESS_PGHOST ?? '127.0.0.1'; +if ( + manifest.synthetic !== true || + manifest.variant === 'get-cards' || + realm.pathname !== '/tessar/' || + !/^sf_run_[a-z0-9_]+$/.test(runtime.databaseName) || + ![realm.hostname, server.hostname, matrix.hostname, pgHost].every((host) => + ['localhost', '127.0.0.1'].includes(host), + ) +) + throw new Error('Tessar restart checks require a synthetic local runtime'); +function processInfo(pid) { + if (!Number.isSafeInteger(pid) || pid < 2) throw new Error('Invalid PID'); + return execFileSync('ps', ['-p', String(pid), '-o', 'ppid=', '-o', 'args='], { + encoding: 'utf8', + }).trim(); +} +const managerPid = runtime.childPids.find((pid) => + processInfo(pid).includes('worker-manager.ts'), +); +if (!managerPid) throw new Error('The recorded harness manager is not alive'); +const { records } = generate(manifest); +const ownerId = 'DaySummary/00000', + ownerURL = new URL(ownerId, realm).href; +const token = buildRealmToken(realm, server); +async function sourceWrite(id, source) { + const response = await fetch(new URL(`${id}.json`, realm), { + method: 'POST', + headers: { + Accept: 'application/vnd.card+source', + 'Content-Type': 'application/vnd.card+source', + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify(source), + signal: AbortSignal.timeout(30000), + }); + await response.arrayBuffer(); + if (!response.ok) throw new Error(`Source write: HTTP ${response.status}`); +} +const config = { + host: pgHost, + port: Number(process.env.TEST_HARNESS_PGPORT ?? '55436'), + user: process.env.TEST_HARNESS_PGUSER ?? 'postgres', + password: process.env.PGPASSWORD, + database: runtime.databaseName, +}; +const db = new Client(config), + barrier = new Client(config); +let browser, + context, + lockHeld = false; +const result = { name: `worker crash ${values.phase}`, valid: false }; +try { + await db.connect(); + await barrier.connect(); + for (const id of [ + 'Observation/00000', + 'Reference/00000', + `Reference/${String(manifest.counts.Reference - 1).padStart(5, '0')}`, + ]) + await sourceWrite(id, records.get(id)); + const before = await ( + await fetch(ownerURL, { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(60000), + }) + ).json(); + validateSummary(before.data.attributes, expectedSummary(records, ownerId)); + if (before.data.meta.tessar?.state !== 'ready') + throw new Error('Setup is pending'); + process.env.TEST_HARNESS_BROWSER_MATRIX_URL = matrix.href; + const { buildBrowserState, installBrowserState } = + await import('../../packages/software-factory/tests/helpers/browser-auth.ts'); + browser = await chromium.launch({ headless: true }); + context = await browser.newContext(); + await installBrowserState( + context, + await buildBrowserState(realm.href, server.href), + ); + const page = await context.newPage(); + // Authenticate a second device during setup, then open its first page only + // after the source acknowledgement. It must not receive stale initial HTML. + const freshContext = await browser.newContext(); + await installBrowserState( + freshContext, + await buildBrowserState(realm.href, server.href), + ); + const freshPage = await freshContext.newPage(); + const errors = [], + inputRequests = []; + for (const observedPage of [page, freshPage]) { + observedPage.on('pageerror', (error) => errors.push(error.message)); + observedPage.on('request', (request) => { + if ( + (request.url().startsWith(realm.href) && + /\/(Student|Staff|Slot|Observation|Report|Reference|Activity)\//.test( + request.url(), + )) || + /\/_(federated-)?search(?:[?/#]|$)/.test(request.url()) + ) + inputRequests.push(request.url()); + }); + } + const url = new URL(ownerURL); + url.searchParams.set( + 'operatorModeState', + JSON.stringify({ + aiAssistantOpen: false, + stacks: [[{ format: 'isolated', id: ownerURL }]], + submode: 'interact', + workspaceChooserOpened: false, + }), + ); + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + await waitForTessarInteractive(page, realm.href); + await waitForDisplay(page, expectedSummary(records, ownerId), 30000); + await page.evaluate((realmURL) => { + window.__tessarCrashRealmEvents = []; + const messages = window._CARDSTACK_REALM_SUBSCRIBE; + window.__tessarCrashConnection = messages?.isTessarConnected; + messages?.subscribe(realmURL, (event) => { + window.__tessarCrashRealmEvents.push({ + name: event.eventName, + at: Date.now(), + origin: event.origin_server_ts, + }); + }); + window.__tessarCrashPending = false; + window.__tessarCrashTransitions = []; + new MutationObserver(() => { + let state = document + .querySelector('[data-tessar-state]') + ?.getAttribute('data-tessar-state'); + let visibleStats = document.querySelectorAll('[data-tessar-stat]').length; + if (window.__tessarCrashTransitions.at(-1)?.state !== state) + window.__tessarCrashTransitions.push({ + state, + visibleStats, + at: Date.now(), + text: document.querySelector('main')?.textContent?.slice(0, 1000), + }); + if ( + document.querySelector('[data-tessar-state="pending"]') && + document.querySelectorAll('[data-tessar-stat]').length === 0 + ) + window.__tessarCrashPending = true; + }).observe(document.body, { + subtree: true, + attributes: true, + childList: true, + }); + }, realm.href); + let barrierPid; + if (values.phase === 'before-publication') { + await barrier.query('BEGIN'); + await barrier.query('SELECT pg_advisory_xact_lock($1::bigint)', [ + hashRealmUrlForAdvisoryLock(`tessar:index:${realm.href}`), + ]); + lockHeld = true; + barrierPid = (await barrier.query('SELECT pg_backend_pid() AS pid')).rows[0] + .pid; + } + const floor = (await db.query('SELECT coalesce(max(id), 0) AS id FROM jobs')) + .rows[0].id; + records.get('Observation/00000').data.attributes.score += 23; + const expected = expectedSummary(records, ownerId); + const started = Date.now(); + await sourceWrite('Observation/00000', records.get('Observation/00000')); + const ack = Date.now(), + deadline = ack + 10000; + result.ackMs = ack - started; + if (values.phase === 'before-publication') { + const navigation = await freshPage.goto(url.href, { + waitUntil: 'domcontentloaded', + timeout: Math.max(1, deadline - Date.now()), + }); + const initialHTML = await navigation.text(); + result.initialHTMLHasOldStats = initialHTML.includes('data-tessar-stat='); + result.initialHTMLCacheControl = navigation.headers()['cache-control']; + if ( + result.initialHTMLHasOldStats || + result.initialHTMLCacheControl !== 'no-store' + ) + throw new Error( + 'A new client received cacheable or stale ready-looking initial HTML', + ); + } + let reservation; + while (Date.now() < deadline) { + const active = await db.query( + ` + SELECT j.id AS job_id, r.id AS reservation_id, r.worker_id + FROM jobs j JOIN job_reservations r ON r.job_id = j.id + WHERE j.id > $1 AND j.args->>'realmURL' = $2 + AND j.job_type = 'incremental-index' AND j.status = 'unfulfilled' + AND r.completed_at IS NULL ORDER BY j.id DESC LIMIT 1`, + [floor, realm.href], + ); + let reached = false; + if (values.phase === 'before-publication') { + reached = + ( + await db.query( + 'SELECT 1 FROM pg_stat_activity WHERE $1::int = ANY(pg_blocking_pids(pid))', + [barrierPid], + ) + ).rows.length > 0; + } else { + const dirty = await db.query( + 'SELECT dirty_generation FROM tessar_owners WHERE realm_url = $1 AND owner_url = $2 AND dirty_generation > $3', + [ + realm.href, + ownerURL + '.json', + before.data.meta.tessar.publishedGeneration, + ], + ); + reached = dirty.rows.length > 0; + if (reached) + result.committedDirtyGeneration = Number( + dirty.rows[0].dirty_generation, + ); + } + if (active.rows.length && reached) { + reservation = active.rows[0]; + break; + } + await delay(20); + } + if (!reservation) throw new Error(`No active worker reached ${values.phase}`); + const match = /^worker-pid-(\d+)$/.exec(reservation.worker_id); + if (!match) throw new Error('Only a local harness worker can be interrupted'); + const pid = Number(match[1]), + info = processInfo(pid); + if ( + Number(info.split(/\s+/)[0]) !== managerPid || + !/\bworker\.ts\b/.test(info) + ) + throw new Error('Worker does not belong to the recorded harness manager'); + result.interruptedReservation = reservation; + result.ackToCrashMs = Date.now() - ack; + process.kill(pid, 'SIGKILL'); + if (lockHeld) { + await barrier.query('ROLLBACK'); + lockHeld = false; + } + if (values.phase === 'after-source-publication') { + const dirty = ( + await db.query( + 'SELECT dirty_generation FROM tessar_owners WHERE realm_url = $1 AND owner_url = $2', + [realm.href, ownerURL + '.json'], + ) + ).rows[0]; + if (dirty?.dirty_generation == null) + throw new Error('The crash missed the durable dirty interval'); + } + await waitForDisplay(page, expected, Math.max(1, deadline - Date.now())); + result.ackToDisplayMs = Date.now() - ack; + if (values.phase === 'before-publication') { + await waitForTessarInteractive( + freshPage, + realm.href, + Math.max(1, deadline - Date.now()), + ); + await waitForDisplay( + freshPage, + expected, + Math.max(1, deadline - Date.now()), + ); + result.newClientAckToDisplayMs = Date.now() - ack; + } + const after = await ( + await fetch(ownerURL, { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())), + }) + ).json(); + validateSummary(after.data.attributes, expected); + if (after.data.meta.tessar?.state !== 'ready') + throw new Error('Recovered view is pending'); + result.ownerRevision = after.data.meta.tessar.publishedGeneration; + result.reservations = ( + await db.query( + 'SELECT worker_id, completion_reason, completed_at FROM job_reservations WHERE job_id = $1 ORDER BY id', + [reservation.job_id], + ) + ).rows; + if ( + !result.reservations.some( + (r) => + r.worker_id === reservation.worker_id && + r.completion_reason === 'interrupted', + ) || + !result.reservations.some( + (r) => + r.worker_id !== reservation.worker_id && + r.completion_reason === 'completed', + ) + ) + throw new Error('The interrupted job was not completed by a replacement'); + result.explicitPending = await page.evaluate( + () => window.__tessarCrashPending, + ); + result.transitions = await page.evaluate( + () => window.__tessarCrashTransitions, + ); + result.realmEvents = await page.evaluate( + () => window.__tessarCrashRealmEvents, + ); + result.initialConnection = await page.evaluate( + () => window.__tessarCrashConnection, + ); + result.pageErrors = errors; + result.inputRequests = inputRequests; + if (!result.explicitPending || errors.length || inputRequests.length) + throw new Error( + 'Recovered browser violated pending, error or graph-fetch gates', + ); + result.valid = true; +} catch (error) { + result.error = error.message; + process.exitCode = 1; +} finally { + if (lockHeld) await barrier.query('ROLLBACK').catch(() => {}); + await barrier.end().catch(() => {}); + await db.end().catch(() => {}); + await context?.close(); + await browser?.close(); + await writeFile( + values.output, + JSON.stringify( + { + version: 2, + readiness: 'subscribed-client-with-server-markup-removed', + manifest, + result, + }, + null, + 2, + ) + '\n', + ); + console.log(JSON.stringify(result)); +} diff --git a/scripts/tessar/resume-html.mjs b/scripts/tessar/resume-html.mjs new file mode 100644 index 00000000000..0b1c8b0484c --- /dev/null +++ b/scripts/tessar/resume-html.mjs @@ -0,0 +1,101 @@ +import { PgAdapter, PgQueuePublisher } from '../../packages/postgres/index.ts'; +import { query, param } from '../../packages/runtime-common/expression.ts'; + +// Diagnostic continuation after source indexing succeeded and HTML failed. +// The source builder remains unchanged, including its failed job history. +export async function resumeTessarHtml({ + sourceDatabase, + runtimeDatabase, + realmURL, +}) { + if ( + !/^sf_bld_[a-z0-9_]+$/.test(sourceDatabase) || + !/^sf_run_[a-z0-9_]+$/.test(runtimeDatabase) || + !['localhost', '127.0.0.1'].includes(new URL(realmURL).hostname) || + new URL(realmURL).pathname !== '/tessar/' + ) + throw new Error('Only an isolated Tessar builder may be resumed'); + process.env.PGHOST = process.env.TEST_HARNESS_PGHOST ?? '127.0.0.1'; + process.env.PGPORT = process.env.TEST_HARNESS_PGPORT ?? '55436'; + process.env.PGUSER = process.env.TEST_HARNESS_PGUSER ?? 'postgres'; + if (!['localhost', '127.0.0.1'].includes(process.env.PGHOST)) + throw new Error('Tessar recovery requires local Postgres'); + process.env.PGDATABASE = sourceDatabase; + let source = new PgAdapter(); + let prior; + try { + [prior] = await query(source, [ + "SELECT args, timeout, priority FROM jobs WHERE job_type = 'prerender_html' AND status = 'rejected' AND args->>'realmURL' =", + param(realmURL), + 'ORDER BY id DESC LIMIT 1', + ]); + let [index] = await query(source, [ + "SELECT status, result->'stats' AS stats FROM jobs WHERE job_type = 'from-scratch-index' AND args->>'realmURL' =", + param(realmURL), + 'ORDER BY id DESC LIMIT 1', + ]); + if ( + !prior || + index?.status !== 'resolved' || + index.stats.instanceErrors || + index.stats.fileErrors + ) + throw new Error( + 'The builder must have successful source indexing and failed HTML', + ); + } finally { + await source.close(); + } + process.env.PGDATABASE = runtimeDatabase; + const target = new PgAdapter(), + queue = new PgQueuePublisher(target); + try { + const [revision] = await query(target, [ + 'SELECT current_generation, loader_epoch FROM realm_generations WHERE realm_url =', + param(realmURL), + ]); + const started = performance.now(); + const job = await queue.publish({ + jobType: 'prerender_html', + concurrencyGroup: `prerender-html:${realmURL}`, + timeout: prior.timeout, + priority: prior.priority, + args: { + ...prior.args, + generation: Number(revision.current_generation), + loaderEpoch: revision.loader_epoch, + spawningJobId: null, + }, + }); + console.log('Tessar: replaying HTML against the committed synthetic index'); + const result = await job.done; + console.log( + JSON.stringify({ + tessarHtmlMs: performance.now() - started, + stats: result.stats, + }), + ); + if (result.stats.instanceErrors || result.stats.fileErrors) { + const failures = await query(target, [ + "SELECT url, error_doc->>'message' AS message FROM prerendered_html WHERE realm_url =", + param(realmURL), + 'AND error_doc IS NOT NULL', + ]); + const error = new Error('Resumed HTML has render errors'); + error.tessarDetails = { + elapsedMs: performance.now() - started, + stats: result.stats, + failures, + }; + throw error; + } + return { + elapsedMs: performance.now() - started, + stats: result.stats, + phaseTimings: result.phaseTimings, + }; + } finally { + await queue.destroy(); + await target.close(); + } +} diff --git a/scripts/tessar/sample-processes.mjs b/scripts/tessar/sample-processes.mjs new file mode 100644 index 00000000000..83ff957cd8a --- /dev/null +++ b/scripts/tessar/sample-processes.mjs @@ -0,0 +1,69 @@ +import { appendFile, writeFile } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { parseArgs } from 'node:util'; +import { setTimeout } from 'node:timers/promises'; + +const { values } = parseArgs({ + options: { + pid: { type: 'string' }, + output: { type: 'string' }, + 'interval-ms': { type: 'string', default: '2000' }, + }, +}); +const root = Number(values.pid), + interval = Number(values['interval-ms']); +if (!Number.isSafeInteger(root) || root < 1 || !values.output || interval < 500) + throw new Error('--pid, --output and an interval >= 500 ms are required'); +await writeFile( + values.output, + JSON.stringify({ + type: 'metadata', + root, + intervalMs: interval, + note: 'Owned process subtree; RSS sums include shared pages. Docker container resources are separate. ps CPU is a smoothed per-process percentage.', + }) + '\n', + { flag: 'wx' }, +); +while (true) { + const rows = execFileSync('ps', ['-axo', 'pid=,ppid=,pcpu=,rss=,comm='], { + encoding: 'utf8', + }) + .trim() + .split('\n') + .map((line) => { + const match = line + .trim() + .match(/^(\d+)\s+(\d+)\s+([\d.]+)\s+(\d+)\s+(.+)$/); + return ( + match && { + pid: Number(match[1]), + parent: Number(match[2]), + cpuPercent: Number(match[3]), + rssKiB: Number(match[4]), + executable: match[5].split('/').at(-1), + } + ); + }) + .filter(Boolean); + if (!rows.some((row) => row.pid === root)) break; + const owned = new Set([root]); + for (let changed = true; changed; ) { + changed = false; + for (const row of rows) + if (owned.has(row.parent) && !owned.has(row.pid)) { + owned.add(row.pid); + changed = true; + } + } + const processes = rows.filter((row) => owned.has(row.pid)); + await appendFile( + values.output, + JSON.stringify({ + at: Date.now(), + rssKiB: processes.reduce((sum, row) => sum + row.rssKiB, 0), + cpuPercent: processes.reduce((sum, row) => sum + row.cpuPercent, 0), + processes, + }) + '\n', + ); + await setTimeout(interval); +} diff --git a/scripts/tessar/switch-variant.mjs b/scripts/tessar/switch-variant.mjs new file mode 100644 index 00000000000..dfd7ec0656f --- /dev/null +++ b/scripts/tessar/switch-variant.mjs @@ -0,0 +1,126 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { buildRealmToken } from '../../packages/realm-test-harness/src/index.ts'; +import { generate, expectedSummary, validateSummary } from './generate.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + variant: { type: 'string' }, + 'manifest-output': { type: 'string' }, + }, +}); +if ( + !values.dataset || + !values.runtime || + !values['manifest-output'] || + !['materialized', 'get-cards'].includes(values.variant) +) + throw new Error( + '--dataset, --runtime, --variant and --manifest-output are required', + ); +const manifest = JSON.parse( + await readFile(join(values.dataset, 'manifest.json'), 'utf8'), +); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const realm = new URL(runtime.realmURL), + server = new URL(runtime.realmServerURL); +if ( + manifest.synthetic !== true || + realm.pathname !== '/tessar/' || + ![realm, server].every((url) => + ['localhost', '127.0.0.1'].includes(url.hostname), + ) +) + throw new Error( + 'Only an isolated synthetic Tessar realm can change variants', + ); +const token = buildRealmToken(realm, server); +const { records } = generate({ ...manifest, variant: values.variant }); +async function post(path, bytes) { + const response = await fetch(new URL(path, realm), { + method: 'POST', + headers: { + Accept: 'application/vnd.card+source', + 'Content-Type': 'application/vnd.card+source', + Authorization: `Bearer ${token}`, + }, + body: bytes, + signal: AbortSignal.timeout(60000), + }); + if (!response.ok) + throw new Error( + `Tessar variant write failed at ${path}: HTTP ${response.status}`, + ); + await response.arrayBuffer(); +} +if (values.variant === 'get-cards') + await post( + 'get-cards.gts', + await readFile(new URL('./realm/get-cards.gts', import.meta.url), 'utf8'), + ); +// Restore the known mutation fixtures without reindexing all unchanged inputs. +for (const id of [ + 'Observation/00000', + 'Reference/00000', + `Reference/${String(manifest.counts.Reference - 1).padStart(5, '0')}`, +]) + await post(`${id}.json`, JSON.stringify(records.get(id))); +const owners = [...records.keys()].filter((id) => id.startsWith('DaySummary/')); +for (const id of owners) + await post(`${id}.json`, JSON.stringify(records.get(id))); +const deadline = Date.now() + 300000; +for (const id of owners) { + let ready = false; + while (Date.now() < deadline && !ready) { + const response = await fetch(new URL(id, realm), { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(60000), + }); + const document = await response.json(); + ready = + response.ok && + document.data?.meta?.adoptsFrom?.name === + (values.variant === 'get-cards' ? 'TessarGetCardsPage' : 'DaySummary'); + if (ready && values.variant === 'materialized') { + ready = document.data.meta.tessar?.state === 'ready'; + if (ready) + validateSummary(document.data.attributes, expectedSummary(records, id)); + } + if (!ready) await delay(250); + } + if (!ready) throw new Error(`Tessar variant did not settle: ${id}`); +} +// This metadata describes only the changed dashboard definition. Input count, +// distribution and seed remain the same. The browser check is still required. +const recordHash = createHash('sha256'); +for (const [id, document] of records) + recordHash + .update(id) + .update('\0') + .update(JSON.stringify(document) + '\n'); +await writeFile( + values['manifest-output'], + JSON.stringify( + { + ...manifest, + sourceDatasetRecordsSha256: manifest.recordsSha256, + recordsSha256: recordHash.digest('hex'), + variant: values.variant, + variantSwitchedInPlace: true, + }, + null, + 2, + ) + '\n', +); +console.log( + JSON.stringify({ + variant: values.variant, + owners: owners.length, + manifest: values['manifest-output'], + }), +); diff --git a/scripts/tessar/verify-owners.mjs b/scripts/tessar/verify-owners.mjs new file mode 100644 index 00000000000..a772eeef859 --- /dev/null +++ b/scripts/tessar/verify-owners.mjs @@ -0,0 +1,67 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { parseArgs } from 'node:util'; +import { validateSummary } from './generate.mjs'; + +const { values } = parseArgs({ + options: { + dataset: { type: 'string' }, + runtime: { type: 'string' }, + output: { type: 'string' }, + }, +}); +if (!values.dataset || !values.runtime || !values.output) + throw new Error('--dataset, --runtime and --output are required'); +const manifest = JSON.parse( + await readFile(`${values.dataset}/manifest.json`, 'utf8'), +); +const expected = JSON.parse( + await readFile(`${values.dataset}/expected.json`, 'utf8'), +); +const runtime = JSON.parse(await readFile(values.runtime, 'utf8')); +const realm = new URL(runtime.realmURL); +if ( + manifest.synthetic !== true || + realm.pathname !== '/tessar/' || + !['localhost', '127.0.0.1'].includes(realm.hostname) +) + throw new Error('Only local synthetic Tessar fixtures may be verified'); +const owners = Object.entries(expected), + results = []; +let next = 0; +await Promise.all( + Array.from({ length: 4 }, async () => { + for (let i = next++; i < owners.length; i = next++) { + const [id, oracle] = owners[i], + result = { id, valid: false }; + try { + const response = await fetch(new URL(id, realm), { + headers: { Accept: 'application/vnd.card+json' }, + signal: AbortSignal.timeout(10000), + }); + const doc = await response.json(); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + if (doc.data.meta.tessar?.state !== 'ready') + throw new Error('View is not ready'); + validateSummary(doc.data.attributes, oracle); + result.valid = true; + result.revision = doc.data.meta.tessar.publishedGeneration; + } catch (error) { + result.error = error.message; + } + results.push(result); + } + }), +); +results.sort((a, b) => a.id.localeCompare(b.id)); +await writeFile( + values.output, + JSON.stringify({ manifest, results }, null, 2) + '\n', +); +console.log( + JSON.stringify({ + owners: results.length, + passed: results.filter((r) => r.valid).length, + failures: results.filter((r) => !r.valid), + }), +); +if (results.some((r) => !r.valid)) process.exitCode = 1;