From 5f0b4d66d410e782609473ce1e16ca8a26019de7 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:10:37 +0200 Subject: [PATCH 01/23] feat(mip8): storage perf-regression suite + consume timing report Runloop-timed SLOAD/SSTORE workloads (NINE vs NEXT): - two-sided MWU p and a sig@0.10 flag - ijson-streamed consume - block diagrams doc Co-Authored-By: Claude --- MIP8_PERF_TESTS_DIAGRAMS.md | 491 +++++++ MONAD_RUNLOOP_TESTING.md | 45 + .../plugins/consume/direct/conftest.py | 33 + .../plugins/consume/direct/test_via_direct.py | 12 +- .../plugins/consume/direct/timing_report.py | 305 +++++ .../client_clis/clis/monad.py | 72 +- .../src/execution_testing/fixtures/consume.py | 25 +- scripts/perf_disjoint_table.py | 401 ++++++ .../test_perf_regression.py | 1129 +++++++++++++++++ 9 files changed, 2503 insertions(+), 10 deletions(-) create mode 100644 MIP8_PERF_TESTS_DIAGRAMS.md create mode 100644 packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py create mode 100644 scripts/perf_disjoint_table.py create mode 100644 tests/monad_ten/mip8_pageified_storage/test_perf_regression.py diff --git a/MIP8_PERF_TESTS_DIAGRAMS.md b/MIP8_PERF_TESTS_DIAGRAMS.md new file mode 100644 index 00000000000..3d424955ea5 --- /dev/null +++ b/MIP8_PERF_TESTS_DIAGRAMS.md @@ -0,0 +1,491 @@ +# MIP-8 perf-regression tests: per-family block diagrams + +What each test-case family in +`tests/monad_ten/mip8_pageified_storage/test_perf_regression.py` actually +does at the SLOAD/SSTORE level inside one block. Numbers (pages per tx, +per-iteration gas) are computed with the module's own sizing helpers at +`BLOCK_GAS_TARGET = 200M`, `REPEATS = 1`. + +## Legend and shared machinery + +Notation used in all diagrams: + +``` +R(o) SLOAD of offset o within a page (result added to a checksum) +W(o)=v SSTORE of value v at offset o within a page +P pages (loop iterations) per transaction +s page-index stride: 1 (contiguous) or 1009 (scattered) +D READ_DOMAIN = 2^40 base page index of pre-populated pools +F FRESH_DOMAIN = 2^52 base page index of fresh-write ranges +g global tx index (unique across all blocks of a fixture) +t local tx index within the block (0..6) +``` + +Storage addressing (MIP-8 page = 128 slots): + +``` +slot = (page_index << 7) + offset + +page D + i*s: + offset: 0 1 ... k-1 k ... 127 + ┌─────┬─────┬─────┬─────┬─────┬─────┬─────┐ + │ 1 │ 1 │ ... │ 1 │ 0 │ ... │ 0 │ + └─────┴─────┴─────┴─────┴─────┴─────┴─────┘ + \__ k pre-populated slots __/ \__ empty __/ +``` + +Block anatomy shared by `test_page_ops`, `test_block_shape` (few_big), +`test_tx_halt`, `test_random_sload` and the two `test_bad_block_*` tests +— one 200M-gas block, 7 equal txs, all from the same sender to the same +workload contract (`test_random_sload` cycles a pool of 8 contracts +instead): + +``` +Block (gas limit 200,000,000) +┌──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┐ +│ tx 0 │ tx 1 │ tx 2 │ tx 3 │ tx 4 │ tx 5 │ tx 6 │ +│ 28.57M │ 28.57M │ 28.57M │ 28.57M │ 28.57M │ 28.57M │ 28.57M │ +└──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┘ +``` + +Per-tx warm/access sets reset between transactions, so every tx of a +block pays **cold** access again even when it re-touches the exact pages +tx 0 touched. This is what lets one pre-populated pool serve all 7 txs +as 7 independent cold passes. + +Every transaction runs the same loop contract: read calldata +`(base, count, g, halt, t)`, execute `count` iterations of the op body, +then finish with one or two bookkeeping SSTOREs on far-away pages: + +``` +tail of every tx: + SSTORE(2^200 + g, 0x1234) success marker (cold, fresh) + SSTORE(2^220 + g, checksum) read ops only; a checksum of 0 still + executes but writes 0 → no state trace +``` + +With `MIP8_PERF_REPEATS > 1` the whole block is emitted again with all +page domains shifted by `r * 2^28`, so every repeat block is a fully +cold copy. Diagrams below show the `r = 0` block. + +--- + +## `test_compute_loop` + +No parameters, one family. The storage-free baseline: one block, one +10M-gas tx running a stack-arithmetic `WhileGas` loop. + +``` +Block +┌───────────────────────────────────────────────┐ +│ tx 0 (10M gas) │ +│ loop: POP(ADD(MUL(NUMBER, GAS), CALLVALUE)) │ no storage access +│ ... repeated until gas nearly spent ... │ +│ SSTORE(slot 1, 0x1234) │ the only write +└───────────────────────────────────────────────┘ +``` + +Only the final marker touches storage, so the block isolates pure +execution overhead from any MIP-8 effect. + +--- + +## `test_page_ops` — one family per storage op + +Family = one `StorageOp`; within a family, `k` (page occupancy) and +`layout` (contiguous / scattered) vary. All variants use the 7-tx full +block above. Per-tx page counts: + +| op | layouts × k | P pages/tx | +|--------------------|--------------------------------|----------------------| +| sload_cold_hit | cont {1,16,128}, scat {1,128} | 3466; k=128: 512¹ | +| sload_cold_miss | cont {1,16,64}, scat {1,64} | 3466; k=64: 1024¹ | +| sload_sweep | cont {2,16,128}, scat {2,128} | k=2:1740 k=16:218 k=128:27 | +| sload_warm_repeat | cont {1}, scat {1} | 158,946 (iterations) | +| sload_empty_page | cont {0}, scat {0} | 3466 | +| sstore_fresh | cont {0}, scat {0} | 1009 | +| sstore_noop | cont {1,16,128}, scat {1,128} | 3429; k=128: 512¹ | +| sstore_grow | cont {1,16,64}, scat {1,64} | 1008 | +| sstore_update | cont {1,16,128}, scat {1,128} | 2562; k=128: 512¹ | +| sstore_clear_keep | cont {8,16,128}, scat {8,128} | 2563; k=128: 512¹ | +| sstore_clear_empty | cont {1}, scat {1} | 2564 | + +¹ capped by `PRE_SLOT_CAP = 65,536` pre-state slots per block +(`P = 65,536 / k`); those blocks do less than 200M of real work by +design — the pre-state, not gas, is the bound. + +Layout changes only the spacing of page indices, never the op: + +``` +contiguous (s = 1): D, D+1, D+2, ..., D+P-1 +scattered (s = 1009): D, D+1009, D+2018, ..., D+(P-1)*1009 +``` + +### Family `sload_cold_hit` + +Pre-state: pool of P pages at domain D, each with slots `0..k-1 = 1`. +Every tx makes one cold pass over the whole pool, reading offset 0 +(always occupied). + +``` +pre-state pool (shared by all 7 txs) + page D page D+s page D+2s page D+(P-1)s +tx 0: R(0) R(0) R(0) ... R(0) all cold +tx 1: R(0) R(0) R(0) ... R(0) cold again + ... (warmth reset) +tx 6: R(0) R(0) R(0) ... R(0) + +each read returns 1 → checksum = P per tx +block total: 7 × P cold page reads of an existing, occupied slot +``` + +### Family `sload_cold_miss` + +Same shared pool as `cold_hit` (k ≤ 64 keeps the last slot empty), but +each read targets offset 127 — the page exists, the slot is zero. + +``` +page D + i*s: + offset: 0 .. k-1 k .. 126 127 + [ 1 .. 1 ] [ 0 .. 0 ] [ 0 ] ◄── R(127) per iteration + occupied empty read misses (returns 0) + +tx 0..6: R(127) on every pool page, all cold +checksum = 0 → the tail checksum SSTORE writes 0 (no state trace) +``` + +Measures "page found, slot not found" lookups, P cold reads per tx. + +### Family `sload_sweep` + +Each loop iteration cold-reads **every occupied slot** of one page, +offsets `0..k-1` in ascending order. + +``` +tx 0..6, iteration i (page D + i*s): + offset: 0 1 2 ... k-1 | k .. 127 + R(0) R(1) R(2) ... R(k-1)| untouched k cold reads + +per tx: P pages × k reads (k=2: 1740×2, k=16: 218×16, k=128: 27×128) +checksum = P × k per tx +``` + +Isolates intra-page locality: same number-ish of cold reads as +`cold_hit`, but bunched k-per-page instead of 1-per-page. + +### Family `sload_warm_repeat` + +No page pool — a single pre-populated slot `W = 2^70` (value 1). The +slot arrives via calldata and is read directly (not page-shifted). + +``` +pre-state: slot W = 1 + +tx 0: SLOAD(W) ×158,946 1st read cold, remaining 158,945 warm +tx 1: SLOAD(W) ×158,946 cold again once (warm set reset), then warm + ... +tx 6: SLOAD(W) ×158,946 + +checksum = 158,946 per tx; block total ≈ 1.11M reads of one hot slot +``` + +The warm-path baseline: page/slot caching should make fork choice +irrelevant here. + +### Family `sload_empty_page` + +Like `cold_hit` but with `k = 0`: the domain-D pages were **never +populated**. Every read is a whole-page miss. + +``` + page D page D+s page D+(P-1)s +tx 0..6: R(0) R(0) ... R(0) page does not exist + +P = 3466 per tx; checksum = 0 (zero write in the tail, no state trace) +``` + +Measures lookups that fall off the page index entirely. + +### Family `sstore_fresh` + +No pre-state. Each tx gets its own disjoint range of never-touched +pages and creates one slot on each: `W(0)=1`, a 0→1 write that brings +a whole new page into existence. + +``` +page-index axis (base F, stride s), tiled per tx: + +F F+P·s F+2P·s F+6P·s F+7P·s +╠═ tx0: P ═╬═ tx1: P ══╬═ tx2: P ══╬═ ... ══╬═ tx6: P ══╣ + pages pages pages pages + +tx t, iteration i: W(0)=1 on page F + (t·P + i)·s state growth + +P = 1009 → block creates 7 × 1009 = 7063 new pages +``` + +### Family `sstore_noop` + +Shared occupied pool (like `cold_hit`); each tx rewrites slot 0 with +its current value — 1→1, no state change ever. + +``` +tx 0..6, iteration i (page D + i*s): + offset 0: [ 1 ] ◄── W(0)=1 cold page access, value unchanged + +post-state == pre-state (plus markers); P = 3429 (512 at k=128) +``` + +Pays the write path without any page mutation. + +### Family `sstore_grow` + +Shared pool with offsets `0..k-1` occupied. Tx `t` writes offset +`k + t` — a 0→1 on an **already-occupied** page (growth within a page, +no page creation). All 7 txs touch the same P pages, each at its own +offset. + +``` +page D + i*s, one column per tx: + offset: 0 .. k-1 | k k+1 k+2 k+3 k+4 k+5 k+6 | k+7 .. 127 + before: [ 1 .. 1 ]| 0 0 0 0 0 0 0 | 0 + writer: | tx0 tx1 tx2 tx3 tx4 tx5 tx6 | + after: [ 1 .. 1 ]| 1 1 1 1 1 1 1 | 0 + +each tx: P = 1008 cold W(k+t)=1 writes, one per pool page +``` + +### Family `sstore_update` + +Shared pool; every tx overwrites the occupied slot 0 with a fresh +nonzero value `2 + t`, so each write is a genuine value change with no +occupancy change. + +``` +page D + i*s, offset 0 over the block: + pre tx0 tx1 tx2 tx3 tx4 tx5 tx6 + 1 ─► W(0)=2 ─► =3 ─► =4 ─► =5 ─► =6 ─► =7 ─► =8 + +each tx: P = 2562 cold writes (512 at k=128); slot 0 ends at 8 +``` + +### Family `sstore_clear_keep` + +Shared pool with `k > 7` occupied slots. Tx `t` clears offset `t` +(1→0). Offsets `7..k-1` stay populated, so no page ever empties. + +``` +page D + i*s: + offset: 0 1 2 3 4 5 6 | 7 .. k-1 | k .. 127 + before: 1 1 1 1 1 1 1 | 1 .. 1 | 0 + clearer: tx0 tx1 tx2 tx3 tx4 tx5 tx6 | untouched | + after: 0 0 0 0 0 0 0 | 1 .. 1 | 0 + +each tx: P = 2563 cold W(t)=0 writes (512 at k=128) +``` + +### Family `sstore_clear_empty` + +The page-removal case. Pre-state gives **each tx its own** range of +single-slot pages (offset 0 = 1). Clearing that slot leaves the page +empty, so every write deletes a page. + +``` +pre-state, tiled like sstore_fresh but pre-populated with k=1: + +F F+P·s F+2P·s F+6P·s F+7P·s +╠═ tx0: P ═╬═ tx1: P ══╬═ ... ══════════╬═ tx6: P ══╣ + 1-slot 1-slot 1-slot + pages pages pages + +tx t, iteration i: W(0)=0 on page F + (t·P + i)·s + before: [1][0..0] → after: [0][0..0] → page removed + +P = 2564 → block removes 7 × 2564 = 17,948 pages +``` + +--- + +## `test_page_spread` — one family + +Op is fixed (`sstore_fresh`, the 0→1 page-creating write); the sweep is +over **where** the writes land: `m` total pages spread evenly across +`n` contracts, in `contiguous` or `scattered` page-index layout. + +Each contract runs the same loop contract with empty storage. Every +contract writes page indices `F + 0·s .. F + (m/n − 1)·s` — the *same* +indices for all contracts, but in n distinct account storages, so +nothing collides. A contract's share becomes one tx while it fits the +30M tx cap (≤ 1059 pages); `m/n = 4096` splits into 4 txs. + +``` +Block (one per fixture; txs sized to the work, not to fill 200M) + +n = 1, m = 4096 (4 txs, one contract): +┌ contract C0 ──────────────────────────────────────────────────────┐ +│ tx0: W(0)=1 on pages F+0·s .. F+1058·s (1059 pages, 30.0M)│ +│ tx1: W(0)=1 on pages F+1059·s .. F+2117·s (1059 pages, 30.0M)│ +│ tx2: W(0)=1 on pages F+2118·s .. F+3176·s (1059 pages, 30.0M)│ +│ tx3: W(0)=1 on pages F+3177·s .. F+4095·s ( 919 pages, 26.0M)│ +└───────────────────────────────────────────────────────────────────┘ + +n = 8, m = 4096 (8 txs, 512 pages each): +┌ C0 ┐┌ C1 ┐┌ C2 ┐┌ C3 ┐┌ C4 ┐┌ C5 ┐┌ C6 ┐┌ C7 ┐ +│tx0 ││tx1 ││tx2 ││tx3 ││tx4 ││tx5 ││tx6 ││tx7 │ each: 512 × W(0)=1 +└────┘└────┘└────┘└────┘└────┘└────┘└────┘└────┘ 14.6M gas each + +n = 512, m = 4096 (512 txs, 8 pages each): +┌C0┐┌C1┐┌C2┐ ... ┌C511┐ each: 8 × W(0)=1, 345,568 gas +└──┘└──┘└──┘ └────┘ block ≈ 177M gas +``` + +Variants: `m ∈ {1,4,16,64,256,1024,4096} × n=1` (total-size sweep, +contiguous), `m=4096 × n ∈ {1,8,64,512}` (distribution sweep), plus +scattered replays of `m4096_n1` and `m4096_n512`. Same total I/O at +`m = 4096` regardless of `n` — only the account fan-out changes. + +--- + +## `test_block_shape` — one family + +Same total work packed as **7 big** txs vs **300 small** txs. Two +workloads, both contiguous: + +| op, k | shape | txs | tx gas | iters/tx | block work | +|-------------------|------------|-----|-----------|----------|--------------------| +| sstore_fresh, k=0 | few_big | 7 | 28.57M | 1009 | 7063 new pages | +| sstore_fresh, k=0 | many_small | 300 | 666,666 | 19 | 5700 new pages | +| sload_cold_hit, 8 | few_big | 7 | 28.57M | 3466 | 7×3466 cold reads | +| sload_cold_hit, 8 | many_small | 300 | 666,666 | 66 | 300×66 cold reads | + +``` +few_big: ┌──────────┬──────────┬──────────┬──── ... ───┬──────────┐ + │ tx 0 │ tx 1 │ tx 2 │ │ tx 6 │ + └──────────┴──────────┴──────────┴──── ... ───┴──────────┘ + +many_small: ┌──┬──┬──┬──┬──┬──┬──┬──┬──┬── ... ──┬──┬──┬──┬──┬──┬──┐ + │t0│t1│t2│t3│t4│t5│t6│t7│t8│ │ │ │ │ │t299│ + └──┴──┴──┴──┴──┴──┴──┴──┴──┴── ... ──┴──┴──┴──┴──┴──┴──┘ +``` + +The reads reuse one shared pre-populated pool sized to a single tx +(3466 pages for few_big, 66 pages for many_small, k = 8 slots each); +every tx is a fresh cold pass over it. The fresh writes tile disjoint +per-tx ranges exactly like `sstore_fresh` above. What varies is per-tx +fixed overhead (intrinsic gas, markers, cold pool re-touch) relative +to loop work. + +--- + +## `test_tx_halt` — one family + +The `sstore_fresh` full block (7 txs × 1009 fresh pages, disjoint +ranges), with a `halt` calldata flag per tx. A halting tx performs all +of its writes **and** its marker, then executes `INVALID`: everything +reverts and the tx consumes its entire 28.57M gas limit. + +``` +mode=success: ┌ tx0 ✓ ┬ tx1 ✓ ┬ tx2 ✓ ┬ tx3 ✓ ┬ tx4 ✓ ┬ tx5 ✓ ┬ tx6 ✓ ┐ + all writes land: 7063 pages + 7 markers + +mode=halt: ┌ tx0 ✗ ┬ tx1 ✗ ┬ tx2 ✗ ┬ tx3 ✗ ┬ tx4 ✗ ┬ tx5 ✗ ┬ tx6 ✗ ┐ + every tx: 1009 × W(0)=1, marker, then INVALID + → post-state empty, block still burns the full 200M + +mode=mix: ┌ tx0 ✓ ┬ tx1 ✗ ┬ tx2 ✓ ┬ tx3 ✗ ┬ tx4 ✓ ┬ tx5 ✗ ┬ tx6 ✓ ┐ + only even txs' pages + markers survive (4 × 1009 pages) +``` + +Measures the cost of executing (and then discarding) storage writes — +the revert path does the same page work as success but must be rolled +back. + +--- + +## `test_random_sload` — one family + +Random-locality reads: no domains, no strides — slot keys are +pseudorandom over the low 2^200 of the slot space, so every slot lands +on its own effectively random page. Params: `slots ∈ {1, 16, 128}` +(size of the per-tx slot set) × `k ∈ {0, 1}` (each read slot's page is +1-slot occupied, or never populated). + +A pool of 8 identical contracts is deployed; tx `g` calls contract +`(3·g) mod 8` and carries its own seed `1 + 1024·g`, giving every tx a +disjoint pseudorandom slot set inside "its" contract: + +``` +Block: tx0→C0 tx1→C3 tx2→C6 tx3→C1 tx4→C4 tx5→C7 tx6→C2 + +tx g: slot set S_g[i] = ((seed_g + i) · MULT) mod 2^200, i < slots + e.g. seed=1: 0x15f39cc0605c..., 0x2be73980c0b9..., ... + + iteration j (of 3462) reads S_g[(5·j) mod slots]: + j: 0 1 2 ... slots−1 │ slots ... 3461 + s_0 s_5 s_10 ... │ (set cycles again) + cold cold cold ... cold │ warm ... warm +``` + +Each tx makes 3462 reads, but only the first pass over the set +(`slots` reads) is genuinely cold — sizing charges every iteration as +cold (8216 gas), so these blocks are heavily gas-underfull by design; +the subject is random-key locality, not I/O volume. With `k = 1` the +set's slots are pre-populated (=1) in that contract's genesis and the +checksum is 3462; with `k = 0` every read is a whole-page miss and the +tail checksum SSTORE writes 0 (no state trace). Markers/checksums land +in the storage of whichever contract the tx called. + +--- + +## `test_bad_block_serial` — one family + +No parameters. The write-conflict adversarial block: all 7 txs +read-then-increment the **same** contiguous slot range, so every tx +depends on the previous tx's writes and the block cannot be executed +in parallel. + +``` +shared slot range: base = 2^40, slots base+0 .. base+782 +(contiguous keys → the whole range is just 7 dense pages) + +per iteration i: SLOAD(base+i), then SSTORE(base+i, read+1) + + slot: base+0 base+1 base+2 ... base+782 +tx 0: 0→1 0→1 0→1 0→1 fresh writes +tx 1: 1→2 1→2 1→2 1→2 ▲ must see the + ... │ previous tx's +tx 6: 6→7 6→7 6→7 6→7 │ writes: serial +``` + +783 read+write pairs per tx (36,294 gas each, sized to the fresh 0→1 +cost — the later increment txs are nonzero→nonzero updates and run +cheaper, leaving the block somewhat gas-underfull). Post-state: every +shared slot ends at 7, plus the 7 markers (no checksum tail here). + +--- + +## `test_bad_block_chained` — one family + +No parameters. The data-dependency adversarial block: the genesis +storage holds a pre-built ring of 3482 pseudorandom slots where each +slot's **value is the address of the next slot**. A tx starts from its +calldata seed and hops the ring with `slot := SLOAD(slot)` — every +read's address comes from the previous read, so the reads serialize +within the tx (no lookahead or batching possible). + +``` +genesis ring (3482 pseudorandom slots, each on its own page): + + ring[0] ──► ring[1] ──► ring[2] ──► ... ──► ring[3481] ──┐ + ▲ │ + └─────────────────────────────────────────────────────┘ + SLOAD(ring[j]) returns ring[j+1] + +tx t (t = 0..6): start at ring[t], then 3482 hops + = one full lap, staggered one step per tx +``` + +All 7 txs traverse the same ring; per-tx warmth reset makes each lap +fully cold, so the block performs 7 × 3482 cold, address-dependent +reads. Nothing is written except the 7 markers — post-state is the +untouched ring plus markers. diff --git a/MONAD_RUNLOOP_TESTING.md b/MONAD_RUNLOOP_TESTING.md index c8d502d1f4d..b2e7ff07155 100644 --- a/MONAD_RUNLOOP_TESTING.md +++ b/MONAD_RUNLOOP_TESTING.md @@ -21,6 +21,7 @@ Requirements: docker, ~10 GB disk for the builder image and build artifacts, ~6 GB RAM for hugepages. ```sh +snap install astral-uv --classic git clone --branch main \ git@github.com:monad-exp/monad-eest-rust-harness.git cd monad-eest-rust-harness @@ -68,6 +69,50 @@ uv run consume direct --input ../fixtures_eestnet \ across ~14 threads), so budget ~5 vCPUs per worker (`-n N` needs roughly `5 * N` cores). +## MIP-8 perf-regression tests + +`tests/monad_ten/mip8_pageified_storage/test_perf_regression.py` fills +SLOAD/SSTORE workloads at both forks and times block execution on the +runloop to compare MONAD_NINE (slot-encoded) vs MONAD_NEXT (page-encoded). + + +### Setup + +```sh +sudo tee /etc/sysctl.d/99-benchmark.conf >/dev/null <<'EOF' +kernel.randomize_va_space = 0 +kernel.perf_event_paranoid = 1 +vm.nr_hugepages = 3072 +EOF +sudo sysctl --system +sudo cpupower idle-set -D 1 +``` + +### Filling & running + +```sh +MIP8_PERF_REPEATS=5 uv run fill --clean -m blockchain_test \ + tests/monad_ten/mip8_pageified_storage/test_perf_regression.py \ + --from MONAD_NINE --until MONAD_NEXT --chain-id 30143 --monad-runloop \ + --output ../fixtures_eestnet -n auto + +uv run consume direct --input ../fixtures_eestnet \ + --bin ../monad-eest-rust-harness/bin/eest-runner \ + --timing-report-dir ../timing +``` + +- Each block is stamped to 200M gas; the same workload is sized to fit + both forks (no gas assertions — post-state is the oracle). +- `MIP8_PERF_REPEATS=N` (default 1) emits N page-disjoint copies of each + workload as successive blocks → N cold timing samples per fixture. +- `MIP8_PERF_BLOCK_GAS=N` shrinks the block for a quick smoke fill. +- Consume writes both `timing_consume.md` and `.csv` by default + (`--timing-report {both,md,csv,none}`, `--timing-report-dir DIR`): one + row per (test, params, fork) — the `min` over the repeat blocks (drops + the warmup block) — with MONAD_NINE, MONAD_NEXT, then a Δ% row. +- Run consume on a quiet host for stable timings; the numbers are noisy + under contention. + ## Behavior and known limits - Fixtures containing expected-invalid blocks are skipped: the ledger diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py index 980c4218768..519ea697cca 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py @@ -17,6 +17,9 @@ from execution_testing.cli.pytest_commands.plugins.consume.consume import ( FixturesSource, ) +from execution_testing.cli.pytest_commands.plugins.consume.direct.timing_report import ( # noqa: E501 + TimingReportPlugin, +) from execution_testing.client_clis.ethereum_cli import EthereumCLI from execution_testing.client_clis.fixture_consumer_tool import ( FixtureConsumerTool, @@ -79,6 +82,29 @@ def pytest_addoption(parser: pytest.Parser) -> None: # noqa: D103 "consumer tool." ), ) + consume_group.addoption( + "--timing-report", + action="store", + dest="timing_report", + choices=["both", "md", "csv", "none"], + default="both", + help=( + "Emit per-block execution timing (from consumers that report " + "it) as a `timing_consume` table. `both` (default) writes " + "Markdown and CSV; `md`/`csv` write one; `none` disables." + ), + ) + consume_group.addoption( + "--timing-report-dir", + action="store", + dest="timing_report_dir", + type=Path, + default=None, + help=( + "Directory for the timing report. Defaults to the HTML report's " + "directory (the fixtures `.meta` directory)." + ), + ) debug_group = parser.getgroup("debug", "Arguments defining debug behavior") debug_group.addoption( "--dump-dir", @@ -123,6 +149,13 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: D103 ) config.fixture_consumers = fixture_consumers # type: ignore[attr-defined] + timing_report = config.getoption("timing_report") + if timing_report != "none": + config.pluginmanager.register( + TimingReportPlugin(config, timing_report), + "consume-timing-report", + ) + @pytest.fixture(scope="function") def test_dump_dir( diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/test_via_direct.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/test_via_direct.py index b16832b8276..07fb8cb382a 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/test_via_direct.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/test_via_direct.py @@ -4,7 +4,11 @@ """ from pathlib import Path +from typing import Callable +from execution_testing.cli.pytest_commands.plugins.consume.direct.timing_report import ( # noqa: E501 + TIMING_PROPERTY, +) from execution_testing.fixtures import FixtureConsumer from execution_testing.fixtures.consume import ( TestCaseIndexFile, @@ -17,14 +21,20 @@ def test_fixture( fixture_consumer: FixtureConsumer, fixture_path: Path, test_dump_dir: Path | None, + record_property: Callable[[str, object], None], ) -> None: """ Generic test function used to call the fixture consumer with a given fixture file path and a fixture name (for a single test run). """ - fixture_consumer.consume_fixture( + block_timings = fixture_consumer.consume_fixture( test_case.format, fixture_path, fixture_name=test_case.id, debug_output_path=test_dump_dir, ) + if block_timings: + record_property( + TIMING_PROPERTY, + {"id": test_case.id, "blocks": [dict(t) for t in block_timings]}, + ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py new file mode 100644 index 00000000000..cc9ba8ec881 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py @@ -0,0 +1,305 @@ +""" +Emit per-block execution timing collected during `consume direct` as +Markdown and/or CSV artifacts alongside the HTML report. + +The consumer (e.g. the monad runloop) returns per-block timing from +`consume_fixture`; `test_via_direct.test_fixture` stashes it on the test's +`user_properties` under `TIMING_PROPERTY`. This plugin writes one part file +per test *in the process that ran it* (so it never depends on xdist +forwarding `user_properties` to the controller), then the controller +aggregates every part into a single table at session end. +""" + +from __future__ import annotations + +import csv +import io +import json +import uuid +from itertools import groupby +from pathlib import Path +from typing import Any, Dict, Generator, List, Tuple + +import pytest + +# Key under which `test_fixture` records the timing payload +# ``{"id": , "blocks": [BlockExecutionTiming, ...]}``. +TIMING_PROPERTY = "consume_block_timing" + +# Column key -> header. Metrics are the per-fixture minimum across blocks +# (repeat samples); the warmup block is naturally discarded by the min. +_COLUMNS: List[Tuple[str, str]] = [ + ("test", "test"), + ("params", "params"), + ("fork", "fork"), + ("tx_count", "tx"), + ("gas", "gas"), + ("tx_exec_us", "tx_exec_us"), + ("state_root_us", "state_root_us"), + ("commit_us", "commit_us"), + ("total_us", "total_us"), +] + +# Numeric metric columns a Δ% row is computed for. +_METRIC_KEYS = ("tx_exec_us", "state_root_us", "commit_us", "total_us") + +# Fork ordering (oldest first) so a Δ% row compares the newer fork against +# the older baseline; unknown forks sort after, alphabetically. +_FORK_ORDER = ["MONAD_EIGHT", "MONAD_NINE", "MONAD_NEXT", "MONAD_TEN"] + +# Fixture-id suffixes identifying the fixture format, not a real parameter. +_FORMAT_TAGS = { + "blockchain_test", + "blockchain_test_engine", + "blockchain_test_sync", + "state_test", +} + + +def _fork_rank(fork: str) -> Tuple[int, str]: + """Sort key placing known forks in release order, others after.""" + if fork in _FORK_ORDER: + return _FORK_ORDER.index(fork), "" + return len(_FORK_ORDER), fork + + +def _delta_row(base: Dict[str, Any], comp: Dict[str, Any]) -> Dict[str, Any]: + """Build a percent-change row comparing `comp` against `base`.""" + + def pct(b: Any, c: Any) -> str: + if not isinstance(b, (int, float)) or b == 0: + return "n/a" + return f"{(c - b) / b * 100:+.1f}%" + + row: Dict[str, Any] = { + "test": "", + "params": "", + "fork": f"Δ% {comp['fork']}/{base['fork']}", + "tx_count": "", + "gas": "", + } + for key in _METRIC_KEYS: + row[key] = pct(base[key], comp[key]) + return row + + +def _aggregate(rows: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int]: + """ + Reduce per-block rows to one row per (test, params, fork). + + Each metric becomes the minimum across the fixture's blocks — repeat + samples on disjoint pages — which discards the warmup block. Returns + the aggregated rows and the largest block count seen (repeat count). + """ + groups: Dict[Tuple[str, str, str], List[Dict[str, Any]]] = {} + order: List[Tuple[str, str, str]] = [] + for row in rows: + key = (row["test"], row["params"], row["fork"]) + if key not in groups: + groups[key] = [] + order.append(key) + groups[key].append(row) + + aggregated: List[Dict[str, Any]] = [] + max_blocks = 1 + for key in order: + members = groups[key] + max_blocks = max(max_blocks, len(members)) + row = dict(members[0]) + for metric in _METRIC_KEYS: + row[metric] = min(m[metric] for m in members) + aggregated.append(row) + return aggregated, max_blocks + + +def _ordered_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Order rows by (test, params) then fork, and append a Δ% row after each + pair of forks sharing a (test, params) group. + """ + + def group_key(row: Dict[str, Any]) -> Tuple[str, str]: + return row["test"], row["params"] + + rows.sort(key=lambda r: (*group_key(r), _fork_rank(r["fork"]))) + ordered: List[Dict[str, Any]] = [] + for _, group in groupby(rows, key=group_key): + members = list(group) + ordered.extend(members) + if len(members) == 2: + ordered.append(_delta_row(members[0], members[1])) + return ordered + + +def _split_fixture_id(fixture_id: str) -> Tuple[str, str, str]: + """ + Split a fixture id into (test, params, fork). + + ``tests/.../test_perf_regression.py::test_compute_loop[ + scheme_a-fork_MONAD_NINE-blockchain_test]`` becomes + ``("test_perf_regression::test_compute_loop", "scheme_a", "MONAD_NINE")``. + Parameters are kept as one ``-``-joined column since their arity varies + per test; the fork token and the trailing format tag are pulled out. + """ + module, _, rest = fixture_id.partition("::") + func = rest.split("[", 1)[0] + test = f"{Path(module).stem}::{func}" if module else func + fork = "" + params: List[str] = [] + if "[" in rest and rest.rstrip().endswith("]"): + inner = rest[rest.index("[") + 1 : rest.rindex("]")] + for token in inner.split("-"): + if token.startswith("fork_"): + fork = token[len("fork_") :] + elif token in _FORMAT_TAGS: + continue + else: + params.append(token) + return test, "-".join(params), fork + + +def _rows_from_payload(payload: Dict[str, Any]) -> List[Dict[str, Any]]: + """Flatten one recorded timing payload into per-block table rows.""" + test, params, fork = _split_fixture_id(payload["id"]) + rows = [] + for block in payload["blocks"]: + rows.append( + {"test": test, "params": params or "-", "fork": fork, **block} + ) + return rows + + +def _render_markdown(rows: List[Dict[str, Any]]) -> str: + """Render rows as a GitHub-flavored Markdown table.""" + headers = [header for _, header in _COLUMNS] + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + for row in rows: + lines.append( + "| " + + " | ".join(str(row.get(key, "")) for key, _ in _COLUMNS) + + " |" + ) + return "\n".join(lines) + "\n" + + +def _render_csv(rows: List[Dict[str, Any]]) -> str: + """Render rows as CSV.""" + buffer = io.StringIO() + writer = csv.writer(buffer) + writer.writerow([header for _, header in _COLUMNS]) + for row in rows: + writer.writerow([row.get(key, "") for key, _ in _COLUMNS]) + return buffer.getvalue() + + +class TimingReportPlugin: + """Collect per-test timing parts and write the aggregated report.""" + + def __init__(self, config: pytest.Config, fmt: str): # noqa: D107 + self.config = config + self.fmt = fmt + self.written: List[Path] = [] + self.repeats = 1 + + def _output_dir(self) -> Path: + """ + Directory the timing artifacts are written to. + + Prefers an explicit ``--timing-report-dir``; otherwise follows the + HTML report's directory (so it lands next to ``report_consume.html``, + which defaults to the fixtures `.meta`). + """ + explicit = self.config.getoption("timing_report_dir", None) + if explicit is not None: + return Path(explicit) + htmlpath = getattr(self.config.option, "htmlpath", None) + if htmlpath: + return Path(htmlpath).parent + source = self.config.fixtures_source # type: ignore[attr-defined] + return Path(source.path) / ".meta" + + def _parts_dir(self) -> Path: + return self._output_dir() / ".timing_parts" + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport( + self, + item: pytest.Item, # noqa: ARG002 + call: pytest.CallInfo[None], + ) -> Generator[None, Any, None]: + """ + Persist this test's timing to a part file, in the running process. + + Reading ``user_properties`` here (rather than on the controller) + avoids relying on xdist to marshal them back; each test writes its + own uniquely-named file, so parallel workers never race. + """ + outcome = yield + if call.when != "call": + return + report = outcome.get_result() + if report.outcome != "passed": + return + payload = dict(report.user_properties).get(TIMING_PROPERTY) + if not payload: + return + parts_dir = self._parts_dir() + parts_dir.mkdir(parents=True, exist_ok=True) + (parts_dir / f"{uuid.uuid4().hex}.json").write_text( + json.dumps(payload) + ) + + def pytest_sessionfinish( + self, + session: pytest.Session, + exitstatus: int, # noqa: ARG002 + ) -> None: + """Aggregate all part files into the report (controller only).""" + if hasattr(session.config, "workerinput"): + return # xdist worker: parts already written by makereport + parts_dir = self._parts_dir() + if not parts_dir.is_dir(): + return + rows: List[Dict[str, Any]] = [] + for part in parts_dir.glob("*.json"): + rows.extend(_rows_from_payload(json.loads(part.read_text()))) + if not rows: + return + aggregated, self.repeats = _aggregate(rows) + rows = _ordered_rows(aggregated) + + output_dir = self._output_dir() + output_dir.mkdir(parents=True, exist_ok=True) + if self.fmt in ("both", "md"): + path = output_dir / "timing_consume.md" + path.write_text(_render_markdown(rows)) + self.written.append(path) + if self.fmt in ("both", "csv"): + path = output_dir / "timing_consume.csv" + path.write_text(_render_csv(rows)) + self.written.append(path) + + for part in parts_dir.glob("*.json"): + part.unlink() + parts_dir.rmdir() + + def pytest_terminal_summary( + self, + terminalreporter: Any, + exitstatus: int, # noqa: ARG002 + config: pytest.Config, # noqa: ARG002 + ) -> None: + """Point the user at the generated timing artifacts.""" + if not self.written: + return + terminalreporter.write_sep("=", "block execution timing report") + if self.repeats > 1: + terminalreporter.write_line( + f"metrics are the min over {self.repeats} repeat block(s) " + "per fixture" + ) + for path in self.written: + terminalreporter.write_line(f"timing report written to: {path}") diff --git a/packages/testing/src/execution_testing/client_clis/clis/monad.py b/packages/testing/src/execution_testing/client_clis/clis/monad.py index 0c5889785e6..36434247242 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/monad.py +++ b/packages/testing/src/execution_testing/client_clis/clis/monad.py @@ -21,9 +21,11 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +import ijson # type: ignore[import-untyped] import pytest from execution_testing.fixtures import BlockchainFixture, FixtureFormat +from execution_testing.fixtures.consume import BlockExecutionTiming from execution_testing.test_types import Transaction from ..fixture_consumer_tool import FixtureConsumerTool @@ -63,6 +65,21 @@ def _set_pdeathsig() -> None: _LIBC.prctl(1, signal.SIGTERM) +def _load_fixture( + fixture_path: Path, fixture_name: Optional[str] +) -> Dict[str, Any]: + """ + Load a single fixture from a (possibly multi-fixture) JSON file. + + ijson for low-memory footprint. + """ + with open(fixture_path, "rb") as f: + for name, fixture in ijson.kvitems(f, ""): + if fixture_name is None or name == fixture_name: + return fixture + raise KeyError(f"fixture {fixture_name!r} not found in {fixture_path}") + + def _hex32(value: str) -> str: """Normalize a hex quantity to a 0x-prefixed 64-nibble word.""" return f"0x{int(value, 16):064x}" @@ -160,6 +177,49 @@ def _compare_account( return mismatches +def _exec_block_row(line: str) -> Optional[BlockExecutionTiming]: + """Parse one `__exec_block` log line, or None if malformed.""" + body = line.split("__exec_block", 1)[1] + fields: Dict[str, str] = {} + for part in body.split(","): + key, sep, value = part.partition("=") + if sep: + fields[key.strip()] = value.strip() + + def us(key: str) -> int: + return int(fields[key].replace("µs", "").strip()) + + try: + return BlockExecutionTiming( + block=int(fields["bl"]), + tx_count=int(fields["tx"]), + gas=int(fields["gas"]), + tx_exec_us=us("txe"), + state_root_us=us("sr"), + commit_us=us("cmt"), + total_us=us("tot"), + ) + except (KeyError, ValueError): + return None + + +def _parse_block_timings(stdout: str) -> List[BlockExecutionTiming]: + """ + Extract per-block timing from the runloop's `__exec_block` log lines. + + The production runloop logs one such line per block, e.g.: + `__exec_block,bl=1,...,tx=1,...,sr=5192µs,txe=14241µs,cmt=879µs, + tot=21153µs,...,gas=10000000,...`. Fields carry leading padding and a + `µs` suffix on durations. Missing/malformed lines are skipped. + """ + rows = [ + _exec_block_row(line) + for line in stdout.splitlines() + if "__exec_block" in line + ] + return [row for row in rows if row is not None] + + class MonadFixtureConsumer( FixtureConsumerTool, fixture_formats=[BlockchainFixture], @@ -299,16 +359,11 @@ def consume_fixture( fixture_path: Path, fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, - ) -> None: + ) -> Optional[List[BlockExecutionTiming]]: """Execute a blockchain fixture on the monad runloop and verify.""" assert fixture_format == BlockchainFixture - with open(fixture_path) as f: - fixtures = json.load(f) - if fixture_name is None: - assert len(fixtures) == 1, "fixture_name required" - fixture_name = next(iter(fixtures)) - fixture = fixtures[fixture_name] + fixture = _load_fixture(fixture_path, fixture_name) network = fixture["network"] assert network in FORK_REVISION_SCHEDULES, ( @@ -362,6 +417,7 @@ def consume_fixture( ) output = json.loads(output_path.read_text()) + block_timings = _parse_block_timings(stdout) actual_post = { address.lower(): account @@ -394,3 +450,5 @@ def consume_fixture( "post-state mismatch on the monad runloop:\n" + "\n".join(mismatches) ) + + return block_timings or None diff --git a/packages/testing/src/execution_testing/fixtures/consume.py b/packages/testing/src/execution_testing/fixtures/consume.py index 499be630dd7..bf66ec6e26f 100644 --- a/packages/testing/src/execution_testing/fixtures/consume.py +++ b/packages/testing/src/execution_testing/fixtures/consume.py @@ -3,7 +3,7 @@ import datetime from abc import ABC, abstractmethod from pathlib import Path -from typing import Iterator, List, Optional, TextIO +from typing import Iterator, List, Optional, TextIO, TypedDict from pydantic import BaseModel, RootModel @@ -14,6 +14,24 @@ from .file import Fixtures +class BlockExecutionTiming(TypedDict): + """ + Per-block execution timing a consumer may optionally report. + + All durations are in microseconds. Consumers that can measure block + processing (e.g. the monad runloop) return these so `consume` can emit + a performance table; consumers that cannot simply return ``None``. + """ + + block: int + tx_count: int + gas: int + tx_exec_us: int + state_root_us: int + commit_us: int + total_us: int + + class FixtureConsumer(ABC): """Abstract class for verifying Ethereum test fixtures.""" @@ -33,10 +51,13 @@ def consume_fixture( fixture_path: Path, fixture_name: str | None = None, debug_output_path: Path | None = None, - ) -> None: + ) -> Optional[List[BlockExecutionTiming]]: """ Test the client with the specified fixture using its direct consumer interface. + + Optionally return per-block execution timing for a performance + report; consumers that do not measure timing return ``None``. """ raise NotImplementedError( "The `consume_fixture()` function is not supported by this tool." diff --git a/scripts/perf_disjoint_table.py b/scripts/perf_disjoint_table.py new file mode 100644 index 00000000000..4aaef03444d --- /dev/null +++ b/scripts/perf_disjoint_table.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +""" +Build a NINE-vs-NEXT significance table from perf timing runs. + +Reads the `timing_consume.csv` produced by `consume direct +--timing-report-dir` for several identical runs (one dir each) and, per +(test, params) case, reports each measure's mean +/- sd over the runs for +both forks, the two-sided Mann-Whitney U p-value comparing the forks, +and (for measures significant at p <= 0.10) the NINE->NEXT average +change. A trailing list describes each significant case's transactions. + +Usage: perf_disjoint_table.py [--html OUT.html] [DIR ...] + (default dirs: ../timing_[0-9]*) +Emits GitHub-flavored Markdown to stdout; with --html also writes a +standalone HTML rendering whose table spans the full window width. +""" + +from __future__ import annotations + +import csv +import glob +import html +import sys +from itertools import combinations +from math import comb, erfc, sqrt +from pathlib import Path +from statistics import mean, stdev + +METRICS = ["tx_exec_us", "state_root_us", "commit_us", "total_us"] +FORKS = ["MONAD_NINE", "MONAD_NEXT"] +ALPHA = 0.10 # a measure is significant at this Mann-Whitney U p-value + +UP = "🔴⬆️" # significant measures all rise NINE->NEXT (NEXT slower) +DOWN = "🟢⬇️" # significant measures all fall NINE->NEXT (NEXT faster) +MIXED = "⚠️" # significant measures move both up and down + + +def _direction(chgs: list[int]) -> str: + """Pick the direction emoji from the significant measures' changes.""" + ups = any(c > 0 for c in chgs) + downs = any(c < 0 for c in chgs) + if ups and downs: + return MIXED + if ups: + return UP + if downs: + return DOWN + return MIXED + + +def parse(report: Path) -> dict: + """Map (test, params, fork) -> {metric: value} from one csv report.""" + rows: dict = {} + with report.open(newline="") as f: + for row in csv.DictReader(f): + if row["fork"] not in FORKS: + continue + try: + mv = {m: int(row[m]) for m in METRICS} + except (ValueError, TypeError): + continue + test = row["test"].split("::")[-1].removeprefix("test_") + rows[(test, row["params"], row["fork"])] = mv + return rows + + +def _stat(values: list[int]) -> str: + """Format a run sample as 'mean ± sd' (µs, rounded).""" + sd = stdev(values) if len(values) > 1 else 0.0 + return f"{round(mean(values))} ± {round(sd)}" + + +def _pfmt(p: float) -> str: + """Format a p-value compactly.""" + return "<0.001" if p < 0.001 else f"{p:.3f}" + + +def _avg_ranks(vals: list[float]) -> list[float]: + """Return 1-based ranks (ties averaged), aligned to `vals`.""" + order = sorted(range(len(vals)), key=lambda i: vals[i]) + ranks = [0.0] * len(vals) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and vals[order[j + 1]] == vals[order[i]]: + j += 1 + avg = (i + j) / 2 + 1 + for k in range(i, j + 1): + ranks[order[k]] = avg + i = j + 1 + return ranks + + +def _mwu_p(a: list[int], b: list[int]) -> float: + """ + Two-sided Mann-Whitney U p-value comparing samples `a` and `b`. + + Exact (permutation of ranks) for small samples; a normal + approximation with continuity correction kicks in past 200k + combinations. Distribution-free — no normality assumption. + """ + n, m = len(a), len(b) + if n == 0 or m == 0: + return 1.0 + ranks = _avg_ranks([*a, *b]) + total = n + m + mu = n * (total + 1) / 2 # null mean of group-a rank sum + d = abs(sum(ranks[:n]) - mu) + if comb(total, n) <= 200_000: + dist = [sum(c) for c in combinations(ranks, n)] + extreme = sum(1 for s in dist if abs(s - mu) >= d - 1e-9) + return extreme / len(dist) + sigma = sqrt(n * m * (total + 1) / 12) + if sigma == 0: + return 1.0 + return erfc(max(0.0, (d - 0.5) / sigma) / sqrt(2)) + + +def _op_phrase(op: str, k: int, layout: str | None) -> str: + """Describe what one block-filling tx does for a storage op.""" + place = f"{layout} pages" if layout else "pages" + page = f"{layout} page" if layout else "page" + occ = f", {k} slot{'' if k == 1 else 's'} occupied per page" + phrases = { + "sload_cold_hit": f"cold-SLOAD one occupied slot on many {place}{occ}", + "sload_cold_miss": f"cold-SLOAD an empty slot on many occupied " + f"{place}{occ}", + "sload_sweep": f"cold-read all {k} occupied slots of each {page}", + "sload_warm_repeat": f"cold-SLOAD a slot of each {page} then " + f"warm-re-read it repeatedly", + "sstore_fresh": f"SSTORE 0->1 into previously-unoccupied slots on " + f"many {place}", + "sstore_noop": f"SSTORE 1->1 (value unchanged) on occupied " + f"{place}{occ}", + "sstore_grow": f"SSTORE 0->1 into a new empty slot of occupied " + f"{place}{occ}", + "sstore_update": f"SSTORE 1->2 (nonzero value change) on occupied " + f"{place}{occ}", + "sstore_clear_keep": f"SSTORE 1->0 clearing one slot of occupied " + f"{place}{occ}, leaving the page populated", + "sstore_clear_empty": f"SSTORE 1->0 clearing the only slot of " + f"single-slot {place}, removing the page", + "sload_empty_page": f"cold-SLOAD a slot on never-populated, empty " + f"{place}", + } + return phrases.get(op, op) + + +def describe(test: str, params: str) -> str: + """One-sentence account of what a case's block transactions do.""" + if test == "page_ops": + op, kpart, layout = params.split("-") + k = int(kpart.removeprefix("k")) + return f"block-filling transactions {_op_phrase(op, k, layout)}" + if test == "block_shape": + op, kpart, shape = params.split("-") + k = int(kpart.removeprefix("k")) + who = "a few big" if shape == "few_big" else "many small (~300)" + return f"{who} transactions each {_op_phrase(op, k, None)}" + if test == "page_spread": + m = params.split("_")[0].removeprefix("m") + n = int(params.split("_")[1].removeprefix("n")) + layout = params.split("_")[2] + target = f"{n} contract" + ("" if n == 1 else "s") + return ( + f"transactions SSTORE 0->1 into {m} fresh slots spread across " + f"{target} ({layout} pages)" + ) + if test == "tx_halt": + mode = params.removeprefix("mode_") + if mode == "success": + return ( + "seven transactions each SSTORE 0->1 into fresh slots and " + "succeed" + ) + if mode == "halt": + return ( + "seven transactions SSTORE 0->1 into fresh slots then hit " + "INVALID, reverting all writes" + ) + return ( + "seven transactions alternate between SSTORE-and-succeed and " + "SSTORE-then-INVALID (halted writes reverted)" + ) + if test == "random_sload": + slots = params.split("-")[0].removeprefix("slots") + k = int(params.split("-")[1].removeprefix("k")) + page = "a 1-element" if k == 1 else "an empty" + noun = "slot" if slots == "1" else "slots" + return ( + f"cold-SLOAD {slots} pseudorandom {noun} (each on {page} page) " + "in a pseudorandom cycle, from a pool of contracts called in a " + "pseudorandom cycle" + ) + if test == "bad_block_serial": + return ( + "every tx SLOAD+SSTORE-increments the same slot sequence, " + "forcing serial execution across the block" + ) + if test == "bad_block_chained": + return ( + "each SLOAD returns the next SLOAD's slot (SLOAD(SLOAD(...))), " + "a data-dependent chain that serialises the reads within a tx" + ) + return f"{test} {params}" + + +def build(runs: list[dict]) -> list[str]: + """Return the markdown lines for the table plus case descriptions.""" + cases: dict = {} + for run in runs: + for (test, params, fork), mv in run.items(): + per_fork = cases.setdefault((test, params), {}) + metrics = per_fork.setdefault(fork, {m: [] for m in METRICS}) + for m in METRICS: + metrics[m].append(mv[m]) + + header = ["test-params"] + for m in METRICS: + header += [f"{m} NINE", f"{m} NEXT", f"{m} p"] + header += ["significant", "Δ avg NINE→NEXT (sig)"] + + lines = [ + f"Mean ± sd over {len(runs)} runs (µs). Bold = measure significant " + f"(Mann–Whitney U p ≤ {ALPHA}). " + f"Significant flag: {UP} NEXT slower, {DOWN} NEXT faster, " + f"{MIXED} mixed.", + "", + "| " + " | ".join(header) + " |", + "| " + " | ".join("---" for _ in header) + " |", + ] + + sig_cases = [] + for test, params in sorted(cases): + forks = cases[(test, params)] + if not all(f in forks for f in FORKS): + continue + cells = [f"{test} {params}"] + deltas = [] + chgs = [] + for m in METRICS: + n, x = forks["MONAD_NINE"][m], forks["MONAD_NEXT"][m] + p = _mwu_p(n, x) + ncell, xcell = _stat(n), _stat(x) + if p <= ALPHA: + ncell, xcell = f"**{ncell}**", f"**{xcell}**" + navg, xavg = mean(n), mean(x) + chg = round((xavg - navg) / navg * 100) if navg else 0 + deltas.append(f"{m} {chg:+d}%") + chgs.append(chg) + cells += [ncell, xcell, _pfmt(p)] + emoji = _direction(chgs) if chgs else "-" + cells.append(emoji) + cells.append(", ".join(deltas) if deltas else "-") + lines.append("| " + " | ".join(cells) + " |") + if chgs: + sig_cases.append((test, params, emoji)) + + lines += [ + "", + "p (MWU) is the two-sided Mann–Whitney U p-value comparing the " + f"NINE and NEXT run samples for that measure; significant at " + f"p ≤ {ALPHA}.", + "", + "Significant cases — what the block's transactions do:", + "", + ] + for test, params, emoji in sig_cases: + lines.append( + f"- **{test} {params}**: {describe(test, params)}. {emoji}" + ) + return lines + + +HTML_TEMPLATE = """ + + + + +MIP-8 perf: NINE vs NEXT significance + + + +__BODY__ + + +""" + + +def _inline(text: str) -> str: + """Render inline **bold** markdown to HTML, escaping the rest.""" + out = [] + for i, part in enumerate(text.split("**")): + esc = html.escape(part) + out.append(f"{esc}" if i % 2 else esc) + return "".join(out) + + +def _table_html(block: list[str]) -> str: + """Render a markdown table (list of `|`-rows) as an HTML table.""" + rows = [ + [c.strip() for c in r.strip().strip("|").split("|")] for r in block + ] + head = "".join(f"{_inline(c)}" for c in rows[0]) + body = [ + "" + "".join(f"{_inline(c)}" for c in row) + "" + for row in rows[2:] + ] + return ( + '
\n' + f"{head}\n\n" + + "\n".join(body) + + "\n
" + ) + + +def md_to_html(md: str) -> str: + """Render the generated markdown report as a standalone HTML page.""" + lines = md.split("\n") + blocks: list[str] = [] + i = 0 + while i < len(lines): + if lines[i].startswith("|"): + table = [] + while i < len(lines) and lines[i].startswith("|"): + table.append(lines[i]) + i += 1 + blocks.append(_table_html(table)) + elif lines[i].startswith("- "): + items = [] + while i < len(lines) and lines[i].startswith("- "): + items.append(f"
  • {_inline(lines[i][2:])}
    • ") + i += 1 + blocks.append("") + else: + if lines[i].strip(): + blocks.append(f"

    {_inline(lines[i])}

    ") + i += 1 + return HTML_TEMPLATE.replace("__BODY__", "\n".join(blocks)) + + +def main() -> None: + """Parse the run dirs, print markdown, optionally write HTML.""" + argv = list(sys.argv[1:]) + html_path = None + if "--html" in argv: + idx = argv.index("--html") + if idx + 1 >= len(argv): + sys.exit("--html requires a path") + html_path = argv[idx + 1] + del argv[idx : idx + 2] + dirs = argv or sorted( + glob.glob("../timing_[0-9]*"), + key=lambda p: int(p.rsplit("_", 1)[-1]), + ) + runs = [ + parse(Path(d) / "timing_consume.csv") + for d in dirs + if (Path(d) / "timing_consume.csv").exists() + ] + if len(runs) < 2: + sys.exit("need >=2 runs with timing_consume.csv") + md = "\n".join(build(runs)) + print(md) + if html_path: + Path(html_path).write_text(md_to_html(md), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py new file mode 100644 index 00000000000..1c1658f588b --- /dev/null +++ b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py @@ -0,0 +1,1129 @@ +""" +MIP-8 performance-regression tests. + +These blockchain tests compare block/transaction execution time on the +monad runloop before MIP-8 (MONAD_NINE, slot-encoded storage) and after +MIP-8 (MONAD_NEXT, page-encoded storage). The same Python builds the +workload for both forks; the `fork` fixture is only consulted where the +two forks genuinely differ (storage-op gas, used to size a workload to a +gas budget). We do not assert gas — the oracle is post-state (success +markers, written slots, and read-checksum slots), so the identical +workload can be timed on both forks via the consume timing report. + +The suite fills the 200M-gas monad block with SLOAD/SSTORE patterns. + +Single-letter test parameters: +- `k`: page occupancy — non-zero slots pre-populated per page (0..128). +- `m`: number of distinct pages a spread test writes in total. +- `n`: number of contracts those `m` pages are spread evenly across. + +Workloads are sized to MONAD_NINE (its cold storage costs >= MONAD_NEXT), +so the same iteration count never out-of-gases on either fork; a +post-fork block may therefore be gas-underfull while doing identical I/O +work, which is exactly the effect being measured. + +`MIP8_PERF_REPEATS` (default 1) emits that many copies of each workload +as successive blocks, each offset to a disjoint page range so every block +is a genuine cold execution. One runloop run then yields N independent +per-block timing samples (the first absorbs process/hugepage warmup); the +consume timing report aggregates them with `min`. The contract bytecode +stays fixed-size regardless of the repeat count — only the pre-state +storage and block list grow. + +Run with `--monad-runloop` and consume via `eest-runner`; see +MONAD_RUNLOOP_TESTING.md. +""" + +import os +from enum import StrEnum, auto +from typing import List, SupportsBytes, Tuple + +import pytest +from execution_testing import ( + Account, + Address, + Alloc, + Block, + BlockchainTestFiller, + Bytecode, + Conditional, + Environment, + Op, + Transaction, + While, + WhileGas, +) +from execution_testing.forks import MONAD_NEXT, MONAD_NINE +from execution_testing.forks.helpers import Fork + +from .helpers import fresh_sstore_cold +from .spec import Spec, ref_spec_8 + +REFERENCE_SPEC_GIT_PATH = ref_spec_8.git_path +REFERENCE_SPEC_VERSION = ref_spec_8.version + +# Key/value type accepted by `deploy_contract`/`Account` storage args; +# pre-built dicts need this annotation (dict is invariant). +StorageDict = dict[ + str | int | bytes | SupportsBytes, str | int | bytes | SupportsBytes +] + +SLOTS_PER_PAGE = Spec.SLOTS_PER_PAGE # 128 + +# The runloop stamps every monad block at 200M gas; the per-tx cap is +# 30M on both forks. Overridable for quick local smoke fills. +BLOCK_GAS_TARGET = int(os.environ.get("MIP8_PERF_BLOCK_GAS", "200000000")) +TX_GAS_CAP = 30_000_000 +# 200M / 30M -> 7 equal txs of ~28.57M each fill a block. +FULL_BLOCK_TXS = 7 +# Workload txs are EIP-1559 with a high max fee and a zero priority tip. +# The high max fee keeps them valid as each full block raises the base +# fee ~12.5%; the zero tip means no fee reaches the block coinbase, so the +# fee routing matches between the fill and the runloop on both forks (a +# nonzero tip is credited to the coinbase by the fill but routed elsewhere +# by the runloop on MONAD_NEXT, which would mismatch the post-state). +MAX_FEE_PER_GAS = 10**6 +# `many_small` block shape: many txs, each still large enough to cover the +# per-tx reserve. The count adapts to the block budget (300 at 200M) so the +# `MIP8_PERF_BLOCK_GAS` smoke knob does not starve individual txs. +MANY_SMALL_TXS = 300 +MANY_SMALL_MIN_TX_GAS = 200_000 + +# Emit this many copies of each workload as successive, page-disjoint +# blocks for repeat timing samples (see module docstring). +REPEATS = int(os.environ.get("MIP8_PERF_REPEATS", "1")) + +# Upper bound on pre-populated storage slots per block (times REPEATS for +# the whole fixture's genesis). +PRE_SLOT_CAP = 65_536 + +# Non-unit stride (in page indices) for the "scattered" layout. +SCATTERED_STRIDE = 1009 + +# Gas of one While control step (JUMPDEST + JUMPI + counter compare); +# a small over-estimate keeps sizing on the safe side (no OOG). +WHILE_CONTROL_GAS = 40 +# Per-tx headroom: intrinsic + calldata + two cold marker SSTOREs + slack. +TX_RESERVE = 120_000 + +# Page/slot domains, chosen far apart so nothing collides. Each repeat r +# shifts a page domain by r * REPEAT_STRIDE, well above any single +# workload's page extent and far below the next domain. +REPEAT_STRIDE = 1 << 28 +READ_DOMAIN = 1 << 40 # page-index base for reused (pre-populated) pages +FRESH_DOMAIN = 1 << 52 # page-index base for fresh-write pages +WARM_BASE = 1 << 70 # per-repeat slot re-read by sload_warm_repeat +MARKER_BASE = 1 << 200 # per-tx success marker slot = MARKER_BASE + global_idx +CKSUM_BASE = 1 << 220 # per-tx read-checksum slot = CKSUM_BASE + global_idx + +# Memory layout inside the workload contract. +M_COUNTER = 0x00 +M_CHECKSUM = 0x20 +M_BASE = 0x40 +M_COUNT = 0x60 +M_GLOBAL = 0x80 # global tx index (marker/checksum slot) +M_PAGE = 0xA0 # sweep page-base scratch +M_LOCAL = 0xC0 # per-block tx index (grow write offset) + +# Calldata layout: base | count | global_idx | halt | local_idx. +CD_BASE = 0x00 +CD_COUNT = 0x20 +CD_GLOBAL = 0x40 +CD_HALT = 0x60 +CD_LOCAL = 0x80 + +slot_code_worked = 0x1 +value_code_worked = 0x1234 + +COMPUTE_GAS = 10_000_000 + + +class StorageOp(StrEnum): + """A storage-access pattern a workload applies to each page.""" + + SLOAD_COLD_HIT = auto() + """Cold SLOAD of an occupied slot (offset 0) on each page.""" + SLOAD_COLD_MISS = auto() + """Cold SLOAD of an empty slot (offset 127) on an occupied page.""" + SLOAD_SWEEP = auto() + """Cold-read every occupied slot of each page.""" + SLOAD_WARM_REPEAT = auto() + """One cold SLOAD of a slot, then repeated warm re-reads of it.""" + SSTORE_FRESH = auto() + """SSTORE 0->1 into empty pages (fresh state growth).""" + SSTORE_NOOP = auto() + """SSTORE 1->1 on occupied pages (rewrite, no value change).""" + SSTORE_GROW = auto() + """SSTORE 0->1 on a new slot of an already-occupied page.""" + SSTORE_UPDATE = auto() + """SSTORE 1->2 on an occupied slot (value change, no growth).""" + SSTORE_CLEAR_KEEP = auto() + """SSTORE 1->0 clearing one slot; the page keeps its other slots.""" + SSTORE_CLEAR_EMPTY = auto() + """SSTORE 1->0 clearing a page's only slot, so the page is removed.""" + SLOAD_EMPTY_PAGE = auto() + """Cold SLOAD of a slot on a never-populated, empty page.""" + + +@pytest.mark.valid_from("MONAD_NINE") +def test_compute_loop( + blockchain_test: BlockchainTestFiller, + pre: Alloc, + fork: Fork, +) -> None: + """ + Run a gas-bounded arithmetic loop, then write a success marker. + + A stack-neutral arithmetic body is repeated until the compute gas + budget is nearly spent, reserving just enough for the trailing + SSTORE. Pure stack arithmetic touches no storage, so the loop costs + the same on both forks and the marker is the only state written. + """ + sender = pre.fund_eoa() + + body = Op.POP(Op.ADD(Op.MUL(Op.NUMBER, Op.GAS), Op.CALLVALUE)) + contract_address = pre.deploy_contract( + WhileGas(body=body, fork=fork, extra_gas=fresh_sstore_cold(fork)) + + Op.SSTORE(slot_code_worked, value_code_worked) + ) + + blocks = [ + Block( + txs=[ + Transaction( + to=contract_address, + sender=sender, + gas_limit=COMPUTE_GAS, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + ), + ], + ), + ] + + blockchain_test( + pre=pre, + blocks=blocks, + post={ + contract_address: Account( + storage={slot_code_worked: value_code_worked} + ), + }, + ) + + +# --- shared bytecode + sizing helpers --------------------------------- + + +def _stride(layout: str) -> int: + """Return the page-index stride for a layout name.""" + return 1 if layout == "contiguous" else SCATTERED_STRIDE + + +def _page_index(stride: int) -> Bytecode: + """Bytecode: base + i*stride (base at M_BASE, i at M_COUNTER).""" + i = Op.MLOAD(M_COUNTER) + return Op.ADD(Op.MLOAD(M_BASE), Op.MUL(i, stride)) + + +def _slot(stride: int, offset: int | Bytecode) -> Bytecode: + """Bytecode: (page_index << 7) + offset.""" + return Op.ADD(offset, Op.SHL(7, _page_index(stride))) + + +def _read_accum(slot: Bytecode | int, *, warm: bool) -> Bytecode: + """Accumulate SLOAD(slot) into the checksum word.""" + load = Op.SLOAD(slot, key_warm=warm, page_load_warm=warm) + return Op.MSTORE(M_CHECKSUM, Op.ADD(Op.MLOAD(M_CHECKSUM), load)) + + +def _sstore( + slot: Bytecode, + value: int | Bytecode, + *, + original: int, + current: int, + new: int, + growth: int, +) -> Bytecode: + """ + Build a cold-page SSTORE(slot, value). + + `value` is what is written at run time; `original`/`current`/`new`/ + `growth` describe the slot transition for gas sizing only. `original` + (the slot's value at tx start) is required so a nonzero-to-nonzero or + nonzero-to-zero reset is priced as a reset on MONAD_NINE, not a no-op. + """ + return Op.SSTORE( + slot, + value, + key_warm=False, + page_load_warm=False, + page_write_warm=False, + original_value=original, + current_value=current, + new_value=new, + current_state_growth=growth, + net_state_growth=growth, + ) + + +def _body(op: StorageOp, k: int, stride: int) -> Bytecode: + """Assemble the per-iteration loop body for an operation.""" + inc = Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) + if op is StorageOp.SLOAD_COLD_HIT: + return _read_accum(_slot(stride, 0), warm=False) + inc + if op is StorageOp.SLOAD_COLD_MISS: + return _read_accum(_slot(stride, SLOTS_PER_PAGE - 1), warm=False) + inc + if op is StorageOp.SLOAD_WARM_REPEAT: + # The slot is passed in calldata (M_BASE) so each block re-reads a + # distinct slot; cold on the first iteration, warm thereafter. + return _read_accum(Op.MLOAD(M_BASE), warm=True) + inc + if op is StorageOp.SLOAD_SWEEP: + code = Op.MSTORE(M_PAGE, Op.SHL(7, _page_index(stride))) + for j in range(k): + code += _read_accum(Op.ADD(Op.MLOAD(M_PAGE), j), warm=False) + return code + inc + if op is StorageOp.SSTORE_FRESH: + return ( + _sstore( + _slot(stride, 0), 1, original=0, current=0, new=1, growth=0 + ) + + inc + ) + if op is StorageOp.SSTORE_NOOP: + return ( + _sstore( + _slot(stride, 0), 1, original=1, current=1, new=1, growth=0 + ) + + inc + ) + if op is StorageOp.SSTORE_GROW: + offset = Op.ADD(k, Op.MLOAD(M_LOCAL)) + return ( + _sstore( + _slot(stride, offset), + 1, + original=0, + current=0, + new=1, + growth=k, + ) + + inc + ) + if op is StorageOp.SSTORE_UPDATE: + # Each tx writes a new nonzero value (2 + its index), so every tx + # is a genuine value change on the still-occupied slot 0. + value = Op.ADD(2, Op.MLOAD(M_LOCAL)) + return ( + _sstore( + _slot(stride, 0), value, original=1, current=1, new=2, growth=0 + ) + + inc + ) + if op is StorageOp.SSTORE_CLEAR_KEEP: + # Tx t clears slot offset t; with k > FULL_BLOCK_TXS the page keeps + # its higher slots, so each clear is a genuine 1->0 on a live page. + offset = Op.MLOAD(M_LOCAL) + return ( + _sstore( + _slot(stride, offset), + 0, + original=1, + current=1, + new=0, + growth=0, + ) + + inc + ) + if op is StorageOp.SSTORE_CLEAR_EMPTY: + return ( + _sstore( + _slot(stride, 0), 0, original=1, current=1, new=0, growth=0 + ) + + inc + ) + if op is StorageOp.SLOAD_EMPTY_PAGE: + return _read_accum(_slot(stride, 0), warm=False) + inc + raise ValueError(f"unknown op {op}") + + +def _is_read(op: StorageOp) -> bool: + """Return whether the operation reads (and writes a checksum).""" + return op.value.startswith("sload") + + +def _contract(op: StorageOp, k: int, stride: int) -> Bytecode: + """ + Build the workload contract (fixed size, independent of REPEATS). + + Reads calldata (base, count, global_idx, halt, local_idx), runs + `count` iterations of the op's body, writes a success marker (and, for + reads, a checksum) keyed by the global index, then STOPs — or, if the + halt flag is set, executes INVALID so the whole transaction reverts. + """ + init = ( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_LOCAL, Op.CALLDATALOAD(CD_LOCAL)) + + Op.MSTORE(M_COUNTER, 0) + ) + loop = While( + body=_body(op, k, stride), + condition=Op.LT(Op.MLOAD(M_COUNTER), Op.MLOAD(M_COUNT)), + ) + markers = Op.SSTORE( + Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked + ) + if _is_read(op): + markers += Op.SSTORE( + Op.ADD(CKSUM_BASE, Op.MLOAD(M_GLOBAL)), Op.MLOAD(M_CHECKSUM) + ) + tail = Conditional( + condition=Op.CALLDATALOAD(CD_HALT), + if_true=Op.INVALID, + if_false=Op.STOP, + ) + return init + loop + markers + tail + + +def _calldata( + base: int, count: int, global_idx: int, halt: int, local_idx: int = 0 +) -> bytes: + """Encode the five-word calldata for one transaction.""" + return b"".join( + v.to_bytes(32, "big") + for v in (base, count, global_idx, halt, local_idx) + ) + + +def _per_iter_gas(op: StorageOp, k: int, stride: int) -> int: + """Gas for one loop iteration, sized to the costlier of both forks.""" + body = _body(op, k, stride) + per_op = max(body.gas_cost(MONAD_NINE), body.gas_cost(MONAD_NEXT)) + return per_op + WHILE_CONTROL_GAS + + +def _iterations(op: StorageOp, k: int, stride: int, budget: int) -> int: + """Loop iterations that fit `budget` gas, leaving tx headroom.""" + return max(1, (budget - TX_RESERVE) // _per_iter_gas(op, k, stride)) + + +def _occupied_prestate( + domain: int, stride: int, pages: int, k: int +) -> StorageDict: + """Pre-populate `pages` pages (from `domain`) with `k` slots each.""" + storage: StorageDict = {} + for i in range(pages): + base_slot = (domain + i * stride) << 7 + for j in range(k): + storage[base_slot + j] = 1 + return storage + + +def _repeat_domains(repeat: int) -> Tuple[int, int, int]: + """Return (read_domain, fresh_domain, warm_slot) for a repeat index.""" + shift = repeat * REPEAT_STRIDE + return READ_DOMAIN + shift, FRESH_DOMAIN + shift, WARM_BASE + repeat + + +# --- dimensions 1, 2, 3, 6: op x occupancy x layout ------------------- + +# (op, layout, page-occupancy k values). Contiguous and scattered are +# listed separately so their k coverage can diverge later (scattered +# fills are far more expensive); for now both share the same k lists. +# cold_miss/grow keep a zero slot free (k < SLOTS_PER_PAGE, and grow +# writes k..k+FULL_BLOCK_TXS-1). clear_keep needs k > FULL_BLOCK_TXS so a +# page still has slots after the block clears offsets 0..FULL_BLOCK_TXS-1. +# clear_empty uses single-slot pages (k=1) that vanish when cleared; +# empty_page reads never-set pages (k=0). +_PAGE_OP_LAYOUT_KS = [ + (StorageOp.SLOAD_COLD_HIT, "contiguous", [1, 16, 128]), + (StorageOp.SLOAD_COLD_MISS, "contiguous", [1, 16, 64]), + (StorageOp.SLOAD_SWEEP, "contiguous", [2, 16, 128]), + (StorageOp.SSTORE_NOOP, "contiguous", [1, 16, 128]), + (StorageOp.SSTORE_GROW, "contiguous", [1, 16, 64]), + (StorageOp.SSTORE_UPDATE, "contiguous", [1, 16, 128]), + (StorageOp.SSTORE_CLEAR_KEEP, "contiguous", [8, 16, 128]), + (StorageOp.SSTORE_CLEAR_EMPTY, "contiguous", [1]), + (StorageOp.SLOAD_EMPTY_PAGE, "contiguous", [0]), + (StorageOp.SLOAD_WARM_REPEAT, "contiguous", [1]), + (StorageOp.SSTORE_FRESH, "contiguous", [0]), + (StorageOp.SLOAD_COLD_HIT, "scattered", [1, 128]), + (StorageOp.SLOAD_COLD_MISS, "scattered", [1, 64]), + (StorageOp.SLOAD_SWEEP, "scattered", [2, 128]), + (StorageOp.SSTORE_NOOP, "scattered", [1, 128]), + (StorageOp.SSTORE_GROW, "scattered", [1, 64]), + (StorageOp.SSTORE_UPDATE, "scattered", [1, 128]), + (StorageOp.SSTORE_CLEAR_KEEP, "scattered", [8, 128]), + (StorageOp.SSTORE_CLEAR_EMPTY, "scattered", [1]), + (StorageOp.SLOAD_EMPTY_PAGE, "scattered", [0]), + (StorageOp.SLOAD_WARM_REPEAT, "scattered", [1]), + (StorageOp.SSTORE_FRESH, "scattered", [0]), +] + +_PAGE_OP_PARAMS = [ + pytest.param(op, k, layout, id=f"{op.value}-k{k}-{layout}") + for op, layout, ks in _PAGE_OP_LAYOUT_KS + for k in ks +] + + +@pytest.mark.parametrize("op, k, layout", _PAGE_OP_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_page_ops( + blockchain_test: BlockchainTestFiller, + pre: Alloc, + op: StorageOp, + k: int, + layout: str, +) -> None: + """ + Fill a block with one storage-op pattern over `pages` pages. + + Occupied-page ops (reads, no-op, growing/updating writes, and the + keep-a-slot clear) reuse a per-block pre-populated page pool: every + transaction re-touches the same pages, paying cold access each time + (per-tx warmth resets). Fresh writes and the page-emptying clear give + each transaction its own disjoint page range; the emptying clear + pre-populates one slot per page so each tx removes whole pages. The + page count is bounded by the tx gas budget and, for pre-populated ops, + by PRE_SLOT_CAP // k — so high-occupancy cold-read blocks are pre-state + bound and run below 200M by design. With REPEATS > 1 each repeat block + is offset to a fresh page range. + """ + stride = _stride(layout) + budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS + pages = _iterations(op, k, stride, budget) + + per_tx_pages = op is StorageOp.SSTORE_CLEAR_EMPTY + occupied = op not in ( + StorageOp.SSTORE_FRESH, + StorageOp.SLOAD_WARM_REPEAT, + StorageOp.SSTORE_CLEAR_EMPTY, + ) + if occupied: + pages = min(pages, PRE_SLOT_CAP // max(k, 1)) + elif per_tx_pages: + pages = min(pages, PRE_SLOT_CAP // FULL_BLOCK_TXS) + + sender = pre.fund_eoa() + + prestate: StorageDict = {} + for r in range(REPEATS): + read_dom, fresh_dom, warm_slot = _repeat_domains(r) + if op is StorageOp.SLOAD_WARM_REPEAT: + prestate[warm_slot] = 1 + elif per_tx_pages: + for t in range(FULL_BLOCK_TXS): + dom = fresh_dom + t * pages * stride + prestate.update(_occupied_prestate(dom, stride, pages, 1)) + elif occupied: + prestate.update(_occupied_prestate(read_dom, stride, pages, k)) + contract = pre.deploy_contract(_contract(op, k, stride), storage=prestate) + + blocks = [] + expected: StorageDict = dict(prestate) + global_idx = 0 + for r in range(REPEATS): + read_dom, fresh_dom, warm_slot = _repeat_domains(r) + txs = [] + for t in range(FULL_BLOCK_TXS): + if op in (StorageOp.SSTORE_FRESH, StorageOp.SSTORE_CLEAR_EMPTY): + base = fresh_dom + t * pages * stride + elif op is StorageOp.SLOAD_WARM_REPEAT: + base = warm_slot + else: + base = read_dom + txs.append( + Transaction( + to=contract, + sender=sender, + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(base, pages, global_idx, 0, t), + ) + ) + expected[MARKER_BASE + global_idx] = value_code_worked + if _is_read(op): + if op is StorageOp.SLOAD_SWEEP: + checksum = pages * k + elif op in ( + StorageOp.SLOAD_COLD_MISS, + StorageOp.SLOAD_EMPTY_PAGE, + ): + checksum = 0 + else: # cold_hit, warm_repeat: each read returns 1 + checksum = pages + if checksum: + expected[CKSUM_BASE + global_idx] = checksum + global_idx += 1 + + if op is StorageOp.SSTORE_GROW: + for i in range(pages): + base_slot = (read_dom + i * stride) << 7 + for t in range(FULL_BLOCK_TXS): + expected[base_slot + (k + t)] = 1 + elif op is StorageOp.SSTORE_FRESH: + for j in range(FULL_BLOCK_TXS * pages): + expected[(fresh_dom + j * stride) << 7] = 1 + elif op is StorageOp.SSTORE_UPDATE: + # Slot 0 is written once per tx (1->2->...); it ends at + # 1 + FULL_BLOCK_TXS. Other occupied slots stay 1. + for i in range(pages): + expected[(read_dom + i * stride) << 7] = 1 + FULL_BLOCK_TXS + elif op is StorageOp.SSTORE_CLEAR_KEEP: + for i in range(pages): + base_slot = (read_dom + i * stride) << 7 + for t in range(FULL_BLOCK_TXS): + expected[base_slot + t] = 0 + elif op is StorageOp.SSTORE_CLEAR_EMPTY: + for t in range(FULL_BLOCK_TXS): + for i in range(pages): + expected[(fresh_dom + (t * pages + i) * stride) << 7] = 0 + + blocks.append(Block(txs=txs)) + + blockchain_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + ) + + +# --- dimension 4: m pages spread across n contracts ------------------- + +_SPREAD_PARAMS = [ + pytest.param(m, n, layout, id=f"m{m}_n{n}_{layout}") + for m, n, layout in [ + (1, 1, "contiguous"), + (4, 1, "contiguous"), + (16, 1, "contiguous"), + (64, 1, "contiguous"), + (256, 1, "contiguous"), + (1024, 1, "contiguous"), + (4096, 1, "contiguous"), + (4096, 8, "contiguous"), + (4096, 64, "contiguous"), + (4096, 512, "contiguous"), + (4096, 1, "scattered"), + (4096, 512, "scattered"), + ] +] + + +@pytest.mark.parametrize("m, n, layout", _SPREAD_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_page_spread( + blockchain_test: BlockchainTestFiller, + pre: Alloc, + m: int, + n: int, + layout: str, +) -> None: + """ + Fresh-write `m` distinct pages spread evenly over `n` contracts. + + Each contract writes `m // n` fresh pages, split into as many + transactions as the per-tx gas cap requires (so large per-contract + shares become several txs, and many contracts become many small + txs). Isolates the effect of write distribution across accounts. Each + repeat block writes a fresh page range. + """ + stride = _stride(layout) + per_iter = _per_iter_gas(StorageOp.SSTORE_FRESH, 0, stride) + max_per_tx = max(1, (TX_GAS_CAP - TX_RESERVE) // per_iter) + pages_per_contract = m // n + + sender = pre.fund_eoa() + contracts: List[Address] = [ + pre.deploy_contract(_contract(StorageOp.SSTORE_FRESH, 0, stride)) + for _ in range(n) + ] + + blocks = [] + contract_storage: dict[Address, StorageDict] = {c: {} for c in contracts} + global_idx = 0 + for r in range(REPEATS): + _, fresh_dom, _ = _repeat_domains(r) + txs = [] + for contract in contracts: + done = 0 + while done < pages_per_contract: + count = min(max_per_tx, pages_per_contract - done) + txs.append( + Transaction( + to=contract, + sender=sender, + gas_limit=count * per_iter + TX_RESERVE, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata( + fresh_dom + done * stride, count, global_idx, 0 + ), + ) + ) + contract_storage[contract][MARKER_BASE + global_idx] = ( + value_code_worked + ) + global_idx += 1 + done += count + for j in range(pages_per_contract): + slot = (fresh_dom + j * stride) << 7 + contract_storage[contract][slot] = 1 + blocks.append(Block(txs=txs)) + + blockchain_test( + pre=pre, + blocks=blocks, + post={ + contract: Account(storage=storage) + for contract, storage in contract_storage.items() + }, + genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + ) + + +# --- dimension 5: few big vs many small transactions ------------------ + +_SHAPE_PARAMS = [ + pytest.param(op, k, shape, id=f"{op.value}-k{k}-{shape}") + for op, k in [(StorageOp.SSTORE_FRESH, 0), (StorageOp.SLOAD_COLD_HIT, 8)] + for shape in ("few_big", "many_small") +] + + +@pytest.mark.parametrize("op, k, shape", _SHAPE_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_block_shape( + blockchain_test: BlockchainTestFiller, + pre: Alloc, + op: StorageOp, + k: int, + shape: str, +) -> None: + """ + Same total work packed as a few big txs vs many small txs. + + `few_big` uses 7 gas-cap-sized transactions; `many_small` uses many + (300 at the full block budget) smaller ones. Reads reuse one + pre-populated page pool (each tx a fresh cold pass); fresh writes give + each tx a disjoint range. Each repeat block is offset to a fresh range. + """ + stride = 1 + if shape == "few_big": + num_txs = FULL_BLOCK_TXS + else: + num_txs = min( + MANY_SMALL_TXS, + max(1, BLOCK_GAS_TARGET // MANY_SMALL_MIN_TX_GAS), + ) + budget = min(TX_GAS_CAP, BLOCK_GAS_TARGET // num_txs) + count = _iterations(op, k, stride, budget) + + occupied = op is not StorageOp.SSTORE_FRESH + if occupied: + count = min(count, PRE_SLOT_CAP // max(k, 1)) + + sender = pre.fund_eoa() + prestate: StorageDict = {} + for r in range(REPEATS): + read_dom, _, _ = _repeat_domains(r) + if occupied: + prestate.update(_occupied_prestate(read_dom, stride, count, k)) + contract = pre.deploy_contract(_contract(op, k, stride), storage=prestate) + + blocks = [] + expected: StorageDict = dict(prestate) + global_idx = 0 + for r in range(REPEATS): + read_dom, fresh_dom, _ = _repeat_domains(r) + txs = [] + for t in range(num_txs): + base = read_dom if occupied else fresh_dom + t * count * stride + txs.append( + Transaction( + to=contract, + sender=sender, + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(base, count, global_idx, 0, t), + ) + ) + expected[MARKER_BASE + global_idx] = value_code_worked + if occupied: # cold_hit reads each return 1 + expected[CKSUM_BASE + global_idx] = count + global_idx += 1 + if op is StorageOp.SSTORE_FRESH: + for j in range(num_txs * count): + expected[(fresh_dom + j * stride) << 7] = 1 + blocks.append(Block(txs=txs)) + + blockchain_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + ) + + +# --- dimension 7: exceptional halt after writes ----------------------- + + +@pytest.mark.parametrize("mode", ["success", "halt", "mix"]) +@pytest.mark.valid_from("MONAD_NINE") +def test_tx_halt( + blockchain_test: BlockchainTestFiller, + pre: Alloc, + mode: str, +) -> None: + """ + Fresh-write block whose transactions succeed, all halt, or alternate. + + A halting tx runs INVALID after its writes and marker, so all of its + state reverts (and it consumes its full gas limit — a perfectly full + block). Only successful txs leave pages and markers behind, giving a + strong post-state oracle for the mixed case. Each repeat block is + offset to a fresh range. + """ + stride = 1 + budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS + count = _iterations(StorageOp.SSTORE_FRESH, 0, stride, budget) + + sender = pre.fund_eoa() + contract = pre.deploy_contract( + _contract(StorageOp.SSTORE_FRESH, 0, stride) + ) + + blocks = [] + expected: StorageDict = {} + global_idx = 0 + for r in range(REPEATS): + _, fresh_dom, _ = _repeat_domains(r) + txs = [] + for t in range(FULL_BLOCK_TXS): + halt = mode == "halt" or (mode == "mix" and t % 2 == 1) + txs.append( + Transaction( + to=contract, + sender=sender, + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata( + fresh_dom + t * count * stride, + count, + global_idx, + int(halt), + t, + ), + ) + ) + if not halt: + for i in range(count): + page = fresh_dom + (t * count + i) * stride + expected[page << 7] = 1 + expected[MARKER_BASE + global_idx] = value_code_worked + global_idx += 1 + blocks.append(Block(txs=txs)) + + blockchain_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + ) + + +# --- random access + adversarial "bad block" cases ------------------- +# +# Additive: the tests, params and StorageOp members above are unchanged, +# so results stay comparable across versions. Slot keys are spread +# pseudo-randomly over the low 2**200 of the slot space (a 256-bit odd +# multiplier + a 200-bit mask), kept below MARKER_BASE so a random key +# never collides with a marker/checksum witness. + +RAND_MULT = 0x9E3779B97F4A7C15F39CC0605CEDC8341082276BF3A27251F86C6A11D0C18E95 +RAND_MASK = (1 << 200) - 1 +RAND_IDX_STEP = 5 # odd: permutes the in-tx access order over the set +RAND_CONTRACTS = 8 # pool of contracts, called in a pseudorandom cycle +RAND_CONTRACT_STEP = 3 # coprime to RAND_CONTRACTS +RAND_SEED_BASE = 1 +RAND_SEED_STRIDE = 1 << 10 # > max slots, so per-tx slot sets are disjoint +M_CHAIN = 0xE0 # chained-sload current slot (memory scratch) +SERIAL_BASE = 1 << 40 +SERIAL_REPEAT_STRIDE = 1 << 24 # >> per-tx slot count +CHAIN_BASE = 1 << 30 +CHAIN_REPEAT_STRIDE = 1 << 20 # >> ring length + + +def _rand_slot(seed: int, idx: int) -> int: + """Python mirror of the contract's pseudorandom slot key.""" + return ((seed + idx) * RAND_MULT) & RAND_MASK + + +def _size_count(body: Bytecode, budget: int) -> int: + """Loop iterations of `body` that fit `budget`, sized to both forks.""" + per = max(body.gas_cost(MONAD_NINE), body.gas_cost(MONAD_NEXT)) + return max(1, (budget - TX_RESERVE) // (per + WHILE_CONTROL_GAS)) + + +def _rand_sload_body(slots: int) -> Bytecode: + """One iteration: cold-SLOAD a pseudorandom slot from the set.""" + idx = Op.AND(Op.MUL(Op.MLOAD(M_COUNTER), RAND_IDX_STEP), slots - 1) + slot = Op.AND(Op.MUL(Op.ADD(Op.MLOAD(M_BASE), idx), RAND_MULT), RAND_MASK) + read = Op.SLOAD(slot, key_warm=False, page_load_warm=False) + return Op.MSTORE( + M_CHECKSUM, Op.ADD(Op.MLOAD(M_CHECKSUM), read) + ) + Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) + + +def _rand_sload_contract(slots: int) -> Bytecode: + """ + SLOAD `slots` pseudorandom slots in a pseudorandom cycle, `count` + times, then write a success marker and the read checksum. + """ + init = ( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # seed + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_COUNTER, 0) + + Op.MSTORE(M_CHECKSUM, 0) + ) + loop = While( + body=_rand_sload_body(slots), + condition=Op.LT(Op.MLOAD(M_COUNTER), Op.MLOAD(M_COUNT)), + ) + tail = ( + Op.SSTORE(Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked) + + Op.SSTORE( + Op.ADD(CKSUM_BASE, Op.MLOAD(M_GLOBAL)), Op.MLOAD(M_CHECKSUM) + ) + + Op.STOP + ) + return init + loop + tail + + +_RAND_PARAMS = [ + pytest.param(slots, k, id=f"slots{slots}-k{k}") + for slots in (1, 16, 128) + for k in (0, 1) +] + + +@pytest.mark.parametrize("slots, k", _RAND_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_random_sload( + blockchain_test: BlockchainTestFiller, + pre: Alloc, + slots: int, + k: int, +) -> None: + """ + Cold-SLOAD a set of `slots` pseudorandom slots (spread over the slot + space) in a pseudorandom cycle, from a pool of contracts called in a + pseudorandom cycle. `k` is the page occupancy of each read slot: 1 + (slot present, reads 1) or 0 (empty page, reads 0). Cycling a small + set means only the first pass is cold, so blocks are gas-underfull by + design; each repeat uses fresh seeds so its slots are disjoint. + """ + budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS + count = _size_count(_rand_sload_body(slots), budget) + code = _rand_sload_contract(slots) + sender = pre.fund_eoa() + + # Plan tx -> (contract index, seed) and per-contract genesis slots. + plan: List[Tuple[int, int]] = [] + genesis: List[StorageDict] = [{} for _ in range(RAND_CONTRACTS)] + g = 0 + for _r in range(REPEATS): + for _t in range(FULL_BLOCK_TXS): + ci = (g * RAND_CONTRACT_STEP) % RAND_CONTRACTS + seed = RAND_SEED_BASE + g * RAND_SEED_STRIDE + if k == 1: + for idx in range(slots): + genesis[ci][_rand_slot(seed, idx)] = 1 + plan.append((ci, seed)) + g += 1 + + contracts: List[Address] = [ + pre.deploy_contract(code, storage=genesis[i]) + for i in range(RAND_CONTRACTS) + ] + + post: dict[Address, StorageDict] = { + contracts[i]: dict(genesis[i]) for i in range(RAND_CONTRACTS) + } + blocks = [] + g = 0 + for _r in range(REPEATS): + txs = [] + for _t in range(FULL_BLOCK_TXS): + ci, seed = plan[g] + contract = contracts[ci] + txs.append( + Transaction( + to=contract, + sender=sender, + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(seed, count, g, 0), + ) + ) + post[contract][MARKER_BASE + g] = value_code_worked + if k == 1: + post[contract][CKSUM_BASE + g] = count + g += 1 + blocks.append(Block(txs=txs)) + + blockchain_test( + pre=pre, + blocks=blocks, + post={c: Account(storage=s) for c, s in post.items()}, + genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + ) + + +@pytest.mark.valid_from("MONAD_NINE") +def test_bad_block_serial( + blockchain_test: BlockchainTestFiller, + pre: Alloc, +) -> None: + """ + Adversarial block forcing serial execution: every tx SLOADs and + SSTOREs (increments) the same contiguous slot sequence, so the txs + conflict and cannot run in parallel; each shared slot ends at + FULL_BLOCK_TXS. Each repeat uses a fresh slot range. + """ + budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS + sender = pre.fund_eoa() + + # Per-iteration body: increment the counter-th shared slot. + sl = Op.ADD(Op.MLOAD(M_BASE), Op.MLOAD(M_COUNTER)) + read = Op.SLOAD(sl, key_warm=False, page_load_warm=False) + body = _sstore( + sl, Op.ADD(read, 1), original=0, current=0, new=1, growth=0 + ) + Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) + count = _size_count(body, budget) + + contract = pre.deploy_contract( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_COUNTER, 0) + + While( + body=body, + condition=Op.LT(Op.MLOAD(M_COUNTER), Op.MLOAD(M_COUNT)), + ) + + Op.SSTORE(Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked) + + Op.STOP + ) + + blocks = [] + expected: StorageDict = {} + g = 0 + for r in range(REPEATS): + base = SERIAL_BASE + r * SERIAL_REPEAT_STRIDE + txs = [] + for _t in range(FULL_BLOCK_TXS): + txs.append( + Transaction( + to=contract, + sender=sender, + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(base, count, g, 0), + ) + ) + expected[MARKER_BASE + g] = value_code_worked + g += 1 + for i in range(count): + expected[base + i] = FULL_BLOCK_TXS + blocks.append(Block(txs=txs)) + + blockchain_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + ) + + +@pytest.mark.valid_from("MONAD_NINE") +def test_bad_block_chained( + blockchain_test: BlockchainTestFiller, + pre: Alloc, +) -> None: + """ + Adversarial block with a data-dependent SLOAD chain: each SLOAD + returns the next SLOAD's slot (SLOAD(SLOAD(...(seed)))), following a + pre-built storage ring, so the reads serialise within a tx. Each + repeat uses a fresh ring. + """ + budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS + sender = pre.fund_eoa() + + # Per-iteration body: SLOAD current slot; its value is the next slot. + nxt = Op.SLOAD(Op.MLOAD(M_CHAIN), key_warm=False, page_load_warm=False) + body = Op.MSTORE(M_CHAIN, nxt) + Op.MSTORE( + M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1) + ) + count = _size_count(body, budget) + + code = ( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # seed slot + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_COUNTER, 0) + + Op.MSTORE(M_CHAIN, Op.MLOAD(M_BASE)) + + While( + body=body, + condition=Op.LT(Op.MLOAD(M_COUNTER), Op.MLOAD(M_COUNT)), + ) + + Op.SSTORE(Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked) + + Op.STOP + ) + + genesis: StorageDict = {} + rings: List[List[int]] = [] + for r in range(REPEATS): + base = CHAIN_BASE + r * CHAIN_REPEAT_STRIDE + ring = [_rand_slot(base, j) for j in range(count)] + for j in range(count): + genesis[ring[j]] = ring[(j + 1) % count] + rings.append(ring) + contract = pre.deploy_contract(code, storage=genesis) + + blocks = [] + expected: StorageDict = dict(genesis) + g = 0 + for r in range(REPEATS): + ring = rings[r] + txs = [] + for t in range(FULL_BLOCK_TXS): + txs.append( + Transaction( + to=contract, + sender=sender, + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(ring[t % count], count, g, 0), + ) + ) + expected[MARKER_BASE + g] = value_code_worked + g += 1 + blocks.append(Block(txs=txs)) + + blockchain_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + ) From 8f5c7cde1d7be8645fae8eb8945c2b1d7c782281 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:13:28 +0000 Subject: [PATCH 02/23] test(mip8): drop layout dimension, random tests hit distinct pages Rework random_sload/bad_block_chained for random file access; refresh disjoint-table descriptions and per-family diagrams. Co-Authored-By: Claude --- MIP8_PERF_TESTS_DIAGRAMS.md | 180 ++++++----- scripts/perf_disjoint_table.py | 28 +- .../test_perf_regression.py | 284 +++++++----------- 3 files changed, 211 insertions(+), 281 deletions(-) diff --git a/MIP8_PERF_TESTS_DIAGRAMS.md b/MIP8_PERF_TESTS_DIAGRAMS.md index 3d424955ea5..29cafe9538c 100644 --- a/MIP8_PERF_TESTS_DIAGRAMS.md +++ b/MIP8_PERF_TESTS_DIAGRAMS.md @@ -14,7 +14,6 @@ Notation used in all diagrams: R(o) SLOAD of offset o within a page (result added to a checksum) W(o)=v SSTORE of value v at offset o within a page P pages (loop iterations) per transaction -s page-index stride: 1 (contiguous) or 1009 (scattered) D READ_DOMAIN = 2^40 base page index of pre-populated pools F FRESH_DOMAIN = 2^52 base page index of fresh-write ranges g global tx index (unique across all blocks of a fixture) @@ -26,7 +25,7 @@ Storage addressing (MIP-8 page = 128 slots): ``` slot = (page_index << 7) + offset -page D + i*s: +page D + i: offset: 0 1 ... k-1 k ... 127 ┌─────┬─────┬─────┬─────┬─────┬─────┬─────┐ │ 1 │ 1 │ ... │ 1 │ 0 │ ... │ 0 │ @@ -92,35 +91,27 @@ execution overhead from any MIP-8 effect. ## `test_page_ops` — one family per storage op -Family = one `StorageOp`; within a family, `k` (page occupancy) and -`layout` (contiguous / scattered) vary. All variants use the 7-tx full -block above. Per-tx page counts: - -| op | layouts × k | P pages/tx | -|--------------------|--------------------------------|----------------------| -| sload_cold_hit | cont {1,16,128}, scat {1,128} | 3466; k=128: 512¹ | -| sload_cold_miss | cont {1,16,64}, scat {1,64} | 3466; k=64: 1024¹ | -| sload_sweep | cont {2,16,128}, scat {2,128} | k=2:1740 k=16:218 k=128:27 | -| sload_warm_repeat | cont {1}, scat {1} | 158,946 (iterations) | -| sload_empty_page | cont {0}, scat {0} | 3466 | -| sstore_fresh | cont {0}, scat {0} | 1009 | -| sstore_noop | cont {1,16,128}, scat {1,128} | 3429; k=128: 512¹ | -| sstore_grow | cont {1,16,64}, scat {1,64} | 1008 | -| sstore_update | cont {1,16,128}, scat {1,128} | 2562; k=128: 512¹ | -| sstore_clear_keep | cont {8,16,128}, scat {8,128} | 2563; k=128: 512¹ | -| sstore_clear_empty | cont {1}, scat {1} | 2564 | +Family = one `StorageOp`; within a family, `k` (page occupancy) varies. +All variants use the 7-tx full block above. Per-tx page counts: + +| op | k values | P pages/tx | +|--------------------|--------------|----------------------------| +| sload_cold_hit | {1,16,128} | 3469; k=128: 512¹ | +| sload_cold_miss | {1,16,64} | 3469; k=64: 1024¹ | +| sload_sweep | {2,16,128} | k=2:1741 k=16:218 k=128:27 | +| sload_warm_repeat | {1} | 158,946 (iterations) | +| sload_empty_page | {0} | 3469 | +| sstore_fresh | {0} | 1009 | +| sstore_noop | {1,16,128} | 3432; k=128: 512¹ | +| sstore_grow | {1,16,64} | 1009 | +| sstore_update | {1,16,128} | 2563; k=128: 512¹ | +| sstore_clear_keep | {8,16,128} | 2565; k=128: 512¹ | +| sstore_clear_empty | {1} | 2565 | ¹ capped by `PRE_SLOT_CAP = 65,536` pre-state slots per block (`P = 65,536 / k`); those blocks do less than 200M of real work by design — the pre-state, not gas, is the bound. -Layout changes only the spacing of page indices, never the op: - -``` -contiguous (s = 1): D, D+1, D+2, ..., D+P-1 -scattered (s = 1009): D, D+1009, D+2018, ..., D+(P-1)*1009 -``` - ### Family `sload_cold_hit` Pre-state: pool of P pages at domain D, each with slots `0..k-1 = 1`. @@ -129,11 +120,11 @@ Every tx makes one cold pass over the whole pool, reading offset 0 ``` pre-state pool (shared by all 7 txs) - page D page D+s page D+2s page D+(P-1)s -tx 0: R(0) R(0) R(0) ... R(0) all cold -tx 1: R(0) R(0) R(0) ... R(0) cold again + page D page D+1 page D+2 page D+(P-1) +tx 0: R(0) R(0) R(0) ... R(0) all cold +tx 1: R(0) R(0) R(0) ... R(0) cold again ... (warmth reset) -tx 6: R(0) R(0) R(0) ... R(0) +tx 6: R(0) R(0) R(0) ... R(0) each read returns 1 → checksum = P per tx block total: 7 × P cold page reads of an existing, occupied slot @@ -145,7 +136,7 @@ Same shared pool as `cold_hit` (k ≤ 64 keeps the last slot empty), but each read targets offset 127 — the page exists, the slot is zero. ``` -page D + i*s: +page D + i: offset: 0 .. k-1 k .. 126 127 [ 1 .. 1 ] [ 0 .. 0 ] [ 0 ] ◄── R(127) per iteration occupied empty read misses (returns 0) @@ -162,11 +153,11 @@ Each loop iteration cold-reads **every occupied slot** of one page, offsets `0..k-1` in ascending order. ``` -tx 0..6, iteration i (page D + i*s): +tx 0..6, iteration i (page D + i): offset: 0 1 2 ... k-1 | k .. 127 R(0) R(1) R(2) ... R(k-1)| untouched k cold reads -per tx: P pages × k reads (k=2: 1740×2, k=16: 218×16, k=128: 27×128) +per tx: P pages × k reads (k=2: 1741×2, k=16: 218×16, k=128: 27×128) checksum = P × k per tx ``` @@ -198,10 +189,10 @@ Like `cold_hit` but with `k = 0`: the domain-D pages were **never populated**. Every read is a whole-page miss. ``` - page D page D+s page D+(P-1)s + page D page D+1 page D+(P-1) tx 0..6: R(0) R(0) ... R(0) page does not exist -P = 3466 per tx; checksum = 0 (zero write in the tail, no state trace) +P = 3469 per tx; checksum = 0 (zero write in the tail, no state trace) ``` Measures lookups that fall off the page index entirely. @@ -213,13 +204,13 @@ pages and creates one slot on each: `W(0)=1`, a 0→1 write that brings a whole new page into existence. ``` -page-index axis (base F, stride s), tiled per tx: +page-index axis (base F), tiled per tx: -F F+P·s F+2P·s F+6P·s F+7P·s +F F+P F+2P F+6P F+7P ╠═ tx0: P ═╬═ tx1: P ══╬═ tx2: P ══╬═ ... ══╬═ tx6: P ══╣ pages pages pages pages -tx t, iteration i: W(0)=1 on page F + (t·P + i)·s state growth +tx t, iteration i: W(0)=1 on page F + t·P + i state growth P = 1009 → block creates 7 × 1009 = 7063 new pages ``` @@ -230,10 +221,10 @@ Shared occupied pool (like `cold_hit`); each tx rewrites slot 0 with its current value — 1→1, no state change ever. ``` -tx 0..6, iteration i (page D + i*s): +tx 0..6, iteration i (page D + i): offset 0: [ 1 ] ◄── W(0)=1 cold page access, value unchanged -post-state == pre-state (plus markers); P = 3429 (512 at k=128) +post-state == pre-state (plus markers); P = 3432 (512 at k=128) ``` Pays the write path without any page mutation. @@ -246,13 +237,13 @@ no page creation). All 7 txs touch the same P pages, each at its own offset. ``` -page D + i*s, one column per tx: +page D + i, one column per tx: offset: 0 .. k-1 | k k+1 k+2 k+3 k+4 k+5 k+6 | k+7 .. 127 before: [ 1 .. 1 ]| 0 0 0 0 0 0 0 | 0 writer: | tx0 tx1 tx2 tx3 tx4 tx5 tx6 | after: [ 1 .. 1 ]| 1 1 1 1 1 1 1 | 0 -each tx: P = 1008 cold W(k+t)=1 writes, one per pool page +each tx: P = 1009 cold W(k+t)=1 writes, one per pool page ``` ### Family `sstore_update` @@ -262,11 +253,11 @@ nonzero value `2 + t`, so each write is a genuine value change with no occupancy change. ``` -page D + i*s, offset 0 over the block: +page D + i, offset 0 over the block: pre tx0 tx1 tx2 tx3 tx4 tx5 tx6 1 ─► W(0)=2 ─► =3 ─► =4 ─► =5 ─► =6 ─► =7 ─► =8 -each tx: P = 2562 cold writes (512 at k=128); slot 0 ends at 8 +each tx: P = 2563 cold writes (512 at k=128); slot 0 ends at 8 ``` ### Family `sstore_clear_keep` @@ -275,13 +266,13 @@ Shared pool with `k > 7` occupied slots. Tx `t` clears offset `t` (1→0). Offsets `7..k-1` stay populated, so no page ever empties. ``` -page D + i*s: +page D + i: offset: 0 1 2 3 4 5 6 | 7 .. k-1 | k .. 127 before: 1 1 1 1 1 1 1 | 1 .. 1 | 0 clearer: tx0 tx1 tx2 tx3 tx4 tx5 tx6 | untouched | after: 0 0 0 0 0 0 0 | 1 .. 1 | 0 -each tx: P = 2563 cold W(t)=0 writes (512 at k=128) +each tx: P = 2565 cold W(t)=0 writes (512 at k=128) ``` ### Family `sstore_clear_empty` @@ -293,15 +284,15 @@ empty, so every write deletes a page. ``` pre-state, tiled like sstore_fresh but pre-populated with k=1: -F F+P·s F+2P·s F+6P·s F+7P·s +F F+P F+2P F+6P F+7P ╠═ tx0: P ═╬═ tx1: P ══╬═ ... ══════════╬═ tx6: P ══╣ 1-slot 1-slot 1-slot pages pages pages -tx t, iteration i: W(0)=0 on page F + (t·P + i)·s +tx t, iteration i: W(0)=0 on page F + t·P + i before: [1][0..0] → after: [0][0..0] → page removed -P = 2564 → block removes 7 × 2564 = 17,948 pages +P = 2565 → block removes 7 × 2565 = 17,955 pages ``` --- @@ -310,38 +301,37 @@ P = 2564 → block removes 7 × 2564 = 17,948 pages Op is fixed (`sstore_fresh`, the 0→1 page-creating write); the sweep is over **where** the writes land: `m` total pages spread evenly across -`n` contracts, in `contiguous` or `scattered` page-index layout. +`n` contracts. Each contract runs the same loop contract with empty storage. Every -contract writes page indices `F + 0·s .. F + (m/n − 1)·s` — the *same* +contract writes page indices `F + 0 .. F + (m/n − 1)` — the *same* indices for all contracts, but in n distinct account storages, so nothing collides. A contract's share becomes one tx while it fits the -30M tx cap (≤ 1059 pages); `m/n = 4096` splits into 4 txs. +30M tx cap (≤ 1060 pages); `m/n = 4096` splits into 4 txs. ``` Block (one per fixture; txs sized to the work, not to fill 200M) n = 1, m = 4096 (4 txs, one contract): ┌ contract C0 ──────────────────────────────────────────────────────┐ -│ tx0: W(0)=1 on pages F+0·s .. F+1058·s (1059 pages, 30.0M)│ -│ tx1: W(0)=1 on pages F+1059·s .. F+2117·s (1059 pages, 30.0M)│ -│ tx2: W(0)=1 on pages F+2118·s .. F+3176·s (1059 pages, 30.0M)│ -│ tx3: W(0)=1 on pages F+3177·s .. F+4095·s ( 919 pages, 26.0M)│ +│ tx0: W(0)=1 on pages F+0 .. F+1059 (1060 pages, 30.0M) │ +│ tx1: W(0)=1 on pages F+1060 .. F+2119 (1060 pages, 30.0M) │ +│ tx2: W(0)=1 on pages F+2120 .. F+3179 (1060 pages, 30.0M) │ +│ tx3: W(0)=1 on pages F+3180 .. F+4095 ( 916 pages, 25.9M) │ └───────────────────────────────────────────────────────────────────┘ n = 8, m = 4096 (8 txs, 512 pages each): ┌ C0 ┐┌ C1 ┐┌ C2 ┐┌ C3 ┐┌ C4 ┐┌ C5 ┐┌ C6 ┐┌ C7 ┐ │tx0 ││tx1 ││tx2 ││tx3 ││tx4 ││tx5 ││tx6 ││tx7 │ each: 512 × W(0)=1 -└────┘└────┘└────┘└────┘└────┘└────┘└────┘└────┘ 14.6M gas each +└────┘└────┘└────┘└────┘└────┘└────┘└────┘└────┘ 14.55M gas each n = 512, m = 4096 (512 txs, 8 pages each): -┌C0┐┌C1┐┌C2┐ ... ┌C511┐ each: 8 × W(0)=1, 345,568 gas +┌C0┐┌C1┐┌C2┐ ... ┌C511┐ each: 8 × W(0)=1, 345,504 gas └──┘└──┘└──┘ └────┘ block ≈ 177M gas ``` -Variants: `m ∈ {1,4,16,64,256,1024,4096} × n=1` (total-size sweep, -contiguous), `m=4096 × n ∈ {1,8,64,512}` (distribution sweep), plus -scattered replays of `m4096_n1` and `m4096_n512`. Same total I/O at +Variants: `m ∈ {1,4,16,64,256,1024,4096} × n=1` (total-size sweep), +`m=4096 × n ∈ {8,64,512}` (distribution sweep). Same total I/O at `m = 4096` regardless of `n` — only the account fan-out changes. --- @@ -349,13 +339,13 @@ scattered replays of `m4096_n1` and `m4096_n512`. Same total I/O at ## `test_block_shape` — one family Same total work packed as **7 big** txs vs **300 small** txs. Two -workloads, both contiguous: +workloads: | op, k | shape | txs | tx gas | iters/tx | block work | |-------------------|------------|-----|-----------|----------|--------------------| | sstore_fresh, k=0 | few_big | 7 | 28.57M | 1009 | 7063 new pages | | sstore_fresh, k=0 | many_small | 300 | 666,666 | 19 | 5700 new pages | -| sload_cold_hit, 8 | few_big | 7 | 28.57M | 3466 | 7×3466 cold reads | +| sload_cold_hit, 8 | few_big | 7 | 28.57M | 3469 | 7×3469 cold reads | | sload_cold_hit, 8 | many_small | 300 | 666,666 | 66 | 300×66 cold reads | ``` @@ -369,7 +359,7 @@ many_small: ┌──┬──┬──┬──┬──┬──┬──┬ ``` The reads reuse one shared pre-populated pool sized to a single tx -(3466 pages for few_big, 66 pages for many_small, k = 8 slots each); +(3469 pages for few_big, 66 pages for many_small, k = 8 slots each); every tx is a fresh cold pass over it. The fresh writes tile disjoint per-tx ranges exactly like `sstore_fresh` above. What varies is per-tx fixed overhead (intrinsic gas, markers, cold pool re-touch) relative @@ -404,36 +394,37 @@ back. ## `test_random_sload` — one family -Random-locality reads: no domains, no strides — slot keys are -pseudorandom over the low 2^200 of the slot space, so every slot lands -on its own effectively random page. Params: `slots ∈ {1, 16, 128}` -(size of the per-tx slot set) × `k ∈ {0, 1}` (each read slot's page is -1-slot occupied, or never populated). +Random file access: each read targets a **distinct page** (`slot = +page_key << 7`), and the MPT hashes every page key (`keccak256`) to an +unpredictable trie/disk position, so consecutive page keys land at +uncorrelated disk locations. Params: `slots ∈ {1, 16, 128}` (size of +the per-tx page set) × `k ∈ {0, 1}` (each read page is 1-slot occupied, +or never populated). A pool of 8 identical contracts is deployed; tx `g` calls contract -`(3·g) mod 8` and carries its own seed `1 + 1024·g`, giving every tx a -disjoint pseudorandom slot set inside "its" contract: +`g mod 8` and carries a base page key `1 + 1024·g`, giving every tx a +disjoint set of pages inside "its" contract: ``` -Block: tx0→C0 tx1→C3 tx2→C6 tx3→C1 tx4→C4 tx5→C7 tx6→C2 +Block: tx0→C0 tx1→C1 tx2→C2 tx3→C3 tx4→C4 tx5→C5 tx6→C6 -tx g: slot set S_g[i] = ((seed_g + i) · MULT) mod 2^200, i < slots - e.g. seed=1: 0x15f39cc0605c..., 0x2be73980c0b9..., ... +tx g: page set = { (base_g + i) << 7 : i < slots }, base_g = 1 + 1024·g + every page key hashed by the MPT → random disk position - iteration j (of 3462) reads S_g[(5·j) mod slots]: - j: 0 1 2 ... slots−1 │ slots ... 3461 - s_0 s_5 s_10 ... │ (set cycles again) - cold cold cold ... cold │ warm ... warm + iteration j (of 3469) reads page (base_g + (j mod slots)) << 7: + j: 0 1 ... slots−1 │ slots ... 3468 + base+0 base+1 ... │ (set cycles again) + cold cold ... cold │ warm ... warm ``` -Each tx makes 3462 reads, but only the first pass over the set +Each tx makes 3469 reads, but only the first pass over the set (`slots` reads) is genuinely cold — sizing charges every iteration as -cold (8216 gas), so these blocks are heavily gas-underfull by design; -the subject is random-key locality, not I/O volume. With `k = 1` the -set's slots are pre-populated (=1) in that contract's genesis and the -checksum is 3462; with `k = 0` every read is a whole-page miss and the -tail checksum SSTORE writes 0 (no state trace). Markers/checksums land -in the storage of whichever contract the tx called. +cold (8160 gas), so these blocks are heavily gas-underfull by design; +the subject is random-locality I/O, not volume. With `k = 1` the set's +pages are pre-populated (one slot = 1) in that contract's genesis and +the checksum is 3469; with `k = 0` every read is a whole-page miss and +the tail checksum SSTORE writes 0 (no state trace). Markers/checksums +land in the storage of whichever contract the tx called. --- @@ -457,7 +448,7 @@ tx 1: 1→2 1→2 1→2 1→2 ▲ must see the tx 6: 6→7 6→7 6→7 6→7 │ writes: serial ``` -783 read+write pairs per tx (36,294 gas each, sized to the fresh 0→1 +783 read+write pairs per tx (36,254 gas each, sized to the fresh 0→1 cost — the later increment txs are nonzero→nonzero updates and run cheaper, leaving the block somewhat gas-underfull). Post-state: every shared slot ends at 7, plus the 7 markers (no checksum tail here). @@ -467,14 +458,15 @@ shared slot ends at 7, plus the 7 markers (no checksum tail here). ## `test_bad_block_chained` — one family No parameters. The data-dependency adversarial block: the genesis -storage holds a pre-built ring of 3482 pseudorandom slots where each -slot's **value is the address of the next slot**. A tx starts from its -calldata seed and hops the ring with `slot := SLOAD(slot)` — every -read's address comes from the previous read, so the reads serialize -within the tx (no lookahead or batching possible). +storage holds a pre-built ring of 3482 **distinct pages** (keys +`(base+j) << 7`) where each page's stored value is the key of the next +page. A tx starts from its calldata base and hops the ring with +`slot := SLOAD(slot)` — every read's address comes from the previous +read, so the reads serialize within the tx, and because the MPT hashes +each page key, every hop is an unpredictable disk position. ``` -genesis ring (3482 pseudorandom slots, each on its own page): +genesis ring (3482 distinct pages, keys (base+j) << 7): ring[0] ──► ring[1] ──► ring[2] ──► ... ──► ring[3481] ──┐ ▲ │ @@ -487,5 +479,5 @@ tx t (t = 0..6): start at ring[t], then 3482 hops All 7 txs traverse the same ring; per-tx warmth reset makes each lap fully cold, so the block performs 7 × 3482 cold, address-dependent -reads. Nothing is written except the 7 markers — post-state is the -untouched ring plus markers. +reads. Nothing is written except the 7 +markers — post-state is the untouched ring plus markers. diff --git a/scripts/perf_disjoint_table.py b/scripts/perf_disjoint_table.py index 4aaef03444d..e132f3abd2f 100644 --- a/scripts/perf_disjoint_table.py +++ b/scripts/perf_disjoint_table.py @@ -116,10 +116,10 @@ def _mwu_p(a: list[int], b: list[int]) -> float: return erfc(max(0.0, (d - 0.5) / sigma) / sqrt(2)) -def _op_phrase(op: str, k: int, layout: str | None) -> str: +def _op_phrase(op: str, k: int) -> str: """Describe what one block-filling tx does for a storage op.""" - place = f"{layout} pages" if layout else "pages" - page = f"{layout} page" if layout else "page" + place = "pages" + page = "page" occ = f", {k} slot{'' if k == 1 else 's'} occupied per page" phrases = { "sload_cold_hit": f"cold-SLOAD one occupied slot on many {place}{occ}", @@ -149,22 +149,21 @@ def _op_phrase(op: str, k: int, layout: str | None) -> str: def describe(test: str, params: str) -> str: """One-sentence account of what a case's block transactions do.""" if test == "page_ops": - op, kpart, layout = params.split("-") + op, kpart = params.split("-") k = int(kpart.removeprefix("k")) - return f"block-filling transactions {_op_phrase(op, k, layout)}" + return f"block-filling transactions {_op_phrase(op, k)}" if test == "block_shape": op, kpart, shape = params.split("-") k = int(kpart.removeprefix("k")) who = "a few big" if shape == "few_big" else "many small (~300)" - return f"{who} transactions each {_op_phrase(op, k, None)}" + return f"{who} transactions each {_op_phrase(op, k)}" if test == "page_spread": m = params.split("_")[0].removeprefix("m") n = int(params.split("_")[1].removeprefix("n")) - layout = params.split("_")[2] target = f"{n} contract" + ("" if n == 1 else "s") return ( f"transactions SSTORE 0->1 into {m} fresh slots spread across " - f"{target} ({layout} pages)" + f"{target}" ) if test == "tx_halt": mode = params.removeprefix("mode_") @@ -185,12 +184,12 @@ def describe(test: str, params: str) -> str: if test == "random_sload": slots = params.split("-")[0].removeprefix("slots") k = int(params.split("-")[1].removeprefix("k")) - page = "a 1-element" if k == 1 else "an empty" - noun = "slot" if slots == "1" else "slots" + page = "1-element" if k == 1 else "empty" + noun = "page" if slots == "1" else "pages" return ( - f"cold-SLOAD {slots} pseudorandom {noun} (each on {page} page) " - "in a pseudorandom cycle, from a pool of contracts called in a " - "pseudorandom cycle" + f"cold-SLOAD {slots} distinct {page} {noun} (random " + "access), " + "from a pool of contracts spread across the block" ) if test == "bad_block_serial": return ( @@ -200,7 +199,8 @@ def describe(test: str, params: str) -> str: if test == "bad_block_chained": return ( "each SLOAD returns the next SLOAD's slot (SLOAD(SLOAD(...))), " - "a data-dependent chain that serialises the reads within a tx" + "a data-dependent pointer chase over distinct pages that " + "serialises the reads within a tx" ) return f"{test} {params}" diff --git a/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py index 1c1658f588b..5eee976770d 100644 --- a/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py +++ b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py @@ -97,9 +97,6 @@ # the whole fixture's genesis). PRE_SLOT_CAP = 65_536 -# Non-unit stride (in page indices) for the "scattered" layout. -SCATTERED_STRIDE = 1009 - # Gas of one While control step (JUMPDEST + JUMPI + counter compare); # a small over-estimate keeps sizing on the safe side (no OOG). WHILE_CONTROL_GAS = 40 @@ -215,20 +212,14 @@ def test_compute_loop( # --- shared bytecode + sizing helpers --------------------------------- -def _stride(layout: str) -> int: - """Return the page-index stride for a layout name.""" - return 1 if layout == "contiguous" else SCATTERED_STRIDE - - -def _page_index(stride: int) -> Bytecode: - """Bytecode: base + i*stride (base at M_BASE, i at M_COUNTER).""" - i = Op.MLOAD(M_COUNTER) - return Op.ADD(Op.MLOAD(M_BASE), Op.MUL(i, stride)) +def _page_index() -> Bytecode: + """Bytecode: base + i (base at M_BASE, i at M_COUNTER).""" + return Op.ADD(Op.MLOAD(M_BASE), Op.MLOAD(M_COUNTER)) -def _slot(stride: int, offset: int | Bytecode) -> Bytecode: +def _slot(offset: int | Bytecode) -> Bytecode: """Bytecode: (page_index << 7) + offset.""" - return Op.ADD(offset, Op.SHL(7, _page_index(stride))) + return Op.ADD(offset, Op.SHL(7, _page_index())) def _read_accum(slot: Bytecode | int, *, warm: bool) -> Bytecode: @@ -268,41 +259,35 @@ def _sstore( ) -def _body(op: StorageOp, k: int, stride: int) -> Bytecode: +def _body(op: StorageOp, k: int) -> Bytecode: """Assemble the per-iteration loop body for an operation.""" inc = Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) if op is StorageOp.SLOAD_COLD_HIT: - return _read_accum(_slot(stride, 0), warm=False) + inc + return _read_accum(_slot(0), warm=False) + inc if op is StorageOp.SLOAD_COLD_MISS: - return _read_accum(_slot(stride, SLOTS_PER_PAGE - 1), warm=False) + inc + return _read_accum(_slot(SLOTS_PER_PAGE - 1), warm=False) + inc if op is StorageOp.SLOAD_WARM_REPEAT: # The slot is passed in calldata (M_BASE) so each block re-reads a # distinct slot; cold on the first iteration, warm thereafter. return _read_accum(Op.MLOAD(M_BASE), warm=True) + inc if op is StorageOp.SLOAD_SWEEP: - code = Op.MSTORE(M_PAGE, Op.SHL(7, _page_index(stride))) + code = Op.MSTORE(M_PAGE, Op.SHL(7, _page_index())) for j in range(k): code += _read_accum(Op.ADD(Op.MLOAD(M_PAGE), j), warm=False) return code + inc if op is StorageOp.SSTORE_FRESH: return ( - _sstore( - _slot(stride, 0), 1, original=0, current=0, new=1, growth=0 - ) - + inc + _sstore(_slot(0), 1, original=0, current=0, new=1, growth=0) + inc ) if op is StorageOp.SSTORE_NOOP: return ( - _sstore( - _slot(stride, 0), 1, original=1, current=1, new=1, growth=0 - ) - + inc + _sstore(_slot(0), 1, original=1, current=1, new=1, growth=0) + inc ) if op is StorageOp.SSTORE_GROW: offset = Op.ADD(k, Op.MLOAD(M_LOCAL)) return ( _sstore( - _slot(stride, offset), + _slot(offset), 1, original=0, current=0, @@ -316,9 +301,7 @@ def _body(op: StorageOp, k: int, stride: int) -> Bytecode: # is a genuine value change on the still-occupied slot 0. value = Op.ADD(2, Op.MLOAD(M_LOCAL)) return ( - _sstore( - _slot(stride, 0), value, original=1, current=1, new=2, growth=0 - ) + _sstore(_slot(0), value, original=1, current=1, new=2, growth=0) + inc ) if op is StorageOp.SSTORE_CLEAR_KEEP: @@ -327,7 +310,7 @@ def _body(op: StorageOp, k: int, stride: int) -> Bytecode: offset = Op.MLOAD(M_LOCAL) return ( _sstore( - _slot(stride, offset), + _slot(offset), 0, original=1, current=1, @@ -338,13 +321,10 @@ def _body(op: StorageOp, k: int, stride: int) -> Bytecode: ) if op is StorageOp.SSTORE_CLEAR_EMPTY: return ( - _sstore( - _slot(stride, 0), 0, original=1, current=1, new=0, growth=0 - ) - + inc + _sstore(_slot(0), 0, original=1, current=1, new=0, growth=0) + inc ) if op is StorageOp.SLOAD_EMPTY_PAGE: - return _read_accum(_slot(stride, 0), warm=False) + inc + return _read_accum(_slot(0), warm=False) + inc raise ValueError(f"unknown op {op}") @@ -353,7 +333,7 @@ def _is_read(op: StorageOp) -> bool: return op.value.startswith("sload") -def _contract(op: StorageOp, k: int, stride: int) -> Bytecode: +def _contract(op: StorageOp, k: int) -> Bytecode: """ Build the workload contract (fixed size, independent of REPEATS). @@ -370,7 +350,7 @@ def _contract(op: StorageOp, k: int, stride: int) -> Bytecode: + Op.MSTORE(M_COUNTER, 0) ) loop = While( - body=_body(op, k, stride), + body=_body(op, k), condition=Op.LT(Op.MLOAD(M_COUNTER), Op.MLOAD(M_COUNT)), ) markers = Op.SSTORE( @@ -398,25 +378,23 @@ def _calldata( ) -def _per_iter_gas(op: StorageOp, k: int, stride: int) -> int: +def _per_iter_gas(op: StorageOp, k: int) -> int: """Gas for one loop iteration, sized to the costlier of both forks.""" - body = _body(op, k, stride) + body = _body(op, k) per_op = max(body.gas_cost(MONAD_NINE), body.gas_cost(MONAD_NEXT)) return per_op + WHILE_CONTROL_GAS -def _iterations(op: StorageOp, k: int, stride: int, budget: int) -> int: +def _iterations(op: StorageOp, k: int, budget: int) -> int: """Loop iterations that fit `budget` gas, leaving tx headroom.""" - return max(1, (budget - TX_RESERVE) // _per_iter_gas(op, k, stride)) + return max(1, (budget - TX_RESERVE) // _per_iter_gas(op, k)) -def _occupied_prestate( - domain: int, stride: int, pages: int, k: int -) -> StorageDict: +def _occupied_prestate(domain: int, pages: int, k: int) -> StorageDict: """Pre-populate `pages` pages (from `domain`) with `k` slots each.""" storage: StorageDict = {} for i in range(pages): - base_slot = (domain + i * stride) << 7 + base_slot = (domain + i) << 7 for j in range(k): storage[base_slot + j] = 1 return storage @@ -428,56 +406,41 @@ def _repeat_domains(repeat: int) -> Tuple[int, int, int]: return READ_DOMAIN + shift, FRESH_DOMAIN + shift, WARM_BASE + repeat -# --- dimensions 1, 2, 3, 6: op x occupancy x layout ------------------- - -# (op, layout, page-occupancy k values). Contiguous and scattered are -# listed separately so their k coverage can diverge later (scattered -# fills are far more expensive); for now both share the same k lists. -# cold_miss/grow keep a zero slot free (k < SLOTS_PER_PAGE, and grow -# writes k..k+FULL_BLOCK_TXS-1). clear_keep needs k > FULL_BLOCK_TXS so a -# page still has slots after the block clears offsets 0..FULL_BLOCK_TXS-1. -# clear_empty uses single-slot pages (k=1) that vanish when cleared; -# empty_page reads never-set pages (k=0). -_PAGE_OP_LAYOUT_KS = [ - (StorageOp.SLOAD_COLD_HIT, "contiguous", [1, 16, 128]), - (StorageOp.SLOAD_COLD_MISS, "contiguous", [1, 16, 64]), - (StorageOp.SLOAD_SWEEP, "contiguous", [2, 16, 128]), - (StorageOp.SSTORE_NOOP, "contiguous", [1, 16, 128]), - (StorageOp.SSTORE_GROW, "contiguous", [1, 16, 64]), - (StorageOp.SSTORE_UPDATE, "contiguous", [1, 16, 128]), - (StorageOp.SSTORE_CLEAR_KEEP, "contiguous", [8, 16, 128]), - (StorageOp.SSTORE_CLEAR_EMPTY, "contiguous", [1]), - (StorageOp.SLOAD_EMPTY_PAGE, "contiguous", [0]), - (StorageOp.SLOAD_WARM_REPEAT, "contiguous", [1]), - (StorageOp.SSTORE_FRESH, "contiguous", [0]), - (StorageOp.SLOAD_COLD_HIT, "scattered", [1, 128]), - (StorageOp.SLOAD_COLD_MISS, "scattered", [1, 64]), - (StorageOp.SLOAD_SWEEP, "scattered", [2, 128]), - (StorageOp.SSTORE_NOOP, "scattered", [1, 128]), - (StorageOp.SSTORE_GROW, "scattered", [1, 64]), - (StorageOp.SSTORE_UPDATE, "scattered", [1, 128]), - (StorageOp.SSTORE_CLEAR_KEEP, "scattered", [8, 128]), - (StorageOp.SSTORE_CLEAR_EMPTY, "scattered", [1]), - (StorageOp.SLOAD_EMPTY_PAGE, "scattered", [0]), - (StorageOp.SLOAD_WARM_REPEAT, "scattered", [1]), - (StorageOp.SSTORE_FRESH, "scattered", [0]), +# --- dimensions 1, 2, 3, 6: op x occupancy ----------------------------- + +# (op, page-occupancy k values). cold_miss/grow keep a zero slot free +# (k < SLOTS_PER_PAGE, and grow writes k..k+FULL_BLOCK_TXS-1). clear_keep +# needs k > FULL_BLOCK_TXS so a page still has slots after the block +# clears offsets 0..FULL_BLOCK_TXS-1. clear_empty uses single-slot pages +# (k=1) that vanish when cleared; empty_page reads never-set pages (k=0). +_PAGE_OP_KS = [ + (StorageOp.SLOAD_COLD_HIT, [1, 16, 128]), + (StorageOp.SLOAD_COLD_MISS, [1, 16, 64]), + (StorageOp.SLOAD_SWEEP, [2, 16, 128]), + (StorageOp.SSTORE_NOOP, [1, 16, 128]), + (StorageOp.SSTORE_GROW, [1, 16, 64]), + (StorageOp.SSTORE_UPDATE, [1, 16, 128]), + (StorageOp.SSTORE_CLEAR_KEEP, [8, 16, 128]), + (StorageOp.SSTORE_CLEAR_EMPTY, [1]), + (StorageOp.SLOAD_EMPTY_PAGE, [0]), + (StorageOp.SLOAD_WARM_REPEAT, [1]), + (StorageOp.SSTORE_FRESH, [0]), ] _PAGE_OP_PARAMS = [ - pytest.param(op, k, layout, id=f"{op.value}-k{k}-{layout}") - for op, layout, ks in _PAGE_OP_LAYOUT_KS + pytest.param(op, k, id=f"{op.value}-k{k}") + for op, ks in _PAGE_OP_KS for k in ks ] -@pytest.mark.parametrize("op, k, layout", _PAGE_OP_PARAMS) +@pytest.mark.parametrize("op, k", _PAGE_OP_PARAMS) @pytest.mark.valid_from("MONAD_NINE") def test_page_ops( blockchain_test: BlockchainTestFiller, pre: Alloc, op: StorageOp, k: int, - layout: str, ) -> None: """ Fill a block with one storage-op pattern over `pages` pages. @@ -493,9 +456,8 @@ def test_page_ops( bound and run below 200M by design. With REPEATS > 1 each repeat block is offset to a fresh page range. """ - stride = _stride(layout) budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS - pages = _iterations(op, k, stride, budget) + pages = _iterations(op, k, budget) per_tx_pages = op is StorageOp.SSTORE_CLEAR_EMPTY occupied = op not in ( @@ -517,11 +479,11 @@ def test_page_ops( prestate[warm_slot] = 1 elif per_tx_pages: for t in range(FULL_BLOCK_TXS): - dom = fresh_dom + t * pages * stride - prestate.update(_occupied_prestate(dom, stride, pages, 1)) + dom = fresh_dom + t * pages + prestate.update(_occupied_prestate(dom, pages, 1)) elif occupied: - prestate.update(_occupied_prestate(read_dom, stride, pages, k)) - contract = pre.deploy_contract(_contract(op, k, stride), storage=prestate) + prestate.update(_occupied_prestate(read_dom, pages, k)) + contract = pre.deploy_contract(_contract(op, k), storage=prestate) blocks = [] expected: StorageDict = dict(prestate) @@ -531,7 +493,7 @@ def test_page_ops( txs = [] for t in range(FULL_BLOCK_TXS): if op in (StorageOp.SSTORE_FRESH, StorageOp.SSTORE_CLEAR_EMPTY): - base = fresh_dom + t * pages * stride + base = fresh_dom + t * pages elif op is StorageOp.SLOAD_WARM_REPEAT: base = warm_slot else: @@ -563,26 +525,26 @@ def test_page_ops( if op is StorageOp.SSTORE_GROW: for i in range(pages): - base_slot = (read_dom + i * stride) << 7 + base_slot = (read_dom + i) << 7 for t in range(FULL_BLOCK_TXS): expected[base_slot + (k + t)] = 1 elif op is StorageOp.SSTORE_FRESH: for j in range(FULL_BLOCK_TXS * pages): - expected[(fresh_dom + j * stride) << 7] = 1 + expected[(fresh_dom + j) << 7] = 1 elif op is StorageOp.SSTORE_UPDATE: # Slot 0 is written once per tx (1->2->...); it ends at # 1 + FULL_BLOCK_TXS. Other occupied slots stay 1. for i in range(pages): - expected[(read_dom + i * stride) << 7] = 1 + FULL_BLOCK_TXS + expected[(read_dom + i) << 7] = 1 + FULL_BLOCK_TXS elif op is StorageOp.SSTORE_CLEAR_KEEP: for i in range(pages): - base_slot = (read_dom + i * stride) << 7 + base_slot = (read_dom + i) << 7 for t in range(FULL_BLOCK_TXS): expected[base_slot + t] = 0 elif op is StorageOp.SSTORE_CLEAR_EMPTY: for t in range(FULL_BLOCK_TXS): for i in range(pages): - expected[(fresh_dom + (t * pages + i) * stride) << 7] = 0 + expected[(fresh_dom + (t * pages + i)) << 7] = 0 blocks.append(Block(txs=txs)) @@ -597,32 +559,29 @@ def test_page_ops( # --- dimension 4: m pages spread across n contracts ------------------- _SPREAD_PARAMS = [ - pytest.param(m, n, layout, id=f"m{m}_n{n}_{layout}") - for m, n, layout in [ - (1, 1, "contiguous"), - (4, 1, "contiguous"), - (16, 1, "contiguous"), - (64, 1, "contiguous"), - (256, 1, "contiguous"), - (1024, 1, "contiguous"), - (4096, 1, "contiguous"), - (4096, 8, "contiguous"), - (4096, 64, "contiguous"), - (4096, 512, "contiguous"), - (4096, 1, "scattered"), - (4096, 512, "scattered"), + pytest.param(m, n, id=f"m{m}_n{n}") + for m, n in [ + (1, 1), + (4, 1), + (16, 1), + (64, 1), + (256, 1), + (1024, 1), + (4096, 1), + (4096, 8), + (4096, 64), + (4096, 512), ] ] -@pytest.mark.parametrize("m, n, layout", _SPREAD_PARAMS) +@pytest.mark.parametrize("m, n", _SPREAD_PARAMS) @pytest.mark.valid_from("MONAD_NINE") def test_page_spread( blockchain_test: BlockchainTestFiller, pre: Alloc, m: int, n: int, - layout: str, ) -> None: """ Fresh-write `m` distinct pages spread evenly over `n` contracts. @@ -633,14 +592,13 @@ def test_page_spread( txs). Isolates the effect of write distribution across accounts. Each repeat block writes a fresh page range. """ - stride = _stride(layout) - per_iter = _per_iter_gas(StorageOp.SSTORE_FRESH, 0, stride) + per_iter = _per_iter_gas(StorageOp.SSTORE_FRESH, 0) max_per_tx = max(1, (TX_GAS_CAP - TX_RESERVE) // per_iter) pages_per_contract = m // n sender = pre.fund_eoa() contracts: List[Address] = [ - pre.deploy_contract(_contract(StorageOp.SSTORE_FRESH, 0, stride)) + pre.deploy_contract(_contract(StorageOp.SSTORE_FRESH, 0)) for _ in range(n) ] @@ -661,9 +619,7 @@ def test_page_spread( gas_limit=count * per_iter + TX_RESERVE, max_fee_per_gas=MAX_FEE_PER_GAS, max_priority_fee_per_gas=0, - data=_calldata( - fresh_dom + done * stride, count, global_idx, 0 - ), + data=_calldata(fresh_dom + done, count, global_idx, 0), ) ) contract_storage[contract][MARKER_BASE + global_idx] = ( @@ -672,7 +628,7 @@ def test_page_spread( global_idx += 1 done += count for j in range(pages_per_contract): - slot = (fresh_dom + j * stride) << 7 + slot = (fresh_dom + j) << 7 contract_storage[contract][slot] = 1 blocks.append(Block(txs=txs)) @@ -713,7 +669,6 @@ def test_block_shape( pre-populated page pool (each tx a fresh cold pass); fresh writes give each tx a disjoint range. Each repeat block is offset to a fresh range. """ - stride = 1 if shape == "few_big": num_txs = FULL_BLOCK_TXS else: @@ -722,7 +677,7 @@ def test_block_shape( max(1, BLOCK_GAS_TARGET // MANY_SMALL_MIN_TX_GAS), ) budget = min(TX_GAS_CAP, BLOCK_GAS_TARGET // num_txs) - count = _iterations(op, k, stride, budget) + count = _iterations(op, k, budget) occupied = op is not StorageOp.SSTORE_FRESH if occupied: @@ -733,8 +688,8 @@ def test_block_shape( for r in range(REPEATS): read_dom, _, _ = _repeat_domains(r) if occupied: - prestate.update(_occupied_prestate(read_dom, stride, count, k)) - contract = pre.deploy_contract(_contract(op, k, stride), storage=prestate) + prestate.update(_occupied_prestate(read_dom, count, k)) + contract = pre.deploy_contract(_contract(op, k), storage=prestate) blocks = [] expected: StorageDict = dict(prestate) @@ -743,7 +698,7 @@ def test_block_shape( read_dom, fresh_dom, _ = _repeat_domains(r) txs = [] for t in range(num_txs): - base = read_dom if occupied else fresh_dom + t * count * stride + base = read_dom if occupied else fresh_dom + t * count txs.append( Transaction( to=contract, @@ -760,7 +715,7 @@ def test_block_shape( global_idx += 1 if op is StorageOp.SSTORE_FRESH: for j in range(num_txs * count): - expected[(fresh_dom + j * stride) << 7] = 1 + expected[(fresh_dom + j) << 7] = 1 blocks.append(Block(txs=txs)) blockchain_test( @@ -790,14 +745,11 @@ def test_tx_halt( strong post-state oracle for the mixed case. Each repeat block is offset to a fresh range. """ - stride = 1 budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS - count = _iterations(StorageOp.SSTORE_FRESH, 0, stride, budget) + count = _iterations(StorageOp.SSTORE_FRESH, 0, budget) sender = pre.fund_eoa() - contract = pre.deploy_contract( - _contract(StorageOp.SSTORE_FRESH, 0, stride) - ) + contract = pre.deploy_contract(_contract(StorageOp.SSTORE_FRESH, 0)) blocks = [] expected: StorageDict = {} @@ -815,7 +767,7 @@ def test_tx_halt( max_fee_per_gas=MAX_FEE_PER_GAS, max_priority_fee_per_gas=0, data=_calldata( - fresh_dom + t * count * stride, + fresh_dom + t * count, count, global_idx, int(halt), @@ -825,7 +777,7 @@ def test_tx_halt( ) if not halt: for i in range(count): - page = fresh_dom + (t * count + i) * stride + page = fresh_dom + (t * count + i) expected[page << 7] = 1 expected[MARKER_BASE + global_idx] = value_code_worked global_idx += 1 @@ -841,19 +793,10 @@ def test_tx_halt( # --- random access + adversarial "bad block" cases ------------------- # -# Additive: the tests, params and StorageOp members above are unchanged, -# so results stay comparable across versions. Slot keys are spread -# pseudo-randomly over the low 2**200 of the slot space (a 256-bit odd -# multiplier + a 200-bit mask), kept below MARKER_BASE so a random key -# never collides with a marker/checksum witness. - -RAND_MULT = 0x9E3779B97F4A7C15F39CC0605CEDC8341082276BF3A27251F86C6A11D0C18E95 -RAND_MASK = (1 << 200) - 1 -RAND_IDX_STEP = 5 # odd: permutes the in-tx access order over the set -RAND_CONTRACTS = 8 # pool of contracts, called in a pseudorandom cycle -RAND_CONTRACT_STEP = 3 # coprime to RAND_CONTRACTS -RAND_SEED_BASE = 1 -RAND_SEED_STRIDE = 1 << 10 # > max slots, so per-tx slot sets are disjoint + +RAND_CONTRACTS = 8 # pool of contracts, spread across the block's txs +RAND_BASE = 1 # first page key of the read set +RAND_STRIDE = 1 << 10 # > max slots, so per-tx page sets are disjoint M_CHAIN = 0xE0 # chained-sload current slot (memory scratch) SERIAL_BASE = 1 << 40 SERIAL_REPEAT_STRIDE = 1 << 24 # >> per-tx slot count @@ -861,11 +804,6 @@ def test_tx_halt( CHAIN_REPEAT_STRIDE = 1 << 20 # >> ring length -def _rand_slot(seed: int, idx: int) -> int: - """Python mirror of the contract's pseudorandom slot key.""" - return ((seed + idx) * RAND_MULT) & RAND_MASK - - def _size_count(body: Bytecode, budget: int) -> int: """Loop iterations of `body` that fit `budget`, sized to both forks.""" per = max(body.gas_cost(MONAD_NINE), body.gas_cost(MONAD_NEXT)) @@ -873,9 +811,9 @@ def _size_count(body: Bytecode, budget: int) -> int: def _rand_sload_body(slots: int) -> Bytecode: - """One iteration: cold-SLOAD a pseudorandom slot from the set.""" - idx = Op.AND(Op.MUL(Op.MLOAD(M_COUNTER), RAND_IDX_STEP), slots - 1) - slot = Op.AND(Op.MUL(Op.ADD(Op.MLOAD(M_BASE), idx), RAND_MULT), RAND_MASK) + """One iteration: cold-SLOAD one distinct page from the read set.""" + idx = Op.AND(Op.MLOAD(M_COUNTER), slots - 1) + slot = Op.SHL(7, Op.ADD(Op.MLOAD(M_BASE), idx)) read = Op.SLOAD(slot, key_warm=False, page_load_warm=False) return Op.MSTORE( M_CHECKSUM, Op.ADD(Op.MLOAD(M_CHECKSUM), read) @@ -884,11 +822,11 @@ def _rand_sload_body(slots: int) -> Bytecode: def _rand_sload_contract(slots: int) -> Bytecode: """ - SLOAD `slots` pseudorandom slots in a pseudorandom cycle, `count` - times, then write a success marker and the read checksum. + SLOAD `slots` distinct pages in a cycle, `count` times, then write a + success marker and the read checksum. """ init = ( - Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # seed + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # base page key + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + Op.MSTORE(M_COUNTER, 0) @@ -924,30 +862,29 @@ def test_random_sload( k: int, ) -> None: """ - Cold-SLOAD a set of `slots` pseudorandom slots (spread over the slot - space) in a pseudorandom cycle, from a pool of contracts called in a - pseudorandom cycle. `k` is the page occupancy of each read slot: 1 - (slot present, reads 1) or 0 (empty page, reads 0). Cycling a small - set means only the first pass is cold, so blocks are gas-underfull by - design; each repeat uses fresh seeds so its slots are disjoint. + Cold-SLOAD a set of `slots` distinct pages, from a pool of contracts + spread across the block's txs. `k` is the page occupancy of each read + page: 1 (slot present, reads 1) or 0 (empty page, reads 0). Cycling a + small set means only the first pass is cold, so blocks are + gas-underfull by design; each repeat uses a fresh page range. """ budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS count = _size_count(_rand_sload_body(slots), budget) code = _rand_sload_contract(slots) sender = pre.fund_eoa() - # Plan tx -> (contract index, seed) and per-contract genesis slots. + # Plan tx -> (contract index, base page key) and genesis slots. plan: List[Tuple[int, int]] = [] genesis: List[StorageDict] = [{} for _ in range(RAND_CONTRACTS)] g = 0 for _r in range(REPEATS): for _t in range(FULL_BLOCK_TXS): - ci = (g * RAND_CONTRACT_STEP) % RAND_CONTRACTS - seed = RAND_SEED_BASE + g * RAND_SEED_STRIDE + ci = g % RAND_CONTRACTS + base = RAND_BASE + g * RAND_STRIDE if k == 1: for idx in range(slots): - genesis[ci][_rand_slot(seed, idx)] = 1 - plan.append((ci, seed)) + genesis[ci][(base + idx) << 7] = 1 + plan.append((ci, base)) g += 1 contracts: List[Address] = [ @@ -963,7 +900,7 @@ def test_random_sload( for _r in range(REPEATS): txs = [] for _t in range(FULL_BLOCK_TXS): - ci, seed = plan[g] + ci, base = plan[g] contract = contracts[ci] txs.append( Transaction( @@ -972,7 +909,7 @@ def test_random_sload( gas_limit=budget, max_fee_per_gas=MAX_FEE_PER_GAS, max_priority_fee_per_gas=0, - data=_calldata(seed, count, g, 0), + data=_calldata(base, count, g, 0), ) ) post[contract][MARKER_BASE + g] = value_code_worked @@ -1062,9 +999,10 @@ def test_bad_block_chained( ) -> None: """ Adversarial block with a data-dependent SLOAD chain: each SLOAD - returns the next SLOAD's slot (SLOAD(SLOAD(...(seed)))), following a - pre-built storage ring, so the reads serialise within a tx. Each - repeat uses a fresh ring. + returns the next SLOAD's slot (SLOAD(SLOAD(...(base)))), following a + pre-built ring of distinct pages, so the reads serialise within a tx + and each hop is a random disk position (pointer chase). Each repeat + uses a fresh ring. """ budget = BLOCK_GAS_TARGET // FULL_BLOCK_TXS sender = pre.fund_eoa() @@ -1077,7 +1015,7 @@ def test_bad_block_chained( count = _size_count(body, budget) code = ( - Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # seed slot + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # base slot + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + Op.MSTORE(M_COUNTER, 0) @@ -1094,7 +1032,7 @@ def test_bad_block_chained( rings: List[List[int]] = [] for r in range(REPEATS): base = CHAIN_BASE + r * CHAIN_REPEAT_STRIDE - ring = [_rand_slot(base, j) for j in range(count)] + ring = [(base + j) << 7 for j in range(count)] for j in range(count): genesis[ring[j]] = ring[(j + 1) % count] rings.append(ring) From 80af283a10335d61d3212aadee05418d2b4217f3 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:17:51 +0000 Subject: [PATCH 03/23] perf(mip8): Various fixes Co-Authored-By: Claude --- MIP8_PERF_TESTS_DIAGRAMS.md | 81 +++++-- MONAD_RUNLOOP_TESTING.md | 46 ++-- .../plugins/consume/direct/conftest.py | 13 +- .../plugins/consume/direct/timing_report.py | 144 +++--------- .../client_clis/clis/monad.py | 39 +++- .../client_clis/tests/test_monad_timing.py | 72 ++++++ scripts/perf_cycle.sh | 57 +++++ ...f_disjoint_table.py => perf_regression.py} | 207 +++++++----------- .../test_perf_regression.py | 57 +++-- whitelist.txt | 1 + 10 files changed, 388 insertions(+), 329 deletions(-) create mode 100644 packages/testing/src/execution_testing/client_clis/tests/test_monad_timing.py create mode 100755 scripts/perf_cycle.sh rename scripts/{perf_disjoint_table.py => perf_regression.py} (62%) diff --git a/MIP8_PERF_TESTS_DIAGRAMS.md b/MIP8_PERF_TESTS_DIAGRAMS.md index 29cafe9538c..f03fdeff32e 100644 --- a/MIP8_PERF_TESTS_DIAGRAMS.md +++ b/MIP8_PERF_TESTS_DIAGRAMS.md @@ -1,6 +1,6 @@ -# MIP-8 perf-regression tests: per-family block diagrams +# MIP-8 perf-regression tests: block diagrams -What each test-case family in +What each test case in `tests/monad_ten/mip8_pageified_storage/test_perf_regression.py` actually does at the SLOAD/SSTORE level inside one block. Numbers (pages per tx, per-iteration gas) are computed with the module's own sizing helpers at @@ -71,7 +71,7 @@ cold copy. Diagrams below show the `r = 0` block. ## `test_compute_loop` -No parameters, one family. The storage-free baseline: one block, one +No parameters. The storage-free baseline: one block, one 10M-gas tx running a stack-arithmetic `WhileGas` loop. ``` @@ -89,10 +89,11 @@ execution overhead from any MIP-8 effect. --- -## `test_page_ops` — one family per storage op +## `test_page_ops` -Family = one `StorageOp`; within a family, `k` (page occupancy) varies. -All variants use the 7-tx full block above. Per-tx page counts: +One section per storage operation (`StorageOp`); within each, `k` (page +occupancy) varies. All variants use the 7-tx full block above. Per-tx +page counts: | op | k values | P pages/tx | |--------------------|--------------|----------------------------| @@ -112,7 +113,9 @@ All variants use the 7-tx full block above. Per-tx page counts: (`P = 65,536 / k`); those blocks do less than 200M of real work by design — the pre-state, not gas, is the bound. -### Family `sload_cold_hit` +### Storage operation `sload_cold_hit` + +Block-filling transactions cold-SLOAD one occupied slot on many pages. Pre-state: pool of P pages at domain D, each with slots `0..k-1 = 1`. Every tx makes one cold pass over the whole pool, reading offset 0 @@ -130,7 +133,9 @@ each read returns 1 → checksum = P per tx block total: 7 × P cold page reads of an existing, occupied slot ``` -### Family `sload_cold_miss` +### Storage operation `sload_cold_miss` + +Block-filling transactions cold-SLOAD an empty slot on many occupied pages. Same shared pool as `cold_hit` (k ≤ 64 keeps the last slot empty), but each read targets offset 127 — the page exists, the slot is zero. @@ -147,7 +152,9 @@ checksum = 0 → the tail checksum SSTORE writes 0 (no state trace) Measures "page found, slot not found" lookups, P cold reads per tx. -### Family `sload_sweep` +### Storage operation `sload_sweep` + +Block-filling transactions cold-read all occupied slots of each page. Each loop iteration cold-reads **every occupied slot** of one page, offsets `0..k-1` in ascending order. @@ -164,7 +171,9 @@ checksum = P × k per tx Isolates intra-page locality: same number-ish of cold reads as `cold_hit`, but bunched k-per-page instead of 1-per-page. -### Family `sload_warm_repeat` +### Storage operation `sload_warm_repeat` + +Block-filling transactions cold-SLOAD a slot of each page then warm-re-read it repeatedly. No page pool — a single pre-populated slot `W = 2^70` (value 1). The slot arrives via calldata and is read directly (not page-shifted). @@ -183,7 +192,9 @@ checksum = 158,946 per tx; block total ≈ 1.11M reads of one hot slot The warm-path baseline: page/slot caching should make fork choice irrelevant here. -### Family `sload_empty_page` +### Storage operation `sload_empty_page` + +Block-filling transactions cold-SLOAD a slot on never-populated, empty pages. Like `cold_hit` but with `k = 0`: the domain-D pages were **never populated**. Every read is a whole-page miss. @@ -197,7 +208,9 @@ P = 3469 per tx; checksum = 0 (zero write in the tail, no state trace) Measures lookups that fall off the page index entirely. -### Family `sstore_fresh` +### Storage operation `sstore_fresh` + +Block-filling transactions SSTORE 0->1 into previously-unoccupied slots on many pages. No pre-state. Each tx gets its own disjoint range of never-touched pages and creates one slot on each: `W(0)=1`, a 0→1 write that brings @@ -215,7 +228,9 @@ tx t, iteration i: W(0)=1 on page F + t·P + i state growth P = 1009 → block creates 7 × 1009 = 7063 new pages ``` -### Family `sstore_noop` +### Storage operation `sstore_noop` + +Block-filling transactions SSTORE 1->1 (value unchanged) on occupied pages. Shared occupied pool (like `cold_hit`); each tx rewrites slot 0 with its current value — 1→1, no state change ever. @@ -229,7 +244,9 @@ post-state == pre-state (plus markers); P = 3432 (512 at k=128) Pays the write path without any page mutation. -### Family `sstore_grow` +### Storage operation `sstore_grow` + +Block-filling transactions SSTORE 0->1 into a new empty slot of occupied pages. Shared pool with offsets `0..k-1` occupied. Tx `t` writes offset `k + t` — a 0→1 on an **already-occupied** page (growth within a page, @@ -246,7 +263,9 @@ page D + i, one column per tx: each tx: P = 1009 cold W(k+t)=1 writes, one per pool page ``` -### Family `sstore_update` +### Storage operation `sstore_update` + +Block-filling transactions SSTORE 1->2 (nonzero value change) on occupied pages. Shared pool; every tx overwrites the occupied slot 0 with a fresh nonzero value `2 + t`, so each write is a genuine value change with no @@ -260,7 +279,9 @@ page D + i, offset 0 over the block: each tx: P = 2563 cold writes (512 at k=128); slot 0 ends at 8 ``` -### Family `sstore_clear_keep` +### Storage operation `sstore_clear_keep` + +Block-filling transactions SSTORE 1->0 clearing one slot of occupied pages, leaving the page populated. Shared pool with `k > 7` occupied slots. Tx `t` clears offset `t` (1→0). Offsets `7..k-1` stay populated, so no page ever empties. @@ -275,7 +296,9 @@ page D + i: each tx: P = 2565 cold W(t)=0 writes (512 at k=128) ``` -### Family `sstore_clear_empty` +### Storage operation `sstore_clear_empty` + +Block-filling transactions SSTORE 1->0 clearing the only slot of single-slot pages, removing the page. The page-removal case. Pre-state gives **each tx its own** range of single-slot pages (offset 0 = 1). Clearing that slot leaves the page @@ -297,7 +320,9 @@ P = 2565 → block removes 7 × 2565 = 17,955 pages --- -## `test_page_spread` — one family +## `test_page_spread` + +Transactions SSTORE 0->1 into `m` fresh slots spread across `n` contracts. Op is fixed (`sstore_fresh`, the 0→1 page-creating write); the sweep is over **where** the writes land: `m` total pages spread evenly across @@ -336,7 +361,9 @@ Variants: `m ∈ {1,4,16,64,256,1024,4096} × n=1` (total-size sweep), --- -## `test_block_shape` — one family +## `test_block_shape` + +A few big vs many small (~300) transactions, each cold-SLOAD one occupied slot or SSTORE 0->1 into fresh slots. Same total work packed as **7 big** txs vs **300 small** txs. Two workloads: @@ -367,7 +394,9 @@ to loop work. --- -## `test_tx_halt` — one family +## `test_tx_halt` + +Seven transactions SSTORE 0->1 into fresh slots and either succeed, hit INVALID reverting all writes, or alternate. The `sstore_fresh` full block (7 txs × 1009 fresh pages, disjoint ranges), with a `halt` calldata flag per tx. A halting tx performs all @@ -392,7 +421,9 @@ back. --- -## `test_random_sload` — one family +## `test_random_sload` + +Cold-SLOAD a set of distinct 1-element or empty pages (random access), from a pool of contracts spread across the block. Random file access: each read targets a **distinct page** (`slot = page_key << 7`), and the MPT hashes every page key (`keccak256`) to an @@ -428,7 +459,9 @@ land in the storage of whichever contract the tx called. --- -## `test_bad_block_serial` — one family +## `test_bad_block_serial` + +Every tx SLOAD+SSTORE-increments the same slot sequence, forcing serial execution across the block. No parameters. The write-conflict adversarial block: all 7 txs read-then-increment the **same** contiguous slot range, so every tx @@ -455,7 +488,9 @@ shared slot ends at 7, plus the 7 markers (no checksum tail here). --- -## `test_bad_block_chained` — one family +## `test_bad_block_chained` + +Each SLOAD returns the next SLOAD's slot (SLOAD(SLOAD(...))), a data-dependent pointer chase over distinct pages that serialises the reads within a tx. No parameters. The data-dependency adversarial block: the genesis storage holds a pre-built ring of 3482 **distinct pages** (keys diff --git a/MONAD_RUNLOOP_TESTING.md b/MONAD_RUNLOOP_TESTING.md index b2e7ff07155..42ef7b0bfcb 100644 --- a/MONAD_RUNLOOP_TESTING.md +++ b/MONAD_RUNLOOP_TESTING.md @@ -10,9 +10,9 @@ executed result. | Repo / branch | Role | |---|---| -| `monad-exp/monad-eest-rust-harness` | `eest-runner` harness: builds consensus blocks from a fixture and runs them on the runloop | -| `monad-bft` @ `execute-with-eestnet` (submodule of the above) | consensus block types + ledger writer; pins monad-execution below | -| `monad` @ `execute-with-eestnet` (submodule of monad-bft) | execution client with the `EestNet` chain (id 30143, per-fixture revision schedule, runtime genesis) and the extended `monad_runloop_*` FFI | +| `monad-exp/monad-eest-rust-harness` @ `perf-regression-eestnet` | `eest-runner` harness: builds consensus blocks from a fixture and runs them on the runloop | +| `monad-bft` @ `perf-regression-eestnet` (submodule of the above) | consensus block types + ledger writer; pins monad-execution below | +| `monad` @ `perf-regression-eestnet` (submodule of monad-bft) | execution client with the `EestNet` chain (id 30143, per-fixture revision schedule, runtime genesis) and the extended `monad_runloop_*` FFI | | this repo | `MonadFixtureConsumer` (`packages/testing/.../client_clis/clis/monad.py`) wired into `consume direct` | ## One-time setup @@ -22,7 +22,7 @@ artifacts, ~6 GB RAM for hugepages. ```sh snap install astral-uv --classic -git clone --branch main \ +git clone --branch perf-regression-eestnet \ git@github.com:monad-exp/monad-eest-rust-harness.git cd monad-eest-rust-harness git submodule update --init --recursive @@ -75,7 +75,6 @@ uv run consume direct --input ../fixtures_eestnet \ SLOAD/SSTORE workloads at both forks and times block execution on the runloop to compare MONAD_NINE (slot-encoded) vs MONAD_NEXT (page-encoded). - ### Setup ```sh @@ -88,30 +87,27 @@ sudo sysctl --system sudo cpupower idle-set -D 1 ``` -### Filling & running +### Run -```sh -MIP8_PERF_REPEATS=5 uv run fill --clean -m blockchain_test \ - tests/monad_ten/mip8_pageified_storage/test_perf_regression.py \ - --from MONAD_NINE --until MONAD_NEXT --chain-id 30143 --monad-runloop \ - --output ../fixtures_eestnet -n auto +From the repo root: -uv run consume direct --input ../fixtures_eestnet \ - --bin ../monad-eest-rust-harness/bin/eest-runner \ - --timing-report-dir ../timing +```sh +tmux new -s perf 'TAG=v4 RUNS=7 scripts/perf_cycle.sh' ``` -- Each block is stamped to 200M gas; the same workload is sized to fit - both forks (no gas assertions — post-state is the oracle). -- `MIP8_PERF_REPEATS=N` (default 1) emits N page-disjoint copies of each - workload as successive blocks → N cold timing samples per fixture. -- `MIP8_PERF_BLOCK_GAS=N` shrinks the block for a quick smoke fill. -- Consume writes both `timing_consume.md` and `.csv` by default - (`--timing-report {both,md,csv,none}`, `--timing-report-dir DIR`): one - row per (test, params, fork) — the `min` over the repeat blocks (drops - the warmup block) — with MONAD_NINE, MONAD_NEXT, then a Δ% row. -- Run consume on a quiet host for stable timings; the numbers are noisy - under contention. +Fills once, consumes `RUNS` times, and writes the NINE-vs-NEXT table to +`../timing_${TAG}__table.{html,md}` (the `.md` is headed with the +cycle time and the four repo SHAs). Knobs: + +- `TAG` (required) names every artifact; use a fresh one per experiment. +- `RUNS` consume passes (samples per fork), `REPEATS` page-disjoint + copies per fixture (cold samples reduced to a `min` within each pass). +- `MIP8_PERF_BLOCK_GAS=N` overrides the block gas target; perf_cycle.sh + fills full 200M blocks by default (the test default, used by release + fills, is a small block). `SKIP_FILL=1` reuses an existing + `../fixtures_${TAG}`. + +Run on a quiet host; timings are noisy under contention. ## Behavior and known limits diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py index 519ea697cca..97629073510 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/conftest.py @@ -84,14 +84,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: # noqa: D103 ) consume_group.addoption( "--timing-report", - action="store", + action="store_true", dest="timing_report", - choices=["both", "md", "csv", "none"], - default="both", + default=False, help=( "Emit per-block execution timing (from consumers that report " - "it) as a `timing_consume` table. `both` (default) writes " - "Markdown and CSV; `md`/`csv` write one; `none` disables." + "it) as a raw `timing_consume.csv`." ), ) consume_group.addoption( @@ -149,10 +147,9 @@ def pytest_configure(config: pytest.Config) -> None: # noqa: D103 ) config.fixture_consumers = fixture_consumers # type: ignore[attr-defined] - timing_report = config.getoption("timing_report") - if timing_report != "none": + if config.getoption("timing_report"): config.pluginmanager.register( - TimingReportPlugin(config, timing_report), + TimingReportPlugin(config), "consume-timing-report", ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py index cc9ba8ec881..71c77003410 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py @@ -1,13 +1,13 @@ """ -Emit per-block execution timing collected during `consume direct` as -Markdown and/or CSV artifacts alongside the HTML report. +Emit per-block execution timing collected during `consume direct` as a +CSV artifact alongside the HTML report. The consumer (e.g. the monad runloop) returns per-block timing from `consume_fixture`; `test_via_direct.test_fixture` stashes it on the test's `user_properties` under `TIMING_PROPERTY`. This plugin writes one part file per test *in the process that ran it* (so it never depends on xdist forwarding `user_properties` to the controller), then the controller -aggregates every part into a single table at session end. +collects every part into a single raw per-block CSV at session end. """ from __future__ import annotations @@ -16,7 +16,6 @@ import io import json import uuid -from itertools import groupby from pathlib import Path from typing import Any, Dict, Generator, List, Tuple @@ -26,12 +25,13 @@ # ``{"id": , "blocks": [BlockExecutionTiming, ...]}``. TIMING_PROPERTY = "consume_block_timing" -# Column key -> header. Metrics are the per-fixture minimum across blocks -# (repeat samples); the warmup block is naturally discarded by the min. +# Column key -> header. One row per block carrying the raw per-block +# measurements. _COLUMNS: List[Tuple[str, str]] = [ ("test", "test"), ("params", "params"), ("fork", "fork"), + ("block", "block"), ("tx_count", "tx"), ("gas", "gas"), ("tx_exec_us", "tx_exec_us"), @@ -40,12 +40,9 @@ ("total_us", "total_us"), ] -# Numeric metric columns a Δ% row is computed for. -_METRIC_KEYS = ("tx_exec_us", "state_root_us", "commit_us", "total_us") - -# Fork ordering (oldest first) so a Δ% row compares the newer fork against -# the older baseline; unknown forks sort after, alphabetically. -_FORK_ORDER = ["MONAD_EIGHT", "MONAD_NINE", "MONAD_NEXT", "MONAD_TEN"] +# Fork ordering (oldest first) so CSV rows group forks in release order; +# unknown forks sort after, alphabetically. +_FORK_ORDER = ["MONAD_EIGHT", "MONAD_NINE", "MONAD_NEXT"] # Fixture-id suffixes identifying the fixture format, not a real parameter. _FORMAT_TAGS = { @@ -63,72 +60,17 @@ def _fork_rank(fork: str) -> Tuple[int, str]: return len(_FORK_ORDER), fork -def _delta_row(base: Dict[str, Any], comp: Dict[str, Any]) -> Dict[str, Any]: - """Build a percent-change row comparing `comp` against `base`.""" - - def pct(b: Any, c: Any) -> str: - if not isinstance(b, (int, float)) or b == 0: - return "n/a" - return f"{(c - b) / b * 100:+.1f}%" - - row: Dict[str, Any] = { - "test": "", - "params": "", - "fork": f"Δ% {comp['fork']}/{base['fork']}", - "tx_count": "", - "gas": "", - } - for key in _METRIC_KEYS: - row[key] = pct(base[key], comp[key]) - return row - - -def _aggregate(rows: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int]: - """ - Reduce per-block rows to one row per (test, params, fork). - - Each metric becomes the minimum across the fixture's blocks — repeat - samples on disjoint pages — which discards the warmup block. Returns - the aggregated rows and the largest block count seen (repeat count). - """ - groups: Dict[Tuple[str, str, str], List[Dict[str, Any]]] = {} - order: List[Tuple[str, str, str]] = [] - for row in rows: - key = (row["test"], row["params"], row["fork"]) - if key not in groups: - groups[key] = [] - order.append(key) - groups[key].append(row) - - aggregated: List[Dict[str, Any]] = [] - max_blocks = 1 - for key in order: - members = groups[key] - max_blocks = max(max_blocks, len(members)) - row = dict(members[0]) - for metric in _METRIC_KEYS: - row[metric] = min(m[metric] for m in members) - aggregated.append(row) - return aggregated, max_blocks - - -def _ordered_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Order rows by (test, params) then fork, and append a Δ% row after each - pair of forks sharing a (test, params) group. - """ - - def group_key(row: Dict[str, Any]) -> Tuple[str, str]: - return row["test"], row["params"] - - rows.sort(key=lambda r: (*group_key(r), _fork_rank(r["fork"]))) - ordered: List[Dict[str, Any]] = [] - for _, group in groupby(rows, key=group_key): - members = list(group) - ordered.extend(members) - if len(members) == 2: - ordered.append(_delta_row(members[0], members[1])) - return ordered +def _sorted_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Order rows by (test, params, fork) then block for readability.""" + rows.sort( + key=lambda r: ( + r["test"], + r["params"], + _fork_rank(r["fork"]), + r["block"], + ) + ) + return rows def _split_fixture_id(fixture_id: str) -> Tuple[str, str, str]: @@ -169,22 +111,6 @@ def _rows_from_payload(payload: Dict[str, Any]) -> List[Dict[str, Any]]: return rows -def _render_markdown(rows: List[Dict[str, Any]]) -> str: - """Render rows as a GitHub-flavored Markdown table.""" - headers = [header for _, header in _COLUMNS] - lines = [ - "| " + " | ".join(headers) + " |", - "| " + " | ".join("---" for _ in headers) + " |", - ] - for row in rows: - lines.append( - "| " - + " | ".join(str(row.get(key, "")) for key, _ in _COLUMNS) - + " |" - ) - return "\n".join(lines) + "\n" - - def _render_csv(rows: List[Dict[str, Any]]) -> str: """Render rows as CSV.""" buffer = io.StringIO() @@ -196,13 +122,11 @@ def _render_csv(rows: List[Dict[str, Any]]) -> str: class TimingReportPlugin: - """Collect per-test timing parts and write the aggregated report.""" + """Collect per-test timing parts and write the raw per-block CSV.""" - def __init__(self, config: pytest.Config, fmt: str): # noqa: D107 + def __init__(self, config: pytest.Config): # noqa: D107 self.config = config - self.fmt = fmt self.written: List[Path] = [] - self.repeats = 1 def _output_dir(self) -> Path: """ @@ -257,7 +181,7 @@ def pytest_sessionfinish( session: pytest.Session, exitstatus: int, # noqa: ARG002 ) -> None: - """Aggregate all part files into the report (controller only).""" + """Collect all part files into the CSV report (controller only).""" if hasattr(session.config, "workerinput"): return # xdist worker: parts already written by makereport parts_dir = self._parts_dir() @@ -268,19 +192,13 @@ def pytest_sessionfinish( rows.extend(_rows_from_payload(json.loads(part.read_text()))) if not rows: return - aggregated, self.repeats = _aggregate(rows) - rows = _ordered_rows(aggregated) + rows = _sorted_rows(rows) output_dir = self._output_dir() output_dir.mkdir(parents=True, exist_ok=True) - if self.fmt in ("both", "md"): - path = output_dir / "timing_consume.md" - path.write_text(_render_markdown(rows)) - self.written.append(path) - if self.fmt in ("both", "csv"): - path = output_dir / "timing_consume.csv" - path.write_text(_render_csv(rows)) - self.written.append(path) + path = output_dir / "timing_consume.csv" + path.write_text(_render_csv(rows)) + self.written.append(path) for part in parts_dir.glob("*.json"): part.unlink() @@ -296,10 +214,8 @@ def pytest_terminal_summary( if not self.written: return terminalreporter.write_sep("=", "block execution timing report") - if self.repeats > 1: - terminalreporter.write_line( - f"metrics are the min over {self.repeats} repeat block(s) " - "per fixture" - ) + terminalreporter.write_line( + "raw per-block timings; aggregate across blocks downstream" + ) for path in self.written: terminalreporter.write_line(f"timing report written to: {path}") diff --git a/packages/testing/src/execution_testing/client_clis/clis/monad.py b/packages/testing/src/execution_testing/client_clis/clis/monad.py index 36434247242..3cc1c606eec 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/monad.py +++ b/packages/testing/src/execution_testing/client_clis/clis/monad.py @@ -26,10 +26,13 @@ from execution_testing.fixtures import BlockchainFixture, FixtureFormat from execution_testing.fixtures.consume import BlockExecutionTiming +from execution_testing.logging import get_logger from execution_testing.test_types import Transaction from ..fixture_consumer_tool import FixtureConsumerTool +logger = get_logger(__name__) + # Monad revision schedule per fixture `network`, as # (monad_revision, activation timestamp) pairs. Non-transition # fixtures run a single revision from genesis; transition fixtures @@ -71,7 +74,7 @@ def _load_fixture( """ Load a single fixture from a (possibly multi-fixture) JSON file. - ijson for low-memory footprint. + ijson streams the file so memory use stays low on large fixtures. """ with open(fixture_path, "rb") as f: for name, fixture in ijson.kvitems(f, ""): @@ -177,8 +180,28 @@ def _compare_account( return mismatches +# A runloop duration: a decimal number and a chrono unit suffix, with +# optional whitespace. +_DURATION_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*(ns|[µμu]s|ms|s)\s*$") +_UNIT_US = {"ns": 1e-3, "us": 1.0, "ms": 1e3, "s": 1e6} + + +def _duration_us(value: str) -> int: + """ + Convert a duration like `5745us`, `5.7 ms` or `0.01s` to integer + microseconds; raise ValueError on any unrecognized format. + """ + match = _DURATION_RE.match(value) + if match is None: + raise ValueError(f"unrecognized duration {value!r}") + number, unit = match.groups() + if unit in ("µs", "μs"): + unit = "us" + return round(float(number) * _UNIT_US[unit]) + + def _exec_block_row(line: str) -> Optional[BlockExecutionTiming]: - """Parse one `__exec_block` log line, or None if malformed.""" + """Parse one `__exec_block` log line, or None (logged) if malformed.""" body = line.split("__exec_block", 1)[1] fields: Dict[str, str] = {} for part in body.split(","): @@ -187,7 +210,7 @@ def _exec_block_row(line: str) -> Optional[BlockExecutionTiming]: fields[key.strip()] = value.strip() def us(key: str) -> int: - return int(fields[key].replace("µs", "").strip()) + return _duration_us(fields[key]) try: return BlockExecutionTiming( @@ -199,7 +222,8 @@ def us(key: str) -> int: commit_us=us("cmt"), total_us=us("tot"), ) - except (KeyError, ValueError): + except (KeyError, ValueError) as e: + logger.error(f"unparsable __exec_block line ({e!r}): {line.strip()}") return None @@ -208,9 +232,10 @@ def _parse_block_timings(stdout: str) -> List[BlockExecutionTiming]: Extract per-block timing from the runloop's `__exec_block` log lines. The production runloop logs one such line per block, e.g.: - `__exec_block,bl=1,...,tx=1,...,sr=5192µs,txe=14241µs,cmt=879µs, - tot=21153µs,...,gas=10000000,...`. Fields carry leading padding and a - `µs` suffix on durations. Missing/malformed lines are skipped. + `__exec_block,bl=1,...,tx=1,...,sr=5192us,txe=14241us,cmt=879us, + tot=21153us,...,gas=10000000,...`. Fields carry leading padding; + durations a chrono unit suffix (see `_duration_us`). Malformed + lines are logged and skipped. """ rows = [ _exec_block_row(line) diff --git a/packages/testing/src/execution_testing/client_clis/tests/test_monad_timing.py b/packages/testing/src/execution_testing/client_clis/tests/test_monad_timing.py new file mode 100644 index 00000000000..0a1cfb3a655 --- /dev/null +++ b/packages/testing/src/execution_testing/client_clis/tests/test_monad_timing.py @@ -0,0 +1,72 @@ +"""Tests for the monad consumer's `__exec_block` timing parsing.""" + +import pytest + +from execution_testing.client_clis.clis.monad import ( + _duration_us, + _parse_block_timings, +) + + +@pytest.mark.parametrize( + "value, us", + [ + ("5745us", 5745), + ("5745µs", 5745), + ("5745μs", 5745), + (" 5.745 ms ", 5745), + ("1200ns", 1), + ("0.013081s", 13081), + ], +) +def test_duration_units(value: str, us: int) -> None: + """Durations convert to integer microseconds whatever the unit.""" + assert _duration_us(value) == us + + +@pytest.mark.parametrize( + "value", + ["", "13081", "5745 sec", "5745usx", "-5us", "5.us", "fast", "us"], +) +def test_duration_unrecognized(value: str) -> None: + """An unrecognized duration format raises instead of guessing.""" + with pytest.raises(ValueError, match="unrecognized duration"): + _duration_us(value) + + +def _line(suffix: str) -> str: + """Return a realistic runloop `__exec_block` log line.""" + return ( + "2026-07-16 12:45:21.263419673 [7] runloop_monad.cpp:385 LOG_INFO" + " __exec_block,bl= 1,id=0x1433,ts=1784205921250," + f"tx= 7,rt= 0,rtp= 0.00%,sr= 5745{suffix}," + f"txe= 6991{suffix},cmt= 169{suffix},tot= 13081{suffix}," + "tpse= 143,tps= 76,gas= 10000000,gpse=1430,gps=764,ae= 2," + "ane= 0,sz= 5,snz= 0,ac= 9,sc= 5 / 1872" + ) + + +@pytest.mark.parametrize("suffix", ["us", "µs"]) +def test_duration_suffix(suffix: str) -> None: + """The duration unit varies with the runloop's toolchain.""" + rows = _parse_block_timings(f"noise\n{_line(suffix)}\n") + assert rows == [ + { + "block": 1, + "tx_count": 7, + "gas": 10_000_000, + "tx_exec_us": 6991, + "state_root_us": 5745, + "commit_us": 169, + "total_us": 13081, + } + ] + + +def test_malformed_line_logged(caplog: pytest.LogCaptureFixture) -> None: + """A malformed line is skipped and reported, not silently dropped.""" + line = "LOG_INFO __exec_block,bl= 1,tx= 7,gas=oops" + with caplog.at_level("ERROR"): + assert _parse_block_timings(line) == [] + assert "unparsable __exec_block line" in caplog.text + assert "gas=oops" in caplog.text diff --git a/scripts/perf_cycle.sh b/scripts/perf_cycle.sh new file mode 100755 index 00000000000..a145298f019 --- /dev/null +++ b/scripts/perf_cycle.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Fill once, consume RUNS times, emit the NINE-vs-NEXT perf table. +set -uo pipefail + +TAG="${TAG:?set TAG (names all artifacts, e.g. TAG=v4)}" +RUNS="${RUNS:-7}" +REPEATS="${REPEATS:-20}" +# Perf runs time full 200M blocks; the test default is a small block. +BLOCK_GAS="${MIP8_PERF_BLOCK_GAS:-200000000}" +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HARNESS="${HARNESS:-$REPO/../monad-eest-rust-harness}" +BIN="${BIN:-$HARNESS/bin/eest-runner}" +TEST="${TEST:-tests/monad_ten/mip8_pageified_storage/test_perf_regression.py}" + +cd "$REPO" +EPOCH="$(date -u +%s)" +NOW="$(date -u -d "@$EPOCH" '+%Y-%m-%dT%H:%M:%SZ')" +STAMP="$(date -u -d "@$EPOCH" '+%y%m%d_%H%M%S')" +FIX="../fixtures_${TAG}" +PREFIX="../timing_${TAG}_${STAMP}" +TABLE="${PREFIX}_table" +[ -x "$BIN" ] || { echo "harness not executable: $BIN" >&2; exit 1; } + +if [ -z "${SKIP_FILL:-}" ]; then + echo "=== fill $FIX (REPEATS=$REPEATS, BLOCK_GAS=$BLOCK_GAS) $NOW ===" + MIP8_PERF_REPEATS="$REPEATS" MIP8_PERF_BLOCK_GAS="$BLOCK_GAS" \ + uv run fill -m blockchain_test "$TEST" \ + --from MONAD_NINE --until MONAD_NEXT --chain-id 30143 --monad-runloop \ + --output "$FIX" -n auto || { + echo "fill failed (a non-empty $FIX aborts fill); rerun with" \ + "SKIP_FILL=1 to reuse it, or delete it to refill" >&2 + exit 1 + } +fi + +for i in $(seq 1 "$RUNS"); do + out="${PREFIX}_${i}" + rm -rf "$out" + echo "=== consume $i/$RUNS -> $out $(date -u +%T)Z ===" + uv run consume direct --input "$FIX" --bin "$BIN" \ + --timing-report --timing-report-dir "$out" + [ -f "$out/timing_consume.csv" ] || { + echo "consume run $i left no $out/timing_consume.csv (eest-runner" \ + "emitted no parseable __exec_block timing lines?)" >&2 + exit 1 + } +done + +sha() { git -C "$1" rev-parse --short HEAD 2>/dev/null || echo '?'; } +python3 scripts/perf_regression.py --md "${TABLE}.md" --html "${TABLE}.html" \ + --now "$NOW" \ + --repo "$(sha "$REPO")" \ + --harness "$(sha "$HARNESS")" \ + --monad-bft "$(sha "$HARNESS/monad-bft")" \ + --monad "$(sha "$HARNESS/monad-bft/monad-execution")" \ + "${PREFIX}"_[0-9]* || { echo "perf_regression.py failed, no table" >&2; exit 1; } +echo "=== table: $(cd .. && pwd)/$(basename "$TABLE").html ===" diff --git a/scripts/perf_disjoint_table.py b/scripts/perf_regression.py similarity index 62% rename from scripts/perf_disjoint_table.py rename to scripts/perf_regression.py index e132f3abd2f..f436d83475f 100644 --- a/scripts/perf_disjoint_table.py +++ b/scripts/perf_regression.py @@ -3,16 +3,18 @@ Build a NINE-vs-NEXT significance table from perf timing runs. Reads the `timing_consume.csv` produced by `consume direct ---timing-report-dir` for several identical runs (one dir each) and, per +--timing-report` for several identical runs (one dir each) and, per (test, params) case, reports each measure's mean +/- sd over the runs for both forks, the two-sided Mann-Whitney U p-value comparing the forks, -and (for measures significant at p <= 0.10) the NINE->NEXT average -change. A trailing list describes each significant case's transactions. - -Usage: perf_disjoint_table.py [--html OUT.html] [DIR ...] - (default dirs: ../timing_[0-9]*) -Emits GitHub-flavored Markdown to stdout; with --html also writes a -standalone HTML rendering whose table spans the full window width. +and (for significant measures) the NINE->NEXT average change. + +Usage: perf_regression.py [--md OUT.md] [--html OUT.html] + [--now TS --repo SHA --harness SHA --monad-bft SHA --monad SHA] + [DIR ...] (default dirs: ../timing_[0-9]*) +Writes GitHub-flavored Markdown to --md (or stdout if omitted); when +--now is given, prefixes a provenance header. With +--html also writes a standalone HTML rendering whose table spans the +full window width. """ from __future__ import annotations @@ -26,14 +28,21 @@ from pathlib import Path from statistics import mean, stdev -METRICS = ["tx_exec_us", "state_root_us", "commit_us", "total_us"] +METRICS = ["tx_exec_us", "commit_us", "total_us"] FORKS = ["MONAD_NINE", "MONAD_NEXT"] -ALPHA = 0.10 # a measure is significant at this Mann-Whitney U p-value +ALPHA = 0.01 # a measure is significant at this Mann-Whitney U p-value UP = "🔴⬆️" # significant measures all rise NINE->NEXT (NEXT slower) DOWN = "🟢⬇️" # significant measures all fall NINE->NEXT (NEXT faster) MIXED = "⚠️" # significant measures move both up and down +# Per-case workload descriptions live here; pinned to the execution-specs +# sha of the run when known, else the repo's default branch (HEAD). +DIAGRAMS_URL = ( + "https://github.com/monad-developers/execution-specs/blob/" + "{ref}/MIP8_PERF_TESTS_DIAGRAMS.md" +) + def _direction(chgs: list[int]) -> str: """Pick the direction emoji from the significant measures' changes.""" @@ -49,7 +58,12 @@ def _direction(chgs: list[int]) -> str: def parse(report: Path) -> dict: - """Map (test, params, fork) -> {metric: value} from one csv report.""" + """ + Map (test, params, fork) -> {metric: value} from one csv report. + + The csv holds one row per block (raw, unaggregated); each metric is + reduced to the minimum across a case's blocks. + """ rows: dict = {} with report.open(newline="") as f: for row in csv.DictReader(f): @@ -60,7 +74,12 @@ def parse(report: Path) -> dict: except (ValueError, TypeError): continue test = row["test"].split("::")[-1].removeprefix("test_") - rows[(test, row["params"], row["fork"])] = mv + key = (test, row["params"], row["fork"]) + prev = rows.get(key) + if prev is None: + rows[key] = mv + else: + rows[key] = {m: min(prev[m], mv[m]) for m in METRICS} return rows @@ -116,97 +135,8 @@ def _mwu_p(a: list[int], b: list[int]) -> float: return erfc(max(0.0, (d - 0.5) / sigma) / sqrt(2)) -def _op_phrase(op: str, k: int) -> str: - """Describe what one block-filling tx does for a storage op.""" - place = "pages" - page = "page" - occ = f", {k} slot{'' if k == 1 else 's'} occupied per page" - phrases = { - "sload_cold_hit": f"cold-SLOAD one occupied slot on many {place}{occ}", - "sload_cold_miss": f"cold-SLOAD an empty slot on many occupied " - f"{place}{occ}", - "sload_sweep": f"cold-read all {k} occupied slots of each {page}", - "sload_warm_repeat": f"cold-SLOAD a slot of each {page} then " - f"warm-re-read it repeatedly", - "sstore_fresh": f"SSTORE 0->1 into previously-unoccupied slots on " - f"many {place}", - "sstore_noop": f"SSTORE 1->1 (value unchanged) on occupied " - f"{place}{occ}", - "sstore_grow": f"SSTORE 0->1 into a new empty slot of occupied " - f"{place}{occ}", - "sstore_update": f"SSTORE 1->2 (nonzero value change) on occupied " - f"{place}{occ}", - "sstore_clear_keep": f"SSTORE 1->0 clearing one slot of occupied " - f"{place}{occ}, leaving the page populated", - "sstore_clear_empty": f"SSTORE 1->0 clearing the only slot of " - f"single-slot {place}, removing the page", - "sload_empty_page": f"cold-SLOAD a slot on never-populated, empty " - f"{place}", - } - return phrases.get(op, op) - - -def describe(test: str, params: str) -> str: - """One-sentence account of what a case's block transactions do.""" - if test == "page_ops": - op, kpart = params.split("-") - k = int(kpart.removeprefix("k")) - return f"block-filling transactions {_op_phrase(op, k)}" - if test == "block_shape": - op, kpart, shape = params.split("-") - k = int(kpart.removeprefix("k")) - who = "a few big" if shape == "few_big" else "many small (~300)" - return f"{who} transactions each {_op_phrase(op, k)}" - if test == "page_spread": - m = params.split("_")[0].removeprefix("m") - n = int(params.split("_")[1].removeprefix("n")) - target = f"{n} contract" + ("" if n == 1 else "s") - return ( - f"transactions SSTORE 0->1 into {m} fresh slots spread across " - f"{target}" - ) - if test == "tx_halt": - mode = params.removeprefix("mode_") - if mode == "success": - return ( - "seven transactions each SSTORE 0->1 into fresh slots and " - "succeed" - ) - if mode == "halt": - return ( - "seven transactions SSTORE 0->1 into fresh slots then hit " - "INVALID, reverting all writes" - ) - return ( - "seven transactions alternate between SSTORE-and-succeed and " - "SSTORE-then-INVALID (halted writes reverted)" - ) - if test == "random_sload": - slots = params.split("-")[0].removeprefix("slots") - k = int(params.split("-")[1].removeprefix("k")) - page = "1-element" if k == 1 else "empty" - noun = "page" if slots == "1" else "pages" - return ( - f"cold-SLOAD {slots} distinct {page} {noun} (random " - "access), " - "from a pool of contracts spread across the block" - ) - if test == "bad_block_serial": - return ( - "every tx SLOAD+SSTORE-increments the same slot sequence, " - "forcing serial execution across the block" - ) - if test == "bad_block_chained": - return ( - "each SLOAD returns the next SLOAD's slot (SLOAD(SLOAD(...))), " - "a data-dependent pointer chase over distinct pages that " - "serialises the reads within a tx" - ) - return f"{test} {params}" - - def build(runs: list[dict]) -> list[str]: - """Return the markdown lines for the table plus case descriptions.""" + """Return the markdown lines for the significance table.""" cases: dict = {} for run in runs: for (test, params, fork), mv in run.items(): @@ -230,7 +160,6 @@ def build(runs: list[dict]) -> list[str]: "| " + " | ".join("---" for _ in header) + " |", ] - sig_cases = [] for test, params in sorted(cases): forks = cases[(test, params)] if not all(f in forks for f in FORKS): @@ -249,26 +178,16 @@ def build(runs: list[dict]) -> list[str]: deltas.append(f"{m} {chg:+d}%") chgs.append(chg) cells += [ncell, xcell, _pfmt(p)] - emoji = _direction(chgs) if chgs else "-" - cells.append(emoji) + cells.append(_direction(chgs) if chgs else "-") cells.append(", ".join(deltas) if deltas else "-") lines.append("| " + " | ".join(cells) + " |") - if chgs: - sig_cases.append((test, params, emoji)) lines += [ "", "p (MWU) is the two-sided Mann–Whitney U p-value comparing the " f"NINE and NEXT run samples for that measure; significant at " f"p ≤ {ALPHA}.", - "", - "Significant cases — what the block's transactions do:", - "", ] - for test, params, emoji in sig_cases: - lines.append( - f"- **{test} {params}**: {describe(test, params)}. {emoji}" - ) return lines @@ -370,16 +289,53 @@ def md_to_html(md: str) -> str: return HTML_TEMPLATE.replace("__BODY__", "\n".join(blocks)) +def _take_opt(argv: list[str], name: str) -> str | None: + """Pop `--name VALUE` out of argv, returning VALUE (or None).""" + if name not in argv: + return None + idx = argv.index(name) + if idx + 1 >= len(argv): + sys.exit(f"{name} requires a value") + value = argv[idx + 1] + del argv[idx : idx + 2] + return value + + +def _provenance(now: str, shas: list[str | None]) -> str: + """Markdown header: descriptions link, cycle time, and repo shas.""" + repos = [ + "execution-specs", + "monad-eest-rust-harness", + "monad-bft", + "monad", + ] + ref = DIAGRAMS_URL.format(ref=shas[0] or "HEAD") + lines = [ + f"Test-case descriptions: {ref}", + "", + f"Cycle {now}", + "", + "| repo | sha |", + "| --- | --- |", + ] + lines += [ + f"| {r} | {s or '?'} |" for r, s in zip(repos, shas, strict=True) + ] + return "\n".join(lines) + + def main() -> None: - """Parse the run dirs, print markdown, optionally write HTML.""" + """Parse the run dirs and write the markdown/HTML report.""" argv = list(sys.argv[1:]) - html_path = None - if "--html" in argv: - idx = argv.index("--html") - if idx + 1 >= len(argv): - sys.exit("--html requires a path") - html_path = argv[idx + 1] - del argv[idx : idx + 2] + md_path = _take_opt(argv, "--md") + html_path = _take_opt(argv, "--html") + now = _take_opt(argv, "--now") + shas = [ + _take_opt(argv, "--repo"), + _take_opt(argv, "--harness"), + _take_opt(argv, "--monad-bft"), + _take_opt(argv, "--monad"), + ] dirs = argv or sorted( glob.glob("../timing_[0-9]*"), key=lambda p: int(p.rsplit("_", 1)[-1]), @@ -392,9 +348,14 @@ def main() -> None: if len(runs) < 2: sys.exit("need >=2 runs with timing_consume.csv") md = "\n".join(build(runs)) - print(md) + if now: + md = f"{_provenance(now, shas)}\n\n{md}" if html_path: Path(html_path).write_text(md_to_html(md), encoding="utf-8") + if md_path: + Path(md_path).write_text(md + "\n", encoding="utf-8") + else: + print(md) if __name__ == "__main__": diff --git a/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py index 5eee976770d..7898816a1a5 100644 --- a/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py +++ b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py @@ -10,7 +10,10 @@ markers, written slots, and read-checksum slots), so the identical workload can be timed on both forks via the consume timing report. -The suite fills the 200M-gas monad block with SLOAD/SSTORE patterns. +Each test fills a monad block with one SLOAD/SSTORE pattern. The block +is small by default so fixture and monad-runloop release fills stay +fast; perf_cycle.sh fills the full 200M-gas block to time execution +(BLOCK_GAS_TARGET, set via MIP8_PERF_BLOCK_GAS). Single-letter test parameters: - `k`: page occupancy — non-zero slots pre-populated per page (0..128). @@ -25,13 +28,9 @@ `MIP8_PERF_REPEATS` (default 1) emits that many copies of each workload as successive blocks, each offset to a disjoint page range so every block is a genuine cold execution. One runloop run then yields N independent -per-block timing samples (the first absorbs process/hugepage warmup); the -consume timing report aggregates them with `min`. The contract bytecode -stays fixed-size regardless of the repeat count — only the pre-state -storage and block list grow. +per-block timing samples (the first absorbs process/hugepage warmup). -Run with `--monad-runloop` and consume via `eest-runner`; see -MONAD_RUNLOOP_TESTING.md. +See MONAD_RUNLOOP_TESTING.md. """ import os @@ -71,10 +70,19 @@ SLOTS_PER_PAGE = Spec.SLOTS_PER_PAGE # 128 # The runloop stamps every monad block at 200M gas; the per-tx cap is -# 30M on both forks. Overridable for quick local smoke fills. -BLOCK_GAS_TARGET = int(os.environ.get("MIP8_PERF_BLOCK_GAS", "200000000")) +# 30M on both forks. perf_cycle.sh fills at FULL_BLOCK_GAS to time real +# blocks; the default is a small block that still exercises every +# workload, keeping fixture/runloop release fills fast. Override with +# MIP8_PERF_BLOCK_GAS. +FULL_BLOCK_GAS = 200_000_000 +SMOKE_BLOCK_GAS = 10_000_000 +BLOCK_GAS_TARGET = int( + os.environ.get("MIP8_PERF_BLOCK_GAS", str(SMOKE_BLOCK_GAS)) +) TX_GAS_CAP = 30_000_000 -# 200M / 30M -> 7 equal txs of ~28.57M each fill a block. +# Fixed tx count per block, sized so a full 200M block splits into 7 +# equal txs of ~28.57M each (near the 30M per-tx cap). A smaller +# BLOCK_GAS_TARGET keeps the 7 txs and shrinks each one's work. FULL_BLOCK_TXS = 7 # Workload txs are EIP-1559 with a high max fee and a zero priority tip. # The high max fee keeps them valid as each full block raises the base @@ -84,13 +92,14 @@ # by the runloop on MONAD_NEXT, which would mismatch the post-state). MAX_FEE_PER_GAS = 10**6 # `many_small` block shape: many txs, each still large enough to cover the -# per-tx reserve. The count adapts to the block budget (300 at 200M) so the -# `MIP8_PERF_BLOCK_GAS` smoke knob does not starve individual txs. +# per-tx reserve. The count adapts to the block budget (300 at the full +# 200M) so a smaller BLOCK_GAS_TARGET does not starve individual txs. MANY_SMALL_TXS = 300 MANY_SMALL_MIN_TX_GAS = 200_000 # Emit this many copies of each workload as successive, page-disjoint -# blocks for repeat timing samples (see module docstring). +# blocks for repeat timing samples (see module docstring). perf_cycle.sh +# raises this; releases fill a single cold copy. REPEATS = int(os.environ.get("MIP8_PERF_REPEATS", "1")) # Upper bound on pre-populated storage slots per block (times REPEATS for @@ -209,9 +218,6 @@ def test_compute_loop( ) -# --- shared bytecode + sizing helpers --------------------------------- - - def _page_index() -> Bytecode: """Bytecode: base + i (base at M_BASE, i at M_COUNTER).""" return Op.ADD(Op.MLOAD(M_BASE), Op.MLOAD(M_COUNTER)) @@ -406,8 +412,6 @@ def _repeat_domains(repeat: int) -> Tuple[int, int, int]: return READ_DOMAIN + shift, FRESH_DOMAIN + shift, WARM_BASE + repeat -# --- dimensions 1, 2, 3, 6: op x occupancy ----------------------------- - # (op, page-occupancy k values). cold_miss/grow keep a zero slot free # (k < SLOTS_PER_PAGE, and grow writes k..k+FULL_BLOCK_TXS-1). clear_keep # needs k > FULL_BLOCK_TXS so a page still has slots after the block @@ -556,8 +560,6 @@ def test_page_ops( ) -# --- dimension 4: m pages spread across n contracts ------------------- - _SPREAD_PARAMS = [ pytest.param(m, n, id=f"m{m}_n{n}") for m, n in [ @@ -591,6 +593,11 @@ def test_page_spread( shares become several txs, and many contracts become many small txs). Isolates the effect of write distribution across accounts. Each repeat block writes a fresh page range. + + Work is sized by `m` and `n`, not by BLOCK_GAS_TARGET: a high fan-out + (n=512) already needs one tx per contract, so these blocks stay large + even under the smoke knob. The block gas limit is the full 200M + ceiling, which covers the largest spread case. """ per_iter = _per_iter_gas(StorageOp.SSTORE_FRESH, 0) max_per_tx = max(1, (TX_GAS_CAP - TX_RESERVE) // per_iter) @@ -639,12 +646,10 @@ def test_page_spread( contract: Account(storage=storage) for contract, storage in contract_storage.items() }, - genesis_environment=Environment(gas_limit=BLOCK_GAS_TARGET), + genesis_environment=Environment(gas_limit=FULL_BLOCK_GAS), ) -# --- dimension 5: few big vs many small transactions ------------------ - _SHAPE_PARAMS = [ pytest.param(op, k, shape, id=f"{op.value}-k{k}-{shape}") for op, k in [(StorageOp.SSTORE_FRESH, 0), (StorageOp.SLOAD_COLD_HIT, 8)] @@ -726,9 +731,6 @@ def test_block_shape( ) -# --- dimension 7: exceptional halt after writes ----------------------- - - @pytest.mark.parametrize("mode", ["success", "halt", "mix"]) @pytest.mark.valid_from("MONAD_NINE") def test_tx_halt( @@ -791,9 +793,6 @@ def test_tx_halt( ) -# --- random access + adversarial "bad block" cases ------------------- -# - RAND_CONTRACTS = 8 # pool of contracts, spread across the block's txs RAND_BASE = 1 # first page key of the read set RAND_STRIDE = 1 << 10 # > max slots, so per-tx page sets are disjoint diff --git a/whitelist.txt b/whitelist.txt index f553ff8b13b..cb9de9c6bbd 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -114,6 +114,7 @@ ae AF af alloc +ane AnnAssign api apis From 22a0f8be4f7ca134cd0ce4a192c81429ad3af6d8 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Thu, 16 Jul 2026 09:36:07 +0000 Subject: [PATCH 04/23] perf(mip8): split sweep into sload_sweep_k and whole-page sload_sweep_page sweep_page reads all 128 offsets at occupancy k in {0,1,64}: k hits + (128-k) misses. Co-Authored-By: Claude --- MIP8_PERF_TESTS_DIAGRAMS.md | 26 +++++++++++++++++-- .../test_perf_regression.py | 25 +++++++++++++----- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/MIP8_PERF_TESTS_DIAGRAMS.md b/MIP8_PERF_TESTS_DIAGRAMS.md index f03fdeff32e..75c6d7e4d75 100644 --- a/MIP8_PERF_TESTS_DIAGRAMS.md +++ b/MIP8_PERF_TESTS_DIAGRAMS.md @@ -99,7 +99,8 @@ page counts: |--------------------|--------------|----------------------------| | sload_cold_hit | {1,16,128} | 3469; k=128: 512¹ | | sload_cold_miss | {1,16,64} | 3469; k=64: 1024¹ | -| sload_sweep | {2,16,128} | k=2:1741 k=16:218 k=128:27 | +| sload_sweep_k | {2,16,128} | k=2:1741 k=16:218 k=128:27 | +| sload_sweep_page | {0,1,64} | 27 | | sload_warm_repeat | {1} | 158,946 (iterations) | | sload_empty_page | {0} | 3469 | | sstore_fresh | {0} | 1009 | @@ -152,7 +153,7 @@ checksum = 0 → the tail checksum SSTORE writes 0 (no state trace) Measures "page found, slot not found" lookups, P cold reads per tx. -### Storage operation `sload_sweep` +### Storage operation `sload_sweep_k` Block-filling transactions cold-read all occupied slots of each page. @@ -171,6 +172,27 @@ checksum = P × k per tx Isolates intra-page locality: same number-ish of cold reads as `cold_hit`, but bunched k-per-page instead of 1-per-page. +### Storage operation `sload_sweep_page` + +Block-filling transactions cold-read all 128 slot offsets of each page, hits and misses. + +Each loop iteration reads offsets 0..127 of one page whose first k +slots are pre-populated: k hits plus 128−k misses. At k=0 the pages +were never populated and every read is a whole-page miss. + +``` +tx 0..6, iteration i (page D + i): + offset: 0 ... k-1 | k ... 127 + R(0) ... R(k-1)| R(k) ... R(127) 128 cold reads + \_ k hits ___/ \_ 128-k misses _/ + +per tx: P = 27 pages × 128 reads; checksum = P × k +(k=0: checksum 0 → the tail SSTORE writes 0, no state trace) +``` + +Fixed whole-page read volume with occupancy as the only variable — +`sload_sweep_k` scales the read count with k instead. + ### Storage operation `sload_warm_repeat` Block-filling transactions cold-SLOAD a slot of each page then warm-re-read it repeatedly. diff --git a/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py index 7898816a1a5..6b2651d6cee 100644 --- a/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py +++ b/tests/monad_ten/mip8_pageified_storage/test_perf_regression.py @@ -151,8 +151,10 @@ class StorageOp(StrEnum): """Cold SLOAD of an occupied slot (offset 0) on each page.""" SLOAD_COLD_MISS = auto() """Cold SLOAD of an empty slot (offset 127) on an occupied page.""" - SLOAD_SWEEP = auto() - """Cold-read every occupied slot of each page.""" + SLOAD_SWEEP_K = auto() + """Cold-read every occupied slot (offsets 0..k-1) of each page.""" + SLOAD_SWEEP_PAGE = auto() + """Cold-read all 128 slot offsets of each page, hits and misses.""" SLOAD_WARM_REPEAT = auto() """One cold SLOAD of a slot, then repeated warm re-reads of it.""" SSTORE_FRESH = auto() @@ -276,11 +278,16 @@ def _body(op: StorageOp, k: int) -> Bytecode: # The slot is passed in calldata (M_BASE) so each block re-reads a # distinct slot; cold on the first iteration, warm thereafter. return _read_accum(Op.MLOAD(M_BASE), warm=True) + inc - if op is StorageOp.SLOAD_SWEEP: + if op is StorageOp.SLOAD_SWEEP_K: code = Op.MSTORE(M_PAGE, Op.SHL(7, _page_index())) for j in range(k): code += _read_accum(Op.ADD(Op.MLOAD(M_PAGE), j), warm=False) return code + inc + if op is StorageOp.SLOAD_SWEEP_PAGE: + code = Op.MSTORE(M_PAGE, Op.SHL(7, _page_index())) + for j in range(SLOTS_PER_PAGE): + code += _read_accum(Op.ADD(Op.MLOAD(M_PAGE), j), warm=False) + return code + inc if op is StorageOp.SSTORE_FRESH: return ( _sstore(_slot(0), 1, original=0, current=0, new=1, growth=0) + inc @@ -416,11 +423,14 @@ def _repeat_domains(repeat: int) -> Tuple[int, int, int]: # (k < SLOTS_PER_PAGE, and grow writes k..k+FULL_BLOCK_TXS-1). clear_keep # needs k > FULL_BLOCK_TXS so a page still has slots after the block # clears offsets 0..FULL_BLOCK_TXS-1. clear_empty uses single-slot pages -# (k=1) that vanish when cleared; empty_page reads never-set pages (k=0). +# (k=1) that vanish when cleared; empty_page reads never-set pages +# (k=0). sweep_k reads only the k occupied slots; sweep_page reads all +# 128 offsets — k hits + (128-k) misses, never-set pages at k=0. _PAGE_OP_KS = [ (StorageOp.SLOAD_COLD_HIT, [1, 16, 128]), (StorageOp.SLOAD_COLD_MISS, [1, 16, 64]), - (StorageOp.SLOAD_SWEEP, [2, 16, 128]), + (StorageOp.SLOAD_SWEEP_K, [2, 16, 128]), + (StorageOp.SLOAD_SWEEP_PAGE, [0, 1, 64]), (StorageOp.SSTORE_NOOP, [1, 16, 128]), (StorageOp.SSTORE_GROW, [1, 16, 64]), (StorageOp.SSTORE_UPDATE, [1, 16, 128]), @@ -514,7 +524,10 @@ def test_page_ops( ) expected[MARKER_BASE + global_idx] = value_code_worked if _is_read(op): - if op is StorageOp.SLOAD_SWEEP: + if op in ( + StorageOp.SLOAD_SWEEP_K, + StorageOp.SLOAD_SWEEP_PAGE, + ): checksum = pages * k elif op in ( StorageOp.SLOAD_COLD_MISS, From 8bba2105c928ae25f964afc5126c927956f318b7 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:53:32 +0000 Subject: [PATCH 05/23] perf(mip8): adapt perf suite to MONAD_TEN Co-Authored-By: Claude --- MONAD_RUNLOOP_TESTING.md | 12 +++++----- .../plugins/consume/direct/timing_report.py | 2 +- scripts/perf_cycle.sh | 4 ++-- scripts/perf_regression.py | 22 +++++++++---------- .../test_perf_regression.py | 12 +++++----- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/MONAD_RUNLOOP_TESTING.md b/MONAD_RUNLOOP_TESTING.md index 42ef7b0bfcb..e722f10bb4b 100644 --- a/MONAD_RUNLOOP_TESTING.md +++ b/MONAD_RUNLOOP_TESTING.md @@ -10,9 +10,9 @@ executed result. | Repo / branch | Role | |---|---| -| `monad-exp/monad-eest-rust-harness` @ `perf-regression-eestnet` | `eest-runner` harness: builds consensus blocks from a fixture and runs them on the runloop | -| `monad-bft` @ `perf-regression-eestnet` (submodule of the above) | consensus block types + ledger writer; pins monad-execution below | -| `monad` @ `perf-regression-eestnet` (submodule of monad-bft) | execution client with the `EestNet` chain (id 30143, per-fixture revision schedule, runtime genesis) and the extended `monad_runloop_*` FFI | +| `monad-exp/monad-eest-rust-harness` @ `perf-regression-eestnet-monad-ten` | `eest-runner` harness: builds consensus blocks from a fixture and runs them on the runloop | +| `monad-bft` @ `perf-regression-eestnet-monad-ten` (submodule of the above) | consensus block types + ledger writer; pins monad-execution below | +| `monad` @ `perf-regression-eestnet-monad-ten` (submodule of monad-bft) | execution client with the `EestNet` chain (id 30143, per-fixture revision schedule, runtime genesis) and the extended `monad_runloop_*` FFI | | this repo | `MonadFixtureConsumer` (`packages/testing/.../client_clis/clis/monad.py`) wired into `consume direct` | ## One-time setup @@ -22,7 +22,7 @@ artifacts, ~6 GB RAM for hugepages. ```sh snap install astral-uv --classic -git clone --branch perf-regression-eestnet \ +git clone --branch perf-regression-eestnet-monad-ten \ git@github.com:monad-exp/monad-eest-rust-harness.git cd monad-eest-rust-harness git submodule update --init --recursive @@ -73,7 +73,7 @@ uv run consume direct --input ../fixtures_eestnet \ `tests/monad_ten/mip8_pageified_storage/test_perf_regression.py` fills SLOAD/SSTORE workloads at both forks and times block execution on the -runloop to compare MONAD_NINE (slot-encoded) vs MONAD_NEXT (page-encoded). +runloop to compare MONAD_NINE (slot-encoded) vs MONAD_TEN (page-encoded). ### Setup @@ -95,7 +95,7 @@ From the repo root: tmux new -s perf 'TAG=v4 RUNS=7 scripts/perf_cycle.sh' ``` -Fills once, consumes `RUNS` times, and writes the NINE-vs-NEXT table to +Fills once, consumes `RUNS` times, and writes the NINE-vs-TEN table to `../timing_${TAG}__table.{html,md}` (the `.md` is headed with the cycle time and the four repo SHAs). Knobs: diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py index 71c77003410..9e28522b4a1 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py @@ -42,7 +42,7 @@ # Fork ordering (oldest first) so CSV rows group forks in release order; # unknown forks sort after, alphabetically. -_FORK_ORDER = ["MONAD_EIGHT", "MONAD_NINE", "MONAD_NEXT"] +_FORK_ORDER = ["MONAD_EIGHT", "MONAD_NINE", "MONAD_TEN", "MONAD_NEXT"] # Fixture-id suffixes identifying the fixture format, not a real parameter. _FORMAT_TAGS = { diff --git a/scripts/perf_cycle.sh b/scripts/perf_cycle.sh index a145298f019..e0d910ba681 100755 --- a/scripts/perf_cycle.sh +++ b/scripts/perf_cycle.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Fill once, consume RUNS times, emit the NINE-vs-NEXT perf table. +# Fill once, consume RUNS times, emit the NINE-vs-TEN perf table. set -uo pipefail TAG="${TAG:?set TAG (names all artifacts, e.g. TAG=v4)}" @@ -25,7 +25,7 @@ if [ -z "${SKIP_FILL:-}" ]; then echo "=== fill $FIX (REPEATS=$REPEATS, BLOCK_GAS=$BLOCK_GAS) $NOW ===" MIP8_PERF_REPEATS="$REPEATS" MIP8_PERF_BLOCK_GAS="$BLOCK_GAS" \ uv run fill -m blockchain_test "$TEST" \ - --from MONAD_NINE --until MONAD_NEXT --chain-id 30143 --monad-runloop \ + --from MONAD_NINE --until MONAD_TEN --chain-id 30143 --monad-runloop \ --output "$FIX" -n auto || { echo "fill failed (a non-empty $FIX aborts fill); rerun with" \ "SKIP_FILL=1 to reuse it, or delete it to refill" >&2 diff --git a/scripts/perf_regression.py b/scripts/perf_regression.py index f436d83475f..430ad4d1e00 100644 --- a/scripts/perf_regression.py +++ b/scripts/perf_regression.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """ -Build a NINE-vs-NEXT significance table from perf timing runs. +Build a NINE-vs-TEN significance table from perf timing runs. Reads the `timing_consume.csv` produced by `consume direct --timing-report` for several identical runs (one dir each) and, per (test, params) case, reports each measure's mean +/- sd over the runs for both forks, the two-sided Mann-Whitney U p-value comparing the forks, -and (for significant measures) the NINE->NEXT average change. +and (for significant measures) the NINE->TEN average change. Usage: perf_regression.py [--md OUT.md] [--html OUT.html] [--now TS --repo SHA --harness SHA --monad-bft SHA --monad SHA] @@ -29,11 +29,11 @@ from statistics import mean, stdev METRICS = ["tx_exec_us", "commit_us", "total_us"] -FORKS = ["MONAD_NINE", "MONAD_NEXT"] +FORKS = ["MONAD_NINE", "MONAD_TEN"] ALPHA = 0.01 # a measure is significant at this Mann-Whitney U p-value -UP = "🔴⬆️" # significant measures all rise NINE->NEXT (NEXT slower) -DOWN = "🟢⬇️" # significant measures all fall NINE->NEXT (NEXT faster) +UP = "🔴⬆️" # significant measures all rise NINE->TEN (TEN slower) +DOWN = "🟢⬇️" # significant measures all fall NINE->TEN (TEN faster) MIXED = "⚠️" # significant measures move both up and down # Per-case workload descriptions live here; pinned to the execution-specs @@ -147,13 +147,13 @@ def build(runs: list[dict]) -> list[str]: header = ["test-params"] for m in METRICS: - header += [f"{m} NINE", f"{m} NEXT", f"{m} p"] - header += ["significant", "Δ avg NINE→NEXT (sig)"] + header += [f"{m} NINE", f"{m} TEN", f"{m} p"] + header += ["significant", "Δ avg NINE→TEN (sig)"] lines = [ f"Mean ± sd over {len(runs)} runs (µs). Bold = measure significant " f"(Mann–Whitney U p ≤ {ALPHA}). " - f"Significant flag: {UP} NEXT slower, {DOWN} NEXT faster, " + f"Significant flag: {UP} TEN slower, {DOWN} TEN faster, " f"{MIXED} mixed.", "", "| " + " | ".join(header) + " |", @@ -168,7 +168,7 @@ def build(runs: list[dict]) -> list[str]: deltas = [] chgs = [] for m in METRICS: - n, x = forks["MONAD_NINE"][m], forks["MONAD_NEXT"][m] + n, x = forks["MONAD_NINE"][m], forks["MONAD_TEN"][m] p = _mwu_p(n, x) ncell, xcell = _stat(n), _stat(x) if p <= ALPHA: @@ -185,7 +185,7 @@ def build(runs: list[dict]) -> list[str]: lines += [ "", "p (MWU) is the two-sided Mann–Whitney U p-value comparing the " - f"NINE and NEXT run samples for that measure; significant at " + f"NINE and TEN run samples for that measure; significant at " f"p ≤ {ALPHA}.", ] return lines @@ -196,7 +196,7 @@ def build(runs: list[dict]) -> list[str]: -MIP-8 perf: NINE vs NEXT significance +MIP-8 perf: NINE vs TEN significance - - -__BODY__ - - -""" - - -def _inline(text: str) -> str: - """Render inline **bold** markdown to HTML, escaping the rest.""" - out = [] - for i, part in enumerate(text.split("**")): - esc = html.escape(part) - out.append(f"{esc}" if i % 2 else esc) - return "".join(out) - - -def _table_html(block: List[str]) -> str: - """Render a markdown table (list of `|`-rows) as an HTML table.""" - rows = [ - [c.strip() for c in r.strip().strip("|").split("|")] for r in block - ] - head = "".join(f"{_inline(c)}" for c in rows[0]) - body = [ - "" + "".join(f"{_inline(c)}" for c in row) + "" - for row in rows[2:] - ] - return ( - '
    \n' - f"{head}\n\n" - + "\n".join(body) - + "\n
    " - ) - - -def md_to_html(md: str) -> str: - """Render the generated markdown report as a standalone HTML page.""" - lines = md.split("\n") - blocks: list[str] = [] - i = 0 - while i < len(lines): - if lines[i].startswith("|"): - table = [] - while i < len(lines) and lines[i].startswith("|"): - table.append(lines[i]) - i += 1 - blocks.append(_table_html(table)) - elif lines[i].startswith("- "): - items = [] - while i < len(lines) and lines[i].startswith("- "): - items.append(f"
  • {_inline(lines[i][2:])}
    • ") - i += 1 - blocks.append("
      \n" + "\n".join(items) + "\n
    ") - else: - if lines[i].strip(): - blocks.append(f"

    {_inline(lines[i])}

    ") - i += 1 - return HTML_TEMPLATE.replace("__BODY__", "\n".join(blocks)) - - def _provenance(now: str, shas: List[Optional[str]]) -> str: """Markdown header: descriptions link, cycle time, and repo shas.""" repos = [ @@ -583,13 +476,6 @@ def report( default=None, help="Write the Markdown report here instead of stdout.", ) -@click.option( - "--html", - "html_path", - type=click.Path(dir_okay=False, path_type=Path), - default=None, - help="Also write a standalone HTML rendering here.", -) @click.option( "--now", default=None, @@ -602,7 +488,6 @@ def report( def main( run_dirs: Tuple[Path, ...], md_path: Optional[Path], - html_path: Optional[Path], now: Optional[str], repo: Optional[str], harness: Optional[str], @@ -616,8 +501,6 @@ def main( directory holding a `timing_consume.csv`. """ md = report(run_dirs, now, [repo, harness, monad_bft, monad]) - if html_path: - html_path.write_text(md_to_html(md), encoding="utf-8") if md_path: md_path.write_text(md + "\n", encoding="utf-8") else: diff --git a/packages/testing/src/execution_testing/cli/tests/test_perf_regression.py b/packages/testing/src/execution_testing/cli/tests/test_perf_regression.py index f13542a036f..3da662b47e2 100644 --- a/packages/testing/src/execution_testing/cli/tests/test_perf_regression.py +++ b/packages/testing/src/execution_testing/cli/tests/test_perf_regression.py @@ -325,7 +325,7 @@ def test_report_warns_when_underpowered(tmp_path: Path) -> None: for i in range(4) ] table = report(tuple(dirs)) - assert "Underpowered for isolated effects" in table + assert "cannot reach q" in table assert _case_row(table).endswith("| - | - |") @@ -336,7 +336,7 @@ def test_report_no_power_warning_when_resolvable(tmp_path: Path) -> None: _run_dir(tmp_path, f"r{i}", nine=100 + i, ten=200 + i) for i in range(10) ] - assert "Underpowered" not in report(tuple(dirs)) + assert "cannot reach q" not in report(tuple(dirs)) def _retry_dir(tmp_path: Path, name: str, nine: int, ten: int) -> Path: @@ -438,22 +438,19 @@ def test_report_warns_about_dirs_without_csv( assert "skipping 1 run dir(s)" in capsys.readouterr().err -def test_cli_writes_markdown_and_html(tmp_path: Path) -> None: - """The CLI writes both renderings and the provenance header.""" +def test_cli_writes_markdown(tmp_path: Path) -> None: + """The CLI writes the report and the provenance header.""" dirs = [ _run_dir(tmp_path, f"r{i}", nine=100 + i, ten=200 + i) for i in range(9) ] md_path = tmp_path / "table.md" - html_path = tmp_path / "table.html" result = CliRunner().invoke( main, [ "--md", str(md_path), - "--html", - str(html_path), "--now", "2026-08-18T00:00:00Z", "--repo", @@ -467,9 +464,6 @@ def test_cli_writes_markdown_and_html(tmp_path: Path) -> None: assert "Cycle 2026-08-18T00:00:00Z" in md assert "| execution-specs | deadbeef |" in md assert "blob/deadbeef/MIP8_PERF_TESTS_DIAGRAMS.md" in md - html = html_path.read_text() - assert "" in html - assert "" in html def test_cli_requires_run_dirs() -> None: diff --git a/scripts/perf_cycle.sh b/scripts/perf_cycle.sh index f915c33d65b..be69a6dda3a 100755 --- a/scripts/perf_cycle.sh +++ b/scripts/perf_cycle.sh @@ -59,11 +59,11 @@ for i in $(seq 1 "$RUNS"); do done sha() { git -C "$1" rev-parse --short HEAD 2>/dev/null || echo '?'; } -uv run perf_regression --md "${TABLE}.md" --html "${TABLE}.html" \ +uv run perf_regression --md "${TABLE}.md" \ --now "$NOW" \ --repo "$(sha "$REPO")" \ --harness "$(sha "$HARNESS")" \ --monad-bft "$(sha "$HARNESS/monad-bft")" \ --monad "$(sha "$HARNESS/monad-bft/monad-execution")" \ "${PREFIX}"_[0-9]* || { echo "perf_regression failed, no table" >&2; exit 1; } -echo "=== table: $(cd .. && pwd)/$(basename "$TABLE").html ===" +echo "=== table: $(cd .. && pwd)/$(basename "$TABLE").md ===" From af3bf26408bb615729ed0f591aa294eeb7a77972 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:52:38 +0000 Subject: [PATCH 21/23] refactor(consume): collect timing rows in memory --timing-report refuses xdist, so the per-test part files were unreachable. Co-Authored-By: Claude --- .../plugins/consume/direct/timing_report.py | 78 ++++--------------- 1 file changed, 14 insertions(+), 64 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py index f7a53a9ca9d..10d403a0a94 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py @@ -4,10 +4,8 @@ A consumer that implements the `BlockTimingReporter` capability (e.g. the monad runloop) exposes per-block timing for the fixture it just ran. This -plugin reads it off the finished test, writes one part file per test *in -the process that ran it* (so it never depends on xdist forwarding data to -the controller), then the controller collects every part into a single raw -per-block CSV at session end. +plugin reads it off the finished test and writes every block it collected +as one raw per-block CSV at session end. The metric columns are whatever keys the consumer reported, so the plugin stays agnostic of which execution phases a given client can measure. @@ -22,8 +20,6 @@ import csv import io -import json -import uuid from pathlib import Path from typing import ( Any, @@ -250,10 +246,11 @@ def _render_csv(rows: List[Dict[str, Any]]) -> str: class TimingReportPlugin: - """Collect per-test timing parts and write the raw per-block CSV.""" + """Collect each test's timing and write the raw per-block CSV.""" def __init__(self, config: pytest.Config): # noqa: D107 self.config = config + self.rows: List[Dict[str, Any]] = [] self.written: List[Path] = [] def _output_dir(self) -> Path: @@ -273,29 +270,6 @@ def _output_dir(self) -> Path: source = self.config.fixtures_source # type: ignore[attr-defined] return Path(source.path) / ".meta" - def _parts_dir(self) -> Path: - """ - Directory holding this session's part files. - - Namespaced by session id so a killed or crashed run's leftovers - are never folded into a later run's CSV. Under xdist the workers - inherit the controller's id through `workerinput`. - """ - return self._output_dir() / ".timing_parts" / self._session_id() - - def _session_id(self) -> str: - """Id shared by the controller and, under xdist, its workers.""" - workerinput = getattr(self.config, "workerinput", None) - if workerinput is not None: - return str(workerinput["timing_report_session"]) - if not hasattr(self.config, "_timing_report_session"): - self.config._timing_report_session = uuid.uuid4().hex # type: ignore[attr-defined] - return str(self.config._timing_report_session) # type: ignore[attr-defined] - - def pytest_configure_node(self, node: Any) -> None: - """Pass the session id to an xdist worker as it starts.""" - node.workerinput["timing_report_session"] = self._session_id() - @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport( self, @@ -303,13 +277,12 @@ def pytest_runtest_makereport( call: pytest.CallInfo[None], ) -> Generator[None, Any, None]: """ - Persist this test's timing to a part file, in the running process. + Keep this test's timing for the report. The consumer and test case are read straight off the finished - item, so the generic test function stays untouched. Writing here - rather than on the controller avoids relying on xdist to marshal - anything back; each test writes its own uniquely-named file, so - parallel workers never race. + item, so the generic test function stays untouched. `--timing-report` + rejects xdist, so one process sees every test and the rows can be + held in memory until session end. """ outcome = yield if call.when != "call": @@ -323,46 +296,23 @@ def pytest_runtest_makereport( if consumer is None or test_case is None: return payload = timing_payload(consumer, test_case) - if not payload: - return - parts_dir = self._parts_dir() - parts_dir.mkdir(parents=True, exist_ok=True) - (parts_dir / f"{uuid.uuid4().hex}.json").write_text( - json.dumps(payload) - ) + if payload: + self.rows.extend(_rows_from_payload(payload)) def pytest_sessionfinish( self, - session: pytest.Session, + session: pytest.Session, # noqa: ARG002 exitstatus: int, # noqa: ARG002 ) -> None: - """Collect all part files into the CSV report (controller only).""" - if hasattr(session.config, "workerinput"): - return # xdist worker: parts already written by makereport - parts_dir = self._parts_dir() - if not parts_dir.is_dir(): - return - rows: List[Dict[str, Any]] = [] - for part in parts_dir.glob("*.json"): - rows.extend(_rows_from_payload(json.loads(part.read_text()))) - if not rows: + """Write the collected rows as the CSV report.""" + if not self.rows: return - rows = _sorted_rows(rows) - output_dir = self._output_dir() output_dir.mkdir(parents=True, exist_ok=True) path = output_dir / "timing_consume.csv" - path.write_text(_render_csv(rows)) + path.write_text(_render_csv(_sorted_rows(self.rows))) self.written.append(path) - for part in parts_dir.glob("*.json"): - part.unlink() - parts_dir.rmdir() - try: - parts_dir.parent.rmdir() - except OSError: - pass # another session's parts are still there - def pytest_terminal_summary( self, terminalreporter: Any, From dc0c87a0fc06aaca490827c5650b39adf1f9f9d5 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:52:38 +0000 Subject: [PATCH 22/23] ci: rehearse the MIP-8 perf suite as its own release feature Filled at the smallest block gas every case supports, outside monad/monad_runloop. Co-Authored-By: Claude --- .github/configs/feature.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index ba528347f43..ecd72f554df 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -6,3 +6,13 @@ monad_runloop: evm-type: eels # Like `monad`, but `--monad-runloop` and eestnet chain id `30143` fill-params: -m blockchain_test --from=MONAD_EIGHT --until=MONAD_TEN --chain-id=30143 --monad-runloop -k "not invalid_header" + +monad_benchmark: + evm-type: eels + # The MIP-8 perf suite, which lives outside the default collection and + # so is named explicitly. Filled at the smallest block gas every case + # supports: below 10M the 128-read sweeps cannot fit one iteration in a + # transaction. The excluded `page_spread` cases size their work from `m` + # rather than the gas budget, so they need ~29M and ~177M whatever is + # passed here. + fill-params: -m blockchain_test tests/benchmark/stateful/mip8_pageified_storage/ --from=MONAD_NINE --until=MONAD_TEN --chain-id=30143 --monad-runloop --gas-benchmark-values=10 -k "not m4096 and not m1024" From e790ffe18dea7f86836b8baf67a1031bbbfb6ba6 Mon Sep 17 00:00:00 2001 From: pdobacz <5735525+pdobacz@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:36:34 +0000 Subject: [PATCH 23/23] fix(monad): verify every fixture when given no name Matches the verification path and `evm blocktest`, which run a whole file. Co-Authored-By: Claude --- .../client_clis/clis/monad.py | 55 +++++++++++--- .../client_clis/tests/test_monad_fixture.py | 73 +++++++++++++++++++ 2 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 packages/testing/src/execution_testing/client_clis/tests/test_monad_fixture.py diff --git a/packages/testing/src/execution_testing/client_clis/clis/monad.py b/packages/testing/src/execution_testing/client_clis/clis/monad.py index 2eb2ff960de..a98fa4dfc48 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/monad.py +++ b/packages/testing/src/execution_testing/client_clis/clis/monad.py @@ -19,7 +19,16 @@ import tempfile import uuid from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Tuple, TypedDict +from typing import ( + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + TypedDict, +) import ijson # type: ignore[import-untyped] import pytest @@ -67,19 +76,28 @@ def _set_pdeathsig() -> None: _LIBC.prctl(1, signal.SIGTERM) -def _load_fixture( +def _iter_fixtures( fixture_path: Path, fixture_name: Optional[str] -) -> Dict[str, Any]: +) -> Iterator[Dict[str, Any]]: """ - Load a single fixture from a (possibly multi-fixture) JSON file. + Yield the fixtures to run from a (possibly multi-fixture) JSON file. - ijson streams the file so memory use stays low on large fixtures. + A `fixture_name` selects one; without it every fixture in the file is + yielded, which is what the fixture-verification path asks for when it + hands over a merged file (`evm blocktest` behaves the same way, by + omitting `--run`). ijson streams the file so memory use stays low on + large fixtures. """ + found = False with open(fixture_path, "rb") as f: for name, fixture in ijson.kvitems(f, ""): if fixture_name is None or name == fixture_name: - return fixture - raise KeyError(f"fixture {fixture_name!r} not found in {fixture_path}") + found = True + yield fixture + if fixture_name is not None: + return + if not found: + raise KeyError(f"fixture {fixture_name!r} not found in {fixture_path}") def _hex32(value: str) -> str: @@ -435,12 +453,25 @@ def consume_fixture( fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, ) -> None: - """Execute a blockchain fixture on the monad runloop and verify.""" + """ + Execute blockchain fixtures on the monad runloop and verify. + + Runs the named fixture, or every fixture in the file when no name + is given, and reports the timing of all their blocks together. + """ assert fixture_format == BlockchainFixture + timings: List[BlockExecutionTiming] = [] self._block_timings = () + for fixture in _iter_fixtures(fixture_path, fixture_name): + timings.extend(self._consume_one(fixture, debug_output_path)) + self._block_timings = timings - fixture = _load_fixture(fixture_path, fixture_name) - + def _consume_one( + self, + fixture: Dict[str, Any], + debug_output_path: Optional[Path], + ) -> Sequence[BlockExecutionTiming]: + """Run one fixture and verify its post state and state root.""" network = fixture["network"] assert network in FORK_REVISION_SCHEDULES, ( f"no monad revision schedule for network {network}" @@ -493,7 +524,7 @@ def consume_fixture( ) output = json.loads(output_path.read_text()) - self._block_timings = _parse_block_timings(stdout) + block_timings = _parse_block_timings(stdout) # The state root below is the authoritative check: it commits to # the whole state, so any divergence changes it. `postState`, when @@ -528,3 +559,5 @@ def consume_fixture( "post-state mismatch on the monad runloop:\n" + "\n".join(mismatches) ) + + return block_timings diff --git a/packages/testing/src/execution_testing/client_clis/tests/test_monad_fixture.py b/packages/testing/src/execution_testing/client_clis/tests/test_monad_fixture.py new file mode 100644 index 00000000000..7d549e211ad --- /dev/null +++ b/packages/testing/src/execution_testing/client_clis/tests/test_monad_fixture.py @@ -0,0 +1,73 @@ +"""Tests for the monad consumer's fixture selection.""" + +import json +from pathlib import Path +from typing import Any, Dict + +import pytest + +from execution_testing.client_clis.clis.monad import _iter_fixtures + + +def _write(path: Path, fixtures: Dict[str, Any]) -> Path: + """Write a fixture file holding the given fixtures.""" + path.write_text(json.dumps(fixtures)) + return path + + +def test_named_fixture_is_the_only_one_yielded(tmp_path: Path) -> None: + """A name selects one fixture out of a file holding several.""" + path = _write( + tmp_path / "f.json", + {"first": {"network": "A"}, "second": {"network": "B"}}, + ) + + assert list(_iter_fixtures(path, "second")) == [{"network": "B"}] + + +def test_no_name_yields_every_fixture(tmp_path: Path) -> None: + """ + Without a name the whole file is run. + + Fixture verification hands over a merged file and expects all of it + checked, so yielding only the first would leave the rest unverified. + """ + path = _write( + tmp_path / "f.json", + {"a": {"network": "A"}, "b": {"network": "B"}, "c": {"network": "C"}}, + ) + + assert list(_iter_fixtures(path, None)) == [ + {"network": "A"}, + {"network": "B"}, + {"network": "C"}, + ] + + +def test_no_name_on_a_sole_fixture(tmp_path: Path) -> None: + """A single-fixture file yields its one fixture.""" + path = _write(tmp_path / "f.json", {"only": {"network": "A"}}) + + assert list(_iter_fixtures(path, None)) == [{"network": "A"}] + + +def test_missing_name_raises(tmp_path: Path) -> None: + """A name absent from the file is an error, not an empty result.""" + path = _write(tmp_path / "f.json", {"only": {"network": "A"}}) + + with pytest.raises(KeyError, match="absent"): + list(_iter_fixtures(path, "absent")) + + +def test_named_lookup_stops_at_the_match(tmp_path: Path) -> None: + """The stream is not read past the fixture that was asked for.""" + path = _write( + tmp_path / "f.json", + {"first": {"network": "A"}, "second": {"network": "B"}}, + ) + + fixtures = _iter_fixtures(path, "first") + + assert next(fixtures) == {"network": "A"} + with pytest.raises(StopIteration): + next(fixtures)