diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index ba528347f43..ecd72f554df 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -6,3 +6,13 @@ monad_runloop: evm-type: eels # Like `monad`, but `--monad-runloop` and eestnet chain id `30143` fill-params: -m blockchain_test --from=MONAD_EIGHT --until=MONAD_TEN --chain-id=30143 --monad-runloop -k "not invalid_header" + +monad_benchmark: + evm-type: eels + # The MIP-8 perf suite, which lives outside the default collection and + # so is named explicitly. Filled at the smallest block gas every case + # supports: below 10M the 128-read sweeps cannot fit one iteration in a + # transaction. The excluded `page_spread` cases size their work from `m` + # rather than the gas budget, so they need ~29M and ~177M whatever is + # passed here. + fill-params: -m blockchain_test tests/benchmark/stateful/mip8_pageified_storage/ --from=MONAD_NINE --until=MONAD_TEN --chain-id=30143 --monad-runloop --gas-benchmark-values=10 -k "not m4096 and not m1024" diff --git a/MIP8_PERF_TESTS_DIAGRAMS.md b/MIP8_PERF_TESTS_DIAGRAMS.md new file mode 100644 index 00000000000..e7b74d7a1a9 --- /dev/null +++ b/MIP8_PERF_TESTS_DIAGRAMS.md @@ -0,0 +1,553 @@ +# MIP-8 perf-regression tests: block diagrams + +What each test case in +`tests/benchmark/stateful/mip8_pageified_storage/test_perf_regression.py` +actually does at the SLOAD/SSTORE level inside one block. Counts stay +symbolic — they follow from `--gas-benchmark-values` and the module's +sizing helpers. Diagrams show the `REPEATS = 1` block. + +## Legend and shared machinery + +Notation used in all diagrams: + +``` +R(o) SLOAD of offset o within a page (result added to a checksum) +W(o)=v SSTORE of value v at offset o within a page +P pages (loop iterations) per transaction +D READ_DOMAIN = 2^40 base page index of pre-populated pools +F FRESH_DOMAIN = 2^52 base page index of fresh-write ranges +g global tx index (unique across all blocks of a fixture) +t local tx index within the block (0..6) +``` + +Storage addressing (MIP-8 page = 128 slots): + +``` +slot = (page_index << 7) + offset + +page D + i: + offset: 0 1 ... k-1 k ... 127 + ┌─────┬─────┬─────┬─────┬─────┬─────┬─────┐ + │ 1 │ 1 │ ... │ 1 │ 0 │ ... │ 0 │ + └─────┴─────┴─────┴─────┴─────┴─────┴─────┘ + \__ k pre-populated slots __/ \__ empty __/ +``` + +Block anatomy shared by `test_page_ops`, `test_block_shape` (few_big), +`test_tx_halt`, `test_random_sload` and the two `test_bad_block_*` tests +— one block, 7 equal txs, each from its own sender, to the same +workload contract (`test_random_sload` cycles a pool of 8 contracts +instead): + +``` +Block (gas limit = the benchmark budget) +┌──────────┬──────────┬──────────┬──────────┬──────────┬──────────┬──────────┐ +│ tx 0 │ tx 1 │ tx 2 │ tx 3 │ tx 4 │ tx 5 │ tx 6 │ +│ budget/7 │ budget/7 │ budget/7 │ budget/7 │ budget/7 │ budget/7 │ budget/7 │ +└──────────┴──────────┴──────────┴──────────┴──────────┴──────────┴──────────┘ +``` + +Per-tx warm/access sets reset between transactions, so every tx of a +block pays **cold** access again even when it re-touches the exact pages +tx 0 touched. This is what lets one pre-populated pool serve all 7 txs +as 7 independent cold passes. + +Each tx also has its own sender. Consecutive nonces from one EOA are a +write-write conflict on that account, which would serialise the block +whatever its storage access looks like; distinct senders leave storage as +the only cross-tx dependency. `test_block_shape` parametrizes +`distinct_senders` so the cost of that chain stays measurable. + +Every transaction runs the same loop contract: read calldata +`(base, count, g, halt, t)`, execute `count` iterations of the op body, +then finish with one or two bookkeeping SSTOREs on far-away pages: + +``` +tail of every tx: + SSTORE(2^200 + g, 0x1234) success marker (cold, fresh) + SSTORE(2^220 + g, checksum) read ops only; a checksum of 0 still + executes but writes 0 → no state trace +``` + +With `MIP8_PERF_REPEATS > 1` the whole block is emitted again with all +page domains shifted by `r * 2^28`, so every repeat block is a fully +cold copy. Diagrams below show the `r = 0` block. + +--- + +## `test_compute_loop` + +No parameters. The storage-free baseline: the same 7-tx full block as +above, each tx running a stack-arithmetic `WhileGas` loop. + +``` +Block (gas limit = the benchmark budget) +┌──────────────────────────────────────────────────┬─── ... ───┬──────────┐ +│ tx 0 (budget/7 gas) │ │ tx 6 │ +│ loop: POP(ADD(MUL(NUMBER, GAS), CALLVALUE)) │ │ (same) │ +│ ... repeated until gas nearly spent ... │ │ │ +│ SSTORE(slot 1, 0x1234) │ │ │ +└──────────────────────────────────────────────────┴─── ... ───┴──────────┘ +``` + +All 7 txs write the same marker slot, so only one slot is touched however +many run. As the suite's control, the loop is sized against a fixed fork, +which keeps the deployed bytecode byte-identical on both sides. + +Only the final marker touches storage, so the block isolates pure +execution overhead from any MIP-8 effect. + +--- + +## `test_page_ops` + +One section per storage operation (`StorageOp`); within each, `k` (page +occupancy) varies. All variants use the 7-tx full block above. Per-tx +page counts: + +| op | k values | P bounded by | +|--------------------|--------------|-----------------------------| +| sload_cold_hit | {1,16,128} | gas; pre-state at high k¹ | +| sload_cold_miss | {1,16,64} | gas; pre-state at high k¹ | +| sload_sweep_k | {2,16,128} | gas (k reads per iteration) | +| sload_sweep_page | {0,1,64} | gas (128 reads per iter) | +| sload_warm_repeat | {1} | gas (one slot, re-read) | +| sload_empty_page | {0} | gas | +| sstore_fresh | {0} | gas | +| sstore_noop | {1,16,128} | gas; pre-state at high k¹ | +| sstore_grow | {1,16,64} | gas | +| sstore_update | {1,16,128} | gas; pre-state at high k¹ | +| sstore_clear_keep | {8,16,128} | gas; pre-state at high k¹ | +| sstore_clear_empty | {1} | pre-state | + +¹ `PRE_SLOT_CAP` bounds the pre-populated slots per block, so at high k +the page count falls to `PRE_SLOT_CAP / k` and the block does less work +than its gas budget allows by design. + +### Storage operation `sload_cold_hit` + +Block-filling transactions cold-SLOAD one occupied slot on many pages. + +Pre-state: pool of P pages at domain D, each with slots `0..k-1 = 1`. +Every tx makes one cold pass over the whole pool, reading offset 0 +(always occupied). + +``` +pre-state pool (shared by all 7 txs) + page D page D+1 page D+2 page D+(P-1) +tx 0: R(0) R(0) R(0) ... R(0) all cold +tx 1: R(0) R(0) R(0) ... R(0) cold again + ... (warmth reset) +tx 6: R(0) R(0) R(0) ... R(0) + +each read returns 1 → checksum = P per tx +block total: 7 × P cold page reads of an existing, occupied slot +``` + +### Storage operation `sload_cold_miss` + +Block-filling transactions cold-SLOAD an empty slot on many occupied pages. + +Same shared pool as `cold_hit` (k ≤ 64 keeps the last slot empty), but +each read targets offset 127 — the page exists, the slot is zero. + +``` +page D + i: + offset: 0 .. k-1 k .. 126 127 + [ 1 .. 1 ] [ 0 .. 0 ] [ 0 ] ◄── R(127) per iteration + occupied empty read misses (returns 0) + +tx 0..6: R(127) on every pool page, all cold +checksum = 0 → the tail checksum SSTORE writes 0 (no state trace) +``` + +Measures "page found, slot not found" lookups, P cold reads per tx. + +### Storage operation `sload_sweep_k` + +Block-filling transactions cold-read all occupied slots of each page. + +Each loop iteration cold-reads **every occupied slot** of one page, +offsets `0..k-1` in ascending order. + +``` +tx 0..6, iteration i (page D + i): + offset: 0 1 2 ... k-1 | k .. 127 + R(0) R(1) R(2) ... R(k-1)| untouched k cold reads + +per tx: P pages × k reads +checksum = P × k per tx +``` + +Isolates intra-page locality: same number-ish of cold reads as +`cold_hit`, but bunched k-per-page instead of 1-per-page. + +### Storage operation `sload_sweep_page` + +Block-filling transactions cold-read all 128 slot offsets of each page, hits and misses. + +Each loop iteration reads offsets 0..127 of one page whose first k +slots are pre-populated: k hits plus 128−k misses. At k=0 the pages +were never populated and every read is a whole-page miss. + +``` +tx 0..6, iteration i (page D + i): + offset: 0 ... k-1 | k ... 127 + R(0) ... R(k-1)| R(k) ... R(127) 128 cold reads + \_ k hits ___/ \_ 128-k misses _/ + +per tx: P pages × 128 reads; checksum = P × k +(k=0: checksum 0 → the tail SSTORE writes 0, no state trace) +``` + +Fixed whole-page read volume with occupancy as the only variable — +`sload_sweep_k` scales the read count with k instead. + +### Storage operation `sload_warm_repeat` + +Block-filling transactions cold-SLOAD a slot of each page then warm-re-read it repeatedly. + +No page pool — a single pre-populated slot `W = 2^70` (value 1). The +slot arrives via calldata and is read directly (not page-shifted). + +``` +pre-state: slot W = 1 + +tx 0: SLOAD(W) ×P 1st read cold, the rest warm +tx 1: SLOAD(W) ×P cold again once (warm set reset), then warm + ... +tx 6: SLOAD(W) ×P + +checksum = P per tx; the block re-reads one hot slot 7 × P times +``` + +The warm-path baseline: page/slot caching should make fork choice +irrelevant here. + +### Storage operation `sload_empty_page` + +Block-filling transactions cold-SLOAD a slot on never-populated, empty pages. + +Like `cold_hit` but with `k = 0`: the domain-D pages were **never +populated**. Every read is a whole-page miss. + +``` + page D page D+1 page D+(P-1) +tx 0..6: R(0) R(0) ... R(0) page does not exist + +checksum = 0 (zero write in the tail, no state trace) +``` + +Measures lookups that fall off the page index entirely. + +### Storage operation `sstore_fresh` + +Block-filling transactions SSTORE 0->1 into previously-unoccupied slots on many pages. + +No pre-state. Each tx gets its own disjoint range of never-touched +pages and creates one slot on each: `W(0)=1`, a 0→1 write that brings +a whole new page into existence. + +``` +page-index axis (base F), tiled per tx: + +F F+P F+2P F+6P F+7P +╠═ tx0: P ═╬═ tx1: P ══╬═ tx2: P ══╬═ ... ══╬═ tx6: P ══╣ + pages pages pages pages + +tx t, iteration i: W(0)=1 on page F + t·P + i state growth + +the block creates 7 × P new pages +``` + +### Storage operation `sstore_noop` + +Block-filling transactions SSTORE 1->1 (value unchanged) on occupied pages. + +Shared occupied pool (like `cold_hit`); each tx rewrites slot 0 with +its current value — 1→1, no state change ever. + +``` +tx 0..6, iteration i (page D + i): + offset 0: [ 1 ] ◄── W(0)=1 cold page access, value unchanged + +post-state == pre-state (plus markers) +``` + +Pays the write path without any page mutation. + +### Storage operation `sstore_grow` + +Block-filling transactions SSTORE 0->1 into a new empty slot of occupied pages. + +Shared pool with offsets `0..k-1` occupied. Tx `t` writes offset +`k + t` — a 0→1 on an **already-occupied** page (growth within a page, +no page creation). All 7 txs touch the same P pages, each at its own +offset. + +``` +page D + i, one column per tx: + offset: 0 .. k-1 | k k+1 k+2 k+3 k+4 k+5 k+6 | k+7 .. 127 + before: [ 1 .. 1 ]| 0 0 0 0 0 0 0 | 0 + writer: | tx0 tx1 tx2 tx3 tx4 tx5 tx6 | + after: [ 1 .. 1 ]| 1 1 1 1 1 1 1 | 0 + +each tx: P cold W(k+t)=1 writes, one per pool page +``` + +### Storage operation `sstore_update` + +Block-filling transactions SSTORE 1->2 (nonzero value change) on occupied pages. + +Shared pool; every tx overwrites the occupied slot 0 with a fresh +nonzero value `2 + t`, so each write is a genuine value change with no +occupancy change. + +``` +page D + i, offset 0 over the block: + pre tx0 tx1 tx2 tx3 tx4 tx5 tx6 + 1 ─► W(0)=2 ─► =3 ─► =4 ─► =5 ─► =6 ─► =7 ─► =8 + +each tx: P cold writes; slot 0 ends at 1 + 7 +``` + +### Storage operation `sstore_clear_keep` + +Block-filling transactions SSTORE 1->0 clearing one slot of occupied pages, leaving the page populated. + +Shared pool with `k > 7` occupied slots. Tx `t` clears offset `t` +(1→0). Offsets `7..k-1` stay populated, so no page ever empties. + +``` +page D + i: + offset: 0 1 2 3 4 5 6 | 7 .. k-1 | k .. 127 + before: 1 1 1 1 1 1 1 | 1 .. 1 | 0 + clearer: tx0 tx1 tx2 tx3 tx4 tx5 tx6 | untouched | + after: 0 0 0 0 0 0 0 | 1 .. 1 | 0 + +each tx: P cold W(t)=0 writes +``` + +### Storage operation `sstore_clear_empty` + +Block-filling transactions SSTORE 1->0 clearing the only slot of single-slot pages, removing the page. + +The page-removal case. Pre-state gives **each tx its own** range of +single-slot pages (offset 0 = 1). Clearing that slot leaves the page +empty, so every write deletes a page. + +``` +pre-state, tiled like sstore_fresh but pre-populated with k=1: + +F F+P F+2P F+6P F+7P +╠═ tx0: P ═╬═ tx1: P ══╬═ ... ══════════╬═ tx6: P ══╣ + 1-slot 1-slot 1-slot + pages pages pages + +tx t, iteration i: W(0)=0 on page F + t·P + i + before: [1][0..0] → after: [0][0..0] → page removed + +the block removes 7 × P pages +``` + +--- + +## `test_page_spread` + +Transactions SSTORE 0->1 into `m` fresh slots spread across `n` contracts. + +Op is fixed (`sstore_fresh`, the 0→1 page-creating write); the sweep is +over **where** the writes land: `m` total pages spread evenly across +`n` contracts. + +Each contract runs the same loop contract with empty storage. Every +contract writes page indices `F + 0 .. F + (m/n − 1)` — the *same* +indices for all contracts, but in n distinct account storages, so +nothing collides. A contract's share becomes one tx while it fits the +per-tx gas cap, and splits into several txs when it does not. + +``` +Block (one per fixture; txs sized to the work, not to the budget) + +n = 1, large m (one contract, share split across txs): +┌ contract C0 ──────────────────────────────────────────────────────┐ +│ tx0: W(0)=1 on pages F+0 .. F+c-1 (a gas-cap-sized share) │ +│ tx1: W(0)=1 on pages F+c .. F+2c-1 │ +│ ... │ +│ txN: W(0)=1 on pages F+Nc .. F+m-1 (the remainder) │ +└───────────────────────────────────────────────────────────────────┘ + +n = 8 (8 txs, m/8 pages each): +┌ C0 ┐┌ C1 ┐┌ C2 ┐┌ C3 ┐┌ C4 ┐┌ C5 ┐┌ C6 ┐┌ C7 ┐ +│tx0 ││tx1 ││tx2 ││tx3 ││tx4 ││tx5 ││tx6 ││tx7 │ each: m/8 × W(0)=1 +└────┘└────┘└────┘└────┘└────┘└────┘└────┘└────┘ + +n = 512 (512 txs, m/512 pages each): +┌C0┐┌C1┐┌C2┐ ... ┌C511┐ each: m/512 × W(0)=1 +└──┘└──┘└──┘ └────┘ +``` + +Variants: `m ∈ {1,4,16,64,256,1024,4096} × n=1` (total-size sweep), +`m=4096 × n ∈ {8,64,512}` (distribution sweep). Same total I/O at +a given `m` regardless of `n` — only the account fan-out changes. + +--- + +## `test_block_shape` + +A few big vs many small transactions, each cold-SLOAD one occupied slot or SSTORE 0->1 into fresh slots. + +Same total work packed as a few big txs vs many small ones. Two +workloads, each filled twice — once with a sender per tx and once with +one sender for the whole block (`distinct_senders`), giving 8 cases. The +sender mode changes no count below, only whether the block also carries +a nonce chain: + +| op, k | shape | txs | tx gas | block work | +|-------------------|------------|------|-----------|-------------------| +| sstore_fresh, k=0 | few_big | 7 | budget/7 | txs × P new pages | +| sstore_fresh, k=0 | many_small | many | budget/N | txs × P new pages | +| sload_cold_hit, 8 | few_big | 7 | budget/7 | txs × P cold reads| +| sload_cold_hit, 8 | many_small | many | budget/N | txs × P cold reads| + +``` +few_big: ┌──────────┬──────────┬──────────┬──── ... ───┬──────────┐ + │ tx 0 │ tx 1 │ tx 2 │ │ tx 6 │ + └──────────┴──────────┴──────────┴──── ... ───┴──────────┘ + +many_small: ┌──┬──┬──┬──┬──┬──┬──┬──┬──┬── ... ──┬──┬──┬──┬──┬──┬──┐ + │t0│t1│t2│t3│t4│t5│t6│t7│t8│ │ │ │ │ │tN-1│ + └──┴──┴──┴──┴──┴──┴──┴──┴──┴── ... ──┴──┴──┴──┴──┴──┴──┘ +``` + +The reads reuse one shared pre-populated pool sized to a single tx +(P pages per tx, k = 8 slots each); +every tx is a fresh cold pass over it. The fresh writes tile disjoint +per-tx ranges exactly like `sstore_fresh` above. What varies is per-tx +fixed overhead (intrinsic gas, markers, cold pool re-touch) relative +to loop work. + +--- + +## `test_tx_halt` + +Seven transactions SSTORE 0->1 into fresh slots and either succeed, hit INVALID reverting all writes, or alternate. + +The `sstore_fresh` full block (7 txs × P fresh pages, disjoint +ranges), with a `halt` calldata flag per tx. A halting tx performs all +of its writes **and** its marker, then executes `INVALID`: everything +reverts and the tx consumes its entire gas limit. + +``` +mode=success: ┌ tx0 ✓ ┬ tx1 ✓ ┬ tx2 ✓ ┬ tx3 ✓ ┬ tx4 ✓ ┬ tx5 ✓ ┬ tx6 ✓ ┐ + all writes land: 7 × P pages + 7 markers + +mode=halt: ┌ tx0 ✗ ┬ tx1 ✗ ┬ tx2 ✗ ┬ tx3 ✗ ┬ tx4 ✗ ┬ tx5 ✗ ┬ tx6 ✗ ┐ + every tx: P × W(0)=1, marker, then INVALID + → post-state empty, block still burns its full budget + +mode=mix: ┌ tx0 ✓ ┬ tx1 ✗ ┬ tx2 ✓ ┬ tx3 ✗ ┬ tx4 ✓ ┬ tx5 ✗ ┬ tx6 ✓ ┐ + only even txs' pages + markers survive (4 × P pages) +``` + +Measures the cost of executing (and then discarding) storage writes — +the revert path does the same page work as success but must be rolled +back. + +--- + +## `test_random_sload` + +Cold-SLOAD a set of distinct 1-element or empty pages (random access), from a pool of contracts spread across the block. + +Random file access: each read targets a **distinct page** (`slot = +page_key << 7`), and the MPT hashes every page key (`keccak256`) to an +unpredictable trie/disk position, so consecutive page keys land at +uncorrelated disk locations. Params: `slots ∈ {1, 16, 128}` (size of +the per-tx page set) × `k ∈ {0, 1}` (each read page is 1-slot occupied, +or never populated). + +A pool of 8 identical contracts is deployed; tx `g` calls contract +`g mod 8` and carries a base page key strided by `g`, giving every tx a +disjoint set of pages inside "its" contract: + +``` +Block: tx0→C0 tx1→C1 tx2→C2 tx3→C3 tx4→C4 tx5→C5 tx6→C6 + +tx g: page set = { (base_g + i) << 7 : i < slots }, base_g strided by g + every page key hashed by the MPT → random disk position + + iteration j (of P) reads page (base_g + (j mod slots)) << 7: + j: 0 1 ... slots−1 │ slots ... P-1 + base+0 base+1 ... │ (set cycles again) + cold cold ... cold │ warm ... warm +``` + +Each tx makes P reads, but only the first pass over the set +(`slots` reads) is genuinely cold — sizing charges every iteration as +cold, so these blocks do far less work than the budget by design; +the subject is random-locality I/O, not volume. With `k = 1` the set's +pages are pre-populated (one slot = 1) in that contract's genesis and +the checksum is P; with `k = 0` every read is a whole-page miss and +the tail checksum SSTORE writes 0 (no state trace). Markers/checksums +land in the storage of whichever contract the tx called. + +--- + +## `test_bad_block_serial` + +Every tx SLOAD+SSTORE-increments the same slot sequence, forcing serial execution across the block. + +No parameters. The write-conflict adversarial block: all 7 txs +read-then-increment the **same** contiguous slot range, so every tx +depends on the previous tx's writes and the block cannot be executed +in parallel. + +``` +shared slot range: base = 2^40, slots base+0 .. base+782 +(contiguous keys → the whole range is just 7 dense pages) + +per iteration i: SLOAD(base+i), then SSTORE(base+i, read+1) + + slot: base+0 base+1 base+2 ... base+P-1 +tx 0: 0→1 0→1 0→1 0→1 fresh writes +tx 1: 1→2 1→2 1→2 1→2 ▲ must see the + ... │ previous tx's +tx 6: 6→7 6→7 6→7 6→7 │ writes: serial +``` + +P read+write pairs per tx (sized to the fresh 0→1 +cost — the later increment txs are nonzero→nonzero updates and run +cheaper, leaving the block somewhat gas-underfull). Post-state: every +shared slot ends at 7, plus the 7 markers (no checksum tail here). + +--- + +## `test_bad_block_chained` + +Each SLOAD returns the next SLOAD's slot (SLOAD(SLOAD(...))), a data-dependent pointer chase over distinct pages that serialises the reads within a tx. + +No parameters. The data-dependency adversarial block: the genesis +storage holds a pre-built ring of P **distinct pages** (keys +`(base+j) << 7`) where each page's stored value is the key of the next +page. A tx starts from its calldata base and hops the ring with +`slot := SLOAD(slot)` — every read's address comes from the previous +read, so the reads serialize within the tx, and because the MPT hashes +each page key, every hop is an unpredictable disk position. + +``` +genesis ring (P distinct pages, keys (base+j) << 7): + + ring[0] ──► ring[1] ──► ring[2] ──► ... ──► ring[P-1] ──┐ + ▲ │ + └─────────────────────────────────────────────────────┘ + SLOAD(ring[j]) returns ring[j+1] + +tx t (t = 0..6): start at ring[t], then P hops + = one full lap, staggered one step per tx +``` + +All 7 txs traverse the same ring; per-tx warmth reset makes each lap +fully cold, so the block performs 7 × P cold, address-dependent +reads. Nothing is written except the 7 +markers — post-state is the untouched ring plus markers. diff --git a/MONAD_RUNLOOP_TESTING.md b/MONAD_RUNLOOP_TESTING.md index c8d502d1f4d..402075a1acd 100644 --- a/MONAD_RUNLOOP_TESTING.md +++ b/MONAD_RUNLOOP_TESTING.md @@ -21,6 +21,7 @@ Requirements: docker, ~10 GB disk for the builder image and build artifacts, ~6 GB RAM for hugepages. ```sh +snap install astral-uv --classic git clone --branch main \ git@github.com:monad-exp/monad-eest-rust-harness.git cd monad-eest-rust-harness @@ -68,6 +69,48 @@ uv run consume direct --input ../fixtures_eestnet \ across ~14 threads), so budget ~5 vCPUs per worker (`-n N` needs roughly `5 * N` cores). +## MIP-8 perf-regression tests + +`tests/benchmark/stateful/mip8_pageified_storage/test_perf_regression.py` +fills SLOAD/SSTORE workloads at both forks and times block execution on +the runloop to compare MONAD_NINE (slot-encoded) vs MONAD_TEN +(page-encoded). + +### Setup + +```sh +sudo tee /etc/sysctl.d/99-benchmark.conf >/dev/null <<'EOF' +kernel.randomize_va_space = 0 +kernel.perf_event_paranoid = 1 +vm.nr_hugepages = 3072 +EOF +sudo sysctl --system +sudo cpupower idle-set -D 1 +``` + +### Run + +From the repo root: + +```sh +tmux new -s perf 'TAG=v4 scripts/perf_cycle.sh' +``` + +Fills once, consumes `RUNS` times, and writes the NINE-vs-TEN table to +`../timing_${TAG}__table.md` (headed with the +cycle time and the four repo SHAs). Knobs: + +- `TAG` (required) names every artifact; use a fresh one per experiment. +- `RUNS` consume passes (default 13), `REPEATS` page-disjoint copies per + fixture (cold samples reduced to a `min` within each pass). +- `BLOCK_GAS_M=N` sets the block gas budget in millions, passed through + as `--gas-benchmark-values`; the default 200 matches the gas the + runloop stamps. Fixtures land under `for_{fork}_at_0200M/`, so the + budget a fixture was built with is visible in its path. + `SKIP_FILL=1` reuses an existing `../fixtures_${TAG}`. + +Run on a quiet host; timings are noisy under contention. + ## Behavior and known limits - Fixtures containing expected-invalid blocks are skipped: the ledger diff --git a/docs/running_tests/consume/direct.md b/docs/running_tests/consume/direct.md index 8cf337ba934..e5f0d791ea6 100644 --- a/docs/running_tests/consume/direct.md +++ b/docs/running_tests/consume/direct.md @@ -8,6 +8,8 @@ uv run consume direct --bin= [OPTIONS] - `--bin EVM_BIN`: Path to an evm executable that can process `StateTestFixture` and/or `BlockTestFixture` formats. - `--traces`: Collect execution traces from the evm executable. +- `--timing-report`: Write per-block execution timing to `timing_consume.csv` (see [Block Execution Timing](#block-execution-timing)). +- `--timing-report-dir DIR`: Directory for that CSV; defaults to the HTML report's directory. !!! warning "Limited Client Support" @@ -35,6 +37,24 @@ uv run consume direct --bin= [OPTIONS] - **Module scope**: Tests EVM, respectively block import, in isolation, not full client behavior. - **Interface dependency**: Requires client-specific test interfaces. +## Block Execution Timing + +`--timing-report` writes a `timing_consume.csv` holding the per-block +measurements the client reported while executing each fixture, one row per +block. + +The `test`, `params` and `fork` columns identify the fixture; the remaining +columns are whatever metrics the client reported, so they vary by client. + +!!! note "Reporting clients only" + + A client only appears in the report if its fixture consumer measures + block execution. + +!!! warning "Run the timed pass serially" + + `--timing-report` is rejected together with xdist (`-n`). + ## Example Usage Only run state tests (by using a mark filter, `-m`) from a local `fixtures` folder with go-ethereum: diff --git a/docs/running_tests/useful_pytest_options.md b/docs/running_tests/useful_pytest_options.md index 6a5c7f6d7d9..423fbdd32e7 100644 --- a/docs/running_tests/useful_pytest_options.md +++ b/docs/running_tests/useful_pytest_options.md @@ -148,3 +148,9 @@ Print relevant test stage timings such as client start-up, payload response time ```bash uv run consume engine --input= --timing-data ``` + +Write the per-block execution timing reported by the client to a CSV (`consume direct` only, see [Block Execution Timing](./consume/direct.md#block-execution-timing)): + +```bash +uv run consume direct --input= --bin= --timing-report +``` diff --git a/packages/testing/pyproject.toml b/packages/testing/pyproject.toml index 5ad9ec0491d..a121bfbd238 100644 --- a/packages/testing/pyproject.toml +++ b/packages/testing/pyproject.toml @@ -102,6 +102,7 @@ groupstats = "execution_testing.cli.show_pre_alloc_group_stats:main" extract_config = "execution_testing.cli.extract_config:extract_config" compare_fixtures = "execution_testing.cli.compare_fixtures:main" benchmark_parser = "execution_testing.cli.benchmark_parser:main" +perf_regression = "execution_testing.cli.perf_regression:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/testing/src/execution_testing/cli/perf_regression.py b/packages/testing/src/execution_testing/cli/perf_regression.py new file mode 100644 index 00000000000..545f9df8c5d --- /dev/null +++ b/packages/testing/src/execution_testing/cli/perf_regression.py @@ -0,0 +1,511 @@ +""" +Build a NINE-vs-TEN significance table from perf timing runs. + +Reads the `timing_consume.csv` produced by `consume direct +--timing-report` for several identical runs (one dir each) and, per +(test, params) case, reports each measure's mean +/- sd over the runs for +both forks, the two-sided Mann-Whitney U p-value comparing the forks, +and (for significant measures) the NINE->TEN average change. + +Writes GitHub-flavored Markdown to `--md`, or stdout if omitted; when +`--now` is given, prefixes a provenance header. +""" + +from __future__ import annotations + +import csv +from itertools import product +from math import erfc, sqrt +from pathlib import Path +from statistics import mean, stdev +from typing import Dict, List, Optional, Tuple + +import click + +# One run's measurements: (test, params, fork) -> {metric: value}. +CaseMetrics = Dict[str, int] +Run = Dict[Tuple[str, str, str], CaseMetrics] + +# A case, identified by (test, params). +CaseKey = Tuple[str, str] + +# One metric's paired samples: [(NINE, TEN) per run measuring both forks]. +PairedSamples = List[Tuple[int, int]] + +METRICS = ["tx_exec_us", "commit_us", "total_us"] +# Work counters, reported alongside the durations but not tested: they say +# whether a slower block did the same work or redid some of it. Taken from +# the block the reduction picked, not reduced themselves — a retry count +# is only meaningful next to the timing of the same block. +COUNTS = ["retries"] +# The measure whose minimum picks a case's representative block. Block +# wall time is what a retry count belongs to: retries are a property of +# executing the whole block, not of one of its phases. +REFERENCE_METRIC = "total_us" +FORKS = ["MONAD_NINE", "MONAD_TEN"] +# A measure is significant at this Benjamini-Hochberg adjusted p-value. +# The table runs one test per (case, measure), so the unadjusted rate +# would produce a steady trickle of false positives across a cycle. The +# adjustment bounds the false discovery rate across the whole table, so +# 5% here caps the expected share of false flags among the measures the +# table does flag. Tightening it costs paired runs, since the floor a +# signed-rank test can reach scales with 2^RUNS. +ALPHA = 0.05 + +UP = "🔴⬆️" # significant measures all rise NINE->TEN (TEN slower) +DOWN = "🟢⬇️" # significant measures all fall NINE->TEN (TEN faster) +MIXED = "⚠️" # significant measures move both up and down + +# Per-case workload descriptions live here; pinned to the execution-specs +# sha of the run when known, else the repo's default branch (HEAD). +DIAGRAMS_URL = ( + "https://github.com/monad-developers/execution-specs/blob/" + "{ref}/MIP8_PERF_TESTS_DIAGRAMS.md" +) + + +def _direction(chgs: List[int]) -> str: + """Pick the direction emoji from the significant measures' changes.""" + ups = any(c > 0 for c in chgs) + downs = any(c < 0 for c in chgs) + if ups and downs: + return MIXED + if ups: + return UP + if downs: + return DOWN + return MIXED + + +def parse(report: Path) -> Run: + """ + Map (test, params, fork) -> {metric: value} from one csv report. + + The csv holds one row per block (raw, unaggregated); each duration is + reduced to the minimum across a case's blocks. Each counter in + `COUNTS` is carried over from the block with the lowest + `REFERENCE_METRIC`, so it describes the block whose timing is + reported rather than a reduction of its own; ties keep the earlier + block. A counter the report predates is treated as zero so older csv + files still parse. + """ + rows: Run = {} + with report.open(newline="") as f: + for row in csv.DictReader(f): + if row["fork"] not in FORKS: + continue + try: + mv = {m: int(row[m]) for m in METRICS} + mv.update({c: int(row.get(c) or 0) for c in COUNTS}) + except (ValueError, TypeError): + continue + test = row["test"].split("::")[-1].removeprefix("test_") + key = (test, row["params"], row["fork"]) + prev = rows.get(key) + if prev is None: + rows[key] = mv + else: + # `prev` already holds the running minimum, so its + # counters belong to the block that set it. + fastest = ( + mv + if mv[REFERENCE_METRIC] < prev[REFERENCE_METRIC] + else prev + ) + rows[key] = { + **{m: min(prev[m], mv[m]) for m in METRICS}, + **{c: fastest[c] for c in COUNTS}, + } + return rows + + +def _stat(values: List[int]) -> str: + """Format a run sample as 'mean ± sd' (µs, rounded).""" + sd = stdev(values) if len(values) > 1 else 0.0 + return f"{round(mean(values))} ± {round(sd)}" + + +def _pfmt(p: float) -> str: + """Format a p-value compactly.""" + return "<0.001" if p < 0.001 else f"{p:.3f}" + + +def _avg_ranks(vals: List[float]) -> List[float]: + """Return 1-based ranks (ties averaged), aligned to `vals`.""" + order = sorted(range(len(vals)), key=lambda i: vals[i]) + ranks = [0.0] * len(vals) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and vals[order[j + 1]] == vals[order[i]]: + j += 1 + avg = (i + j) / 2 + 1 + for k in range(i, j + 1): + ranks[order[k]] = avg + i = j + 1 + return ranks + + +def _wilcoxon_p(diffs: List[int]) -> float: + """ + Two-sided Wilcoxon signed-rank p-value for paired differences. + + Both forks are measured inside the same consume pass, so run `i` + yields a matched (NINE, TEN) pair. Testing the differences cancels + whatever drifted between runs — host thermals, page cache, other load + — which an unpaired test would instead charge to both samples' + variance. + + Zero differences carry no sign information and are dropped, as in the + standard procedure. Exact (enumerating the sign assignments, + conditional on the observed ranks, so averaged ties are handled) up to + 200k of them; past that a normal approximation with a continuity and + tie correction. Distribution-free — no normality assumption. + + Follows Wilcoxon, F. (1945), "Individual Comparisons by Ranking + Methods", Biometrics Bulletin 1(6), 80-83. On tie-free samples both + branches reproduce `scipy.stats.wilcoxon` exactly, and the exact + branch reproduces the published two-sided critical values; see + `test_perf_regression.py`. + + With ties in |difference| there is no single standard answer, because + the exact null distribution assumes distinct ranks. This enumerates + signs over the observed averaged ranks (the conditional exact test); + scipy's exact branch instead ranks 1..n and ignores ties. The two + differ by a few percent on tied samples. + """ + nonzero = [d for d in diffs if d != 0] + n = len(nonzero) + if n == 0: + return 1.0 + ranks = _avg_ranks([abs(d) for d in nonzero]) + positive = sum(r for r, d in zip(ranks, nonzero, strict=True) if d > 0) + mu = sum(ranks) / 2 # null mean of the positive-rank sum + d = abs(positive - mu) + if 2**n <= 200_000: + extreme = 0 + for signs in product((0, 1), repeat=n): + total = sum(r for r, s in zip(ranks, signs, strict=True) if s) + if abs(total - mu) >= d - 1e-9: + extreme += 1 + return extreme / 2**n + tie_correction = sum(t**3 - t for t in _tie_sizes(ranks)) / 48 + variance = n * (n + 1) * (2 * n + 1) / 24 - tie_correction + if variance <= 0: + return 1.0 + return erfc(max(0.0, (d - 0.5) / sqrt(variance)) / sqrt(2)) + + +def _tie_sizes(ranks: List[float]) -> List[int]: + """Group sizes of tied ranks, for the signed-rank tie correction.""" + counts: Dict[float, int] = {} + for rank in ranks: + counts[rank] = counts.get(rank, 0) + 1 + return [count for count in counts.values() if count > 1] + + +def _bh_adjust(ps: List[float]) -> List[float]: + """ + Benjamini-Hochberg adjusted p-values, aligned to `ps`. + + The table tests every (case, measure) pair, so raw p-values would let + a handful of false positives through every cycle. Comparing the + adjusted value against ALPHA controls the false discovery rate across + the whole table instead. + """ + m = len(ps) + if m == 0: + return [] + order = sorted(range(m), key=lambda i: ps[i]) + adjusted = [1.0] * m + running = 1.0 + for rank, i in reversed(list(enumerate(order, start=1))): + running = min(running, ps[i] * m / rank) + adjusted[i] = min(1.0, running) + return adjusted + + +def _paired_samples( + runs: List[Run], +) -> Tuple[Dict[CaseKey, Dict[str, PairedSamples]], Dict[CaseKey, int]]: + """ + Pair each case's per-run measurements across the two forks. + + Returns the per-case, per-metric pairs and, per case, how many runs + were dropped because they measured only one of the two forks. + """ + cases = sorted({(test, params) for run in runs for test, params, _ in run}) + samples: Dict[CaseKey, Dict[str, PairedSamples]] = {} + dropped: Dict[CaseKey, int] = {} + for case in cases: + per_metric: Dict[str, PairedSamples] = { + m: [] for m in (*METRICS, *COUNTS) + } + missing = 0 + for run in runs: + nine = run.get((*case, FORKS[0])) + ten = run.get((*case, FORKS[1])) + if nine is None or ten is None: + missing += 1 + continue + for m in (*METRICS, *COUNTS): + per_metric[m].append((nine[m], ten[m])) + samples[case] = per_metric + if missing: + dropped[case] = missing + return samples, dropped + + +def build(runs: List[Run]) -> List[str]: + """Return the markdown lines for the significance table.""" + samples, dropped = _paired_samples(runs) + + # Test every (case, measure) first, so the significance threshold can + # be adjusted across the whole table before anything is rendered. + tested = [ + (case, metric) + for case in sorted(samples) + for metric in METRICS + if samples[case][metric] + ] + ps = [_wilcoxon_p([t - n for n, t in samples[c][m]]) for c, m in tested] + adjusted = dict(zip(tested, _bh_adjust(ps), strict=True)) + raw = dict(zip(tested, ps, strict=True)) + + header = ["test-params"] + for m in METRICS: + header += [f"{m} NINE", f"{m} TEN", f"{m} q"] + for c in COUNTS: + header += [f"{c} NINE", f"{c} TEN"] + header += ["significant", "Δ avg NINE→TEN (sig)"] + + lines = [ + f"Mean ± sd over up to {len(runs)} runs; durations in µs, " + f"{'/'.join(COUNTS)} a count. Bold = measure significant (paired " + f"Wilcoxon signed-rank, Benjamini–Hochberg adjusted q ≤ {ALPHA}). " + f"Significant flag: {UP} TEN slower, {DOWN} TEN faster, " + f"{MIXED} mixed.", + "", + "| " + " | ".join(header) + " |", + "| " + " | ".join("---" for _ in header) + " |", + ] + + unpaired = [] + for case in sorted(samples): + test, params = case + if not samples[case][METRICS[0]]: + unpaired.append(f"{test} {params}") + continue + cells = [f"{test} {params}"] + deltas = [] + chgs = [] + for m in METRICS: + pairs = samples[case][m] + n = [nine for nine, _ in pairs] + x = [ten for _, ten in pairs] + q = adjusted[(case, m)] + ncell, xcell = _stat(n), _stat(x) + if q <= ALPHA: + ncell, xcell = f"**{ncell}**", f"**{xcell}**" + navg, xavg = mean(n), mean(x) + chg = round((xavg - navg) / navg * 100) if navg else 0 + deltas.append(f"{m} {chg:+d}%") + chgs.append(chg) + cells += [ncell, xcell, _pfmt(q)] + for c in COUNTS: + pairs = samples[case][c] + cells += [ + _stat([nine for nine, _ in pairs]), + _stat([ten for _, ten in pairs]), + ] + cells.append(_direction(chgs) if chgs else "-") + cells.append(", ".join(deltas) if deltas else "-") + lines.append("| " + " | ".join(cells) + " |") + + lines += [ + "", + "q is the Benjamini–Hochberg adjusted p-value of a two-sided " + "paired Wilcoxon signed-rank test on the per-run NINE→TEN " + f"differences for that measure; significant at q ≤ {ALPHA}. " + f"Smallest raw p in this table: {_pfmt(min(ps)) if ps else 'n/a'}. " + f"{'/'.join(COUNTS)} is reported, not tested: it tells a block " + "that did the same work more slowly from one that redid work.", + ] + + lines += _power_note(samples, len(raw)) + lines += _coverage_notes(runs, unpaired, dropped, raw) + return lines + + +def _power_note( + samples: Dict[CaseKey, Dict[str, PairedSamples]], tests: int +) -> List[str]: + """ + Warn when the run count cannot resolve an isolated effect. + + A signed-rank test on `n` pairs has 2^n sign assignments, so no raw p + can fall below 2/2^n whatever the effect size; times the table's test + count that is a floor on q. + """ + paired = [len(m[METRICS[0]]) for m in samples.values() if m[METRICS[0]]] + if not paired or not tests: + return [] + n = max(paired) + if 2 / 2**n * tests <= ALPHA: + return [] + needed = 1 + while 2 / 2**needed * tests > ALPHA: + needed += 1 + return [ + "", + f"⚠️ {n} paired runs cannot reach q ≤ {ALPHA} for an effect in one " + f"measure alone across {tests} tests; that needs ≥ {needed} runs.", + ] + + +def _coverage_notes( + runs: List[Run], + unpaired: List[str], + dropped: Dict[CaseKey, int], + raw: Dict[Tuple[CaseKey, str], float], +) -> List[str]: + """ + Report what the table could not cover. + + A case only appears above when a run measured both forks; a case or + run that failed or was skipped would otherwise vanish from the report + without a trace. + """ + notes: List[str] = [] + if unpaired: + notes += [ + "", + f"**{len(unpaired)} case(s) not compared** — no run measured " + "both forks (fixture failed, was skipped, or was not filled):", + "", + ] + notes += [f"- {case}" for case in sorted(unpaired)] + partial = { + c: n for c, n in dropped.items() if f"{c[0]} {c[1]}" not in unpaired + } + if partial: + notes += [ + "", + f"**{len(partial)} case(s) compared on fewer than {len(runs)} " + "runs** — the remaining runs measured only one fork:", + "", + ] + notes += [ + f"- {test} {params}: {len(runs) - missing} of {len(runs)} runs" + for (test, params), missing in sorted(partial.items()) + ] + if raw: + notes += [ + "", + f"{len(raw)} hypothesis tests in this table " + f"({len({c for c, _ in raw})} cases × {len(METRICS)} measures).", + ] + return notes + + +def _provenance(now: str, shas: List[Optional[str]]) -> str: + """Markdown header: descriptions link, cycle time, and repo shas.""" + repos = [ + "execution-specs", + "monad-eest-rust-harness", + "monad-bft", + "monad", + ] + ref = DIAGRAMS_URL.format(ref=shas[0] or "HEAD") + lines = [ + f"Test-case descriptions: {ref}", + "", + f"Cycle {now}", + "", + "| repo | sha |", + "| --- | --- |", + ] + lines += [ + f"| {r} | {s or '?'} |" for r, s in zip(repos, shas, strict=True) + ] + return "\n".join(lines) + + +def report( + run_dirs: Tuple[Path, ...], + now: Optional[str] = None, + shas: Optional[List[Optional[str]]] = None, +) -> str: + """ + Render the significance table for the given run directories. + + Raises `click.ClickException` if fewer than two of them hold a + `timing_consume.csv`, since a comparison needs at least two samples + per fork. + """ + reports = [Path(d) / "timing_consume.csv" for d in run_dirs] + missing = [str(p.parent) for p in reports if not p.exists()] + runs = [parse(p) for p in reports if p.exists()] + if missing: + click.echo( + f"skipping {len(missing)} run dir(s) without a " + f"timing_consume.csv: {', '.join(missing)}", + err=True, + ) + if len(runs) < 2: + raise click.ClickException( + f"need >=2 runs with timing_consume.csv, found {len(runs)}" + ) + md = "\n".join(build(runs)) + if now: + md = f"{_provenance(now, shas or [None] * 4)}\n\n{md}" + return md + + +@click.command() +@click.argument( + "run_dirs", + nargs=-1, + required=True, + type=click.Path(exists=True, file_okay=False, path_type=Path), +) +@click.option( + "--md", + "md_path", + type=click.Path(dir_okay=False, path_type=Path), + default=None, + help="Write the Markdown report here instead of stdout.", +) +@click.option( + "--now", + default=None, + help="Cycle timestamp; prefixes the report with a provenance header.", +) +@click.option("--repo", default=None, help="execution-specs sha.") +@click.option("--harness", default=None, help="monad-eest-rust-harness sha.") +@click.option("--monad-bft", default=None, help="monad-bft sha.") +@click.option("--monad", default=None, help="monad-execution sha.") +def main( + run_dirs: Tuple[Path, ...], + md_path: Optional[Path], + now: Optional[str], + repo: Optional[str], + harness: Optional[str], + monad_bft: Optional[str], + monad: Optional[str], +) -> None: + """ + Build a NINE-vs-TEN significance table from perf timing runs. + + Each RUN_DIRS argument is one `consume direct --timing-report` output + directory holding a `timing_consume.csv`. + """ + md = report(run_dirs, now, [repo, harness, monad_bft, monad]) + if md_path: + md_path.write_text(md + "\n", encoding="utf-8") + else: + click.echo(md) + + +if __name__ == "__main__": + main() diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py new file mode 100644 index 00000000000..10d403a0a94 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/direct/timing_report.py @@ -0,0 +1,330 @@ +""" +Emit per-block execution timing collected during `consume direct` as a +CSV artifact alongside the HTML report. + +A consumer that implements the `BlockTimingReporter` capability (e.g. the +monad runloop) exposes per-block timing for the fixture it just ran. This +plugin reads it off the finished test and writes every block it collected +as one raw per-block CSV at session end. + +The metric columns are whatever keys the consumer reported, so the plugin +stays agnostic of which execution phases a given client can measure. + +Everything the feature needs lives here — the capability protocol, the +command-line options and the reporting — so the generic `consume` modules +stay byte-identical to upstream. `pytest-consume.ini` registers it with a +single `-p` line. +""" + +from __future__ import annotations + +import csv +import io +from pathlib import Path +from typing import ( + Any, + Dict, + Generator, + List, + Mapping, + Optional, + Protocol, + Sequence, + Tuple, + runtime_checkable, +) + +import pytest + +from execution_testing.fixtures.consume import ( + TestCaseIndexFile, + TestCaseStream, +) +from execution_testing.forks import Fork, TransitionFork +from execution_testing.forks.helpers import get_forks + +BlockTiming = Mapping[str, int] +""" +One block's measurements, keyed by metric name. + +The metric names are the reporting consumer's own: it decides which +phases it can measure and how they are labelled. Consumers must use the +same keys, in the same order, for every block they report. +""" + + +@runtime_checkable +class BlockTimingReporter(Protocol): + """ + Optional consumer capability: per-block timing of the last fixture. + + A consumer that can measure block processing implements this so + `consume direct --timing-report` can tabulate it. + """ + + def last_block_timings(self) -> Sequence[BlockTiming]: + """ + Return per-block measurements from the most recent + `consume_fixture` call, empty if it measured none. + """ + ... + + +# Fixtures the report hook reads off the finished test. Both are +# parametrized by the consume plugins, so they are present in +# `item.funcargs` for the call phase. +_CONSUMER_FIXTURE = "fixture_consumer" +_TEST_CASE_FIXTURE = "test_case" + +# Columns identifying a row; the reported metric keys follow them. +_ID_COLUMNS = ["test", "params", "fork"] + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add the timing report options to the consume command group.""" + group = parser.getgroup( + "consume_direct", + "Arguments related to consuming fixtures via a client", + ) + group.addoption( + "--timing-report", + action="store_true", + dest="timing_report", + default=False, + help=( + "Emit per-block execution timing (from consumers that report " + "it) as a raw `timing_consume.csv`." + ), + ) + group.addoption( + "--timing-report-dir", + action="store", + dest="timing_report_dir", + type=Path, + default=None, + help=( + "Directory for the timing report. Defaults to the HTML report's " + "directory (the fixtures `.meta` directory)." + ), + ) + + +def pytest_configure(config: pytest.Config) -> None: + """Register the report plugin, or refuse a run that cannot time.""" + if not config.getoption("timing_report", False): + return + # `--bin` is registered by the `consume direct` conftest only, so its + # absence means a hive simulator, where no consumer reports timing. + # A hive command usually fails its own setup check before reaching + # here; this keeps the flag from being silently accepted if it does. + if config.getoption("fixture_consumer_bin", None) is None: + raise pytest.UsageError( + "--timing-report is only available for `consume direct`." + ) + if (config.getoption("numprocesses", None) or 0) != 0: + raise pytest.UsageError( + "--timing-report cannot be combined with xdist." + ) + config.pluginmanager.register( + TimingReportPlugin(config), "consume-timing-report" + ) + + +def _fork_rank(fork: Fork | TransitionFork | None) -> int: + """ + Position of `fork` in the framework's chronological fork list. + + Used to group CSV rows in release order without hardcoding a fork + list. Transition and unknown forks sort after all plain forks. + """ + forks = get_forks() + try: + return forks.index(fork) # type: ignore[arg-type] + except ValueError: + return len(forks) + + +def _sorted_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Order rows by (test, params, fork) then block for readability.""" + rows.sort( + key=lambda r: ( + r["test"], + r["params"], + r["fork_rank"], + r["fork"], + r.get("block", 0), + ) + ) + return rows + + +def _split_fixture_id( + fixture_id: str, fork: str, fixture_format: str +) -> Tuple[str, str]: + """ + Split a fixture id into (test, params). + + ``tests/.../test_perf_regression.py::test_page_ops[ + sstore_fresh-k0-fork_MONAD_NINE-blockchain_test]`` becomes + ``("test_perf_regression::test_page_ops", "sstore_fresh-k0")``. The + fork and fixture-format tokens are dropped by value (both are known + from the test case), and the remaining parameters are kept as one + ``-``-joined column since their arity varies per test. + """ + module, _, rest = fixture_id.partition("::") + func = rest.split("[", 1)[0] + test = f"{Path(module).stem}::{func}" if module else func + params: List[str] = [] + if "[" in rest and rest.rstrip().endswith("]"): + inner = rest[rest.index("[") + 1 : rest.rindex("]")] + params = [token for token in inner.split("-") if token] + for known in (f"fork_{fork}", fixture_format): + if known in params: + params.remove(known) + return test, "-".join(params) + + +def timing_payload( + consumer: object, + test_case: TestCaseIndexFile | TestCaseStream, +) -> Optional[Dict[str, Any]]: + """ + Build one test's timing payload from the consumer that ran it. + + Returns None for a consumer without the `BlockTimingReporter` + capability and for a fixture the consumer measured nothing for. + """ + if not isinstance(consumer, BlockTimingReporter): + return None + timings = consumer.last_block_timings() + if not timings: + return None + fork = test_case.fork + fork_name = fork.name() if fork is not None else "" + test, params = _split_fixture_id( + test_case.id, fork_name, test_case.format.format_name + ) + return { + "test": test, + "params": params or "-", + "fork": fork_name, + "fork_rank": _fork_rank(fork), + "blocks": [dict(timing) for timing in timings], + } + + +def _rows_from_payload(payload: Dict[str, Any]) -> List[Dict[str, Any]]: + """Flatten one recorded timing payload into per-block table rows.""" + identity = {key: payload[key] for key in (*_ID_COLUMNS, "fork_rank")} + return [{**identity, **block} for block in payload["blocks"]] + + +def _metric_columns(rows: List[Dict[str, Any]]) -> List[str]: + """ + Metric columns, in the order the consumer first reported them. + + Consumers reporting different metrics in one session contribute their + own columns; a row missing a column is written blank. + """ + columns: List[str] = [] + for row in rows: + for key in row: + if key not in columns and key not in (*_ID_COLUMNS, "fork_rank"): + columns.append(key) + return columns + + +def _render_csv(rows: List[Dict[str, Any]]) -> str: + """Render rows as CSV.""" + columns = [*_ID_COLUMNS, *_metric_columns(rows)] + buffer = io.StringIO() + writer = csv.writer(buffer) + writer.writerow(columns) + for row in rows: + writer.writerow([row.get(column, "") for column in columns]) + return buffer.getvalue() + + +class TimingReportPlugin: + """Collect each test's timing and write the raw per-block CSV.""" + + def __init__(self, config: pytest.Config): # noqa: D107 + self.config = config + self.rows: List[Dict[str, Any]] = [] + self.written: List[Path] = [] + + def _output_dir(self) -> Path: + """ + Directory the timing artifacts are written to. + + Prefers an explicit ``--timing-report-dir``; otherwise follows the + HTML report's directory (so it lands next to ``report_consume.html``, + which defaults to the fixtures `.meta`). + """ + explicit = self.config.getoption("timing_report_dir", None) + if explicit is not None: + return Path(explicit) + htmlpath = getattr(self.config.option, "htmlpath", None) + if htmlpath: + return Path(htmlpath).parent + source = self.config.fixtures_source # type: ignore[attr-defined] + return Path(source.path) / ".meta" + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport( + self, + item: pytest.Item, + call: pytest.CallInfo[None], + ) -> Generator[None, Any, None]: + """ + Keep this test's timing for the report. + + The consumer and test case are read straight off the finished + item, so the generic test function stays untouched. `--timing-report` + rejects xdist, so one process sees every test and the rows can be + held in memory until session end. + """ + outcome = yield + if call.when != "call": + return + report = outcome.get_result() + if report.outcome != "passed": + return + funcargs = getattr(item, "funcargs", None) or {} + consumer = funcargs.get(_CONSUMER_FIXTURE) + test_case = funcargs.get(_TEST_CASE_FIXTURE) + if consumer is None or test_case is None: + return + payload = timing_payload(consumer, test_case) + if payload: + self.rows.extend(_rows_from_payload(payload)) + + def pytest_sessionfinish( + self, + session: pytest.Session, # noqa: ARG002 + exitstatus: int, # noqa: ARG002 + ) -> None: + """Write the collected rows as the CSV report.""" + if not self.rows: + return + output_dir = self._output_dir() + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / "timing_consume.csv" + path.write_text(_render_csv(_sorted_rows(self.rows))) + self.written.append(path) + + def pytest_terminal_summary( + self, + terminalreporter: Any, + exitstatus: int, # noqa: ARG002 + config: pytest.Config, # noqa: ARG002 + ) -> None: + """Point the user at the generated timing artifacts.""" + if not self.written: + return + terminalreporter.write_sep("=", "block execution timing report") + terminalreporter.write_line( + "raw per-block timings; aggregate across blocks downstream" + ) + for path in self.written: + terminalreporter.write_line(f"timing report written to: {path}") diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_timing_report.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_timing_report.py new file mode 100644 index 00000000000..dce7ae69241 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_timing_report.py @@ -0,0 +1,252 @@ +"""Tests for the `consume direct` block execution timing report.""" + +from pathlib import Path +from typing import Any, List, Sequence, cast + +import pytest + +from execution_testing.cli.pytest_commands.plugins.consume.direct.timing_report import ( # noqa: E501 + BlockTiming, + BlockTimingReporter, + _fork_rank, + _render_csv, + _rows_from_payload, + _sorted_rows, + _split_fixture_id, + pytest_configure, + timing_payload, +) +from execution_testing.fixtures import BlockchainFixture +from execution_testing.fixtures.consume import TestCaseIndexFile +from execution_testing.forks import MONAD_NINE, MONAD_TEN + +FIXTURE_ID = ( + "tests/benchmark/stateful/mip8_pageified_storage/test_perf_regression.py" + "::test_page_ops[sstore_fresh-k0-fork_MONAD_NINE-blockchain_test]" +) + + +class TimingConsumer: + """A consumer implementing the `BlockTimingReporter` capability.""" + + def __init__(self, timings: Sequence[BlockTiming]): # noqa: D107 + self.timings = timings + + def last_block_timings(self) -> Sequence[BlockTiming]: + """Return the canned timings.""" + return self.timings + + +class SilentConsumer: + """A consumer that does not measure block timing.""" + + +def _test_case(fixture_id: str = FIXTURE_ID) -> TestCaseIndexFile: + """Build an index test case for `fixture_id` at MONAD_NINE.""" + return TestCaseIndexFile( + id=fixture_id, + fork=MONAD_NINE, + format=BlockchainFixture, + json_path=Path("dummy.json"), + ) + + +class _StubPluginManager: + """Records what the plugin registers.""" + + def __init__(self) -> None: # noqa: D107 + self.registered: List[str] = [] + + def register(self, plugin: object, name: str) -> None: # noqa: ARG002 + """Record a registration.""" + self.registered.append(name) + + +class _StubConfig: + """Just enough config surface for `pytest_configure`.""" + + def __init__(self, **options: Any) -> None: # noqa: D107 + self._options = options + self.pluginmanager = _StubPluginManager() + + def getoption(self, name: str, default: Any = None) -> Any: + """Return the option, or `default` when it was never registered.""" + return self._options.get(name, default) + + +def _configure(**options: Any) -> _StubConfig: + """Run `pytest_configure` against a stub config.""" + config = _StubConfig(**options) + pytest_configure(cast(pytest.Config, config)) + return config + + +def test_configure_ignores_a_run_without_the_flag() -> None: + """Nothing is registered unless --timing-report was passed.""" + assert _configure().pluginmanager.registered == [] + + +def test_configure_registers_for_consume_direct() -> None: + """A direct run with the flag gets the reporting plugin.""" + config = _configure(timing_report=True, fixture_consumer_bin=[]) + assert config.pluginmanager.registered == ["consume-timing-report"] + + +def test_configure_rejects_a_non_direct_command() -> None: + """Without `--bin` registered the command cannot report timing.""" + with pytest.raises(pytest.UsageError, match="consume direct"): + _configure(timing_report=True) + + +@pytest.mark.parametrize("numprocesses", [2, "auto", "logical"]) +def test_configure_rejects_xdist(numprocesses: Any) -> None: + """Parallel workers make the measurements meaningless.""" + with pytest.raises(pytest.UsageError, match="xdist"): + _configure( + timing_report=True, + fixture_consumer_bin=[], + numprocesses=numprocesses, + ) + + +def test_configure_allows_explicitly_disabled_xdist() -> None: + """`-n 0` is serial, so it is allowed.""" + config = _configure( + timing_report=True, fixture_consumer_bin=[], numprocesses=0 + ) + assert config.pluginmanager.registered == ["consume-timing-report"] + + +def test_split_fixture_id_drops_fork_and_format() -> None: + """The fork and format tokens are removed, the parameters kept.""" + test, params = _split_fixture_id( + FIXTURE_ID, "MONAD_NINE", "blockchain_test" + ) + assert test == "test_perf_regression::test_page_ops" + assert params == "sstore_fresh-k0" + + +def test_split_fixture_id_keeps_hyphenated_params() -> None: + """A parameter that merely looks like a format tag is kept.""" + fixture_id = "a/b/test_mod.py::test_f[blockchain_test_sync-fork_Prague-blockchain_test]" # noqa: E501 + test, params = _split_fixture_id(fixture_id, "Prague", "blockchain_test") + assert test == "test_mod::test_f" + assert params == "blockchain_test_sync" + + +def test_split_fixture_id_without_params() -> None: + """An unparametrized fixture id yields empty parameters.""" + test, params = _split_fixture_id( + "a/test_mod.py::test_f", "Prague", "blockchain_test" + ) + assert test == "test_mod::test_f" + assert params == "" + + +def test_fork_rank_orders_by_release() -> None: + """Known forks rank chronologically, unknown ones sort last.""" + assert _fork_rank(MONAD_NINE) < _fork_rank(MONAD_TEN) + assert _fork_rank(None) > _fork_rank(MONAD_TEN) + + +def test_timing_payload_from_reporting_consumer() -> None: + """A reporting consumer's timings become one test's payload.""" + consumer = TimingConsumer([{"block": 1, "total_us": 10}]) + + assert timing_payload(consumer, _test_case()) == { + "test": "test_perf_regression::test_page_ops", + "params": "sstore_fresh-k0", + "fork": "MONAD_NINE", + "fork_rank": _fork_rank(MONAD_NINE), + "blocks": [{"block": 1, "total_us": 10}], + } + + +@pytest.mark.parametrize( + "consumer", + [SilentConsumer(), TimingConsumer([])], + ids=["no_capability", "nothing_measured"], +) +def test_timing_payload_none_without_timings(consumer: object) -> None: + """No payload when the consumer reports no timing.""" + assert timing_payload(consumer, _test_case()) is None + + +def test_capability_probe_is_structural() -> None: + """The capability is detected by shape, without inheritance.""" + assert isinstance(TimingConsumer([]), BlockTimingReporter) + assert not isinstance(SilentConsumer(), BlockTimingReporter) + + +def test_render_csv_uses_reported_metric_keys() -> None: + """Metric columns follow the consumer's own keys and order.""" + payload = { + "test": "test_mod::test_f", + "params": "-", + "fork": "MONAD_NINE", + "fork_rank": 0, + "blocks": [ + {"block": 1, "gas": 100, "total_us": 10}, + {"block": 2, "gas": 200, "total_us": 20}, + ], + } + csv = _render_csv(_rows_from_payload(payload)) + assert csv.splitlines()[0] == "test,params,fork,block,gas,total_us" + assert csv.splitlines()[1] == "test_mod::test_f,-,MONAD_NINE,1,100,10" + + +def test_render_csv_blanks_missing_metrics() -> None: + """A consumer reporting other metrics contributes its own columns.""" + rows = _rows_from_payload( + { + "test": "t", + "params": "-", + "fork": "MONAD_NINE", + "fork_rank": 0, + "blocks": [{"block": 1, "total_us": 10}], + } + ) + _rows_from_payload( + { + "test": "t", + "params": "-", + "fork": "MONAD_NINE", + "fork_rank": 0, + "blocks": [{"block": 1, "other_us": 5}], + } + ) + lines = _render_csv(rows).splitlines() + assert lines[0] == "test,params,fork,block,total_us,other_us" + assert lines[1] == "t,-,MONAD_NINE,1,10," + assert lines[2] == "t,-,MONAD_NINE,1,,5" + + +def test_sorted_rows_groups_forks_in_release_order() -> None: + """Rows group by case, then fork release order, then block.""" + rows = [ + { + "test": "t", + "params": "p", + "fork": "MONAD_TEN", + "fork_rank": 1, + "block": 1, + }, + { + "test": "t", + "params": "p", + "fork": "MONAD_NINE", + "fork_rank": 0, + "block": 2, + }, + { + "test": "t", + "params": "p", + "fork": "MONAD_NINE", + "fork_rank": 0, + "block": 1, + }, + ] + assert [(r["fork"], r["block"]) for r in _sorted_rows(rows)] == [ + ("MONAD_NINE", 1), + ("MONAD_NINE", 2), + ("MONAD_TEN", 1), + ] diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-consume.ini b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-consume.ini index 9641d987419..2480f1d7c1b 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-consume.ini +++ b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-consume.ini @@ -10,4 +10,5 @@ addopts = -p no:logging -p execution_testing.cli.pytest_commands.plugins.custom_logging.plugin_logging -p execution_testing.cli.pytest_commands.plugins.consume.consume + -p execution_testing.cli.pytest_commands.plugins.consume.direct.timing_report -p execution_testing.cli.pytest_commands.plugins.help.help \ No newline at end of file diff --git a/packages/testing/src/execution_testing/cli/tests/test_perf_regression.py b/packages/testing/src/execution_testing/cli/tests/test_perf_regression.py new file mode 100644 index 00000000000..3da662b47e2 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/tests/test_perf_regression.py @@ -0,0 +1,472 @@ +"""Tests for the `perf_regression` NINE-vs-TEN significance table.""" + +from pathlib import Path +from typing import Dict, List, Sequence + +import pytest +from click import ClickException +from click.testing import CliRunner + +from execution_testing.cli.perf_regression import ( + ALPHA, + DOWN, + MIXED, + UP, + _avg_ranks, + _bh_adjust, + _wilcoxon_p, + main, + parse, + report, +) + +HEADER = ( + "test,params,fork,block,tx_count,gas,retries," + "tx_exec_us,state_root_us,commit_us,total_us" +) + + +def _csv(rows: Sequence[Dict[str, object]]) -> str: + """Render timing rows the way the consume timing report does.""" + lines = [HEADER] + for row in rows: + lines.append( + ",".join(str(row.get(column, "")) for column in HEADER.split(",")) + ) + return "\n".join(lines) + "\n" + + +def _row(fork: str, total: int, block: int = 1) -> Dict[str, object]: + """One per-block row for the `page_ops` case at `fork`.""" + return { + "test": "test_perf_regression::test_page_ops", + "params": "sstore_fresh-k0", + "fork": fork, + "block": block, + "tx_count": 7, + "gas": 200_000_000, + "retries": 0, + "tx_exec_us": total // 2, + "state_root_us": total // 4, + "commit_us": total // 4, + "total_us": total, + } + + +def _run_dir(tmp_path: Path, name: str, nine: int, ten: int) -> Path: + """Write a one-case run directory with the given per-fork totals.""" + path = tmp_path / name + path.mkdir() + (path / "timing_consume.csv").write_text( + _csv([_row("MONAD_NINE", nine), _row("MONAD_TEN", ten)]) + ) + return path + + +def test_parse_reduces_blocks_to_minimum(tmp_path: Path) -> None: + """Each metric is reduced to its minimum across a case's blocks.""" + path = tmp_path / "timing_consume.csv" + path.write_text( + _csv( + [ + _row("MONAD_NINE", 300, block=1), + _row("MONAD_NINE", 100, block=2), + _row("MONAD_NINE", 200, block=3), + ] + ) + ) + parsed = parse(path) + assert ( + parsed[("page_ops", "sstore_fresh-k0", "MONAD_NINE")]["total_us"] + == 100 + ) + + +def test_parse_ignores_other_forks(tmp_path: Path) -> None: + """Rows for forks outside the comparison are dropped.""" + path = tmp_path / "timing_consume.csv" + path.write_text(_csv([_row("Prague", 100), _row("MONAD_TEN", 200)])) + assert list(parse(path)) == [("page_ops", "sstore_fresh-k0", "MONAD_TEN")] + + +def test_parse_skips_unparsable_metrics(tmp_path: Path) -> None: + """A row with a non-numeric metric is skipped, not fatal.""" + bad = _row("MONAD_NINE", 100) + bad["total_us"] = "oops" + path = tmp_path / "timing_consume.csv" + path.write_text(_csv([bad, _row("MONAD_TEN", 200)])) + assert list(parse(path)) == [("page_ops", "sstore_fresh-k0", "MONAD_TEN")] + + +def test_avg_ranks_averages_ties() -> None: + """Tied values share the average of their ranks.""" + assert _avg_ranks([10.0, 20.0, 20.0, 30.0]) == [1.0, 2.5, 2.5, 4.0] + + +def test_wilcoxon_p_no_differences_is_one() -> None: + """Identical paired samples carry no sign information.""" + assert _wilcoxon_p([0, 0, 0]) == 1.0 + assert _wilcoxon_p([]) == 1.0 + + +def test_wilcoxon_p_drops_zero_differences() -> None: + """Zero differences are excluded from the rank count.""" + assert _wilcoxon_p([0, 0, 1, 2, 3]) == _wilcoxon_p([1, 2, 3]) + + +def test_wilcoxon_p_consistent_sign_is_the_floor() -> None: + """All-positive differences hit the smallest attainable p, 2/2**n.""" + assert _wilcoxon_p([1, 2, 3, 4, 5, 6, 7]) == pytest.approx(2 / 2**7) + assert _wilcoxon_p([5] * 7) == pytest.approx(2 / 2**7) + + +def test_wilcoxon_p_mixed_signs_is_not_significant() -> None: + """Differences that change sign give no evidence of a shift.""" + assert _wilcoxon_p([1, -2, 3, -4, 2, -1, 1]) > ALPHA + + +def test_wilcoxon_p_is_paired_not_pooled() -> None: + """A consistent shift is detected even when the levels overlap.""" + # Both forks drift upward across runs, but TEN is always the slower + # of the pair; an unpaired test would drown this in the drift. + nine = [100, 200, 300, 400, 500, 600, 700, 800] + ten = [110, 210, 310, 410, 510, 610, 710, 810] + diffs = [t - n for n, t in zip(nine, ten, strict=True)] + assert _wilcoxon_p(diffs) == pytest.approx(2 / 2**8) + + +# Two-sided critical values of the signed-rank statistic at alpha = 0.05, +# from the published table: reject when W+ <= W_crit. Reproducing the whole +# table pins the exact branch against a source outside this repo. +WILCOXON_CRITICAL_05 = {6: 0, 7: 2, 8: 3, 9: 5, 10: 8, 11: 10, 12: 13, 13: 17} + + +def _diffs_with_positive_rank_sum(n: int, target: int) -> List[int]: + """Signed differences over ranks 1..n whose positive ranks sum to it.""" + positive = set() + remaining = target + for rank in range(n, 0, -1): + if rank <= remaining: + positive.add(rank) + remaining -= rank + assert remaining == 0, f"cannot hit W+={target} with ranks 1..{n}" + return [r if r in positive else -r for r in range(1, n + 1)] + + +@pytest.mark.parametrize("n, w_crit", sorted(WILCOXON_CRITICAL_05.items())) +def test_wilcoxon_p_matches_published_critical_values( + n: int, w_crit: int +) -> None: + """The published rejection boundary falls where the table says.""" + at = _wilcoxon_p(_diffs_with_positive_rank_sum(n, w_crit)) + above = _wilcoxon_p(_diffs_with_positive_rank_sum(n, w_crit + 1)) + + assert at <= 0.05 < above + + +@pytest.mark.parametrize( + "diffs, expected", + [ + ([1, 2, 3, 4, 5, 6, 7], 2 / 2**7), + ([-3, 8, -1, 12, 5, -20, 2, 30], 0.3828125), + ([10, -20, 30, -40, 50, -60, 70, -80, 90, -100], 0.845703125), + ], + ids=["all_positive_n7", "mixed_n8", "mixed_n10"], +) +def test_wilcoxon_p_tie_free_reference_values( + diffs: List[int], expected: float +) -> None: + """Tie-free samples agree with `scipy.stats.wilcoxon` (1.18.1).""" + assert _wilcoxon_p(diffs) == pytest.approx(expected) + + +def test_wilcoxon_p_averages_tied_ranks() -> None: + """ + Pin the tie convention on a published worked example. + + The differences are the Wikipedia signed-rank example: one zero, which + drops to n=9, and |5| twice. With ties the exact null distribution is + not defined, so implementations disagree — this enumerates signs over + the averaged ranks, giving 324 of 2**9 sign assignments at least as + extreme. scipy's exact branch ranks 1..n instead and reports 0.6523; + the article reports 0.6113. + """ + diffs = [15, -7, 5, 20, 0, -9, 17, -12, 5, -10] + + assert _wilcoxon_p(diffs) == pytest.approx(324 / 2**9) + + +def test_wilcoxon_p_normal_approximation_for_large_samples() -> None: + """Past the exact-enumeration cap the normal fallback still ranks.""" + assert _wilcoxon_p(list(range(1, 25))) < ALPHA + + +def test_bh_adjust_matches_step_up_procedure() -> None: + """Adjusted values match the Benjamini-Hochberg step-up example.""" + ps = [0.001, 0.008, 0.039, 0.041, 0.042, 0.06, 0.074, 0.205] + adjusted = _bh_adjust(ps) + assert adjusted[0] == pytest.approx(0.008) + assert adjusted[1] == pytest.approx(0.032) + assert adjusted[-1] == pytest.approx(0.205) + + +def test_bh_adjust_is_monotone_and_bounded() -> None: + """Adjusted values never decrease with p, and never exceed 1.""" + ps = [0.2, 0.9, 0.01, 0.5, 0.99] + adjusted = _bh_adjust(ps) + assert all(q <= 1.0 for q in adjusted) + by_p = [q for _, q in sorted(zip(ps, adjusted, strict=True))] + assert by_p == sorted(by_p) + + +def test_bh_adjust_empty() -> None: + """An empty table adjusts to nothing.""" + assert _bh_adjust([]) == [] + + +def _case_row(table: str) -> str: + """The table's data row for the single `page_ops` case.""" + rows = [ + line for line in table.splitlines() if line.startswith("| page_ops") + ] + assert len(rows) == 1, table + return rows[0] + + +def test_build_flags_significant_slowdown(tmp_path: Path) -> None: + """A clean slowdown is flagged, bolded and quantified.""" + dirs = [ + _run_dir(tmp_path, f"r{i}", nine=100 + i, ten=200 + i) + for i in range(9) + ] + row = _case_row(report(tuple(dirs))) + assert UP in row + assert "**" in row + assert "total_us +96%" in row + + +def test_build_flags_significant_speedup(tmp_path: Path) -> None: + """A clean speedup is flagged in the other direction.""" + dirs = [ + _run_dir(tmp_path, f"r{i}", nine=200 + i, ten=100 + i) + for i in range(9) + ] + row = _case_row(report(tuple(dirs))) + assert DOWN in row + assert "total_us -49%" in row + + +def test_build_marks_noise_insignificant(tmp_path: Path) -> None: + """Interleaved samples are reported without a significance flag.""" + totals = [ + (100, 101), + (102, 99), + (98, 103), + (101, 100), + (99, 102), + (103, 98), + (97, 104), + (104, 97), + (96, 105), + ] + dirs = [ + _run_dir(tmp_path, f"r{i}", nine=n, ten=t) + for i, (n, t) in enumerate(totals) + ] + row = _case_row(report(tuple(dirs))) + assert UP not in row + assert DOWN not in row + assert MIXED not in row + assert row.endswith("| - | - |") + assert "**" not in row + + +def _one_fork_dir(tmp_path: Path, name: str, fork: str, total: int) -> Path: + """Write a run directory that measured only one of the two forks.""" + path = tmp_path / name + path.mkdir() + (path / "timing_consume.csv").write_text(_csv([_row(fork, total)])) + return path + + +def test_report_names_cases_never_compared(tmp_path: Path) -> None: + """A case no run measured at both forks is named, not dropped.""" + dirs = [ + _one_fork_dir(tmp_path, f"r{i}", "MONAD_NINE", 100 + i) + for i in range(3) + ] + table = report(tuple(dirs)) + assert "1 case(s) not compared" in table + assert "- page_ops sstore_fresh-k0" in table + + +def test_report_names_partially_compared_cases(tmp_path: Path) -> None: + """A case missing from some runs reports its reduced run count.""" + dirs = [ + _run_dir(tmp_path, f"r{i}", nine=100 + i, ten=200 + i) + for i in range(3) + ] + dirs.append(_one_fork_dir(tmp_path, "partial", "MONAD_NINE", 100)) + table = report(tuple(dirs)) + assert "compared on fewer than 4 runs" in table + assert "3 of 4 runs" in table + + +def test_report_counts_hypothesis_tests(tmp_path: Path) -> None: + """The report states how many tests the adjustment covers.""" + dirs = [_run_dir(tmp_path, f"r{i}", nine=100, ten=200) for i in range(3)] + assert "3 hypothesis tests in this table" in report(tuple(dirs)) + + +def test_report_warns_when_underpowered(tmp_path: Path) -> None: + """Too few pairs to clear the floor is called out, not hidden.""" + dirs = [ + _run_dir(tmp_path, f"r{i}", nine=100 + i, ten=200 + i) + for i in range(4) + ] + table = report(tuple(dirs)) + assert "cannot reach q" in table + assert _case_row(table).endswith("| - | - |") + + +def test_report_no_power_warning_when_resolvable(tmp_path: Path) -> None: + """Enough pairs to clear the floor leaves the warning out.""" + # 3 measures at 2/2**10 clears q <= 0.05; 6 runs would not. + dirs = [ + _run_dir(tmp_path, f"r{i}", nine=100 + i, ten=200 + i) + for i in range(10) + ] + assert "cannot reach q" not in report(tuple(dirs)) + + +def _retry_dir(tmp_path: Path, name: str, nine: int, ten: int) -> Path: + """Write a run directory whose blocks report differing retry counts.""" + path = tmp_path / name + path.mkdir() + rows = [] + for fork, retries in (("MONAD_NINE", nine), ("MONAD_TEN", ten)): + for block, r in enumerate((0, retries), start=1): + row = _row(fork, 100, block=block) + row["retries"] = r + rows.append(row) + (path / "timing_consume.csv").write_text(_csv(rows)) + return path + + +def test_report_shows_retries_of_the_reported_block(tmp_path: Path) -> None: + """Retries come from the block whose timing the row reports.""" + dirs = [_retry_dir(tmp_path, f"r{i}", nine=0, ten=4) for i in range(3)] + table = report(tuple(dirs)) + assert "retries NINE" in table + assert "retries TEN" in table + cells = [c.strip() for c in _case_row(table).split("|")] + # Both blocks tie on total_us, so the earlier one (retries 0) stands. + assert cells.count("0 ± 0") >= 2 + + +def test_parse_takes_the_counter_from_the_fastest_block( + tmp_path: Path, +) -> None: + """The counter follows argmin(REFERENCE_METRIC), not its own extremum.""" + rows = [] + for block, (total, retries) in enumerate( + [(300, 9), (100, 2), (200, 5)], start=1 + ): + row = _row("MONAD_NINE", total, block=block) + row["retries"] = retries + rows.append(row) + path = tmp_path / "timing_consume.csv" + path.write_text(_csv(rows)) + + parsed = parse(path)[("page_ops", "sstore_fresh-k0", "MONAD_NINE")] + + assert parsed["total_us"] == 100 + assert parsed["retries"] == 2 # not 9 (max) and not 5 + + +def test_parse_counter_ignores_a_slower_block_with_more_retries( + tmp_path: Path, +) -> None: + """A later, slower block does not drag its counter into the row.""" + rows = [] + for block, (total, retries) in enumerate([(100, 1), (500, 99)], start=1): + row = _row("MONAD_NINE", total, block=block) + row["retries"] = retries + rows.append(row) + path = tmp_path / "timing_consume.csv" + path.write_text(_csv(rows)) + + parsed = parse(path)[("page_ops", "sstore_fresh-k0", "MONAD_NINE")] + + assert parsed["total_us"] == 100 + assert parsed["retries"] == 1 + + +def test_parse_tolerates_csv_without_retries(tmp_path: Path) -> None: + """A csv written before the counter existed still parses.""" + legacy = HEADER.replace(",retries", "") + row = _row("MONAD_NINE", 100) + path = tmp_path / "timing_consume.csv" + path.write_text( + legacy + + "\n" + + ",".join(str(row.get(c, "")) for c in legacy.split(",")) + + "\n" + ) + parsed = parse(path) + assert ( + parsed[("page_ops", "sstore_fresh-k0", "MONAD_NINE")]["retries"] == 0 + ) + + +def test_report_needs_two_runs(tmp_path: Path) -> None: + """A single run cannot support a comparison.""" + with pytest.raises(ClickException, match="need >=2 runs"): + report((_run_dir(tmp_path, "only", nine=100, ten=200),)) + + +def test_report_warns_about_dirs_without_csv( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A run dir without a CSV is named rather than silently dropped.""" + dirs = [_run_dir(tmp_path, f"r{i}", nine=100, ten=200) for i in range(2)] + empty = tmp_path / "empty" + empty.mkdir() + + report((*dirs, empty)) + + assert "skipping 1 run dir(s)" in capsys.readouterr().err + + +def test_cli_writes_markdown(tmp_path: Path) -> None: + """The CLI writes the report and the provenance header.""" + dirs = [ + _run_dir(tmp_path, f"r{i}", nine=100 + i, ten=200 + i) + for i in range(9) + ] + md_path = tmp_path / "table.md" + + result = CliRunner().invoke( + main, + [ + "--md", + str(md_path), + "--now", + "2026-08-18T00:00:00Z", + "--repo", + "deadbeef", + *[str(d) for d in dirs], + ], + ) + + assert result.exit_code == 0, result.output + md = md_path.read_text() + assert "Cycle 2026-08-18T00:00:00Z" in md + assert "| execution-specs | deadbeef |" in md + assert "blob/deadbeef/MIP8_PERF_TESTS_DIAGRAMS.md" in md + + +def test_cli_requires_run_dirs() -> None: + """Invoking without run directories is a usage error.""" + result = CliRunner().invoke(main, []) + assert result.exit_code != 0 diff --git a/packages/testing/src/execution_testing/client_clis/clis/monad.py b/packages/testing/src/execution_testing/client_clis/clis/monad.py index 0c5889785e6..a98fa4dfc48 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/monad.py +++ b/packages/testing/src/execution_testing/client_clis/clis/monad.py @@ -19,15 +19,28 @@ import tempfile import uuid from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - +from typing import ( + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + TypedDict, +) + +import ijson # type: ignore[import-untyped] import pytest from execution_testing.fixtures import BlockchainFixture, FixtureFormat +from execution_testing.logging import get_logger from execution_testing.test_types import Transaction from ..fixture_consumer_tool import FixtureConsumerTool +logger = get_logger(__name__) + # Monad revision schedule per fixture `network`, as # (monad_revision, activation timestamp) pairs. Non-transition # fixtures run a single revision from genesis; transition fixtures @@ -63,6 +76,30 @@ def _set_pdeathsig() -> None: _LIBC.prctl(1, signal.SIGTERM) +def _iter_fixtures( + fixture_path: Path, fixture_name: Optional[str] +) -> Iterator[Dict[str, Any]]: + """ + Yield the fixtures to run from a (possibly multi-fixture) JSON file. + + A `fixture_name` selects one; without it every fixture in the file is + yielded, which is what the fixture-verification path asks for when it + hands over a merged file (`evm blocktest` behaves the same way, by + omitting `--run`). ijson streams the file so memory use stays low on + large fixtures. + """ + found = False + with open(fixture_path, "rb") as f: + for name, fixture in ijson.kvitems(f, ""): + if fixture_name is None or name == fixture_name: + found = True + yield fixture + if fixture_name is not None: + return + if not found: + raise KeyError(f"fixture {fixture_name!r} not found in {fixture_path}") + + def _hex32(value: str) -> str: """Normalize a hex quantity to a 0x-prefixed 64-nibble word.""" return f"0x{int(value, 16):064x}" @@ -160,6 +197,99 @@ def _compare_account( return mismatches +class BlockExecutionTiming(TypedDict): + """ + Per-block execution timing reported by the monad runloop. + + All durations are in microseconds. The keys name the runloop's own + execution phases, and are the columns `consume direct + --timing-report` writes for this consumer. + """ + + block: int + tx_count: int + gas: int + retries: int + tx_exec_us: int + state_root_us: int + commit_us: int + total_us: int + + +# A runloop duration: a decimal number and a chrono unit suffix, with +# optional whitespace. +_DURATION_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*(ns|[µμu]s|ms|s)\s*$") +_UNIT_US = {"ns": 1e-3, "us": 1.0, "ms": 1e3, "s": 1e6} + + +def _duration_us(value: str) -> int: + """ + Convert a duration like `5745us`, `5.7 ms` or `0.01s` to integer + microseconds; raise ValueError on any unrecognized format. + """ + match = _DURATION_RE.match(value) + if match is None: + raise ValueError(f"unrecognized duration {value!r}") + number, unit = match.groups() + if unit in ("µs", "μs"): + unit = "us" + return round(float(number) * _UNIT_US[unit]) + + +def _exec_block_row(line: str) -> Optional[BlockExecutionTiming]: + """Parse one `__exec_block` log line, or None (logged) if malformed.""" + body = line.split("__exec_block", 1)[1] + fields: Dict[str, str] = {} + for part in body.split(","): + key, sep, value = part.partition("=") + if sep: + fields[key.strip()] = value.strip() + + def us(key: str) -> int: + return _duration_us(fields[key]) + + try: + return BlockExecutionTiming( + block=int(fields["bl"]), + tx_count=int(fields["tx"]), + gas=int(fields["gas"]), + retries=int(fields.get("rt", 0)), + tx_exec_us=us("txe"), + state_root_us=us("sr"), + commit_us=us("cmt"), + total_us=us("tot"), + ) + except (KeyError, ValueError) as e: + logger.error(f"unparsable __exec_block line ({e!r}): {line.strip()}") + return None + + +def _parse_block_timings(stdout: str) -> List[BlockExecutionTiming]: + """ + Extract per-block timing from the runloop's `__exec_block` log lines. + + The production runloop logs one such line per block, e.g.: + `__exec_block,bl=1,...,tx=1,rt=0,...,sr=5192us,txe=14241us,cmt=879us, + tot=21153us,...,gas=10000000,...`. Fields carry leading padding; + durations a chrono unit suffix (see `_duration_us`). `rt` is the + optimistic-execution retry count, reported as `retries`: it separates + a block that did the same work more slowly from one that redid work. + A runloop that does not log `rt` still yields timings, with `retries` + reported as zero and one warning naming how many blocks it covered. + Malformed lines are logged and skipped. + """ + lines = [line for line in stdout.splitlines() if "__exec_block" in line] + without_retries = sum(1 for line in lines if "rt=" not in line) + if without_retries: + logger.warning( + f"{without_retries} of {len(lines)} __exec_block lines carry no " + "`rt` field; retries are reported as 0 for those blocks and " + "cannot be told apart from a block that really retried none" + ) + rows = [_exec_block_row(line) for line in lines] + return [row for row in rows if row is not None] + + class MonadFixtureConsumer( FixtureConsumerTool, fixture_formats=[BlockchainFixture], @@ -178,6 +308,17 @@ def __init__( """Initialize the MonadFixtureConsumer.""" super().__init__(binary=binary) self.trace = trace + self._block_timings: Sequence[BlockExecutionTiming] = () + + def last_block_timings(self) -> Sequence[BlockExecutionTiming]: + """ + Return the per-block timing parsed from the last consumed fixture. + + Implements the `BlockTimingReporter` capability. Reset at the start + of every `consume_fixture` call, so a fixture the runloop reported + nothing for never inherits the previous one's timing. + """ + return self._block_timings def _init_triedb( self, db_path: Path, schedule: List[Tuple[int, int]] @@ -191,6 +332,11 @@ def _init_triedb( (e.g. a MONAD_NINE->MONAD_TEN transition) needs both: a slot-encoded primary plus an activated page-encoded secondary timeline, so the runloop can dual-write across the fork. + + Only the secondary's kind is decided here. `Db::Db` re-stamps the + primary to whatever state machine the runloop opens it with, so the + `--state-machine` passed for the primary below does not survive + into the run. """ monad_mpt = self.binary.parent / "monad-mpt" revisions = [revision for revision, _ in schedule] @@ -203,7 +349,14 @@ def _init_triedb( # Shrunk chunk capacity / history ring keep per-test time at ~2s # (the production defaults dominate runtime). `monad` is the # page-encoded state machine, `ethereum` the slot-encoded one. - primary = "monad" if uses_page and not uses_slot else "ethereum" + # This only sets the kind `--create` stamps; the runloop re-stamps + # the primary when it opens the db, so the value is inert. Kept + # matched to the schedule so the two agree on disk; it can collapse + # to one constant once a TEN-only run confirms nothing reads the + # db in between. + initial_primary = ( + "monad" if uses_page and not uses_slot else "ethereum" + ) subprocess.run( [ str(monad_mpt), @@ -215,7 +368,7 @@ def _init_triedb( "--root-offsets-chunk-count", "2", "--state-machine", - primary, + initial_primary, ], capture_output=True, text=True, @@ -300,16 +453,25 @@ def consume_fixture( fixture_name: Optional[str] = None, debug_output_path: Optional[Path] = None, ) -> None: - """Execute a blockchain fixture on the monad runloop and verify.""" - assert fixture_format == BlockchainFixture + """ + Execute blockchain fixtures on the monad runloop and verify. - with open(fixture_path) as f: - fixtures = json.load(f) - if fixture_name is None: - assert len(fixtures) == 1, "fixture_name required" - fixture_name = next(iter(fixtures)) - fixture = fixtures[fixture_name] + Runs the named fixture, or every fixture in the file when no name + is given, and reports the timing of all their blocks together. + """ + assert fixture_format == BlockchainFixture + timings: List[BlockExecutionTiming] = [] + self._block_timings = () + for fixture in _iter_fixtures(fixture_path, fixture_name): + timings.extend(self._consume_one(fixture, debug_output_path)) + self._block_timings = timings + def _consume_one( + self, + fixture: Dict[str, Any], + debug_output_path: Optional[Path], + ) -> Sequence[BlockExecutionTiming]: + """Run one fixture and verify its post state and state root.""" network = fixture["network"] assert network in FORK_REVISION_SCHEDULES, ( f"no monad revision schedule for network {network}" @@ -362,20 +524,23 @@ def consume_fixture( ) output = json.loads(output_path.read_text()) + block_timings = _parse_block_timings(stdout) - actual_post = { - address.lower(): account - for address, account in output["post_state"].items() - } - post_state = fixture.get("postState") - assert post_state is not None, ( - "fixture has no postState (hash-only fixtures not supported)" - ) - + # The state root below is the authoritative check: it commits to + # the whole state, so any divergence changes it. `postState`, when + # the fixture carries it, only adds per-account detail to the + # failure message; fixtures that omit it (benchmark fixtures do, + # keeping them small) are verified by the root alone. mismatches = [] - for address, expected in post_state.items(): - actual = actual_post.get(address.lower()) - mismatches.extend(_compare_account(address, expected, actual)) + post_state = fixture.get("postState") + if post_state is not None: + actual_post = { + address.lower(): account + for address, account in output["post_state"].items() + } + for address, expected in post_state.items(): + actual = actual_post.get(address.lower()) + mismatches.extend(_compare_account(address, expected, actual)) # Assert the final state root, the last executed block's root. Under # monad's synchronous execution it equals the fixture's last block @@ -394,3 +559,5 @@ def consume_fixture( "post-state mismatch on the monad runloop:\n" + "\n".join(mismatches) ) + + return block_timings diff --git a/packages/testing/src/execution_testing/client_clis/tests/test_monad_fixture.py b/packages/testing/src/execution_testing/client_clis/tests/test_monad_fixture.py new file mode 100644 index 00000000000..7d549e211ad --- /dev/null +++ b/packages/testing/src/execution_testing/client_clis/tests/test_monad_fixture.py @@ -0,0 +1,73 @@ +"""Tests for the monad consumer's fixture selection.""" + +import json +from pathlib import Path +from typing import Any, Dict + +import pytest + +from execution_testing.client_clis.clis.monad import _iter_fixtures + + +def _write(path: Path, fixtures: Dict[str, Any]) -> Path: + """Write a fixture file holding the given fixtures.""" + path.write_text(json.dumps(fixtures)) + return path + + +def test_named_fixture_is_the_only_one_yielded(tmp_path: Path) -> None: + """A name selects one fixture out of a file holding several.""" + path = _write( + tmp_path / "f.json", + {"first": {"network": "A"}, "second": {"network": "B"}}, + ) + + assert list(_iter_fixtures(path, "second")) == [{"network": "B"}] + + +def test_no_name_yields_every_fixture(tmp_path: Path) -> None: + """ + Without a name the whole file is run. + + Fixture verification hands over a merged file and expects all of it + checked, so yielding only the first would leave the rest unverified. + """ + path = _write( + tmp_path / "f.json", + {"a": {"network": "A"}, "b": {"network": "B"}, "c": {"network": "C"}}, + ) + + assert list(_iter_fixtures(path, None)) == [ + {"network": "A"}, + {"network": "B"}, + {"network": "C"}, + ] + + +def test_no_name_on_a_sole_fixture(tmp_path: Path) -> None: + """A single-fixture file yields its one fixture.""" + path = _write(tmp_path / "f.json", {"only": {"network": "A"}}) + + assert list(_iter_fixtures(path, None)) == [{"network": "A"}] + + +def test_missing_name_raises(tmp_path: Path) -> None: + """A name absent from the file is an error, not an empty result.""" + path = _write(tmp_path / "f.json", {"only": {"network": "A"}}) + + with pytest.raises(KeyError, match="absent"): + list(_iter_fixtures(path, "absent")) + + +def test_named_lookup_stops_at_the_match(tmp_path: Path) -> None: + """The stream is not read past the fixture that was asked for.""" + path = _write( + tmp_path / "f.json", + {"first": {"network": "A"}, "second": {"network": "B"}}, + ) + + fixtures = _iter_fixtures(path, "first") + + assert next(fixtures) == {"network": "A"} + with pytest.raises(StopIteration): + next(fixtures) diff --git a/packages/testing/src/execution_testing/client_clis/tests/test_monad_timing.py b/packages/testing/src/execution_testing/client_clis/tests/test_monad_timing.py new file mode 100644 index 00000000000..2cc382aa256 --- /dev/null +++ b/packages/testing/src/execution_testing/client_clis/tests/test_monad_timing.py @@ -0,0 +1,96 @@ +"""Tests for the monad consumer's `__exec_block` timing parsing.""" + +import pytest + +from execution_testing.client_clis.clis.monad import ( + _duration_us, + _parse_block_timings, +) + + +@pytest.mark.parametrize( + "value, us", + [ + ("5745us", 5745), + ("5745µs", 5745), + ("5745μs", 5745), + (" 5.745 ms ", 5745), + ("1200ns", 1), + ("0.013081s", 13081), + ], +) +def test_duration_units(value: str, us: int) -> None: + """Durations convert to integer microseconds whatever the unit.""" + assert _duration_us(value) == us + + +@pytest.mark.parametrize( + "value", + ["", "13081", "5745 sec", "5745usx", "-5us", "5.us", "fast", "us"], +) +def test_duration_unrecognized(value: str) -> None: + """An unrecognized duration format raises instead of guessing.""" + with pytest.raises(ValueError, match="unrecognized duration"): + _duration_us(value) + + +def _line(suffix: str) -> str: + """Return a realistic runloop `__exec_block` log line.""" + return ( + "2026-07-16 12:45:21.263419673 [7] runloop_monad.cpp:385 LOG_INFO" + " __exec_block,bl= 1,id=0x1433,ts=1784205921250," + f"tx= 7,rt= 0,rtp= 0.00%,sr= 5745{suffix}," + f"txe= 6991{suffix},cmt= 169{suffix},tot= 13081{suffix}," + "tpse= 143,tps= 76,gas= 10000000,gpse=1430,gps=764,ae= 2," + "ane= 0,sz= 5,snz= 0,ac= 9,sc= 5 / 1872" + ) + + +@pytest.mark.parametrize("suffix", ["us", "µs"]) +def test_duration_suffix(suffix: str) -> None: + """The duration unit varies with the runloop's toolchain.""" + rows = _parse_block_timings(f"noise\n{_line(suffix)}\n") + assert rows == [ + { + "block": 1, + "tx_count": 7, + "gas": 10_000_000, + "retries": 0, + "tx_exec_us": 6991, + "state_root_us": 5745, + "commit_us": 169, + "total_us": 13081, + } + ] + + +def test_missing_retry_field_degrades_with_a_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """A runloop without `rt` still yields timings, and says so once.""" + line = _line("us").replace("rt= 0,", "") + with caplog.at_level("WARNING"): + rows = _parse_block_timings(f"{line}\n{line}\n") + + assert [row["retries"] for row in rows] == [0, 0] + assert "2 of 2 __exec_block lines carry no `rt` field" in caplog.text + + +def test_retry_field_present_logs_no_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """A complete line reports its retries without a warning.""" + with caplog.at_level("WARNING"): + rows = _parse_block_timings(_line("us")) + + assert rows[0]["retries"] == 0 + assert caplog.text == "" + + +def test_malformed_line_logged(caplog: pytest.LogCaptureFixture) -> None: + """A malformed line is skipped and reported, not silently dropped.""" + line = "LOG_INFO __exec_block,bl= 1,tx= 7,gas=oops" + with caplog.at_level("ERROR"): + assert _parse_block_timings(line) == [] + assert "unparsable __exec_block line" in caplog.text + assert "gas=oops" in caplog.text diff --git a/scripts/perf_cycle.sh b/scripts/perf_cycle.sh new file mode 100755 index 00000000000..be69a6dda3a --- /dev/null +++ b/scripts/perf_cycle.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Fill once, consume RUNS times, emit the NINE-vs-TEN perf table. +set -euo pipefail + +TAG="${TAG:?set TAG (names all artifacts, e.g. TAG=v4)}" +# The table pairs runs and adjusts for its own test count, so no raw +# p-value can fall below 2/2^RUNS. At the table's current size 13 is the +# fewest that lets an isolated effect clear the 5% threshold; below it +# only measures that move together reach significance (the table says so +# when it applies). +RUNS="${RUNS:-13}" +REPEATS="${REPEATS:-20}" +# Block gas budget in millions, as --gas-benchmark-values takes it. The +# runloop stamps every monad block at 200M. +BLOCK_GAS_M="${BLOCK_GAS_M:-200}" +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HARNESS="${HARNESS:-$REPO/../monad-eest-rust-harness}" +BIN="${BIN:-$HARNESS/bin/eest-runner}" +TEST="${TEST:-tests/benchmark/stateful/mip8_pageified_storage/test_perf_regression.py}" + +cd "$REPO" || exit 1 +EPOCH="$(date -u +%s)" +NOW="$(date -u -d "@$EPOCH" '+%Y-%m-%dT%H:%M:%SZ')" +STAMP="$(date -u -d "@$EPOCH" '+%y%m%d_%H%M%S')" +FIX="../fixtures_${TAG}" +PREFIX="../timing_${TAG}_${STAMP}" +TABLE="${PREFIX}_table" +[ -x "$BIN" ] || { echo "harness not executable: $BIN" >&2; exit 1; } + +if [ -z "${SKIP_FILL:-}" ]; then + echo "=== fill $FIX (REPEATS=$REPEATS, ${BLOCK_GAS_M}M blocks) $NOW ===" + MIP8_PERF_REPEATS="$REPEATS" \ + uv run fill -m blockchain_test "$TEST" \ + --from MONAD_NINE --until MONAD_TEN --chain-id 30143 --monad-runloop \ + --gas-benchmark-values "$BLOCK_GAS_M" \ + --output "$FIX" -n auto || { + echo "fill failed (a non-empty $FIX aborts fill); rerun with" \ + "SKIP_FILL=1 to reuse it, or delete it to refill" >&2 + exit 1 + } +fi + +for i in $(seq 1 "$RUNS"); do + out="${PREFIX}_${i}" + rm -rf "$out" + echo "=== consume $i/$RUNS -> $out $(date -u +%T)Z ===" + # A failing fixture must not silently drop its case from the table. + uv run consume direct --input "$FIX" --bin "$BIN" \ + --timing-report --timing-report-dir "$out" || { + echo "consume run $i reported failures; the table would silently omit" \ + "the affected cases. Fix them or drop them from $TEST." >&2 + exit 1 + } + [ -f "$out/timing_consume.csv" ] || { + echo "consume run $i left no $out/timing_consume.csv (eest-runner" \ + "emitted no parseable __exec_block timing lines?)" >&2 + exit 1 + } +done + +sha() { git -C "$1" rev-parse --short HEAD 2>/dev/null || echo '?'; } +uv run perf_regression --md "${TABLE}.md" \ + --now "$NOW" \ + --repo "$(sha "$REPO")" \ + --harness "$(sha "$HARNESS")" \ + --monad-bft "$(sha "$HARNESS/monad-bft")" \ + --monad "$(sha "$HARNESS/monad-bft/monad-execution")" \ + "${PREFIX}"_[0-9]* || { echo "perf_regression failed, no table" >&2; exit 1; } +echo "=== table: $(cd .. && pwd)/$(basename "$TABLE").md ===" diff --git a/tests/benchmark/stateful/mip8_pageified_storage/__init__.py b/tests/benchmark/stateful/mip8_pageified_storage/__init__.py new file mode 100644 index 00000000000..90fe0f534cc --- /dev/null +++ b/tests/benchmark/stateful/mip8_pageified_storage/__init__.py @@ -0,0 +1 @@ +"""MIP-8 page-encoded storage performance benchmarks.""" diff --git a/tests/benchmark/stateful/mip8_pageified_storage/test_perf_regression.py b/tests/benchmark/stateful/mip8_pageified_storage/test_perf_regression.py new file mode 100644 index 00000000000..10ddf17d68b --- /dev/null +++ b/tests/benchmark/stateful/mip8_pageified_storage/test_perf_regression.py @@ -0,0 +1,1233 @@ +""" +MIP-8 performance-regression tests. + +These blockchain tests compare block/transaction execution time on the +monad runloop before MIP-8 (MONAD_NINE, slot-encoded storage) and after +MIP-8 (MONAD_TEN, page-encoded storage). The same Python builds the +workload for both forks; the `fork` fixture is only consulted where the +two forks genuinely differ (storage-op gas, used to size a workload to a +gas budget). We do not assert gas — the oracle is post-state (success +markers, written slots, and read-checksum slots), so the identical +workload can be timed on both forks via the consume timing report. + +Each test fills a monad block with one SLOAD/SSTORE pattern, sized to the +`gas_benchmark_value` the framework injects from +`--gas-benchmark-values` (in millions). perf_cycle.sh passes 200, the gas +the runloop stamps every monad block with. + +Single-letter test parameters: +- `k`: page occupancy — non-zero slots pre-populated per page (0..128). +- `m`: number of distinct pages a spread test writes in total. +- `n`: number of contracts those `m` pages are spread evenly across. + +Every transaction gets its own sender, so no block carries a sender nonce +chain that would serialise it regardless of storage access. The block-shape +sweep parametrizes this to measure what the chain alone costs. + +Workloads are sized to MONAD_NINE (its cold storage costs >= MONAD_TEN), +so the same iteration count never out-of-gases on either fork; a +post-fork block may therefore be gas-underfull while doing identical I/O +work, which is exactly the effect being measured. + +That is also why these tests pass `skip_gas_used_validation`: the gas a +block actually uses differs between the forks by construction, so no +single `expected_benchmark_gas_used` can match both. The framework still +enforces that a block stays within its budget, `_assert_tx_within_budget` +checks the plan against it, and `_per_iter_gas` bounds its own estimate, +so a block cannot quietly end up sized for less work than the budget. + +Benchmark fixtures omit the full post state, which keeps them small and +quick to consume; the monad consumer then verifies each run against the +block's state root, which commits to the whole state. The `post` argument +still drives what the fill asserts, so the markers and checksums below +remain the oracle at fill time. + +`MIP8_PERF_REPEATS` (default 1) emits that many copies of each workload +as successive blocks, each offset to a disjoint page range so every block +is a genuine cold execution. One runloop run then yields N independent +per-block timing samples (the first absorbs process/hugepage warmup). + +See MONAD_RUNLOOP_TESTING.md. +""" + +import os +from enum import StrEnum, auto +from typing import Iterator, List, SupportsBytes, Tuple + +import pytest +from execution_testing import ( + EOA, + Account, + Address, + Alloc, + BenchmarkTestFiller, + Block, + Bytecode, + Conditional, + Environment, + Op, + Transaction, + While, + WhileGas, +) +from execution_testing.forks import MONAD_NINE, MONAD_TEN +from execution_testing.forks.helpers import Fork + +from tests.monad_ten.mip8_pageified_storage.helpers import fresh_sstore_cold +from tests.monad_ten.mip8_pageified_storage.spec import Spec, ref_spec_8 + +REFERENCE_SPEC_GIT_PATH = ref_spec_8.git_path +REFERENCE_SPEC_VERSION = ref_spec_8.version + +# Key/value type accepted by `deploy_contract`/`Account` storage args; +# pre-built dicts need this annotation (dict is invariant). +StorageDict = dict[ + str | int | bytes | SupportsBytes, str | int | bytes | SupportsBytes +] + +SLOTS_PER_PAGE = Spec.SLOTS_PER_PAGE # 128 + +# Every test takes its block gas budget from `gas_benchmark_value`, set +# by `--gas-benchmark-values` (in millions). The runloop stamps every +# monad block at 200M gas, so `--gas-benchmark-values 200` is the setting +# that times real blocks; smaller values shrink the same workloads. +TX_GAS_CAP = 30_000_000 +# Fixed tx count per block, sized so a full 200M block splits into 7 +# equal txs of ~28.57M each (near the 30M per-tx cap). A smaller budget +# keeps the 7 txs and shrinks each one's work. +FULL_BLOCK_TXS = 7 +# Workload txs are EIP-1559 with a high max fee and a zero priority tip. +# The high max fee keeps them valid as each full block raises the base +# fee ~12.5%. +MAX_FEE_PER_GAS = 10**6 +# `many_small` block shape: many txs, each still large enough to cover the +# per-tx reserve. The count adapts to the block budget (300 at the full +# 200M) so a smaller budget does not starve individual txs. +MANY_SMALL_TXS = 300 +MANY_SMALL_MIN_TX_GAS = 200_000 + +# Emit this many copies of each workload as successive, page-disjoint +# blocks for repeat timing samples (see module docstring). perf_cycle.sh +# raises this; releases fill a single cold copy. +REPEATS = int(os.environ.get("MIP8_PERF_REPEATS", "1")) + +# Upper bound on pre-populated storage slots per block (times REPEATS for +# the whole fixture's genesis). +PRE_SLOT_CAP = 65_536 + +# Gas of one While control step (JUMPDEST + JUMPI + the jump back). +# Deliberately above the real cost so sizing never over-commits a tx into +# an OOG; `_while_control_gas` measures the real cost and +# `_per_iter_gas` fails if this drifts out of MAX_CONTROL_SLACK of it. +# Every gas of slack here is workload the block does not do, so the bound +# is checked rather than trusted. +WHILE_CONTROL_GAS = 40 +MAX_CONTROL_SLACK = 24 +# Per-tx headroom: intrinsic + calldata + two cold marker SSTOREs + slack. +TX_RESERVE = 120_000 + +# Page/slot domains, chosen far apart so nothing collides. Each repeat r +# shifts a page domain by r * REPEAT_STRIDE, well above any single +# workload's page extent and far below the next domain. +REPEAT_STRIDE = 1 << 28 +READ_DOMAIN = 1 << 40 # page-index base for reused (pre-populated) pages +FRESH_DOMAIN = 1 << 52 # page-index base for fresh-write pages +WARM_BASE = 1 << 70 # per-repeat slot re-read by sload_warm_repeat +MARKER_BASE = 1 << 200 # per-tx success marker slot = MARKER_BASE + global_idx +CKSUM_BASE = 1 << 220 # per-tx read-checksum slot = CKSUM_BASE + global_idx + + +# Loop condition shared by every workload: iterate while the counter is +# below the requested count. Shared so `_while_control_gas` measures the +# same control overhead the workloads pay. +def _loop_condition() -> Bytecode: + """Bytecode: counter < count.""" + return Op.LT(Op.MLOAD(M_COUNTER), Op.MLOAD(M_COUNT)) + + +# Memory layout inside the workload contract. +M_COUNTER = 0x00 +M_CHECKSUM = 0x20 +M_BASE = 0x40 +M_COUNT = 0x60 +M_GLOBAL = 0x80 # global tx index (marker/checksum slot) +M_PAGE = 0xA0 # sweep page-base scratch +M_LOCAL = 0xC0 # per-block tx index (grow write offset) + +# Calldata layout: base | count | global_idx | halt | local_idx. +CD_BASE = 0x00 +CD_COUNT = 0x20 +CD_GLOBAL = 0x40 +CD_HALT = 0x60 +CD_LOCAL = 0x80 + +slot_code_worked = 0x1 +value_code_worked = 0x1234 + + +class StorageOp(StrEnum): + """A storage-access pattern a workload applies to each page.""" + + SLOAD_COLD_HIT = auto() + """Cold SLOAD of an occupied slot (offset 0) on each page.""" + SLOAD_COLD_MISS = auto() + """Cold SLOAD of an empty slot (offset 127) on an occupied page.""" + SLOAD_SWEEP_K = auto() + """Cold-read every occupied slot (offsets 0..k-1) of each page.""" + SLOAD_SWEEP_PAGE = auto() + """Cold-read all 128 slot offsets of each page, hits and misses.""" + SLOAD_WARM_REPEAT = auto() + """One cold SLOAD of a slot, then repeated warm re-reads of it.""" + SSTORE_FRESH = auto() + """SSTORE 0->1 into empty pages (fresh state growth).""" + SSTORE_NOOP = auto() + """SSTORE 1->1 on occupied pages (rewrite, no value change).""" + SSTORE_GROW = auto() + """SSTORE 0->1 on a new slot of an already-occupied page.""" + SSTORE_UPDATE = auto() + """SSTORE 1->2 on an occupied slot (value change, no growth).""" + SSTORE_CLEAR_KEEP = auto() + """SSTORE 1->0 clearing one slot; the page keeps its other slots.""" + SSTORE_CLEAR_EMPTY = auto() + """SSTORE 1->0 clearing a page's only slot, so the page is removed.""" + SLOAD_EMPTY_PAGE = auto() + """Cold SLOAD of a slot on a never-populated, empty page.""" + + +def _senders(pre: Alloc, distinct: bool = True) -> Iterator[EOA]: + """ + Yield one sender per transaction. + + Consecutive nonces from one EOA conflict on that account, so a block + whose transactions all share a sender carries a dependency chain that + serialises it in a parallel executor — which would mask whatever the + storage encoding does. Distinct senders remove that chain, leaving + storage access as the only cross-transaction dependency. `distinct` + stays a parameter where the sender chain is itself the subject. + """ + if distinct: + while True: + yield pre.fund_eoa() + shared = pre.fund_eoa() + while True: + yield shared + + +@pytest.mark.valid_from("MONAD_NINE") +def test_compute_loop( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, +) -> None: + """ + Run a gas-bounded arithmetic loop, then write a success marker. + + A stack-neutral arithmetic body is repeated until the compute gas + budget is nearly spent, reserving just enough for the trailing + SSTORE. Pure stack arithmetic touches no storage, so the loop costs + the same on both forks and the marker is the only state written. + + This is the suite's control, so the contract is built from a fixed + fork rather than the one under test: sizing the loop against the + running fork would emit different bytecode either side of MIP-8, and + a control has to run the same code. MONAD_NINE prices the trailing + cold SSTORE at least as high as MONAD_TEN, so its reserve is safe on + both. + """ + senders = _senders(pre) + budget = gas_benchmark_value // FULL_BLOCK_TXS + + body = Op.POP(Op.ADD(Op.MUL(Op.NUMBER, Op.GAS), Op.CALLVALUE)) + contract_address = pre.deploy_contract( + WhileGas( + body=body, + fork=MONAD_NINE, + extra_gas=fresh_sstore_cold(MONAD_NINE), + ) + + Op.SSTORE(slot_code_worked, value_code_worked) + ) + + blocks = [ + Block( + txs=[ + Transaction( + to=contract_address, + sender=next(senders), + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + ) + for _ in range(FULL_BLOCK_TXS) + ], + ), + ] + + benchmark_test( + pre=pre, + blocks=blocks, + post={ + contract_address: Account( + storage={slot_code_worked: value_code_worked} + ), + }, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) + + +def _page_index() -> Bytecode: + """Bytecode: base + i (base at M_BASE, i at M_COUNTER).""" + return Op.ADD(Op.MLOAD(M_BASE), Op.MLOAD(M_COUNTER)) + + +def _slot(offset: int | Bytecode) -> Bytecode: + """Bytecode: (page_index << 7) + offset.""" + return Op.ADD(offset, Op.SHL(7, _page_index())) + + +def _read_accum(slot: Bytecode | int, *, warm: bool) -> Bytecode: + """Accumulate SLOAD(slot) into the checksum word.""" + load = Op.SLOAD(slot, key_warm=warm, page_load_warm=warm) + return Op.MSTORE(M_CHECKSUM, Op.ADD(Op.MLOAD(M_CHECKSUM), load)) + + +def _sstore( + slot: Bytecode, + value: int | Bytecode, + *, + original: int, + current: int, + new: int, + growth: int, +) -> Bytecode: + """ + Build a cold-page SSTORE(slot, value). + + `value` is what is written at run time; `original`/`current`/`new`/ + `growth` describe the slot transition for gas sizing only. `original` + (the slot's value at tx start) is required so a nonzero-to-nonzero or + nonzero-to-zero reset is priced as a reset on MONAD_NINE, not a no-op. + """ + return Op.SSTORE( + slot, + value, + key_warm=False, + page_load_warm=False, + page_write_warm=False, + original_value=original, + current_value=current, + new_value=new, + current_state_growth=growth, + net_state_growth=growth, + ) + + +def _body(op: StorageOp, k: int) -> Bytecode: + """Assemble the per-iteration loop body for an operation.""" + inc = Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) + if op is StorageOp.SLOAD_COLD_HIT: + return _read_accum(_slot(0), warm=False) + inc + if op is StorageOp.SLOAD_COLD_MISS: + return _read_accum(_slot(SLOTS_PER_PAGE - 1), warm=False) + inc + if op is StorageOp.SLOAD_WARM_REPEAT: + # The slot is passed in calldata (M_BASE) so each block re-reads a + # distinct slot; cold on the first iteration, warm thereafter. + return _read_accum(Op.MLOAD(M_BASE), warm=True) + inc + if op is StorageOp.SLOAD_SWEEP_K: + code = Op.MSTORE(M_PAGE, Op.SHL(7, _page_index())) + for j in range(k): + code += _read_accum(Op.ADD(Op.MLOAD(M_PAGE), j), warm=False) + return code + inc + if op is StorageOp.SLOAD_SWEEP_PAGE: + code = Op.MSTORE(M_PAGE, Op.SHL(7, _page_index())) + for j in range(SLOTS_PER_PAGE): + code += _read_accum(Op.ADD(Op.MLOAD(M_PAGE), j), warm=False) + return code + inc + if op is StorageOp.SSTORE_FRESH: + return ( + _sstore(_slot(0), 1, original=0, current=0, new=1, growth=0) + inc + ) + if op is StorageOp.SSTORE_NOOP: + return ( + _sstore(_slot(0), 1, original=1, current=1, new=1, growth=0) + inc + ) + if op is StorageOp.SSTORE_GROW: + offset = Op.ADD(k, Op.MLOAD(M_LOCAL)) + return ( + _sstore( + _slot(offset), + 1, + original=0, + current=0, + new=1, + growth=k, + ) + + inc + ) + if op is StorageOp.SSTORE_UPDATE: + # Each tx writes a new nonzero value (2 + its index), so every tx + # is a genuine value change on the still-occupied slot 0. + value = Op.ADD(2, Op.MLOAD(M_LOCAL)) + return ( + _sstore(_slot(0), value, original=1, current=1, new=2, growth=0) + + inc + ) + if op is StorageOp.SSTORE_CLEAR_KEEP: + # Tx t clears slot offset t; with k > FULL_BLOCK_TXS the page keeps + # its higher slots, so each clear is a genuine 1->0 on a live page. + offset = Op.MLOAD(M_LOCAL) + return ( + _sstore( + _slot(offset), + 0, + original=1, + current=1, + new=0, + growth=0, + ) + + inc + ) + if op is StorageOp.SSTORE_CLEAR_EMPTY: + return ( + _sstore(_slot(0), 0, original=1, current=1, new=0, growth=0) + inc + ) + if op is StorageOp.SLOAD_EMPTY_PAGE: + return _read_accum(_slot(0), warm=False) + inc + raise ValueError(f"unknown op {op}") + + +def _is_read(op: StorageOp) -> bool: + """Return whether the operation reads (and writes a checksum).""" + return op.value.startswith("sload") + + +def _contract(op: StorageOp, k: int) -> Bytecode: + """ + Build the workload contract (fixed size, independent of REPEATS). + + Reads calldata (base, count, global_idx, halt, local_idx), runs + `count` iterations of the op's body, writes a success marker (and, for + reads, a checksum) keyed by the global index, then STOPs — or, if the + halt flag is set, executes INVALID so the whole transaction reverts. + """ + init = ( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_LOCAL, Op.CALLDATALOAD(CD_LOCAL)) + + Op.MSTORE(M_COUNTER, 0) + ) + loop = While( + body=_body(op, k), + condition=_loop_condition(), + ) + markers = Op.SSTORE( + Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked + ) + if _is_read(op): + markers += Op.SSTORE( + Op.ADD(CKSUM_BASE, Op.MLOAD(M_GLOBAL)), Op.MLOAD(M_CHECKSUM) + ) + tail = Conditional( + condition=Op.CALLDATALOAD(CD_HALT), + if_true=Op.INVALID, + if_false=Op.STOP, + ) + return init + loop + markers + tail + + +def _calldata( + base: int, count: int, global_idx: int, halt: int, local_idx: int = 0 +) -> bytes: + """Encode the five-word calldata for one transaction.""" + return b"".join( + v.to_bytes(32, "big") + for v in (base, count, global_idx, halt, local_idx) + ) + + +def _while_control_gas(fork: Fork) -> int: + """ + Gas the `While` wrapper adds around one body iteration. + + Taken from the framework's own accounting rather than counted by hand, + so a change to how `While` is assembled surfaces as a failed bound in + `_per_iter_gas` instead of as quietly smaller workloads. + """ + body = Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) + condition = _loop_condition() + loop = While(body=body, condition=condition) + return loop.gas_cost(fork) - body.gas_cost(fork) - condition.gas_cost(fork) + + +def _per_iter_gas(op: StorageOp, k: int) -> int: + """Gas for one loop iteration, sized to the costlier of both forks.""" + body = _body(op, k) + per_op = max(body.gas_cost(MONAD_NINE), body.gas_cost(MONAD_TEN)) + control = max( + _while_control_gas(MONAD_NINE), _while_control_gas(MONAD_TEN) + ) + assert control <= WHILE_CONTROL_GAS <= control + MAX_CONTROL_SLACK, ( + f"WHILE_CONTROL_GAS is {WHILE_CONTROL_GAS} but a While control step " + f"actually costs {control}: below it every workload risks an OOG, " + f"more than {MAX_CONTROL_SLACK} above it and every block is sized " + "for measurably less work than its gas budget allows" + ) + return per_op + WHILE_CONTROL_GAS + + +def _iterations(op: StorageOp, k: int, budget: int) -> int: + """Loop iterations that fit `budget` gas, leaving tx headroom.""" + return max(1, (budget - TX_RESERVE) // _per_iter_gas(op, k)) + + +def _assert_tx_within_budget(per_iter: int, count: int, budget: int) -> None: + """ + Fail if a workload transaction plans more gas than its budget. + + Catches a `count` that was not derived from `budget` and a `TX_RESERVE` + that outgrew a small budget; either would put more gas in the block + than the runloop allows. + + How tightly a gas-bound tx *fills* its budget is not checked here — it + follows from `_per_iter_gas`, whose estimate is bounded against the + framework's own accounting at the point it is built. + """ + planned = count * per_iter + TX_RESERVE + assert planned <= budget, ( + f"workload tx plans {planned} gas but its budget is {budget}; " + "the block would exceed the gas the runloop allows" + ) + + +def _occupied_prestate(domain: int, pages: int, k: int) -> StorageDict: + """Pre-populate `pages` pages (from `domain`) with `k` slots each.""" + storage: StorageDict = {} + for i in range(pages): + base_slot = (domain + i) << 7 + for j in range(k): + storage[base_slot + j] = 1 + return storage + + +def _repeat_domains(repeat: int) -> Tuple[int, int, int]: + """Return (read_domain, fresh_domain, warm_slot) for a repeat index.""" + shift = repeat * REPEAT_STRIDE + return READ_DOMAIN + shift, FRESH_DOMAIN + shift, WARM_BASE + repeat + + +# (op, page-occupancy k values). cold_miss/grow keep a zero slot free +# (k < SLOTS_PER_PAGE, and grow writes k..k+FULL_BLOCK_TXS-1). clear_keep +# needs k > FULL_BLOCK_TXS so a page still has slots after the block +# clears offsets 0..FULL_BLOCK_TXS-1. clear_empty uses single-slot pages +# (k=1) that vanish when cleared; empty_page reads never-set pages +# (k=0). sweep_k reads only the k occupied slots; sweep_page reads all +# 128 offsets — k hits + (128-k) misses, never-set pages at k=0. +_PAGE_OP_KS = [ + (StorageOp.SLOAD_COLD_HIT, [1, 16, 128]), + (StorageOp.SLOAD_COLD_MISS, [1, 16, 64]), + (StorageOp.SLOAD_SWEEP_K, [2, 16, 128]), + (StorageOp.SLOAD_SWEEP_PAGE, [0, 1, 64]), + (StorageOp.SSTORE_NOOP, [1, 16, 128]), + (StorageOp.SSTORE_GROW, [1, 16, 64]), + (StorageOp.SSTORE_UPDATE, [1, 16, 128]), + (StorageOp.SSTORE_CLEAR_KEEP, [8, 16, 128]), + (StorageOp.SSTORE_CLEAR_EMPTY, [1]), + (StorageOp.SLOAD_EMPTY_PAGE, [0]), + (StorageOp.SLOAD_WARM_REPEAT, [1]), + (StorageOp.SSTORE_FRESH, [0]), +] + +_PAGE_OP_PARAMS = [ + pytest.param(op, k, id=f"{op.value}-k{k}") + for op, ks in _PAGE_OP_KS + for k in ks +] + + +@pytest.mark.parametrize("op, k", _PAGE_OP_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_page_ops( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, + op: StorageOp, + k: int, +) -> None: + """ + Fill a block with one storage-op pattern over `pages` pages. + + Occupied-page ops (reads, no-op, growing/updating writes, and the + keep-a-slot clear) reuse a per-block pre-populated page pool: every + transaction re-touches the same pages, paying cold access each time + (per-tx warmth resets). Fresh writes and the page-emptying clear give + each transaction its own disjoint page range; the emptying clear + pre-populates one slot per page so each tx removes whole pages. The + page count is bounded by the tx gas budget and, for pre-populated ops, + by PRE_SLOT_CAP // k — so high-occupancy cold-read blocks are pre-state + bound and run below 200M by design. With REPEATS > 1 each repeat block + is offset to a fresh page range. + """ + budget = gas_benchmark_value // FULL_BLOCK_TXS + pages = _iterations(op, k, budget) + + per_tx_pages = op is StorageOp.SSTORE_CLEAR_EMPTY + occupied = op not in ( + StorageOp.SSTORE_FRESH, + StorageOp.SLOAD_WARM_REPEAT, + StorageOp.SSTORE_CLEAR_EMPTY, + ) + if occupied: + pages = min(pages, PRE_SLOT_CAP // max(k, 1)) + elif per_tx_pages: + pages = min(pages, PRE_SLOT_CAP // FULL_BLOCK_TXS) + _assert_tx_within_budget(_per_iter_gas(op, k), pages, budget) + + senders = _senders(pre) + + prestate: StorageDict = {} + for r in range(REPEATS): + read_dom, fresh_dom, warm_slot = _repeat_domains(r) + if op is StorageOp.SLOAD_WARM_REPEAT: + prestate[warm_slot] = 1 + elif per_tx_pages: + for t in range(FULL_BLOCK_TXS): + dom = fresh_dom + t * pages + prestate.update(_occupied_prestate(dom, pages, 1)) + elif occupied: + prestate.update(_occupied_prestate(read_dom, pages, k)) + contract = pre.deploy_contract(_contract(op, k), storage=prestate) + + blocks = [] + expected: StorageDict = dict(prestate) + global_idx = 0 + for r in range(REPEATS): + read_dom, fresh_dom, warm_slot = _repeat_domains(r) + txs = [] + for t in range(FULL_BLOCK_TXS): + if op in (StorageOp.SSTORE_FRESH, StorageOp.SSTORE_CLEAR_EMPTY): + base = fresh_dom + t * pages + elif op is StorageOp.SLOAD_WARM_REPEAT: + base = warm_slot + else: + base = read_dom + txs.append( + Transaction( + to=contract, + sender=next(senders), + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(base, pages, global_idx, 0, t), + ) + ) + expected[MARKER_BASE + global_idx] = value_code_worked + if _is_read(op): + if op in ( + StorageOp.SLOAD_SWEEP_K, + StorageOp.SLOAD_SWEEP_PAGE, + ): + checksum = pages * k + elif op in ( + StorageOp.SLOAD_COLD_MISS, + StorageOp.SLOAD_EMPTY_PAGE, + ): + checksum = 0 + else: # cold_hit, warm_repeat: each read returns 1 + checksum = pages + if checksum: + expected[CKSUM_BASE + global_idx] = checksum + global_idx += 1 + + if op is StorageOp.SSTORE_GROW: + for i in range(pages): + base_slot = (read_dom + i) << 7 + for t in range(FULL_BLOCK_TXS): + expected[base_slot + (k + t)] = 1 + elif op is StorageOp.SSTORE_FRESH: + for j in range(FULL_BLOCK_TXS * pages): + expected[(fresh_dom + j) << 7] = 1 + elif op is StorageOp.SSTORE_UPDATE: + # Slot 0 is written once per tx (1->2->...); it ends at + # 1 + FULL_BLOCK_TXS. Other occupied slots stay 1. + for i in range(pages): + expected[(read_dom + i) << 7] = 1 + FULL_BLOCK_TXS + elif op is StorageOp.SSTORE_CLEAR_KEEP: + for i in range(pages): + base_slot = (read_dom + i) << 7 + for t in range(FULL_BLOCK_TXS): + expected[base_slot + t] = 0 + elif op is StorageOp.SSTORE_CLEAR_EMPTY: + for t in range(FULL_BLOCK_TXS): + for i in range(pages): + expected[(fresh_dom + (t * pages + i)) << 7] = 0 + + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) + + +_SPREAD_PARAMS = [ + pytest.param(m, n, id=f"m{m}_n{n}") + for m, n in [ + (1, 1), + (4, 1), + (16, 1), + (64, 1), + (256, 1), + (1024, 1), + (4096, 1), + (4096, 8), + (4096, 64), + (4096, 512), + ] +] + + +@pytest.mark.parametrize("m, n", _SPREAD_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_page_spread( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, + m: int, + n: int, +) -> None: + """ + Fresh-write `m` distinct pages spread evenly over `n` contracts. + + Each contract writes `m // n` fresh pages, split into as many + transactions as the per-tx gas cap requires (so large per-contract + shares become several txs, and many contracts become many small + txs). Isolates the effect of write distribution across accounts. Each + repeat block writes a fresh page range. + + Work is sized by `m` and `n`, not by the gas budget, so a high fan-out + (n=512) stays large however small the budget is. The budget only has + to be large enough to hold the resulting block, which the sizing + check below enforces. + """ + per_iter = _per_iter_gas(StorageOp.SSTORE_FRESH, 0) + max_per_tx = max(1, (TX_GAS_CAP - TX_RESERVE) // per_iter) + pages_per_contract = m // n + + txs_per_contract = -(-pages_per_contract // max_per_tx) + planned = n * ( + pages_per_contract * per_iter + txs_per_contract * TX_RESERVE + ) + assert planned <= gas_benchmark_value, ( + f"m={m} n={n} needs {planned} gas but the budget is " + f"{gas_benchmark_value}; raise --gas-benchmark-values or drop the case" + ) + + senders = _senders(pre) + contracts: List[Address] = [ + pre.deploy_contract(_contract(StorageOp.SSTORE_FRESH, 0)) + for _ in range(n) + ] + + blocks = [] + contract_storage: dict[Address, StorageDict] = {c: {} for c in contracts} + global_idx = 0 + for r in range(REPEATS): + _, fresh_dom, _ = _repeat_domains(r) + txs = [] + for contract in contracts: + done = 0 + while done < pages_per_contract: + count = min(max_per_tx, pages_per_contract - done) + txs.append( + Transaction( + to=contract, + sender=next(senders), + gas_limit=count * per_iter + TX_RESERVE, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(fresh_dom + done, count, global_idx, 0), + ) + ) + contract_storage[contract][MARKER_BASE + global_idx] = ( + value_code_worked + ) + global_idx += 1 + done += count + for j in range(pages_per_contract): + slot = (fresh_dom + j) << 7 + contract_storage[contract][slot] = 1 + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + post={ + contract: Account(storage=storage) + for contract, storage in contract_storage.items() + }, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) + + +_SHAPE_PARAMS = [ + pytest.param( + op, + k, + shape, + distinct, + id=f"{op.value}-k{k}-{shape}-" + f"{'distinct' if distinct else 'shared'}_senders", + ) + for op, k in [(StorageOp.SSTORE_FRESH, 0), (StorageOp.SLOAD_COLD_HIT, 8)] + for shape in ("few_big", "many_small") + for distinct in (True, False) +] + + +@pytest.mark.parametrize("op, k, shape, distinct_senders", _SHAPE_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_block_shape( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, + op: StorageOp, + k: int, + shape: str, + distinct_senders: bool, +) -> None: + """ + Same total work packed as a few big txs vs many small txs. + + `few_big` uses 7 gas-cap-sized transactions; `many_small` uses many + (300 at the full block budget) smaller ones. Reads reuse one + pre-populated page pool (each tx a fresh cold pass); fresh writes give + each tx a disjoint range. Each repeat block is offset to a fresh range. + + `distinct_senders` decides whether the block also carries a sender + nonce chain: shared senders serialise the transactions independently + of their storage access, which bounds how much the packing can matter. + """ + if shape == "few_big": + num_txs = FULL_BLOCK_TXS + else: + num_txs = min( + MANY_SMALL_TXS, + max(1, gas_benchmark_value // MANY_SMALL_MIN_TX_GAS), + ) + budget = min(TX_GAS_CAP, gas_benchmark_value // num_txs) + count = _iterations(op, k, budget) + + occupied = op is not StorageOp.SSTORE_FRESH + if occupied: + count = min(count, PRE_SLOT_CAP // max(k, 1)) + _assert_tx_within_budget(_per_iter_gas(op, k), count, budget) + + senders = _senders(pre, distinct_senders) + prestate: StorageDict = {} + for r in range(REPEATS): + read_dom, _, _ = _repeat_domains(r) + if occupied: + prestate.update(_occupied_prestate(read_dom, count, k)) + contract = pre.deploy_contract(_contract(op, k), storage=prestate) + + blocks = [] + expected: StorageDict = dict(prestate) + global_idx = 0 + for r in range(REPEATS): + read_dom, fresh_dom, _ = _repeat_domains(r) + txs = [] + for t in range(num_txs): + base = read_dom if occupied else fresh_dom + t * count + txs.append( + Transaction( + to=contract, + sender=next(senders), + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(base, count, global_idx, 0, t), + ) + ) + expected[MARKER_BASE + global_idx] = value_code_worked + if occupied: # cold_hit reads each return 1 + expected[CKSUM_BASE + global_idx] = count + global_idx += 1 + if op is StorageOp.SSTORE_FRESH: + for j in range(num_txs * count): + expected[(fresh_dom + j) << 7] = 1 + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) + + +@pytest.mark.parametrize("mode", ["success", "halt", "mix"]) +@pytest.mark.valid_from("MONAD_NINE") +def test_tx_halt( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, + mode: str, +) -> None: + """ + Fresh-write block whose transactions succeed, all halt, or alternate. + + A halting tx runs INVALID after its writes and marker, so all of its + state reverts (and it consumes its full gas limit — a perfectly full + block). Only successful txs leave pages and markers behind, giving a + strong post-state oracle for the mixed case. Each repeat block is + offset to a fresh range. + """ + budget = gas_benchmark_value // FULL_BLOCK_TXS + count = _iterations(StorageOp.SSTORE_FRESH, 0, budget) + _assert_tx_within_budget( + _per_iter_gas(StorageOp.SSTORE_FRESH, 0), count, budget + ) + + senders = _senders(pre) + contract = pre.deploy_contract(_contract(StorageOp.SSTORE_FRESH, 0)) + + blocks = [] + expected: StorageDict = {} + global_idx = 0 + for r in range(REPEATS): + _, fresh_dom, _ = _repeat_domains(r) + txs = [] + for t in range(FULL_BLOCK_TXS): + halt = mode == "halt" or (mode == "mix" and t % 2 == 1) + txs.append( + Transaction( + to=contract, + sender=next(senders), + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata( + fresh_dom + t * count, + count, + global_idx, + int(halt), + t, + ), + ) + ) + if not halt: + for i in range(count): + page = fresh_dom + (t * count + i) + expected[page << 7] = 1 + expected[MARKER_BASE + global_idx] = value_code_worked + global_idx += 1 + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) + + +RAND_CONTRACTS = 8 # pool of contracts, spread across the block's txs +RAND_BASE = 1 # first page key of the read set +RAND_STRIDE = 1 << 10 # > max slots, so per-tx page sets are disjoint +M_CHAIN = 0xE0 # chained-sload current slot (memory scratch) +SERIAL_BASE = 1 << 40 +SERIAL_REPEAT_STRIDE = 1 << 24 # >> per-tx slot count +CHAIN_BASE = 1 << 30 +CHAIN_REPEAT_STRIDE = 1 << 20 # >> ring length + + +def _size_count(body: Bytecode, budget: int) -> int: + """Loop iterations of `body` that fit `budget`, sized to both forks.""" + per_iter = ( + max(body.gas_cost(MONAD_NINE), body.gas_cost(MONAD_TEN)) + + WHILE_CONTROL_GAS + ) + count = max(1, (budget - TX_RESERVE) // per_iter) + _assert_tx_within_budget(per_iter, count, budget) + return count + + +def _rand_sload_body(slots: int) -> Bytecode: + """One iteration: cold-SLOAD one distinct page from the read set.""" + idx = Op.AND(Op.MLOAD(M_COUNTER), slots - 1) + slot = Op.SHL(7, Op.ADD(Op.MLOAD(M_BASE), idx)) + read = Op.SLOAD(slot, key_warm=False, page_load_warm=False) + return Op.MSTORE( + M_CHECKSUM, Op.ADD(Op.MLOAD(M_CHECKSUM), read) + ) + Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) + + +def _rand_sload_contract(slots: int) -> Bytecode: + """ + SLOAD `slots` distinct pages in a cycle, `count` times, then write a + success marker and the read checksum. + """ + init = ( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # base page key + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_COUNTER, 0) + + Op.MSTORE(M_CHECKSUM, 0) + ) + loop = While( + body=_rand_sload_body(slots), + condition=_loop_condition(), + ) + tail = ( + Op.SSTORE(Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked) + + Op.SSTORE( + Op.ADD(CKSUM_BASE, Op.MLOAD(M_GLOBAL)), Op.MLOAD(M_CHECKSUM) + ) + + Op.STOP + ) + return init + loop + tail + + +_RAND_PARAMS = [ + pytest.param(slots, k, id=f"slots{slots}-k{k}") + for slots in (1, 16, 128) + for k in (0, 1) +] + + +@pytest.mark.parametrize("slots, k", _RAND_PARAMS) +@pytest.mark.valid_from("MONAD_NINE") +def test_random_sload( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, + slots: int, + k: int, +) -> None: + """ + Cold-SLOAD a set of `slots` distinct pages, from a pool of contracts + spread across the block's txs. `k` is the page occupancy of each read + page: 1 (slot present, reads 1) or 0 (empty page, reads 0). Cycling a + small set means only the first pass is cold, so blocks are + gas-underfull by design; each repeat uses a fresh page range. + """ + budget = gas_benchmark_value // FULL_BLOCK_TXS + count = _size_count(_rand_sload_body(slots), budget) + code = _rand_sload_contract(slots) + senders = _senders(pre) + + # Plan tx -> (contract index, base page key) and genesis slots. + plan: List[Tuple[int, int]] = [] + genesis: List[StorageDict] = [{} for _ in range(RAND_CONTRACTS)] + g = 0 + for _r in range(REPEATS): + for _t in range(FULL_BLOCK_TXS): + ci = g % RAND_CONTRACTS + base = RAND_BASE + g * RAND_STRIDE + if k == 1: + for idx in range(slots): + genesis[ci][(base + idx) << 7] = 1 + plan.append((ci, base)) + g += 1 + + contracts: List[Address] = [ + pre.deploy_contract(code, storage=genesis[i]) + for i in range(RAND_CONTRACTS) + ] + + post: dict[Address, StorageDict] = { + contracts[i]: dict(genesis[i]) for i in range(RAND_CONTRACTS) + } + blocks = [] + g = 0 + for _r in range(REPEATS): + txs = [] + for _t in range(FULL_BLOCK_TXS): + ci, base = plan[g] + contract = contracts[ci] + txs.append( + Transaction( + to=contract, + sender=next(senders), + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(base, count, g, 0), + ) + ) + post[contract][MARKER_BASE + g] = value_code_worked + if k == 1: + post[contract][CKSUM_BASE + g] = count + g += 1 + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + post={c: Account(storage=s) for c, s in post.items()}, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) + + +@pytest.mark.valid_from("MONAD_NINE") +def test_bad_block_serial( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, +) -> None: + """ + Adversarial block forcing serial execution: every tx SLOADs and + SSTOREs (increments) the same contiguous slot sequence, so the txs + conflict and cannot run in parallel; each shared slot ends at + FULL_BLOCK_TXS. Each repeat uses a fresh slot range. + + Senders are distinct, so the storage conflict is the only thing + serialising the block. + """ + budget = gas_benchmark_value // FULL_BLOCK_TXS + senders = _senders(pre) + + # Per-iteration body: increment the counter-th shared slot. + sl = Op.ADD(Op.MLOAD(M_BASE), Op.MLOAD(M_COUNTER)) + read = Op.SLOAD(sl, key_warm=False, page_load_warm=False) + body = _sstore( + sl, Op.ADD(read, 1), original=0, current=0, new=1, growth=0 + ) + Op.MSTORE(M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1)) + count = _size_count(body, budget) + + contract = pre.deploy_contract( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_COUNTER, 0) + + While( + body=body, + condition=_loop_condition(), + ) + + Op.SSTORE(Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked) + + Op.STOP + ) + + blocks = [] + expected: StorageDict = {} + g = 0 + for r in range(REPEATS): + base = SERIAL_BASE + r * SERIAL_REPEAT_STRIDE + txs = [] + for _t in range(FULL_BLOCK_TXS): + txs.append( + Transaction( + to=contract, + sender=next(senders), + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(base, count, g, 0), + ) + ) + expected[MARKER_BASE + g] = value_code_worked + g += 1 + for i in range(count): + expected[base + i] = FULL_BLOCK_TXS + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) + + +@pytest.mark.valid_from("MONAD_NINE") +def test_bad_block_chained( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + gas_benchmark_value: int, +) -> None: + """ + Adversarial block with a data-dependent SLOAD chain: each SLOAD + returns the next SLOAD's slot (SLOAD(SLOAD(...(base)))), following a + pre-built ring of distinct pages, so the reads serialise within a tx + and each hop is a random disk position (pointer chase). Each repeat + uses a fresh ring. + """ + budget = gas_benchmark_value // FULL_BLOCK_TXS + senders = _senders(pre) + + # Per-iteration body: SLOAD current slot; its value is the next slot. + nxt = Op.SLOAD(Op.MLOAD(M_CHAIN), key_warm=False, page_load_warm=False) + body = Op.MSTORE(M_CHAIN, nxt) + Op.MSTORE( + M_COUNTER, Op.ADD(Op.MLOAD(M_COUNTER), 1) + ) + count = _size_count(body, budget) + + code = ( + Op.MSTORE(M_BASE, Op.CALLDATALOAD(CD_BASE)) # base slot + + Op.MSTORE(M_COUNT, Op.CALLDATALOAD(CD_COUNT)) + + Op.MSTORE(M_GLOBAL, Op.CALLDATALOAD(CD_GLOBAL)) + + Op.MSTORE(M_COUNTER, 0) + + Op.MSTORE(M_CHAIN, Op.MLOAD(M_BASE)) + + While( + body=body, + condition=_loop_condition(), + ) + + Op.SSTORE(Op.ADD(MARKER_BASE, Op.MLOAD(M_GLOBAL)), value_code_worked) + + Op.STOP + ) + + genesis: StorageDict = {} + rings: List[List[int]] = [] + for r in range(REPEATS): + base = CHAIN_BASE + r * CHAIN_REPEAT_STRIDE + ring = [(base + j) << 7 for j in range(count)] + for j in range(count): + genesis[ring[j]] = ring[(j + 1) % count] + rings.append(ring) + contract = pre.deploy_contract(code, storage=genesis) + + blocks = [] + expected: StorageDict = dict(genesis) + g = 0 + for r in range(REPEATS): + ring = rings[r] + txs = [] + for t in range(FULL_BLOCK_TXS): + txs.append( + Transaction( + to=contract, + sender=next(senders), + gas_limit=budget, + max_fee_per_gas=MAX_FEE_PER_GAS, + max_priority_fee_per_gas=0, + data=_calldata(ring[t % count], count, g, 0), + ) + ) + expected[MARKER_BASE + g] = value_code_worked + g += 1 + blocks.append(Block(txs=txs)) + + benchmark_test( + pre=pre, + blocks=blocks, + post={contract: Account(storage=expected)}, + env=Environment(gas_limit=gas_benchmark_value), + gas_benchmark_value=gas_benchmark_value, + skip_gas_used_validation=True, + ) diff --git a/whitelist.txt b/whitelist.txt index f553ff8b13b..cb9de9c6bbd 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -114,6 +114,7 @@ ae AF af alloc +ane AnnAssign api apis