From 146270ac2c20d3153829f8bf87af91e65eba6be2 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 26 Sep 2026 10:34:41 -0700 Subject: [PATCH 1/2] perf: keep shared-memory locking out of line with_memory! locked a shared memory inline and held the guard to the end of the access, so every load and store handler contained the lock and unlock calls and saved extra callee-saved registers on every access, shared memory or not. Run the shared-memory body in a cold, out-of-line helper instead; the ordinary-memory path stays inline. The three bodies that used `?` for a unit result (the memory.copy range checks and active data segment initialization) now return the Result and apply `?` outside the macro, so a body means the same thing in both paths. --- crates/tinywasm/src/store/mod.rs | 7 ++-- crates/tinywasm/src/store/state.rs | 55 ++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index e89650a2..af63a334 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -34,6 +34,8 @@ pub(crate) use memory::{MemValue, MemoryInstance}; pub use memory::{MemoryShared, MemorySharedGuard}; pub(crate) use state::State; pub(crate) use state::with_memory; +#[cfg(feature = "std")] +pub(crate) use state::with_shared_memory; pub(crate) use types::{canonicalize_ref_type, canonicalize_value_type}; pub(crate) use {data::*, element::*, function::*, global::*, table::*, tag::*}; @@ -822,9 +824,8 @@ impl Store { }; let offset = usize::try_from(offset).unwrap_or(usize::MAX); with_memory!(self.state, *mem_addr, |mem, kind| { - mem.write_all(offset, &data.data) - .ok_or_else(|| memory::memory_oob(offset, data.data.len(), mem.len()))?; - }); + mem.write_all(offset, &data.data).ok_or_else(|| memory::memory_oob(offset, data.data.len(), mem.len())) + })?; self.state.data[data_addrs[i] as usize].drop(); } tinywasm_types::DataKind::Passive => {} diff --git a/crates/tinywasm/src/store/state.rs b/crates/tinywasm/src/store/state.rs index 50262014..0f688320 100644 --- a/crates/tinywasm/src/store/state.rs +++ b/crates/tinywasm/src/store/state.rs @@ -26,35 +26,54 @@ pub(crate) struct State { pub(crate) roots: gc::Roots, } -// Dispatch once per operation, keeping ordinary memory on the direct-access path +// Dispatch once per operation, keeping ordinary memory on the direct-access path. +// A shared memory runs the body out of line, under its lock, so the handlers of +// ordinary memory accesses contain no lock/unlock calls and spill no registers for them. macro_rules! with_memory { ($state:expr, $addr:expr, |$memory:ident, $kind:ident| $body:block) => {{ let state = &mut $state; let addr = $addr; #[cfg(feature = "std")] - let mut guard; - #[cfg(feature = "std")] - let (kind, bytes) = if addr & $crate::store::SHARED_MEM_BIT != 0 { - core::hint::cold_path(); - guard = state.shared_memories[(addr & !$crate::store::SHARED_MEM_BIT) as usize].lock(); - (guard.kind, &mut *guard.inner) + let result = if addr & $crate::store::SHARED_MEM_BIT != 0 { + $crate::store::with_shared_memory(state, addr, |$memory, kind| { + #[allow(unused_variables)] + let $kind = kind; + $body + }) } else { let ordinary = &mut state.memories[addr as usize]; - (ordinary.kind, &mut ordinary.inner) + #[allow(unused_variables)] + let $kind = ordinary.kind; + let $memory = &mut ordinary.inner; + $body }; #[cfg(not(feature = "std"))] - let (kind, bytes) = { + let result = { let ordinary = state.get_mem_mut(addr); - (ordinary.kind, &mut ordinary.inner) + #[allow(unused_variables)] + let $kind = ordinary.kind; + let $memory = &mut ordinary.inner; + $body }; - #[allow(unused_variables)] - let $kind = kind; - let $memory = bytes; - $body + result }}; } pub(crate) use with_memory; +/// Runs `body` on a shared memory's bytes while holding its lock. +#[cfg(feature = "std")] +#[cold] +#[inline(never)] +pub(crate) fn with_shared_memory( + state: &mut State, + addr: MemAddr, + body: impl FnOnce(&mut super::memory::MemoryStorage, MemoryType) -> R, +) -> R { + let mut guard = state.shared_memories[(addr & !SHARED_MEM_BIT) as usize].lock(); + let kind = guard.kind; + body(&mut guard.inner, kind) +} + impl State { /// Returns the immutable memory type without taking a shared-memory lock. pub(crate) fn memory_type(&self, addr: MemAddr) -> MemoryType { @@ -120,11 +139,11 @@ impl State { // Never hold two backing locks, since another store may copy in the opposite direction. // Growth cannot invalidate these ranges. Check both before changing the destination. with_memory!(*self, src_addr, |source, kind| { - source.checked_range(src, size).ok_or_else(|| memory_oob(src, size, source.len()))?; - }); + source.checked_range(src, size).ok_or_else(|| memory_oob(src, size, source.len())) + })?; with_memory!(*self, dst_addr, |destination, kind| { - destination.checked_range(dst, size).ok_or_else(|| memory_oob(dst, size, destination.len()))?; - }); + destination.checked_range(dst, size).ok_or_else(|| memory_oob(dst, size, destination.len())) + })?; let mut bytes = [0u8; 4096]; for offset in (0..size).step_by(bytes.len()) { let chunk = bytes.len().min(size - offset); From 6c4df078330d751546641ba6be023d3ff3ca92d5 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 26 Sep 2026 11:40:00 -0700 Subject: [PATCH 2/2] perf: keep the shared-memory lock inline for atomics Atomic operations usually target a shared memory, so for them the out-of-line path was the common one and cost an extra call: about 1-2% more cycles per atomic read-modify-write. They now use with_memory!(@lock_inline ...), the previous inline form without the cold_path() hint. Per access, shared memory, M4 (P-cores / E-cores), against next: atomic RMW -5.6% instructions, -3.4% / -7.8% cycles; plain load and store -3.2% instructions, +1.9% / -6.9% cycles. Ordinary memory: -5.8% instructions, -5.3% / -5.0% cycles. --- .../tinywasm/src/interpreter/executor/mod.rs | 4 +-- crates/tinywasm/src/store/state.rs | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 4f2c27fa..c68e0aa8 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -1432,7 +1432,7 @@ impl<'store> Executor<'store> { let memory = arg.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); let width = if op == AtomicWaitOp::Wait64 { 8 } else { 4 }; - let addr = crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + let addr = crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { let base = self.store.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::<1>(base, memory.offset()))?; if addr % width != 0 { @@ -1484,7 +1484,7 @@ impl<'store> Executor<'store> { let memory = arg.memory.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); - crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { + crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { let base = self.store.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; if addr % N != 0 { diff --git a/crates/tinywasm/src/store/state.rs b/crates/tinywasm/src/store/state.rs index 0f688320..e433bddd 100644 --- a/crates/tinywasm/src/store/state.rs +++ b/crates/tinywasm/src/store/state.rs @@ -29,7 +29,32 @@ pub(crate) struct State { // Dispatch once per operation, keeping ordinary memory on the direct-access path. // A shared memory runs the body out of line, under its lock, so the handlers of // ordinary memory accesses contain no lock/unlock calls and spill no registers for them. +// Atomic operations, whose memory is usually shared, keep the lock inline with +// `with_memory!(@lock_inline ...)`. macro_rules! with_memory { + (@lock_inline $state:expr, $addr:expr, |$memory:ident, $kind:ident| $body:block) => {{ + let state = &mut $state; + let addr = $addr; + #[cfg(feature = "std")] + let mut guard; + #[cfg(feature = "std")] + let (kind, bytes) = if addr & $crate::store::SHARED_MEM_BIT != 0 { + guard = state.shared_memories[(addr & !$crate::store::SHARED_MEM_BIT) as usize].lock(); + (guard.kind, &mut *guard.inner) + } else { + let ordinary = &mut state.memories[addr as usize]; + (ordinary.kind, &mut ordinary.inner) + }; + #[cfg(not(feature = "std"))] + let (kind, bytes) = { + let ordinary = state.get_mem_mut(addr); + (ordinary.kind, &mut ordinary.inner) + }; + #[allow(unused_variables)] + let $kind = kind; + let $memory = bytes; + $body + }}; ($state:expr, $addr:expr, |$memory:ident, $kind:ident| $body:block) => {{ let state = &mut $state; let addr = $addr;