From 485c5ca7123f05a7045bbf267eca6f1fd5573dfe Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 26 Sep 2026 16:26:43 -0700 Subject: [PATCH 1/2] perf: skip unused value-stack lanes when entering a function Entering a function reserved and zero-filled all three value-stack lanes even when the function never touches the 64- or 128-bit lane, which is the common case. Skip a lane whose locals and operand-stack depth are both zero. A single-result return now moves the result down and truncates once, instead of popping, truncating and pushing it back. --- .../src/interpreter/stack/value_stack.rs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index b3ac91a0..f1d94022 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -135,12 +135,15 @@ impl Stack { self.data.truncate(n); } + /// Moves the top value down to index `n` and drops everything above it (a function's result + /// replacing its frame). #[inline(always)] pub(crate) fn truncate_to_one_tail(&mut self, n: usize) { - debug_assert!(n < self.data.len()); - let last = self.pop(); - self.data.truncate(n); - self.data.push(last); + let len = self.data.len(); + debug_assert!(n < len); + let last = self.data[len.wrapping_sub(1)]; + self.data[n] = last; + self.data.truncate(n + 1); } /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves @@ -276,10 +279,17 @@ impl ValueStack { ) -> Result { let locals_base32 = self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize, max_stack.c32 as usize)?; - let locals_base64 = - self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)?; - let locals_base128 = - self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)?; + // Most functions use only the 32-bit lane; an unused lane needs no reservation. + let locals_base64 = if locals.c64 | max_stack.c64 == 0 { + self.stack_64.len() as u32 + } else { + self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)? + }; + let locals_base128 = if locals.c128 | max_stack.c128 == 0 { + self.stack_128.len() as u32 + } else { + self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)? + }; Ok(StackBase { s32: locals_base32, s64: locals_base64, s128: locals_base128 }) } From 8370a09225ff1d2e6ef50f32a2e0fc10df949676 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 26 Sep 2026 16:26:43 -0700 Subject: [PATCH 2/2] perf: borrow the executing function from its module instance Every call and return between two functions cloned the callee's `Shared` and dropped the previous one, two refcount updates each way, and looked the function up in the store. A module instance now keeps its own functions, which the store allocates contiguously. The interpreter holds the instance for the whole run, so the executor borrows the executing function and its module from it: a call or return within the instance switches a reference, and a direct call to one of the module's own functions skips the address table, the host check and the owner check. Execution that continues in another instance's function (a call through an import, table or reference, a return, or an exception unwinding into it) ends the run, and `InterpreterRuntime` resumes that frame with an executor for its instance. Fuel and time budgets carry over, so a run suspends at the same points as before. --- crates/tinywasm/src/instance.rs | 26 +- .../src/interpreter/executor/dispatch.rs | 53 ++-- .../interpreter/executor/dispatch_become.rs | 94 ++++--- .../tinywasm/src/interpreter/executor/mod.rs | 247 ++++++++++++------ crates/tinywasm/src/interpreter/mod.rs | 46 +++- crates/tinywasm/src/store/mod.rs | 4 +- crates/tinywasm/tests/cross_instance_calls.rs | 173 ++++++++++++ 7 files changed, 498 insertions(+), 145 deletions(-) create mode 100644 crates/tinywasm/tests/cross_instance_calls.rs diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index f92765dd..45666a62 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -60,6 +60,11 @@ struct ModuleInstanceInner { id: ModuleInstanceId, type_addrs: Box<[TypeAddr]>, func_addrs: Box<[FuncAddr]>, + /// The module's own (non-imported) functions, in index order. The store allocates them + /// contiguously from `local_func_base`, so an executor can borrow any of them for a whole run. + local_funcs: Box<[Shared]>, + local_func_base: FuncAddr, + imported_funcs: u32, table_addrs: Box<[TableAddr]>, mem_addrs: Box<[MemAddr]>, #[cfg(feature = "std")] @@ -84,6 +89,20 @@ impl ModuleInstance { self.0.func_addrs[addr as usize] } + /// The store address and body of the module's own function `idx`, or `None` for an import. + #[inline(always)] + pub(crate) fn local_func_by_index(&self, idx: FuncAddr) -> Option<(FuncAddr, &WasmFunction)> { + let local = idx.wrapping_sub(self.0.imported_funcs); + let func = self.0.local_funcs.get(local as usize)?; + Some((self.0.local_func_base + local, func)) + } + + /// The body of the function at store address `addr`, if this instance owns it. + #[inline(always)] + pub(crate) fn local_func(&self, addr: FuncAddr) -> Option<&WasmFunction> { + self.0.local_funcs.get(addr.wrapping_sub(self.0.local_func_base) as usize).map(|func| &**func) + } + /// resolve a table address to the global store address #[inline] pub(crate) fn resolve_table_addr(&self, addr: TableAddr) -> TableAddr { @@ -211,7 +230,9 @@ impl ModuleInstance { let id = store.next_module_instance_id(); let mut addrs = crate::imports::ResolvedImports::new(store, module, &type_addrs, imports)?; let imported_funcs = addrs.funcs.len(); - addrs.funcs.extend(store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs)); + let local_funcs = store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs); + let local_func_base = local_funcs.start; + addrs.funcs.extend(local_funcs); addrs.tags.extend(store.init_tags(&module.tags, &type_addrs)); let limiter = store.engine.config().resource_limiter.clone(); if !module.skip_local_memory_allocation { @@ -238,6 +259,9 @@ impl ModuleInstance { id, type_addrs, func_addrs: addrs.funcs.into_boxed_slice(), + local_funcs: module.funcs.clone(), + local_func_base, + imported_funcs: imported_funcs as u32, table_addrs: addrs.tables.into_boxed_slice(), mem_addrs: addrs.memories.into_boxed_slice(), #[cfg(feature = "std")] diff --git a/crates/tinywasm/src/interpreter/executor/dispatch.rs b/crates/tinywasm/src/interpreter/executor/dispatch.rs index ac520475..0a2c4fed 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch.rs @@ -24,67 +24,76 @@ macro_rules! define_stable_dispatch { }; } -impl Executor<'_> { +impl Executor<'_, '_> { instruction_handlers!(define_stable_dispatch); + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + pub(crate) fn run_to_completion(mut self) -> Result> { let mut instr_ptr = self.cf.instr_ptr; loop { match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return cold!(Ok(())), + None => return cold!(Ok(self.left())), } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { let mut instr_ptr = self.cf.instr_ptr; loop { - for _ in 0..CHECKPOINT_INTERVAL { + while chunk_left != 0 { + chunk_left -= 1; match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - let mut instr_ptr = self.cf.instr_ptr; loop { - for _ in 0..CHECKPOINT_INTERVAL { + while chunk_left != 0 { + chunk_left -= 1; match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + + fn run_end(&self, chunk_left: u32) -> RunEnd { + match self.left() { + Some(frame) => RunEnd::Left(frame, chunk_left), + None => RunEnd::State(ExecState::Completed), + } + } } diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index d5c86442..dcfab3e1 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,8 +3,10 @@ use super::*; struct Unbudgeted; struct Bounded; -type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; -type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; +type UnbudgetedHandler = + for<'store, 'module> fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction) -> ExecResult<()>; +type BoundedHandler = + for<'store, 'module> fn(&mut Executor<'store, 'module>, usize, Instruction, u32) -> ExecResult<()>; #[cold] #[inline(never)] @@ -28,7 +30,7 @@ macro_rules! define_unbudgeted_tail_dispatch { $( #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, instructions: &[Instruction], $instr_ptr: usize, instruction: Instruction, @@ -49,7 +51,12 @@ macro_rules! define_unbudgeted_tail_dispatch { $executor.cf.instr_ptr = next_instr_ptr; return Ok(()); }, - ExecFlow::Complete => return cold!({ $executor.completed = true; Ok(()) }), + ExecFlow::Complete => return cold!({ + if !$executor.left { + $executor.completed = true; + } + Ok(()) + }), } }}; } @@ -83,7 +90,7 @@ macro_rules! define_bounded_tail_dispatch { $( #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, @@ -108,7 +115,11 @@ macro_rules! define_bounded_tail_dispatch { match $flow.next_instr_ptr() { Some(next_instr_ptr) => $dispatch_next!(next_instr_ptr), None => return cold!({ - $executor.completed = true; + if $executor.left { + $executor.chunk_left = instructions_until_checkpoint; + } else { + $executor.completed = true; + } Ok(()) }), } @@ -135,69 +146,82 @@ impl Unbudgeted { impl Bounded { instruction_handlers!(define_bounded_tail_dispatch); + /// Runs up to `chunk_left` (at least 1) instructions from `executor.cf`. #[inline(always)] - fn run(executor: &mut Executor<'_>) -> ExecResult<()> { + fn run(executor: &mut Executor<'_, '_>, chunk_left: u32) -> ExecResult<()> { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1) + handler(executor, instr_ptr, instruction, chunk_left - 1) } } -impl<'store> Executor<'store> { +impl Executor<'_, '_> { + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + pub(crate) fn run_to_completion(mut self) -> Result> { loop { - let func = self.func.clone(); + let func = self.func; let instructions = &func.instructions; let instr_ptr = self.cf.instr_ptr; let instruction = instructions[instr_ptr]; let handler = Unbudgeted::handler_for(instruction.opcode()); handler(&mut self, instructions, instr_ptr, instruction)?; - if self.completed { - return Ok(()); + if self.completed || self.left { + return Ok(self.left()); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); - + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + + /// How a bounded chain that stopped ended, unless it stopped at a checkpoint. + #[inline(always)] + fn run_end(&self) -> Option { + if self.completed { + return cold!(Some(RunEnd::State(ExecState::Completed))); + } + self.left().map(|frame| RunEnd::Left(frame, self.chunk_left)) + } } diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 149ebc3a..ff2032aa 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -26,7 +26,7 @@ mod dispatch; #[cfg(feature = "nightly-tail-calls")] mod dispatch_become; -const CHECKPOINT_INTERVAL: u32 = 128; +pub(crate) const CHECKPOINT_INTERVAL: u32 = 128; const FUEL_COST_CALL_TOTAL: u32 = 5; struct ExecError(Box); @@ -71,49 +71,80 @@ impl ExecFlow { } } -pub(crate) struct Executor<'store> { +/// Runs frames of one module instance. The executing function is borrowed from the instance, which +/// the caller keeps alive for the whole run, so calls and returns within the instance switch a +/// reference instead of cloning a `Shared` handle. Execution that continues in a frame of another +/// instance ends the run with [`Executor::left`] set; [`InterpreterRuntime`](super::InterpreterRuntime) +/// resumes that frame with an executor for its instance. +pub(crate) struct Executor<'store, 'module> { cf: CallFrame, - func: Shared, - module: ModuleInstance, + func: &'module WasmFunction, + module: &'module ModuleInstance, store: &'store mut Store, call_stack_base: u32, mem0: MemAddr, fuel_metered: bool, + /// Execution continues in `cf`, which belongs to another module instance. + left: bool, + /// Budgeted runs: instructions left before the next checkpoint when the run left. + #[cfg(feature = "nightly-tail-calls")] + chunk_left: u32, #[cfg(feature = "nightly-tail-calls")] completed: bool, } -impl<'store> Executor<'store> { - pub(crate) fn new(store: &'store mut Store, cf: CallFrame, call_stack_base: u32) -> Self { - let wasm_func = store.state.funcs.wasm(cf.func_addr); - let module = store.get_module_instance(wasm_func.owner).expect("invalid module instance").clone(); - let mem0 = module.mem0_addr(); +/// How a budgeted run of one executor ended. +pub(crate) enum RunEnd { + State(ExecState), + /// Execution continues in this frame of another module instance, with this many instructions + /// left before the next checkpoint. + Left(CallFrame, u32), +} + +impl<'store, 'module> Executor<'store, 'module> { + /// `module` must own the function of `cf`. + pub(crate) fn new( + store: &'store mut Store, + module: &'module ModuleInstance, + cf: CallFrame, + call_stack_base: u32, + ) -> Self { + let func = module.local_func(cf.func_addr).expect("call frame from another module instance"); Self { module, cf, - func: wasm_func.func.clone(), + func, store, call_stack_base, - mem0, + mem0: module.mem0_addr(), fuel_metered: false, + left: false, + #[cfg(feature = "nightly-tail-calls")] + chunk_left: 0, #[cfg(feature = "nightly-tail-calls")] completed: false, } } + /// The frame to resume in another module instance, if execution left this one. + #[inline(always)] + pub(crate) fn left(&self) -> Option { + self.left.then_some(self.cf) + } + + /// Ends the run because `self.cf` belongs to another module instance. + #[cold] + fn leave(&mut self) -> ExecFlow { + self.left = true; + ExecFlow::Complete + } + /// Resolves a module-local memory index to its store address, caching the common memory-0 case. #[inline(always)] fn mem_addr(&self, idx: MemAddr) -> MemAddr { if idx == 0 { self.mem0 } else { self.module.resolve_mem_addr(idx) } } - /// Switches the executor to another module, keeping the cached memory-0 address in sync. - #[inline] - fn set_module(&mut self, owner: ModuleInstanceId) { - self.module = self.store.get_module_instance(owner).expect("invalid module instance").clone(); - self.mem0 = self.module.mem0_addr(); - } - #[inline(always)] fn charge_call_fuel(&mut self, total_fuel_cost: u32) { if self.fuel_metered { @@ -564,8 +595,13 @@ impl<'store> Executor<'store> { } } - fn matching_catch(&self, protected_ip: usize, tag_addr: TagAddr) -> Option { - let handlers = &self.func.data.exception_handlers; + fn matching_catch( + func: &WasmFunction, + module: &ModuleInstance, + protected_ip: usize, + tag_addr: TagAddr, + ) -> Option { + let handlers = &func.data.exception_handlers; let end = handlers.partition_point(|handler| handler.start_ip as usize <= protected_ip); handlers[..end] .iter() @@ -573,35 +609,27 @@ impl<'store> Executor<'store> { .filter(|handler| protected_ip < handler.end_ip as usize) .flat_map(|handler| handler.catches.iter().copied()) .find(|catch| match catch { - ExceptionCatch::Tag { tag, .. } => self.module.resolve_tag_addr(*tag) == tag_addr, + ExceptionCatch::Tag { tag, .. } => module.resolve_tag_addr(*tag) == tag_addr, ExceptionCatch::All { .. } => true, }) } - #[inline(always)] - fn switch_to_frame(&mut self, frame: CallFrame) { - let previous = core::mem::replace(&mut self.cf, frame); - if previous.func_addr == self.cf.func_addr { - return; - } - - let wasm_func = self.store.state.funcs.wasm(self.cf.func_addr); - if !Shared::ptr_eq(&self.func, &wasm_func.func) { - self.func = wasm_func.func.clone(); - } - if wasm_func.owner != self.module.id() { - self.set_module(wasm_func.owner); - } - } - + /// Unwinds to the innermost handler that catches `exception`. Returns the flow to its landing + /// pad, or `None` if no frame of this run catches it. fn dispatch_exception(&mut self, exception: ValueRef, mut protected_ip: usize) -> Result, Trap> { let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; let crate::store::GcObjectKind::Exception(tag_addr) = object.kind else { return Err(Trap::InvalidReference); }; - let mut switched = false; + let entry_func_addr = self.cf.func_addr; + // The function and instance of the frame being searched while it belongs to another instance. + let mut foreign: Option<(Shared, ModuleInstance)> = None; loop { - if let Some(catch) = self.matching_catch(protected_ip, tag_addr) { + let catch = match &foreign { + None => Self::matching_catch(self.func, self.module, protected_ip, tag_addr), + Some((func, module)) => Self::matching_catch(func, module, protected_ip, tag_addr), + }; + if let Some(catch) = catch { let (landing_pad, base, with_ref, include_payload) = match catch { ExceptionCatch::Tag { landing_pad, base, with_ref, .. } => (landing_pad, base, with_ref, true), ExceptionCatch::All { landing_pad, base, with_ref } => (landing_pad, base, with_ref, false), @@ -623,10 +651,15 @@ impl<'store> Executor<'store> { if with_ref { ValueRef::stack_push(&mut self.store.value_stack, exception)?; } - return Ok(Some(if switched { - ExecFlow::Switch(landing_pad as usize) + if foreign.is_some() { + self.cf.instr_ptr = landing_pad as usize; + return Ok(Some(self.leave())); + } + let landing_pad = landing_pad as usize; + return Ok(Some(if self.cf.func_addr == entry_func_addr { + ExecFlow::Next(landing_pad) } else { - ExecFlow::Next(landing_pad as usize) + ExecFlow::Switch(landing_pad) })); } @@ -634,8 +667,21 @@ impl<'store> Executor<'store> { let Some(caller) = self.store.call_stack.pop_frame(self.call_stack_base) else { return Ok(None); }; - switched = true; - self.switch_to_frame(caller); + if caller.func_addr != self.cf.func_addr { + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + foreign = None; + } + None => { + let wasm_func = self.store.state.funcs.wasm(caller.func_addr); + let owner = self.store.get_module_instance(wasm_func.owner).expect("invalid module instance"); + foreign = Some((wasm_func.func.clone(), owner.clone())); + } + } + } + self.cf = caller; protected_ip = self.cf.instr_ptr.checked_sub(1).expect("invalid caller IP"); } } @@ -682,27 +728,34 @@ impl<'store> Executor<'store> { fn exec_call_direct(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let addr = self.module.resolve_func_addr(v); - if self.store.state.funcs.is_host(addr) { - let host_func = self.store.state.funcs.host(addr); - let type_addr = host_func.type_addr; - let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, return_instr_ptr) - } else { - self.exec_call_wasm::(addr, return_instr_ptr) + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, return_instr_ptr), + None => self.exec_call_import::(v, return_instr_ptr), } } fn exec_return_call_direct(&mut self, v: u32) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, 0), + None => self.exec_call_import::(v, 0), + } + } + + /// Calls an imported function: a host function or another instance's Wasm function. + #[inline(always)] + fn exec_call_import(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { let addr = self.module.resolve_func_addr(v); if self.store.state.funcs.is_host(addr) { let host_func = self.store.state.funcs.host(addr); let type_addr = host_func.type_addr; let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, 0) + self.exec_call_host::(host_func, type_addr, return_instr_ptr) } else { - self.exec_call_wasm::(addr, 0) + // An instance cannot import its own functions. + self.exec_call_foreign::(addr, return_instr_ptr) } } @@ -774,35 +827,65 @@ impl<'store> Executor<'store> { self.exec_call_wasm::(func_addr, return_instr_ptr) } + /// Calls the Wasm function at store address `func_addr`. #[inline(always)] fn exec_call_wasm( &mut self, func_addr: FuncAddr, return_instr_ptr: usize, ) -> ExecResult { - let wasm_func = self.store.state.funcs.wasm(func_addr); - let (params, locals, max_stack, owner, next_func) = { - let next_func = (!Shared::ptr_eq(&self.func, &wasm_func.func)).then(|| wasm_func.func.clone()); - (wasm_func.func.params, wasm_func.func.locals, wasm_func.func.max_stack, wasm_func.owner, next_func) - }; - if TAIL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); + let module = self.module; + match module.local_func(func_addr) { + Some(func) => self.exec_call_local::(func_addr, func, return_instr_ptr), + None => self.exec_call_foreign::(func_addr, return_instr_ptr), } - let locals_base = self.store.value_stack.enter_locals(¶ms, &locals, &max_stack)?; - if TAIL { - self.cf = CallFrame::new(func_addr, locals_base, locals); - } else { - self.store.call_stack.push(self.cf, return_instr_ptr)?; - self.cf = CallFrame::new(func_addr, locals_base, locals); + } + + /// Enters `func`, which this run's module instance owns. + #[inline(always)] + fn exec_call_local( + &mut self, + func_addr: FuncAddr, + func: &'module WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult { + self.enter_frame::(func_addr, func, return_instr_ptr)?; + if core::ptr::eq(self.func, func) { + return Ok(ExecFlow::Next(0)); } - let switched = next_func.is_some(); - if let Some(next_func) = next_func { - self.func = next_func; + self.func = func; + Ok(ExecFlow::Switch(0)) + } + + /// Enters a function of another module instance, which ends this run. + #[inline(never)] + fn exec_call_foreign( + &mut self, + func_addr: FuncAddr, + return_instr_ptr: usize, + ) -> ExecResult { + let func = self.store.state.funcs.wasm(func_addr).func.clone(); + self.enter_frame::(func_addr, &func, return_instr_ptr)?; + Ok(self.leave()) + } + + /// Sets up the frame of a call to `func`, replacing the current one for a tail call. + #[inline(always)] + fn enter_frame( + &mut self, + func_addr: FuncAddr, + func: &WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult<()> { + if TAIL { + self.store.value_stack.truncate_keep_counts(self.cf.locals_base, func.params); } - if owner != self.module.id() { - self.set_module(owner); + let locals_base = self.store.value_stack.enter_locals(&func.params, &func.locals, &func.max_stack)?; + if !TAIL { + self.store.call_stack.push(self.cf, return_instr_ptr)?; } - Ok(if switched { ExecFlow::Switch(0) } else { ExecFlow::Next(0) }) + self.cf = CallFrame::new(func_addr, locals_base, func.locals); + Ok(()) } fn exec_call_ref(&mut self, type_addr: u32, return_instr_ptr: usize) -> ExecResult { @@ -826,12 +909,18 @@ impl<'store> Executor<'store> { return ExecFlow::Complete; }; let instr_ptr = caller.instr_ptr; - if caller.func_addr == self.cf.func_addr { - self.cf = caller; - ExecFlow::Next(instr_ptr) - } else { - self.switch_to_frame(caller); - ExecFlow::Switch(instr_ptr) + let switched = caller.func_addr != self.cf.func_addr; + self.cf = caller; + if !switched { + return ExecFlow::Next(instr_ptr); + } + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + ExecFlow::Switch(instr_ptr) + } + None => self.leave(), } } diff --git a/crates/tinywasm/src/interpreter/mod.rs b/crates/tinywasm/src/interpreter/mod.rs index 0f7ae20d..3400a94e 100644 --- a/crates/tinywasm/src/interpreter/mod.rs +++ b/crates/tinywasm/src/interpreter/mod.rs @@ -26,20 +26,54 @@ pub(crate) enum ExecState { pub(crate) struct InterpreterRuntime; impl InterpreterRuntime { - pub(crate) fn exec(store: &mut Store, cf: CallFrame, call_stack_base: u32) -> Result<()> { - executor::Executor::new(store, cf, call_stack_base).run_to_completion() + pub(crate) fn exec(store: &mut Store, mut cf: CallFrame, call_stack_base: u32) -> Result<()> { + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, call_stack_base).run_to_completion()? { + None => return Ok(()), + Some(frame) => cf = frame, + } + } } - pub(crate) fn exec_with_fuel(store: &mut Store, cf: CallFrame, fuel: u32) -> Result { - executor::Executor::new(store, cf, 0).run_with_fuel(fuel) + pub(crate) fn exec_with_fuel(store: &mut Store, mut cf: CallFrame, fuel: u32) -> Result { + store.execution_fuel = fuel; + if fuel == 0 { + return Ok(ExecState::Suspended(cf)); + } + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_fuel(chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } } #[cfg(feature = "std")] pub(crate) fn exec_with_time_budget( store: &mut Store, - cf: CallFrame, + mut cf: CallFrame, time_budget: core::time::Duration, ) -> Result { - executor::Executor::new(store, cf, 0).run_with_time_budget(time_budget) + if time_budget.is_zero() { + return Ok(ExecState::Suspended(cf)); + } + let start = crate::std::time::Instant::now(); + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_time_budget(start, time_budget, chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } + } + + /// The module instance that owns the function of `cf`. + fn frame_module(store: &Store, cf: &CallFrame) -> crate::ModuleInstance { + let owner = store.state.funcs.wasm(cf.func_addr).owner; + store.get_module_instance(owner).expect("invalid module instance").clone() } } diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index 7f5ae61d..9b547ec9 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -573,14 +573,14 @@ impl Store { result } - /// Add functions to the store, returning their addresses in the store + /// Add functions to the store, returning their (contiguous) addresses in the store pub(crate) fn init_funcs( &mut self, funcs: &[Shared], owner: ModuleInstanceId, module_type_idxs: &[TypeAddr], type_addrs: &[TypeAddr], - ) -> impl ExactSizeIterator { + ) -> core::ops::Range { debug_assert_eq!(funcs.len(), module_type_idxs.len()); let funcs = funcs.iter().cloned().zip(module_type_idxs).map(|(func, &type_idx)| (type_addrs[type_idx as usize], func)); diff --git a/crates/tinywasm/tests/cross_instance_calls.rs b/crates/tinywasm/tests/cross_instance_calls.rs new file mode 100644 index 00000000..1a3054f0 --- /dev/null +++ b/crates/tinywasm/tests/cross_instance_calls.rs @@ -0,0 +1,173 @@ +//! Calls, returns and exceptions that move between module instances. The interpreter runs one +//! instance's frames at a time and hands over to the other instance at each crossing. + +use tinywasm::{ExecProgress, Imports, ModuleInstance, Store}; + +type TestResult = Result<(), Box>; + +const MODULE_A: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (tag $e (export "e") (param i32)) + (table $t (export "t") 4 funcref) + (func $add1 (export "add1") (param i32) (result i32) + (i32.add (local.get 0) (i32.const 1))) + (func (export "throw") (param i32) + (throw $e (local.get 0))) + ;; Calls table slot 0, which the other instance fills. + (func (export "apply") (param i32) (result i32) + (i32.add (call_indirect $t (type $i2i) (local.get 0) (i32.const 0)) (i32.const 100))) + ;; Same, catching `$e` from the callee. + (func (export "apply_catching") (param i32) (result i32) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 0))) + (return)) + (i32.add (i32.const 1000))) + (elem (table $t) (i32.const 1) func $add1)) +"#; + +const MODULE_B: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (import "a" "e" (tag $e (param i32))) + (import "a" "t" (table $t 4 funcref)) + (import "a" "add1" (func $add1 (param i32) (result i32))) + (import "a" "throw" (func $throw (param i32))) + (import "a" "apply" (func $apply (param i32) (result i32))) + (import "a" "apply_catching" (func $apply_catching (param i32) (result i32))) + (func $double (param i32) (result i32) + (i32.mul (local.get 0) (i32.const 2))) + (func $throws (param i32) (result i32) + (throw $e (local.get 0))) + (elem declare func $double $throws) + + (func (export "direct") (param i32) (result i32) + (call $add1 (call $add1 (local.get 0)))) + (func (export "tail") (param i32) (result i32) + (return_call $add1 (local.get 0))) + (func (export "indirect") (param i32) (result i32) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 1))) + ;; B -> A -> B -> A -> B + (func (export "callback") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $double)) + (call $apply (local.get 0))) + (func (export "catch_from_import") (param i32) (result i32) + (block $caught (result i32) + (try_table (catch $e $caught) + (call $throw (local.get 0))) + (i32.const -1))) + ;; Thrown in B, unwinds through A, caught in B. + (func (export "catch_through_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call $apply (local.get 0))) + (return)) + (i32.add (i32.const 2000))) + ;; Thrown in B, caught in A. + (func (export "caught_in_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (call $apply_catching (local.get 0))) + ;; Four crossings per iteration. + (func (export "crossings") (param $n i32) (result i32) (local $sum i32) + (table.set $t (i32.const 0) (ref.func $double)) + (loop $next + (local.set $sum (i32.add (local.get $sum) (call $apply (local.get $n)))) + (local.set $n (i32.sub (local.get $n) (i32.const 1))) + (br_if $next (local.get $n))) + (local.get $sum))) +"#; + +fn instantiate(store: &mut Store) -> Result> { + let a = tinywasm::parse_bytes(&wat::parse_str(MODULE_A)?)?; + let b = tinywasm::parse_bytes(&wat::parse_str(MODULE_B)?)?; + let a = ModuleInstance::instantiate(store, &a, None)?; + let mut imports = Imports::new(); + imports.link_module("a", a)?; + Ok(ModuleInstance::instantiate(store, &b, Some(&imports))?) +} + +fn call( + store: &mut Store, + instance: &ModuleInstance, + name: &str, + arg: i32, +) -> Result> { + Ok(instance.func::(store, name)?.call(store, arg)?) +} + +#[test] +fn calls_and_returns_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "direct", 5)?, 7); + assert_eq!(call(&mut store, &b, "tail", 5)?, 6); + assert_eq!(call(&mut store, &b, "indirect", 5)?, 6); + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn exceptions_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "catch_from_import", 7)?, 7); + assert_eq!(call(&mut store, &b, "catch_through_import", 9)?, 2009); + assert_eq!(call(&mut store, &b, "caught_in_import", 9)?, 1009); + // The instances keep working after each unwind. + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn budgeted_runs_cross_instances() -> TestResult { + const N: i32 = 1000; + const EXPECTED: i32 = N * (N + 1) + 100 * N; + + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "crossings", N)?, EXPECTED); + + for fuel in [1, 3, 64, 1000] { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let mut suspensions = 0; + let result = loop { + match exec.resume_with_fuel(fuel)? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => suspensions += 1, + } + }; + assert_eq!(result, EXPECTED, "fuel {fuel}"); + assert!(suspensions > 0, "fuel {fuel}"); + } + + #[cfg(feature = "std")] + { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let result = loop { + match exec.resume_with_time_budget(std::time::Duration::from_micros(10))? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => {} + } + }; + assert_eq!(result, EXPECTED); + + // An effectively unlimited budget completes in one resume. + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + match exec.resume_with_time_budget(std::time::Duration::MAX)? { + ExecProgress::Completed(value) => assert_eq!(value, EXPECTED), + ExecProgress::Suspended => panic!("suspended with an unlimited time budget"), + } + } + Ok(()) +}