diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index f92765d..45666a6 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -60,6 +60,11 @@ struct ModuleInstanceInner { id: ModuleInstanceId, type_addrs: Box<[TypeAddr]>, func_addrs: Box<[FuncAddr]>, + /// The module's own (non-imported) functions, in index order. The store allocates them + /// contiguously from `local_func_base`, so an executor can borrow any of them for a whole run. + local_funcs: Box<[Shared]>, + local_func_base: FuncAddr, + imported_funcs: u32, table_addrs: Box<[TableAddr]>, mem_addrs: Box<[MemAddr]>, #[cfg(feature = "std")] @@ -84,6 +89,20 @@ impl ModuleInstance { self.0.func_addrs[addr as usize] } + /// The store address and body of the module's own function `idx`, or `None` for an import. + #[inline(always)] + pub(crate) fn local_func_by_index(&self, idx: FuncAddr) -> Option<(FuncAddr, &WasmFunction)> { + let local = idx.wrapping_sub(self.0.imported_funcs); + let func = self.0.local_funcs.get(local as usize)?; + Some((self.0.local_func_base + local, func)) + } + + /// The body of the function at store address `addr`, if this instance owns it. + #[inline(always)] + pub(crate) fn local_func(&self, addr: FuncAddr) -> Option<&WasmFunction> { + self.0.local_funcs.get(addr.wrapping_sub(self.0.local_func_base) as usize).map(|func| &**func) + } + /// resolve a table address to the global store address #[inline] pub(crate) fn resolve_table_addr(&self, addr: TableAddr) -> TableAddr { @@ -211,7 +230,9 @@ impl ModuleInstance { let id = store.next_module_instance_id(); let mut addrs = crate::imports::ResolvedImports::new(store, module, &type_addrs, imports)?; let imported_funcs = addrs.funcs.len(); - addrs.funcs.extend(store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs)); + let local_funcs = store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs); + let local_func_base = local_funcs.start; + addrs.funcs.extend(local_funcs); addrs.tags.extend(store.init_tags(&module.tags, &type_addrs)); let limiter = store.engine.config().resource_limiter.clone(); if !module.skip_local_memory_allocation { @@ -238,6 +259,9 @@ impl ModuleInstance { id, type_addrs, func_addrs: addrs.funcs.into_boxed_slice(), + local_funcs: module.funcs.clone(), + local_func_base, + imported_funcs: imported_funcs as u32, table_addrs: addrs.tables.into_boxed_slice(), mem_addrs: addrs.memories.into_boxed_slice(), #[cfg(feature = "std")] diff --git a/crates/tinywasm/src/interpreter/executor/dispatch.rs b/crates/tinywasm/src/interpreter/executor/dispatch.rs index ac52047..0a2c4fe 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch.rs @@ -24,67 +24,76 @@ macro_rules! define_stable_dispatch { }; } -impl Executor<'_> { +impl Executor<'_, '_> { instruction_handlers!(define_stable_dispatch); + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + pub(crate) fn run_to_completion(mut self) -> Result> { let mut instr_ptr = self.cf.instr_ptr; loop { match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return cold!(Ok(())), + None => return cold!(Ok(self.left())), } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { let mut instr_ptr = self.cf.instr_ptr; loop { - for _ in 0..CHECKPOINT_INTERVAL { + while chunk_left != 0 { + chunk_left -= 1; match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - let mut instr_ptr = self.cf.instr_ptr; loop { - for _ in 0..CHECKPOINT_INTERVAL { + while chunk_left != 0 { + chunk_left -= 1; match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + + fn run_end(&self, chunk_left: u32) -> RunEnd { + match self.left() { + Some(frame) => RunEnd::Left(frame, chunk_left), + None => RunEnd::State(ExecState::Completed), + } + } } diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index f790d63..03ec2b8 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,8 +3,10 @@ use super::*; struct Unbudgeted; struct Bounded; -type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; -type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; +type UnbudgetedHandler = + for<'store, 'module> fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction) -> ExecResult<()>; +type BoundedHandler = + for<'store, 'module> fn(&mut Executor<'store, 'module>, usize, Instruction, u32) -> ExecResult<()>; macro_rules! define_unbudgeted_tail_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; @@ -22,7 +24,7 @@ macro_rules! define_unbudgeted_tail_dispatch { $( #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, instructions: &[Instruction], $instr_ptr: usize, instruction: Instruction, @@ -45,7 +47,12 @@ macro_rules! define_unbudgeted_tail_dispatch { $executor.cf.instr_ptr = next_instr_ptr; return Ok(()); }, - ExecFlow::Complete => return cold!({ $executor.completed = true; Ok(()) }), + ExecFlow::Complete => return cold!({ + if !$executor.left { + $executor.completed = true; + } + Ok(()) + }), } }}; } @@ -79,7 +86,7 @@ macro_rules! define_bounded_tail_dispatch { $( #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, @@ -106,7 +113,11 @@ macro_rules! define_bounded_tail_dispatch { match $flow.next_instr_ptr() { Some(next_instr_ptr) => $dispatch_next!(next_instr_ptr), None => return cold!({ - $executor.completed = true; + if $executor.left { + $executor.chunk_left = instructions_until_checkpoint; + } else { + $executor.completed = true; + } Ok(()) }), } @@ -134,13 +145,13 @@ impl Unbudgeted { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_, '_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { unreachable!("instruction handler mismatch") } #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } } @@ -152,79 +163,92 @@ impl Bounded { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { unreachable!("instruction handler mismatch") } #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } + /// Runs up to `chunk_left` (at least 1) instructions from `executor.cf`. #[inline(always)] - fn run(executor: &mut Executor<'_>) -> ExecResult<()> { + fn run(executor: &mut Executor<'_, '_>, chunk_left: u32) -> ExecResult<()> { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1) + handler(executor, instr_ptr, instruction, chunk_left - 1) } } -impl<'store> Executor<'store> { +impl Executor<'_, '_> { + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + pub(crate) fn run_to_completion(mut self) -> Result> { loop { - let func = self.func.clone(); + let func = self.func; let instructions = &func.instructions; let instr_ptr = self.cf.instr_ptr; let instruction = instructions[instr_ptr]; let handler = Unbudgeted::handler_for(instruction.opcode()); handler(&mut self, instructions, instr_ptr, instruction)?; - if self.completed { - return Ok(()); + if self.completed || self.left { + return Ok(self.left()); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); - + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + + /// How a bounded chain that stopped ended, unless it stopped at a checkpoint. + #[inline(always)] + fn run_end(&self) -> Option { + if self.completed { + return cold!(Some(RunEnd::State(ExecState::Completed))); + } + self.left().map(|frame| RunEnd::Left(frame, self.chunk_left)) + } } diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index decc92b..33f88b6 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -26,7 +26,7 @@ mod dispatch; #[cfg(feature = "nightly-tail-calls")] mod dispatch_become; -const CHECKPOINT_INTERVAL: u32 = 128; +pub(crate) const CHECKPOINT_INTERVAL: u32 = 128; const FUEL_COST_CALL_TOTAL: u32 = 5; struct ExecError(Box); @@ -71,49 +71,80 @@ impl ExecFlow { } } -pub(crate) struct Executor<'store> { +/// Runs frames of one module instance. The executing function is borrowed from the instance, which +/// the caller keeps alive for the whole run, so calls and returns within the instance switch a +/// reference instead of cloning a `Shared` handle. Execution that continues in a frame of another +/// instance ends the run with [`Executor::left`] set; [`InterpreterRuntime`](super::InterpreterRuntime) +/// resumes that frame with an executor for its instance. +pub(crate) struct Executor<'store, 'module> { cf: CallFrame, - func: Shared, - module: ModuleInstance, + func: &'module WasmFunction, + module: &'module ModuleInstance, store: &'store mut Store, call_stack_base: u32, mem0: MemAddr, fuel_metered: bool, + /// Execution continues in `cf`, which belongs to another module instance. + left: bool, + /// Budgeted runs: instructions left before the next checkpoint when the run left. + #[cfg(feature = "nightly-tail-calls")] + chunk_left: u32, #[cfg(feature = "nightly-tail-calls")] completed: bool, } -impl<'store> Executor<'store> { - pub(crate) fn new(store: &'store mut Store, cf: CallFrame, call_stack_base: u32) -> Self { - let wasm_func = store.state.funcs.wasm(cf.func_addr); - let module = store.get_module_instance(wasm_func.owner).expect("invalid module instance").clone(); - let mem0 = module.mem0_addr(); +/// How a budgeted run of one executor ended. +pub(crate) enum RunEnd { + State(ExecState), + /// Execution continues in this frame of another module instance, with this many instructions + /// left before the next checkpoint. + Left(CallFrame, u32), +} + +impl<'store, 'module> Executor<'store, 'module> { + /// `module` must own the function of `cf`. + pub(crate) fn new( + store: &'store mut Store, + module: &'module ModuleInstance, + cf: CallFrame, + call_stack_base: u32, + ) -> Self { + let func = module.local_func(cf.func_addr).expect("call frame from another module instance"); Self { module, cf, - func: wasm_func.func.clone(), + func, store, call_stack_base, - mem0, + mem0: module.mem0_addr(), fuel_metered: false, + left: false, + #[cfg(feature = "nightly-tail-calls")] + chunk_left: 0, #[cfg(feature = "nightly-tail-calls")] completed: false, } } + /// The frame to resume in another module instance, if execution left this one. + #[inline(always)] + pub(crate) fn left(&self) -> Option { + self.left.then_some(self.cf) + } + + /// Ends the run because `self.cf` belongs to another module instance. + #[cold] + fn leave(&mut self) -> ExecFlow { + self.left = true; + ExecFlow::Complete + } + /// Resolves a module-local memory index to its store address, caching the common memory-0 case. #[inline(always)] fn mem_addr(&self, idx: MemAddr) -> MemAddr { if idx == 0 { self.mem0 } else { self.module.resolve_mem_addr(idx) } } - /// Switches the executor to another module, keeping the cached memory-0 address in sync. - #[inline] - fn set_module(&mut self, owner: ModuleInstanceId) { - self.module = self.store.get_module_instance(owner).expect("invalid module instance").clone(); - self.mem0 = self.module.mem0_addr(); - } - #[inline(always)] fn charge_call_fuel(&mut self, total_fuel_cost: u32) { if self.fuel_metered { @@ -577,8 +608,13 @@ impl<'store> Executor<'store> { } } - fn matching_catch(&self, protected_ip: usize, tag_addr: TagAddr) -> Option { - let handlers = &self.func.data.exception_handlers; + fn matching_catch( + func: &WasmFunction, + module: &ModuleInstance, + protected_ip: usize, + tag_addr: TagAddr, + ) -> Option { + let handlers = &func.data.exception_handlers; let end = handlers.partition_point(|handler| handler.start_ip as usize <= protected_ip); handlers[..end] .iter() @@ -586,35 +622,27 @@ impl<'store> Executor<'store> { .filter(|handler| protected_ip < handler.end_ip as usize) .flat_map(|handler| handler.catches.iter().copied()) .find(|catch| match catch { - ExceptionCatch::Tag { tag, .. } => self.module.resolve_tag_addr(*tag) == tag_addr, + ExceptionCatch::Tag { tag, .. } => module.resolve_tag_addr(*tag) == tag_addr, ExceptionCatch::All { .. } => true, }) } - #[inline(always)] - fn switch_to_frame(&mut self, frame: CallFrame) { - let previous = core::mem::replace(&mut self.cf, frame); - if previous.func_addr == self.cf.func_addr { - return; - } - - let wasm_func = self.store.state.funcs.wasm(self.cf.func_addr); - if !Shared::ptr_eq(&self.func, &wasm_func.func) { - self.func = wasm_func.func.clone(); - } - if wasm_func.owner != self.module.id() { - self.set_module(wasm_func.owner); - } - } - + /// Unwinds to the innermost handler that catches `exception`. Returns the flow to its landing + /// pad, or `None` if no frame of this run catches it. fn dispatch_exception(&mut self, exception: ValueRef, mut protected_ip: usize) -> Result, Trap> { let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; let crate::store::GcObjectKind::Exception(tag_addr) = object.kind else { return Err(Trap::InvalidReference); }; - let mut switched = false; + let entry_func_addr = self.cf.func_addr; + // The function and instance of the frame being searched while it belongs to another instance. + let mut foreign: Option<(Shared, ModuleInstance)> = None; loop { - if let Some(catch) = self.matching_catch(protected_ip, tag_addr) { + let catch = match &foreign { + None => Self::matching_catch(self.func, self.module, protected_ip, tag_addr), + Some((func, module)) => Self::matching_catch(func, module, protected_ip, tag_addr), + }; + if let Some(catch) = catch { let (landing_pad, base, with_ref, include_payload) = match catch { ExceptionCatch::Tag { landing_pad, base, with_ref, .. } => (landing_pad, base, with_ref, true), ExceptionCatch::All { landing_pad, base, with_ref } => (landing_pad, base, with_ref, false), @@ -636,10 +664,15 @@ impl<'store> Executor<'store> { if with_ref { ValueRef::stack_push(&mut self.store.value_stack, exception); } - return Ok(Some(if switched { - ExecFlow::Switch(landing_pad as usize) + if foreign.is_some() { + self.cf.instr_ptr = landing_pad as usize; + return Ok(Some(self.leave())); + } + let landing_pad = landing_pad as usize; + return Ok(Some(if self.cf.func_addr == entry_func_addr { + ExecFlow::Next(landing_pad) } else { - ExecFlow::Next(landing_pad as usize) + ExecFlow::Switch(landing_pad) })); } @@ -647,8 +680,21 @@ impl<'store> Executor<'store> { let Some(caller) = self.store.call_stack.pop_frame(self.call_stack_base) else { return Ok(None); }; - switched = true; - self.switch_to_frame(caller); + if caller.func_addr != self.cf.func_addr { + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + foreign = None; + } + None => { + let wasm_func = self.store.state.funcs.wasm(caller.func_addr); + let owner = self.store.get_module_instance(wasm_func.owner).expect("invalid module instance"); + foreign = Some((wasm_func.func.clone(), owner.clone())); + } + } + } + self.cf = caller; protected_ip = self.cf.instr_ptr.checked_sub(1).expect("invalid caller IP"); } } @@ -695,27 +741,34 @@ impl<'store> Executor<'store> { fn exec_call_direct(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let addr = self.module.resolve_func_addr(v); - if self.store.state.funcs.is_host(addr) { - let host_func = self.store.state.funcs.host(addr); - let type_addr = host_func.type_addr; - let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, return_instr_ptr) - } else { - self.exec_call_wasm::(addr, return_instr_ptr) + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, return_instr_ptr), + None => self.exec_call_import::(v, return_instr_ptr), } } fn exec_return_call_direct(&mut self, v: u32) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, 0), + None => self.exec_call_import::(v, 0), + } + } + + /// Calls an imported function: a host function or another instance's Wasm function. + #[inline(always)] + fn exec_call_import(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { let addr = self.module.resolve_func_addr(v); if self.store.state.funcs.is_host(addr) { let host_func = self.store.state.funcs.host(addr); let type_addr = host_func.type_addr; let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, 0) + self.exec_call_host::(host_func, type_addr, return_instr_ptr) } else { - self.exec_call_wasm::(addr, 0) + // An instance cannot import its own functions. + self.exec_call_foreign::(addr, return_instr_ptr) } } @@ -787,35 +840,65 @@ impl<'store> Executor<'store> { self.exec_call_wasm::(func_addr, return_instr_ptr) } + /// Calls the Wasm function at store address `func_addr`. #[inline(always)] fn exec_call_wasm( &mut self, func_addr: FuncAddr, return_instr_ptr: usize, ) -> ExecResult { - let wasm_func = self.store.state.funcs.wasm(func_addr); - let (params, locals, max_stack, owner, next_func) = { - let next_func = (!Shared::ptr_eq(&self.func, &wasm_func.func)).then(|| wasm_func.func.clone()); - (wasm_func.func.params, wasm_func.func.locals, wasm_func.func.max_stack, wasm_func.owner, next_func) - }; - if TAIL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); + let module = self.module; + match module.local_func(func_addr) { + Some(func) => self.exec_call_local::(func_addr, func, return_instr_ptr), + None => self.exec_call_foreign::(func_addr, return_instr_ptr), } - let locals_base = self.store.value_stack.enter_locals(¶ms, &locals, &max_stack)?; - if TAIL { - self.cf = CallFrame::new(func_addr, locals_base, locals); - } else { - self.store.call_stack.push(self.cf, return_instr_ptr)?; - self.cf = CallFrame::new(func_addr, locals_base, locals); + } + + /// Enters `func`, which this run's module instance owns. + #[inline(always)] + fn exec_call_local( + &mut self, + func_addr: FuncAddr, + func: &'module WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult { + self.enter_frame::(func_addr, func, return_instr_ptr)?; + if core::ptr::eq(self.func, func) { + return Ok(ExecFlow::Next(0)); } - let switched = next_func.is_some(); - if let Some(next_func) = next_func { - self.func = next_func; + self.func = func; + Ok(ExecFlow::Switch(0)) + } + + /// Enters a function of another module instance, which ends this run. + #[inline(never)] + fn exec_call_foreign( + &mut self, + func_addr: FuncAddr, + return_instr_ptr: usize, + ) -> ExecResult { + let func = self.store.state.funcs.wasm(func_addr).func.clone(); + self.enter_frame::(func_addr, &func, return_instr_ptr)?; + Ok(self.leave()) + } + + /// Sets up the frame of a call to `func`, replacing the current one for a tail call. + #[inline(always)] + fn enter_frame( + &mut self, + func_addr: FuncAddr, + func: &WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult<()> { + if TAIL { + self.store.value_stack.truncate_keep_counts(self.cf.locals_base, func.params); } - if owner != self.module.id() { - self.set_module(owner); + let locals_base = self.store.value_stack.enter_locals(&func.params, &func.locals, &func.max_stack)?; + if !TAIL { + self.store.call_stack.push(self.cf, return_instr_ptr)?; } - Ok(if switched { ExecFlow::Switch(0) } else { ExecFlow::Next(0) }) + self.cf = CallFrame::new(func_addr, locals_base, func.locals); + Ok(()) } fn exec_call_ref(&mut self, type_addr: u32, return_instr_ptr: usize) -> ExecResult { @@ -839,12 +922,18 @@ impl<'store> Executor<'store> { return ExecFlow::Complete; }; let instr_ptr = caller.instr_ptr; - if caller.func_addr == self.cf.func_addr { - self.cf = caller; - ExecFlow::Next(instr_ptr) - } else { - self.switch_to_frame(caller); - ExecFlow::Switch(instr_ptr) + let switched = caller.func_addr != self.cf.func_addr; + self.cf = caller; + if !switched { + return ExecFlow::Next(instr_ptr); + } + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + ExecFlow::Switch(instr_ptr) + } + None => self.leave(), } } diff --git a/crates/tinywasm/src/interpreter/mod.rs b/crates/tinywasm/src/interpreter/mod.rs index 0f7ae20..3400a94 100644 --- a/crates/tinywasm/src/interpreter/mod.rs +++ b/crates/tinywasm/src/interpreter/mod.rs @@ -26,20 +26,54 @@ pub(crate) enum ExecState { pub(crate) struct InterpreterRuntime; impl InterpreterRuntime { - pub(crate) fn exec(store: &mut Store, cf: CallFrame, call_stack_base: u32) -> Result<()> { - executor::Executor::new(store, cf, call_stack_base).run_to_completion() + pub(crate) fn exec(store: &mut Store, mut cf: CallFrame, call_stack_base: u32) -> Result<()> { + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, call_stack_base).run_to_completion()? { + None => return Ok(()), + Some(frame) => cf = frame, + } + } } - pub(crate) fn exec_with_fuel(store: &mut Store, cf: CallFrame, fuel: u32) -> Result { - executor::Executor::new(store, cf, 0).run_with_fuel(fuel) + pub(crate) fn exec_with_fuel(store: &mut Store, mut cf: CallFrame, fuel: u32) -> Result { + store.execution_fuel = fuel; + if fuel == 0 { + return Ok(ExecState::Suspended(cf)); + } + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_fuel(chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } } #[cfg(feature = "std")] pub(crate) fn exec_with_time_budget( store: &mut Store, - cf: CallFrame, + mut cf: CallFrame, time_budget: core::time::Duration, ) -> Result { - executor::Executor::new(store, cf, 0).run_with_time_budget(time_budget) + if time_budget.is_zero() { + return Ok(ExecState::Suspended(cf)); + } + let start = crate::std::time::Instant::now(); + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_time_budget(start, time_budget, chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } + } + + /// The module instance that owns the function of `cf`. + fn frame_module(store: &Store, cf: &CallFrame) -> crate::ModuleInstance { + let owner = store.state.funcs.wasm(cf.func_addr).owner; + store.get_module_instance(owner).expect("invalid module instance").clone() } } diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 1c40e31..3a9936e 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -135,12 +135,15 @@ impl Stack { self.data.truncate(n); } + /// Moves the top value down to index `n` and drops everything above it (a function's result + /// replacing its frame). #[inline(always)] pub(crate) fn truncate_to_one_tail(&mut self, n: usize) { - debug_assert!(n < self.data.len()); - let last = self.pop(); - self.data.truncate(n); - self.data.push(last); + let len = self.data.len(); + debug_assert!(n < len); + let last = self.data[len.wrapping_sub(1)]; + self.data[n] = last; + self.data.truncate(n + 1); } /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves @@ -276,10 +279,17 @@ impl ValueStack { ) -> Result { let locals_base32 = self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize, max_stack.c32 as usize)?; - let locals_base64 = - self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)?; - let locals_base128 = - self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)?; + // Most functions use only the 32-bit lane; an unused lane needs no reservation. + let locals_base64 = if locals.c64 | max_stack.c64 == 0 { + self.stack_64.len() as u32 + } else { + self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)? + }; + let locals_base128 = if locals.c128 | max_stack.c128 == 0 { + self.stack_128.len() as u32 + } else { + self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)? + }; Ok(StackBase { s32: locals_base32, s64: locals_base64, s128: locals_base128 }) } diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index 7f5ae61..9b547ec 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -573,14 +573,14 @@ impl Store { result } - /// Add functions to the store, returning their addresses in the store + /// Add functions to the store, returning their (contiguous) addresses in the store pub(crate) fn init_funcs( &mut self, funcs: &[Shared], owner: ModuleInstanceId, module_type_idxs: &[TypeAddr], type_addrs: &[TypeAddr], - ) -> impl ExactSizeIterator { + ) -> core::ops::Range { debug_assert_eq!(funcs.len(), module_type_idxs.len()); let funcs = funcs.iter().cloned().zip(module_type_idxs).map(|(func, &type_idx)| (type_addrs[type_idx as usize], func)); diff --git a/crates/tinywasm/tests/cross_instance_calls.rs b/crates/tinywasm/tests/cross_instance_calls.rs new file mode 100644 index 0000000..1a3054f --- /dev/null +++ b/crates/tinywasm/tests/cross_instance_calls.rs @@ -0,0 +1,173 @@ +//! Calls, returns and exceptions that move between module instances. The interpreter runs one +//! instance's frames at a time and hands over to the other instance at each crossing. + +use tinywasm::{ExecProgress, Imports, ModuleInstance, Store}; + +type TestResult = Result<(), Box>; + +const MODULE_A: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (tag $e (export "e") (param i32)) + (table $t (export "t") 4 funcref) + (func $add1 (export "add1") (param i32) (result i32) + (i32.add (local.get 0) (i32.const 1))) + (func (export "throw") (param i32) + (throw $e (local.get 0))) + ;; Calls table slot 0, which the other instance fills. + (func (export "apply") (param i32) (result i32) + (i32.add (call_indirect $t (type $i2i) (local.get 0) (i32.const 0)) (i32.const 100))) + ;; Same, catching `$e` from the callee. + (func (export "apply_catching") (param i32) (result i32) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 0))) + (return)) + (i32.add (i32.const 1000))) + (elem (table $t) (i32.const 1) func $add1)) +"#; + +const MODULE_B: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (import "a" "e" (tag $e (param i32))) + (import "a" "t" (table $t 4 funcref)) + (import "a" "add1" (func $add1 (param i32) (result i32))) + (import "a" "throw" (func $throw (param i32))) + (import "a" "apply" (func $apply (param i32) (result i32))) + (import "a" "apply_catching" (func $apply_catching (param i32) (result i32))) + (func $double (param i32) (result i32) + (i32.mul (local.get 0) (i32.const 2))) + (func $throws (param i32) (result i32) + (throw $e (local.get 0))) + (elem declare func $double $throws) + + (func (export "direct") (param i32) (result i32) + (call $add1 (call $add1 (local.get 0)))) + (func (export "tail") (param i32) (result i32) + (return_call $add1 (local.get 0))) + (func (export "indirect") (param i32) (result i32) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 1))) + ;; B -> A -> B -> A -> B + (func (export "callback") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $double)) + (call $apply (local.get 0))) + (func (export "catch_from_import") (param i32) (result i32) + (block $caught (result i32) + (try_table (catch $e $caught) + (call $throw (local.get 0))) + (i32.const -1))) + ;; Thrown in B, unwinds through A, caught in B. + (func (export "catch_through_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call $apply (local.get 0))) + (return)) + (i32.add (i32.const 2000))) + ;; Thrown in B, caught in A. + (func (export "caught_in_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (call $apply_catching (local.get 0))) + ;; Four crossings per iteration. + (func (export "crossings") (param $n i32) (result i32) (local $sum i32) + (table.set $t (i32.const 0) (ref.func $double)) + (loop $next + (local.set $sum (i32.add (local.get $sum) (call $apply (local.get $n)))) + (local.set $n (i32.sub (local.get $n) (i32.const 1))) + (br_if $next (local.get $n))) + (local.get $sum))) +"#; + +fn instantiate(store: &mut Store) -> Result> { + let a = tinywasm::parse_bytes(&wat::parse_str(MODULE_A)?)?; + let b = tinywasm::parse_bytes(&wat::parse_str(MODULE_B)?)?; + let a = ModuleInstance::instantiate(store, &a, None)?; + let mut imports = Imports::new(); + imports.link_module("a", a)?; + Ok(ModuleInstance::instantiate(store, &b, Some(&imports))?) +} + +fn call( + store: &mut Store, + instance: &ModuleInstance, + name: &str, + arg: i32, +) -> Result> { + Ok(instance.func::(store, name)?.call(store, arg)?) +} + +#[test] +fn calls_and_returns_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "direct", 5)?, 7); + assert_eq!(call(&mut store, &b, "tail", 5)?, 6); + assert_eq!(call(&mut store, &b, "indirect", 5)?, 6); + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn exceptions_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "catch_from_import", 7)?, 7); + assert_eq!(call(&mut store, &b, "catch_through_import", 9)?, 2009); + assert_eq!(call(&mut store, &b, "caught_in_import", 9)?, 1009); + // The instances keep working after each unwind. + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn budgeted_runs_cross_instances() -> TestResult { + const N: i32 = 1000; + const EXPECTED: i32 = N * (N + 1) + 100 * N; + + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "crossings", N)?, EXPECTED); + + for fuel in [1, 3, 64, 1000] { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let mut suspensions = 0; + let result = loop { + match exec.resume_with_fuel(fuel)? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => suspensions += 1, + } + }; + assert_eq!(result, EXPECTED, "fuel {fuel}"); + assert!(suspensions > 0, "fuel {fuel}"); + } + + #[cfg(feature = "std")] + { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let result = loop { + match exec.resume_with_time_budget(std::time::Duration::from_micros(10))? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => {} + } + }; + assert_eq!(result, EXPECTED); + + // An effectively unlimited budget completes in one resume. + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + match exec.resume_with_time_budget(std::time::Duration::MAX)? { + ExecProgress::Completed(value) => assert_eq!(value, EXPECTED), + ExecProgress::Suspended => panic!("suspended with an unlimited time budget"), + } + } + Ok(()) +}