From b7734436d6b2858a0461546645490993dc0ee743 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 01:23:11 +0800 Subject: [PATCH 01/19] perf(jit): avoid redundant write-protect on trace entry --- src/vm/jit/runtime.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index 997bcaea..c40cead3 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -504,7 +504,6 @@ impl Vm { self.native_trace_state(current_trace_id)?; loop { native::clear_bridge_error(); - unsafe { crate::vm::native::prepare_for_execution() }; let status = unsafe { entry(self as *mut Vm) }; self.native_trace_exec_count = self.native_trace_exec_count.saturating_add(1); self.jit.mark_trace_executed(current_trace_id); From c3fa7cb88b87ba7b691ef2e95a99662371b9d365 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 01:26:02 +0800 Subject: [PATCH 02/19] perf(jit): index native traces by dense id --- src/vm/jit/runtime.rs | 93 +++++++++++++++++++++++-------------------- src/vm/mod.rs | 4 +- 2 files changed, 51 insertions(+), 46 deletions(-) diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index c40cead3..f379c049 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -420,16 +420,15 @@ impl Vm { out.push_str(&format!(" bridge {}: {}\n", name, count)); } } - if self.native_traces.is_empty() { + let native_trace_count = self.native_traces.iter().flatten().count(); + if native_trace_count == 0 { out.push_str(" native traces: 0\n"); return out; } - out.push_str(&format!(" native traces: {}\n", self.native_traces.len())); - let mut ids: Vec = self.native_traces.keys().copied().collect(); - ids.sort_unstable(); - for id in ids { - if let Some(native) = self.native_traces.get(&id) { + out.push_str(&format!(" native traces: {}\n", native_trace_count)); + for (id, native) in self.native_traces.iter().enumerate() { + if let Some(native) = native { out.push_str(&format!( " native trace#{} entry=0x{:X} code_bytes={} lowering={}\n", id, @@ -651,9 +650,13 @@ impl Vm { &self, trace_id: usize, ) -> VmResult<(NativeTraceEntry, usize, JitTraceTerminal, bool, bool)> { - let native = self.native_traces.get(&trace_id).ok_or_else(|| { - VmError::JitNative(format!("native trace entry for id {} missing", trace_id)) - })?; + let native = self + .native_traces + .get(trace_id) + .and_then(Option::as_ref) + .ok_or_else(|| { + VmError::JitNative(format!("native trace entry for id {} missing", trace_id)) + })?; Ok(( native.entry, native.root_ip, @@ -692,14 +695,16 @@ impl Vm { compile_profile: native::NativeCompileProfile, interrupt_settings: Option, ) -> VmResult<()> { - if let Some(native) = self.native_traces.get(&trace_id) + if let Some(native) = self.native_traces.get(trace_id).and_then(Option::as_ref) && native.interrupt_settings == interrupt_settings && compile_profile_satisfies(native.compile_profile, compile_profile) && native.drop_contract_events_enabled == self.drop_contract_events_enabled() { return Ok(()); } - self.native_traces.remove(&trace_id); + if let Some(slot) = self.native_traces.get_mut(trace_id) { + *slot = None; + } let program_cache_key = self.ensure_program_cache_key(); let trace = self.jit.trace_clone(trace_id).ok_or_else(|| { @@ -723,22 +728,22 @@ impl Vm { None }); if let Some(cached) = cached { - self.native_traces.insert( - trace_id, - NativeTrace { - _keepalive: cached.keepalive, - entry: cached.entry, - code: cached.code, - root_ip: trace.root_ip, - terminal: trace.terminal, - has_call: trace.has_call, - has_yielding_call: trace.has_yielding_call, - lowering_kind: cached.lowering_kind, - interrupt_settings, - compile_profile: cached.compile_profile, - drop_contract_events_enabled, - }, - ); + if self.native_traces.len() <= trace_id { + self.native_traces.resize_with(trace_id + 1, || None); + } + self.native_traces[trace_id] = Some(NativeTrace { + _keepalive: cached.keepalive, + entry: cached.entry, + code: cached.code, + root_ip: trace.root_ip, + terminal: trace.terminal, + has_call: trace.has_call, + has_yielding_call: trace.has_yielding_call, + lowering_kind: cached.lowering_kind, + interrupt_settings, + compile_profile: cached.compile_profile, + drop_contract_events_enabled, + }); return Ok(()); } @@ -765,27 +770,27 @@ impl Vm { } cache.entries.insert(key, cached); }); - self.native_traces.insert( - trace_id, - NativeTrace { - _keepalive: keepalive, - entry, - code, - root_ip: trace.root_ip, - terminal: trace.terminal, - has_call: trace.has_call, - has_yielding_call: trace.has_yielding_call, - lowering_kind: compiled.lowering_kind, - interrupt_settings, - compile_profile, - drop_contract_events_enabled, - }, - ); + if self.native_traces.len() <= trace_id { + self.native_traces.resize_with(trace_id + 1, || None); + } + self.native_traces[trace_id] = Some(NativeTrace { + _keepalive: keepalive, + entry, + code, + root_ip: trace.root_ip, + terminal: trace.terminal, + has_call: trace.has_call, + has_yielding_call: trace.has_yielding_call, + lowering_kind: compiled.lowering_kind, + interrupt_settings, + compile_profile, + drop_contract_events_enabled, + }); Ok(()) } pub fn jit_native_trace_count(&self) -> usize { - self.native_traces.len() + self.native_traces.iter().flatten().count() } pub fn jit_native_exec_count(&self) -> u64 { diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 19aea3fc..15e5378b 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -239,7 +239,7 @@ pub struct Vm { aot_program: Option, aot_exec_count: u64, jit: jit::TraceJitEngine, - native_traces: HashMap, + native_traces: Vec>, native_trace_exec_count: u64, jit_trace_exit_count: u64, jit_native_loop_back_count: u64, @@ -496,7 +496,7 @@ impl Vm { aot_program: None, aot_exec_count: 0, jit: jit::TraceJitEngine::new(jit_config), - native_traces: HashMap::new(), + native_traces: Vec::new(), native_trace_exec_count: 0, jit_trace_exit_count: 0, jit_native_loop_back_count: 0, From d9912118b974e6b690e5934bdc35ea04615dd50c Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 01:28:10 +0800 Subject: [PATCH 03/19] perf(jit): use dense trace entry lookup --- src/vm/jit/trace.rs | 61 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 39441bdd..243647e8 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -173,7 +173,7 @@ pub struct JitNyiDoc { pub struct TraceJitEngine { config: JitConfig, hot_counts: HashMap, - compiled_by_entry: HashMap, + compiled_by_ip: Vec>, blocked_entries: HashSet, loop_headers: Option>, traces: Vec, @@ -191,7 +191,7 @@ impl TraceJitEngine { Self { config, hot_counts: HashMap::new(), - compiled_by_entry: HashMap::new(), + compiled_by_ip: Vec::new(), blocked_entries: HashSet::new(), loop_headers: None, traces: Vec::new(), @@ -206,7 +206,7 @@ impl TraceJitEngine { pub fn set_config(&mut self, config: JitConfig) { self.config = config; self.hot_counts.clear(); - self.compiled_by_entry.clear(); + self.compiled_by_ip.clear(); self.blocked_entries.clear(); self.loop_headers = None; self.traces.clear(); @@ -230,7 +230,7 @@ impl TraceJitEngine { root_ip: ip, stack_depth, }; - if let Some(&trace_id) = self.compiled_by_entry.get(&key) { + if let Some(trace_id) = self.compiled_trace_for_key(key) { return Some(trace_id); } if self.blocked_entries.contains(&key) || !self.is_loop_header(program, ip) { @@ -279,7 +279,7 @@ impl TraceJitEngine { root_ip: ip, stack_depth, }; - if let Some(&trace_id) = self.compiled_by_entry.get(&key) { + if let Some(trace_id) = self.compiled_trace_for_key(key) { return Some(trace_id); } if self.blocked_entries.contains(&key) { @@ -309,12 +309,10 @@ impl TraceJitEngine { } pub(crate) fn compiled_trace_for_entry(&self, ip: usize, stack_depth: usize) -> Option { - self.compiled_by_entry - .get(&TraceEntryKey { - root_ip: ip, - stack_depth, - }) - .copied() + self.compiled_trace_for_key(TraceEntryKey { + root_ip: ip, + stack_depth, + }) } pub fn mark_trace_executed(&mut self, trace_id: usize) { @@ -329,7 +327,7 @@ impl TraceJitEngine { root_ip: trace.root_ip, stack_depth: trace.entry_stack_depth, }; - self.compiled_by_entry.remove(&key); + self.remove_compiled_trace(key); self.blocked_entries.insert(key); } } @@ -452,7 +450,7 @@ impl TraceJitEngine { line, result: Ok(trace_id), }); - self.compiled_by_entry.insert(key, trace_id); + self.insert_compiled_trace(key, trace_id); Some(trace_id) } Err(reason) => { @@ -490,6 +488,43 @@ impl TraceJitEngine { Ok(id) } + #[inline(always)] + fn compiled_trace_for_key(&self, key: TraceEntryKey) -> Option { + self.compiled_by_ip + .get(key.root_ip)? + .iter() + .find_map(|(stack_depth, trace_id)| { + (*stack_depth == key.stack_depth).then_some(*trace_id) + }) + } + + fn insert_compiled_trace(&mut self, key: TraceEntryKey, trace_id: usize) { + if self.compiled_by_ip.len() <= key.root_ip { + self.compiled_by_ip.resize_with(key.root_ip + 1, Vec::new); + } + let entries = &mut self.compiled_by_ip[key.root_ip]; + if let Some((_, existing_trace_id)) = entries + .iter_mut() + .find(|(stack_depth, _)| *stack_depth == key.stack_depth) + { + *existing_trace_id = trace_id; + } else { + entries.push((key.stack_depth, trace_id)); + } + } + + fn remove_compiled_trace(&mut self, key: TraceEntryKey) { + let Some(entries) = self.compiled_by_ip.get_mut(key.root_ip) else { + return; + }; + if let Some(index) = entries + .iter() + .position(|(stack_depth, _)| *stack_depth == key.stack_depth) + { + entries.swap_remove(index); + } + } + fn is_loop_header(&mut self, program: &Program, ip: usize) -> bool { if self.loop_headers.is_none() { self.loop_headers = Some(scan_loop_headers(program)); From 4438670dc24d1fd96d1e3ef7864eda18660de898 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 01:33:18 +0800 Subject: [PATCH 04/19] perf(jit): use dense loop header lookup --- src/vm/jit/trace.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 243647e8..e4c690dd 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -175,7 +175,7 @@ pub struct TraceJitEngine { hot_counts: HashMap, compiled_by_ip: Vec>, blocked_entries: HashSet, - loop_headers: Option>, + loop_headers: Option>, traces: Vec, attempts: Vec, } @@ -233,7 +233,9 @@ impl TraceJitEngine { if let Some(trace_id) = self.compiled_trace_for_key(key) { return Some(trace_id); } - if self.blocked_entries.contains(&key) || !self.is_loop_header(program, ip) { + if (!self.blocked_entries.is_empty() && self.blocked_entries.contains(&key)) + || !self.is_loop_header(program, ip) + { return None; } @@ -282,7 +284,7 @@ impl TraceJitEngine { if let Some(trace_id) = self.compiled_trace_for_key(key) { return Some(trace_id); } - if self.blocked_entries.contains(&key) { + if !self.blocked_entries.is_empty() && self.blocked_entries.contains(&key) { return None; } @@ -531,7 +533,9 @@ impl TraceJitEngine { } self.loop_headers .as_ref() - .is_some_and(|headers| headers.contains(&ip)) + .and_then(|headers| headers.get(ip)) + .copied() + .unwrap_or(false) } fn aggregate_metrics(&self, mut runtime_metrics: JitMetrics) -> JitMetrics { @@ -589,9 +593,9 @@ fn to_nyi(err: TraceRecordError) -> JitNyiReason { } } -fn scan_loop_headers(program: &Program) -> HashSet { - let mut headers = HashSet::new(); +fn scan_loop_headers(program: &Program) -> Vec { let code = &program.code; + let mut headers = vec![false; code.len()]; let mut ip = 0usize; while ip < code.len() { @@ -609,8 +613,8 @@ fn scan_loop_headers(program: &Program) -> HashSet { break; }; let target = target_u32 as usize; - if target <= instr_ip { - headers.insert(target); + if target <= instr_ip && target < headers.len() { + headers[target] = true; } } x if x == OpCode::Ldloc as u8 || x == OpCode::Stloc as u8 => { @@ -685,7 +689,7 @@ mod tests { let program = Program::new(vec![Value::Int(1)], bc.finish()); let headers = scan_loop_headers(&program); - assert!(headers.contains(&(root_ip as usize))); - assert!(!headers.contains(&(branch_ip as usize))); + assert!(headers[root_ip as usize]); + assert!(!headers[branch_ip as usize]); } } From 960fb2aee54a1f9c832cee1a7a7a9853a81ebb21 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 01:53:34 +0800 Subject: [PATCH 05/19] perf(jit): keep non-yielding host calls in native traces --- src/vm/host.rs | 75 +++++++++++++++++++++++++++++-- src/vm/jit/ir.rs | 8 ++++ src/vm/jit/native/lower.rs | 80 +++++++++++++++++++++++++++++++++ src/vm/jit/recorder.rs | 90 ++++++++++++++++++++++++++++++++++---- src/vm/jit/trace.rs | 17 +++++++ src/vm/mod.rs | 1 + src/vm/native/bridge.rs | 71 +++++++++++++++++++++++++++++- src/vm/native/codegen.rs | 15 +++++++ src/vm/native/mod.rs | 4 +- tests/jit/jit_tests.rs | 56 +++++++++++++++++++++++- 10 files changed, 400 insertions(+), 17 deletions(-) diff --git a/src/vm/host.rs b/src/vm/host.rs index c1c7cbba..5a031dcc 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -429,6 +429,7 @@ pub(super) enum VmHostFunction { StackStatic(StaticHostStackFunction), ArgsDynamic(Box), ArgsStatic(StaticHostArgsFunction), + ArgsStaticNonYielding(StaticHostArgsFunction), } pub(super) enum HostCallExecOutcome { @@ -515,6 +516,20 @@ impl Vm { index } + /// Registers a static args-only host function that always returns synchronously. + /// + /// Violating this contract by returning `Halt`, `Yield`, or `Pending` is a host error. + pub fn register_static_non_yielding_args_function( + &mut self, + function: StaticHostArgsFunction, + ) -> u16 { + let index = self.host_functions.len() as u16; + self.host_functions + .push(VmHostFunction::ArgsStaticNonYielding(function)); + self.resolved_calls_dirty = true; + index + } + pub fn bind_function(&mut self, name: impl Into, function: Box) { let name = name.into(); if let Some(builtin) = builtin_for_binding_name(&name) { @@ -650,6 +665,37 @@ impl Vm { self.resolved_calls_dirty = true; } + /// Binds a static args-only host function that always returns synchronously. + /// + /// This is equivalent to [`Vm::bind_static_args_function`] except that the VM may keep + /// native JIT traces active across the call boundary. Returning `Halt`, `Yield`, or + /// `Pending` violates the contract and is reported as a host error. + pub fn bind_static_non_yielding_args_function( + &mut self, + name: impl Into, + function: StaticHostArgsFunction, + ) { + let name = name.into(); + if let Some(builtin) = builtin_for_binding_name(&name) { + self.bind_builtin_overrideslot( + builtin.call_index(), + VmHostFunction::ArgsStaticNonYielding(function), + ); + return; + } + if let Some(&index) = self.host_function_symbols.get(&name) + && let Some(slot) = self.host_functions.get_mut(index as usize) + { + *slot = VmHostFunction::ArgsStaticNonYielding(function); + self.resolved_calls_dirty = true; + return; + } + + let index = self.register_static_non_yielding_args_function(function); + self.host_function_symbols.insert(name, index); + self.resolved_calls_dirty = true; + } + pub fn bind_builtin_override( &mut self, name: impl Into, @@ -1295,7 +1341,8 @@ impl Vm { VmHostFunction::StackDynamic(_) | VmHostFunction::StackStatic(_) | VmHostFunction::ArgsDynamic(_) - | VmHostFunction::ArgsStatic(_) => unreachable!(), + | VmHostFunction::ArgsStatic(_) + | VmHostFunction::ArgsStaticNonYielding(_) => unreachable!(), } }; self.call_depth = self.call_depth.saturating_sub(1); @@ -1350,7 +1397,9 @@ impl Vm { .ok_or(VmError::InvalidCall(resolved_index))?; Ok(matches!( function, - VmHostFunction::ArgsDynamic(_) | VmHostFunction::ArgsStatic(_) + VmHostFunction::ArgsDynamic(_) + | VmHostFunction::ArgsStatic(_) + | VmHostFunction::ArgsStaticNonYielding(_) )) } @@ -1385,7 +1434,8 @@ impl Vm { .ok_or(VmError::InvalidCall(resolved_index))?; match function { VmHostFunction::ArgsDynamic(function) => function.call(args), - VmHostFunction::ArgsStatic(function) => function(args), + VmHostFunction::ArgsStatic(function) + | VmHostFunction::ArgsStaticNonYielding(function) => function(args), VmHostFunction::Dynamic(_) | VmHostFunction::Static(_) | VmHostFunction::StackDynamic(_) @@ -1446,7 +1496,8 @@ impl Vm { VmHostFunction::Dynamic(_) | VmHostFunction::Static(_) | VmHostFunction::ArgsDynamic(_) - | VmHostFunction::ArgsStatic(_) => unreachable!(), + | VmHostFunction::ArgsStatic(_) + | VmHostFunction::ArgsStaticNonYielding(_) => unreachable!(), } }; self.call_depth = self.call_depth.saturating_sub(1); @@ -1588,6 +1639,22 @@ impl Vm { Ok(()) } + pub(super) fn sync_jit_non_yielding_host_imports(&mut self) { + let imports = self + .resolved_calls + .iter() + .map(|&slot| { + matches!( + self.host_functions.get(usize::from(slot)), + Some(VmHostFunction::ArgsStaticNonYielding(_)) + ) + }) + .collect(); + if self.jit.set_non_yielding_host_imports(imports) { + self.native_traces.clear(); + } + } + pub(super) fn resolve_call_target(&mut self, index: u16, argc: u8) -> VmResult { if self.program.imports.is_empty() { return Ok(index); diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index 429e8e5a..9e374385 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -210,6 +210,10 @@ pub(crate) enum SsaInstKind { MapIterTakeValue { slot: SsaValueId, }, + HostCall { + import: u16, + args: Vec, + }, IntNeg { input: SsaValueId, }, @@ -349,6 +353,7 @@ impl SsaInstKind { fn inputs(&self) -> Vec { match self { Self::Constant(_) => Vec::new(), + Self::HostCall { args, .. } => args.clone(), Self::UnboxInt { input } | Self::UnboxFloat { input } | Self::UnboxBool { input } @@ -1011,6 +1016,9 @@ fn render_inst_kind(kind: &SsaInstKind) -> String { SsaInstKind::MapIterNext { slot } => format!("map_iter_next {slot}"), SsaInstKind::MapIterTakeKey { slot } => format!("map_iter_take_key {slot}"), SsaInstKind::MapIterTakeValue { slot } => format!("map_iter_take_value {slot}"), + SsaInstKind::HostCall { import, args } => { + format!("host_call {import}({})", render_value_list(args)) + } SsaInstKind::IntNeg { input } => format!("ineg {input}"), SsaInstKind::IntAdd { lhs, rhs } => format!("iadd {lhs}, {rhs}"), SsaInstKind::IntAddImm { lhs, imm } => format!("iadd_imm {lhs}, {imm}"), diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index b72ac48d..44d06c7b 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -19,6 +19,7 @@ use crate::vm::native::{ init_null_value_slot_entry_address, jump_with_status, map_get_entry_address, map_has_entry_address, map_iter_next_entry_address, map_iter_next_signature, map_iter_take_key_entry_address, map_iter_take_signature, map_iter_take_value_entry_address, + non_yielding_host_call_entry_address, non_yielding_host_call_signature, pack_shared_signature, restore_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, sparse_restore_exit_signature, @@ -97,6 +98,7 @@ fn try_compile_ssa_trace( let mut ctx = module.make_context(); ctx.func.signature = entry_signature(pointer_type, call_conv); let clone_value_sig = clone_value_signature(pointer_type, call_conv); + let non_yielding_host_call_sig = non_yielding_host_call_signature(pointer_type, call_conv); let value_slot_sig = value_slot_signature(pointer_type, call_conv); let box_heap_value_sig = box_heap_value_signature(pointer_type, call_conv); let alloc_buffer_sig = alloc_buffer_signature(pointer_type, call_conv); @@ -159,6 +161,7 @@ fn try_compile_ssa_trace( }; let deopt_refs = SsaDeoptHelperRefs { clone_value_ref: b.import_signature(clone_value_sig), + non_yielding_host_call_ref: b.import_signature(non_yielding_host_call_sig), init_null_slot_ref: b.import_signature(value_slot_sig.clone()), clear_value_slot_ref: b.import_signature(value_slot_sig), box_heap_value_ref: b.import_signature(box_heap_value_sig), @@ -174,6 +177,7 @@ fn try_compile_ssa_trace( }; let deopt_addrs = SsaDeoptHelperAddrs { clone_value: clone_value_to_slot_entry_address(), + non_yielding_host_call: non_yielding_host_call_entry_address(), init_null_slot: init_null_value_slot_entry_address(), clear_value_slot: clear_value_slot_entry_address(), box_heap_value: write_heap_value_to_slot_entry_address(), @@ -420,6 +424,7 @@ struct SsaExitLowering { #[derive(Clone, Copy)] struct SsaDeoptHelperRefs { clone_value_ref: cranelift_codegen::ir::SigRef, + non_yielding_host_call_ref: cranelift_codegen::ir::SigRef, init_null_slot_ref: cranelift_codegen::ir::SigRef, clear_value_slot_ref: cranelift_codegen::ir::SigRef, box_heap_value_ref: cranelift_codegen::ir::SigRef, @@ -437,6 +442,7 @@ struct SsaDeoptHelperRefs { #[derive(Clone, Copy)] struct SsaDeoptHelperAddrs { clone_value: usize, + non_yielding_host_call: usize, init_null_slot: usize, clear_value_slot: usize, box_heap_value: usize, @@ -470,6 +476,7 @@ struct SsaStringHelperAddrs { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] enum SsaTempValueSlotKey { Output(SsaValueId), + HostArgs(SsaValueId), MapKeyBox(SsaValueId), MutationArgBox(SsaValueId, u8), } @@ -598,6 +605,7 @@ fn ssa_trace_supported(ssa: &SsaTrace) -> bool { | SsaInstKind::IntCmpLtImm { .. } | SsaInstKind::IntCmpGt { .. } | SsaInstKind::IntCmpGtImm { .. } + | SsaInstKind::HostCall { .. } ) { return false; } @@ -681,6 +689,22 @@ fn allocate_owned_value_temps( ordered.push(slot); slots.insert(SsaTempValueSlotKey::Output(output.id), slot); } + if let SsaInstKind::HostCall { args, .. } = &inst.kind { + let arg_bytes = usize::try_from(value_size) + .ok() + .and_then(|value_size| value_size.checked_mul(args.len().max(1))) + .and_then(|bytes| u32::try_from(bytes).ok()) + .ok_or_else(|| { + VmError::JitNative("SSA host-call argument storage too large".to_string()) + })?; + let align_shift = std::mem::align_of::().trailing_zeros() as u8; + let args_slot = b.create_sized_stack_slot(StackSlotData::new( + StackSlotKind::ExplicitSlot, + arg_bytes, + align_shift, + )); + slots.insert(SsaTempValueSlotKey::HostArgs(output.id), args_slot); + } if matches!( inst.kind, SsaInstKind::MapGet { .. } | SsaInstKind::MapHas { .. } @@ -726,6 +750,7 @@ fn ssa_inst_requires_owned_value_slot(kind: &SsaInstKind) -> bool { | SsaInstKind::BytesToArrayU8 { .. } | SsaInstKind::StringConcat { .. } | SsaInstKind::BytesConcat { .. } + | SsaInstKind::HostCall { .. } ) } @@ -2316,6 +2341,61 @@ fn lower_ssa_inst( b.switch_to_block(cont); out } + SsaInstKind::HostCall { import, args } => { + let out = owned_value_temp_slot_addr( + b, + pointer_type, + owned_value_temps, + SsaTempValueSlotKey::Output(output.id), + )?; + let arg_values = owned_value_temp_slot_addr( + b, + pointer_type, + owned_value_temps, + SsaTempValueSlotKey::HostArgs(output.id), + )?; + for (index, arg) in args.iter().copied().enumerate() { + let repr = *value_reprs.get(&arg).ok_or_else(|| { + VmError::JitNative("SSA host-call argument representation missing".to_string()) + })?; + let value = values[&arg]; + let index_value = b.ins().iconst( + pointer_type, + i64::try_from(index).map_err(|_| { + VmError::JitNative("SSA host-call argument index out of range".to_string()) + })?, + ); + let addr = + ssa_value_addr(b, pointer_type, arg_values, index_value, layout.value.size); + match repr { + SsaValueRepr::Tagged => ssa_copy_value_bytes(b, value, addr, layout.value.size), + SsaValueRepr::I64 => ssa_store_int_in_value(b, layout.value, addr, value), + SsaValueRepr::F64 => ssa_store_float_in_value(b, layout.value, addr, value), + SsaValueRepr::Bool => ssa_store_bool_in_value(b, layout.value, addr, value), + SsaValueRepr::HeapPtr(tag) => { + let tag = ssa_heap_tag(layout.value, tag)?; + ssa_store_heap_ptr_in_value(b, layout.value, addr, tag, value); + } + } + } + clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; + let import = b.ins().iconst(pointer_type, i64::from(*import)); + let argc = b.ins().iconst( + pointer_type, + i64::try_from(args.len()).map_err(|_| { + VmError::JitNative("SSA host-call argument count out of range".to_string()) + })?, + ); + ssa_call_status_helper( + b, + exit_block, + pointer_type, + helper_refs.non_yielding_host_call_ref, + helper_addrs.non_yielding_host_call, + &[vm_ptr, import, arg_values, argc, out], + )?; + out + } SsaInstKind::IntNeg { input } => b.ins().ineg(values[input]), SsaInstKind::IntAdd { lhs, rhs } => b.ins().iadd(values[lhs], values[rhs]), SsaInstKind::IntAddImm { lhs, imm } => b.ins().iadd_imm(values[lhs], *imm), diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index fe35ae31..75ec511b 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -353,6 +353,7 @@ enum DecodedOp { }, Call { ip: usize, + index: u16, builtin: Option, argc: u8, yields: bool, @@ -636,6 +637,7 @@ impl<'a> TraceCursor<'a> { } DecodedOp::Call { ip: instr_ip, + index, builtin: Some(builtin), argc, yields: false, @@ -643,6 +645,7 @@ impl<'a> TraceCursor<'a> { } else { DecodedOp::Call { ip: instr_ip, + index, builtin: None, argc, yields: true, @@ -661,9 +664,15 @@ pub(crate) fn record_trace( root_ip: usize, entry_stack_depth: usize, max_trace_len: usize, + non_yielding_host_imports: &[bool], ) -> Result { - let loop_header_plan = - infer_loop_header_plan(program, root_ip, entry_stack_depth, max_trace_len)?; + let loop_header_plan = infer_loop_header_plan( + program, + root_ip, + entry_stack_depth, + max_trace_len, + non_yielding_host_imports, + )?; let mut builder = SsaTraceBuilder::new(root_ip, entry_stack_depth); let entry = builder.entry(); @@ -1096,6 +1105,7 @@ pub(crate) fn record_trace( } DecodedOp::Call { ip, + index, builtin, argc, yields, @@ -1133,6 +1143,44 @@ pub(crate) fn record_trace( continue; } } + if builtin.is_none() + && non_yielding_host_imports + .get(usize::from(index)) + .copied() + .unwrap_or(false) + { + let args = call_arg_slice(&frame.stack, usize::from(argc))? + .iter() + .map(|arg| arg.value.id) + .collect::>(); + for _ in 0..argc { + let _ = frame.pop()?; + } + let return_type = program + .imports + .get(usize::from(index)) + .map(|import| import.return_type) + .unwrap_or(ValueType::Unknown); + let value = builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::HostCall { + import: index, + args, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + frame.push(SymbolicValue { + value, + info: ValueInfo::tagged_typed(return_type), + }); + has_call = true; + op_names.push("host_call".to_string()); + has_useful_native_computation = true; + continue; + } if !has_useful_native_computation { return Err(TraceRecordError::UnsupportedTrace( "zero-benefit call-boundary trace".to_string(), @@ -1176,6 +1224,7 @@ fn infer_loop_header_plan( root_ip: usize, entry_stack_depth: usize, max_trace_len: usize, + non_yielding_host_imports: &[bool], ) -> Result, TraceRecordError> { let mut cursor = TraceCursor::new(program, root_ip, max_trace_len); let mut frame = AnalysisFrame::new(program, entry_stack_depth); @@ -1315,6 +1364,26 @@ fn infer_loop_header_plan( }; let _ = analyze_specialized_builtin_call(&mut frame, kind)?; } + DecodedOp::Call { + index, + builtin: None, + argc, + .. + } if non_yielding_host_imports + .get(usize::from(index)) + .copied() + .unwrap_or(false) => + { + for _ in 0..argc { + let _ = frame.pop()?; + } + let return_type = program + .imports + .get(usize::from(index)) + .map(|import| import.return_type) + .unwrap_or(ValueType::Unknown); + frame.push(ValueInfo::tagged_typed(return_type)); + } DecodedOp::Call { .. } => return Ok(None), DecodedOp::Brfalse { target, @@ -4101,7 +4170,7 @@ mod tests { bc.ret(); let program = Program::new(vec![Value::Int(1)], bc.finish()).with_local_count(1); - let recorded = record_trace(&program, 0, 0, 32).expect("recorded trace"); + let recorded = record_trace(&program, 0, 0, 32, &[]).expect("recorded trace"); assert_eq!(recorded.terminal, JitTraceTerminal::Halt); assert!(recorded.op_names.iter().any(|name| name == "iadd_imm")); assert!(matches!( @@ -4121,7 +4190,7 @@ mod tests { bc.ret(); let program = Program::new(Vec::new(), bc.finish()).with_local_count(2); - let recorded = record_trace(&program, 0, 0, 16).expect("recorded trace"); + let recorded = record_trace(&program, 0, 0, 16, &[]).expect("recorded trace"); assert_eq!(recorded.ssa.exits[0].dirty_locals, vec![false, false]); assert!(recorded.ssa.render_text().contains("dirty_locals=[]")); @@ -4135,7 +4204,7 @@ mod tests { bc.ret(); let program = Program::new(vec![Value::Int(7)], bc.finish()).with_local_count(4); - let recorded = record_trace(&program, 0, 0, 16).expect("recorded trace"); + let recorded = record_trace(&program, 0, 0, 16, &[]).expect("recorded trace"); assert_eq!( recorded.ssa.exits[0].dirty_locals, @@ -4162,7 +4231,7 @@ mod tests { patch_branch_target(&mut code, guard_ip, exit_ip); let program = Program::new(vec![Value::Int(5), Value::Int(10)], code).with_local_count(2); - let recorded = record_trace(&program, 0, 0, 32).expect("recorded trace"); + let recorded = record_trace(&program, 0, 0, 32, &[]).expect("recorded trace"); assert_eq!(recorded.ssa.exits.len(), 2); assert_eq!(recorded.ssa.exits[0].dirty_locals, vec![false, true]); @@ -4192,7 +4261,8 @@ mod tests { let program = Program::new(vec![Value::Int(0), Value::Int(1), Value::Int(4)], code) .with_local_count(1); - let recorded = record_trace(&program, root_ip as usize, 0, 64).expect("recorded trace"); + let recorded = + record_trace(&program, root_ip as usize, 0, 64, &[]).expect("recorded trace"); assert_eq!(recorded.terminal, JitTraceTerminal::BranchExit); assert!(recorded.op_names.iter().any(|name| name == "loop_if_false")); assert_eq!(recorded.ssa.blocks.len(), 2); @@ -4232,7 +4302,8 @@ mod tests { let program = Program::new(vec![Value::Int(1), Value::Int(10), Value::Int(3)], code) .with_local_count(2); - let recorded = record_trace(&program, root_ip as usize, 0, 128).expect("recorded trace"); + let recorded = + record_trace(&program, root_ip as usize, 0, 128, &[]).expect("recorded trace"); assert_eq!(recorded.terminal, JitTraceTerminal::BranchExit); assert_eq!(recorded.ssa.exits.len(), 2); for exit in &recorded.ssa.exits { @@ -4271,7 +4342,8 @@ mod tests { let program = Program::new(vec![Value::Int(0), Value::Int(6), Value::Int(1)], code) .with_local_count(2); - let recorded = record_trace(&program, root_ip as usize, 0, 128).expect("recorded trace"); + let recorded = + record_trace(&program, root_ip as usize, 0, 128, &[]).expect("recorded trace"); assert_eq!(recorded.terminal, JitTraceTerminal::LoopBack); assert!(recorded.op_names.iter().any(|name| name == "guard_false")); assert!(recorded.op_names.iter().any(|name| name == "jump_root")); diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index e4c690dd..ca626c35 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -176,6 +176,7 @@ pub struct TraceJitEngine { compiled_by_ip: Vec>, blocked_entries: HashSet, loop_headers: Option>, + non_yielding_host_imports: Vec, traces: Vec, attempts: Vec, } @@ -194,6 +195,7 @@ impl TraceJitEngine { compiled_by_ip: Vec::new(), blocked_entries: HashSet::new(), loop_headers: None, + non_yielding_host_imports: Vec::new(), traces: Vec::new(), attempts: Vec::new(), } @@ -213,6 +215,20 @@ impl TraceJitEngine { self.attempts.clear(); } + pub(crate) fn set_non_yielding_host_imports(&mut self, imports: Vec) -> bool { + if self.non_yielding_host_imports == imports { + return false; + } + self.non_yielding_host_imports = imports; + self.hot_counts.clear(); + self.compiled_by_ip.clear(); + self.blocked_entries.clear(); + self.loop_headers = None; + self.traces.clear(); + self.attempts.clear(); + true + } + pub fn observe_hot_ip(&mut self, ip: usize, program: &Program) -> Option { self.observe_hot_entry(ip, 0, program) } @@ -478,6 +494,7 @@ impl TraceJitEngine { key.root_ip, key.stack_depth, self.config.max_trace_len, + &self.non_yielding_host_imports, ) .map_err(to_nyi)?; let id = self.traces.len(); diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 15e5378b..e9379f35 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -1352,6 +1352,7 @@ impl Vm { allow_jit: bool, ) -> VmResult { self.ensure_call_bindings()?; + self.sync_jit_non_yielding_host_imports(); if let Some(waiting) = self.waiting_host_op { self.last_yield_reason = None; let status = VmStatus::Waiting(waiting.op_id); diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index e357b0fd..da94f4ce 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -1,7 +1,10 @@ #![allow(dead_code)] use crate::builtins::BuiltinFunction; use crate::bytecode::{Value, ValueType, VmMap}; -use crate::vm::{HostCallExecOutcome, NumericValue, Vm, VmError, VmResult, logical_shr_i64}; +use crate::vm::{ + CallOutcome, CallReturn, HostCallExecOutcome, NumericValue, Vm, VmError, VmHostFunction, + VmResult, logical_shr_i64, +}; use std::sync::Arc; use std::sync::{Mutex, OnceLock}; @@ -277,6 +280,10 @@ pub(crate) fn array_push_entry_address() -> usize { pd_vm_native_array_push as *const () as usize } +pub(crate) fn non_yielding_host_call_entry_address() -> usize { + pd_vm_native_non_yielding_host_call as *const () as usize +} + pub(crate) fn helper_entry_offset() -> i32 { i32::try_from(std::mem::offset_of!(Vm, native_helper_fn)) .expect("Vm::native_helper_fn offset must fit i32") @@ -802,6 +809,68 @@ pub(crate) extern "C" fn pd_vm_native_array_push( } } +pub(crate) extern "C" fn pd_vm_native_non_yielding_host_call( + vm: *mut Vm, + import: usize, + args: *const Value, + argc: usize, + out: *mut Value, +) -> i32 { + const MAX_ARGS: usize = u8::MAX as usize; + let Some(vm) = (unsafe { vm.as_mut() }) else { + store_bridge_error(VmError::JitNative( + "native host-call helper received null vm pointer".to_string(), + )); + return STATUS_ERROR; + }; + if out.is_null() || (argc != 0 && args.is_null()) || argc > MAX_ARGS { + store_bridge_error(VmError::JitNative( + "native host-call helper received invalid argument storage".to_string(), + )); + return STATUS_ERROR; + } + let Some(&resolved) = vm.resolved_calls.get(import) else { + store_bridge_error(VmError::InvalidCall(import as u16)); + return STATUS_ERROR; + }; + let Some(VmHostFunction::ArgsStaticNonYielding(function)) = + vm.host_functions.get(usize::from(resolved)) + else { + store_bridge_error(VmError::JitNative( + "native host-call binding changed after trace compilation".to_string(), + )); + return STATUS_ERROR; + }; + + let args = unsafe { std::slice::from_raw_parts(args, argc) }; + vm.call_depth = vm.call_depth.saturating_add(1); + let outcome = function(args); + vm.call_depth = vm.call_depth.saturating_sub(1); + match outcome { + Ok(CallOutcome::Return(CallReturn::One(value))) => { + unsafe { std::ptr::write(out, value) }; + STATUS_CONTINUE + } + Ok(CallOutcome::Return(CallReturn::None)) => { + store_bridge_error(VmError::HostError( + "non-yielding native host call returned no value".to_string(), + )); + STATUS_ERROR + } + Ok(CallOutcome::Halt | CallOutcome::Yield | CallOutcome::Pending(_)) => { + store_bridge_error(VmError::HostError( + "non-yielding native host call violated its synchronous return contract" + .to_string(), + )); + STATUS_ERROR + } + Err(err) => { + store_bridge_error(err); + STATUS_ERROR + } + } +} + pub(crate) extern "C" fn pd_vm_native_step(vm: *mut Vm, op: i64, a: i64, b: i64, c: i64) -> i32 { run_step(vm, "step", |vm| { if op == OP_BUILTIN_CALL { diff --git a/src/vm/native/codegen.rs b/src/vm/native/codegen.rs index c366ded7..c06d18ee 100644 --- a/src/vm/native/codegen.rs +++ b/src/vm/native/codegen.rs @@ -134,6 +134,21 @@ pub(crate) fn clone_value_signature( sig } +#[cfg(feature = "cranelift-jit")] +pub(crate) fn non_yielding_host_call_signature( + pointer_type: cranelift_codegen::ir::Type, + call_conv: cranelift_codegen::isa::CallConv, +) -> Signature { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.returns.push(AbiParam::new(types::I32)); + sig +} + #[cfg(feature = "cranelift-jit")] pub(crate) fn collection_predicate_signature( pointer_type: cranelift_codegen::ir::Type, diff --git a/src/vm/native/mod.rs b/src/vm/native/mod.rs index 0cba7c39..7535c19a 100644 --- a/src/vm/native/mod.rs +++ b/src/vm/native/mod.rs @@ -15,7 +15,8 @@ pub(crate) use bridge::{ helper_entry_address, helper_entry_offset, init_null_value_slot_entry_address, interrupt_helper_entry_address, interrupt_helper_entry_offset, map_get_entry_address, map_has_entry_address, map_iter_next_entry_address, map_iter_take_key_entry_address, - map_iter_take_value_entry_address, restore_exit_state_entry_address, + map_iter_take_value_entry_address, non_yielding_host_call_entry_address, + restore_exit_state_entry_address, restore_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, store_bridge_error, string_contains_entry_address, string_lower_ascii_entry_address, @@ -28,6 +29,7 @@ pub(crate) use codegen::{ collection_get_signature, collection_mutation_signature, collection_predicate_signature, copy_bytes_signature, entry_signature, free_buffer_signature, helper_signature, jump_with_status, map_iter_next_signature, map_iter_take_signature, pack_shared_signature, + non_yielding_host_call_signature, restore_exit_signature, sparse_restore_exit_signature, string_binary_transform_signature, string_contains_signature, string_replace_signature, string_unary_transform_signature, value_eq_signature, value_slot_signature, diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index c40c026d..c07dc983 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -1,6 +1,6 @@ use vm::{ - BytecodeBuilder, CallOutcome, HostFunction, JitConfig, JitTraceTerminal, OpCode, Program, - Value, ValueType, Vm, VmStatus, VmYieldReason, builtin_call_index, compile_source, + BytecodeBuilder, CallOutcome, CallReturn, HostFunction, JitConfig, JitTraceTerminal, OpCode, + Program, Value, ValueType, Vm, VmStatus, VmYieldReason, builtin_call_index, compile_source, disassemble_program, }; @@ -1722,6 +1722,58 @@ fn trace_jit_supports_host_call_loops_with_branch_exit_traces() { } } +fn increment_non_yielding(args: &[Value]) -> Result { + let Value::Int(value) = args[0] else { + return Err(vm::VmError::TypeMismatch("int")); + }; + Ok(CallOutcome::Return(CallReturn::one(Value::Int(value + 1)))) +} + +#[test] +fn trace_jit_keeps_non_yielding_static_args_calls_inside_loop_trace() { + if !native_jit_supported() { + return; + } + let compiled = compile_source( + r#" + fn increment(value) -> int; + let mut i = 0; + while i < 100 { + i = increment(i); + } + i; + "#, + ) + .expect("compile should succeed"); + let mut vm = Vm::new(compiled.program); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + vm.bind_static_non_yielding_args_function("increment", increment_non_yielding); + + let status = vm.run(); + assert!( + status.is_ok(), + "vm should run: {status:?}\n{}", + vm.dump_jit_info() + ); + assert_eq!(status.unwrap(), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(100)]); + let snapshot = vm.jit_snapshot(); + assert!( + snapshot.traces.iter().any(|trace| { + trace.terminal == JitTraceTerminal::LoopBack + && trace.op_names().iter().any(|op| op == "host_call") + && trace.ssa_text().contains("host_call") + }), + "non-yielding call should remain inside a loop-back trace, dump:\n{}", + vm.dump_jit_info() + ); + assert!(vm.jit_native_exec_count() > 0); +} + #[test] fn trace_jit_sparse_exit_preserves_clean_scalar_and_heap_locals() { if !native_jit_supported() { From 4fcd2df05ae1ba85f4ca3f6e07191704e4e8cdec Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 01:58:58 +0800 Subject: [PATCH 06/19] perf(jit): link hot non-yielding side exits --- src/vm/jit/native/lower.rs | 2 +- src/vm/jit/runtime.rs | 18 ++++++------------ src/vm/jit/trace.rs | 9 ++++++--- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 44d06c7b..7f77715b 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -191,7 +191,7 @@ fn try_compile_ssa_trace( sparse_restore_exit: restore_sparse_exit_state_entry_address(), resume_linked_trace: resume_linked_trace_entry_address(), }; - let allow_exit_link_handoff = !trace.has_call && !trace.has_yielding_call; + let allow_exit_link_handoff = !trace.has_yielding_call; let mut value_reprs = HashMap::new(); for block in &ssa.blocks { diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index f379c049..a5866abe 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -252,7 +252,7 @@ impl Vm { return Err(err); } - let (mut entry, mut root_ip, mut terminal, mut has_call, mut has_yielding_call) = + let (mut entry, mut root_ip, mut terminal, _, mut has_yielding_call) = self.native_trace_state(current_trace_id)?; loop { @@ -281,7 +281,7 @@ impl Vm { } return Err(err); } - (entry, root_ip, terminal, has_call, has_yielding_call) = + (entry, root_ip, terminal, _, has_yielding_call) = self.native_trace_state(current_trace_id)?; continue; } @@ -289,9 +289,6 @@ impl Vm { } native::STATUS_TRACE_EXIT => { self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1); - if has_call { - return Ok(native::STATUS_LINKED_CONTINUE); - } if !has_yielding_call && terminal == JitTraceTerminal::LoopBack && self.ip == root_ip @@ -325,7 +322,7 @@ impl Vm { } return Err(err); } - (entry, root_ip, terminal, has_call, has_yielding_call) = + (entry, root_ip, terminal, _, has_yielding_call) = self.native_trace_state(current_trace_id)?; continue; } @@ -499,7 +496,7 @@ impl Vm { } return Err(err); } - let (mut entry, mut root_ip, mut terminal, mut has_call, mut has_yielding_call) = + let (mut entry, mut root_ip, mut terminal, _, mut has_yielding_call) = self.native_trace_state(current_trace_id)?; loop { native::clear_bridge_error(); @@ -526,7 +523,7 @@ impl Vm { } return Err(err); } - (entry, root_ip, terminal, has_call, has_yielding_call) = + (entry, root_ip, terminal, _, has_yielding_call) = self.native_trace_state(current_trace_id)?; continue; } @@ -534,9 +531,6 @@ impl Vm { } native::STATUS_TRACE_EXIT => { self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1); - if has_call { - return Ok(ExecOutcome::Continue); - } // Fast path: if this trace looped back to its own root and cannot yield via host // calls, keep executing in native mode without bouncing through the interpreter. if !has_yielding_call @@ -572,7 +566,7 @@ impl Vm { } return Err(err); } - (entry, root_ip, terminal, has_call, has_yielding_call) = + (entry, root_ip, terminal, _, has_yielding_call) = self.native_trace_state(current_trace_id)?; continue; } diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index ca626c35..3ca42a55 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -290,9 +290,6 @@ impl TraceJitEngine { if !self.config.enabled || !native_jit_supported() { return None; } - if self.config.hot_loop_threshold > 1 { - return None; - } let key = TraceEntryKey { root_ip: ip, stack_depth, @@ -304,6 +301,12 @@ impl TraceJitEngine { return None; } + let count = self.hot_counts.entry(key).or_insert(0); + *count = count.saturating_add(1); + if *count < self.config.hot_loop_threshold { + return None; + } + let line = program .debug .as_ref() From c509091c2199dc23e8115a805905d4fd458d41c8 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:06:30 +0800 Subject: [PATCH 07/19] perf(jit): inline scalar-only side-exit restores --- src/vm/jit/native/lower.rs | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 7f77715b..278e0696 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -252,6 +252,7 @@ fn try_compile_ssa_trace( pointer_type, layout, offsets, + entry_stack_depth: ssa.entry_stack_depth, heap_refs, heap_addrs, string_refs, @@ -494,6 +495,7 @@ struct SsaLowerCtx<'a> { pointer_type: cranelift_codegen::ir::Type, layout: crate::vm::native::NativeStackLayout, offsets: ResolvedOffsets, + entry_stack_depth: usize, heap_refs: HeapIntrinsicRefs, heap_addrs: HeapIntrinsicAddrs, string_refs: SsaStringHelperRefs, @@ -982,6 +984,7 @@ fn lower_ssa_inst( owned_value_temps, value_reprs, tagged_constant_addrs, + .. } = ctx; let Some(output) = inst.output else { return Err(VmError::JitNative( @@ -2757,6 +2760,8 @@ fn lower_ssa_exit_block( exit_block, pointer_type, layout, + offsets, + entry_stack_depth, helper_refs: deopt_refs, helper_addrs: deopt_addrs, .. @@ -2783,6 +2788,66 @@ fn lower_ssa_exit_block( deopt_addrs, }; + let inline_scalar_restore = entry_stack_depth == 0 + && exit.stack.is_empty() + && exit + .locals + .iter() + .zip(&exit.dirty_locals) + .all(|(materialization, dirty)| { + !*dirty + || matches!( + materialization, + SsaMaterialization::BoxInt(_) + | SsaMaterialization::BoxBool(_) + | SsaMaterialization::BoxFloat(_) + ) + }); + if inline_scalar_restore { + let vm_locals_ptr = b + .ins() + .load(pointer_type, MemFlags::new(), vm_ptr, offsets.locals_ptr); + for (local_index, materialization) in + exit.locals + .iter() + .enumerate() + .filter_map(|(local_index, materialization)| { + exit.dirty_locals[local_index].then_some((local_index, materialization)) + }) + { + let index = b.ins().iconst( + pointer_type, + i64::try_from(local_index).map_err(|_| { + VmError::JitNative("SSA dirty local index out of range".to_string()) + })?, + ); + let dst_addr = ssa_value_addr(b, pointer_type, vm_locals_ptr, index, layout.value.size); + clear_owned_value_temp_slot(b, pointer_type, deopt_refs, deopt_addrs, dst_addr)?; + ssa_materialize_slot(b, materialize_ctx, materialization, dst_addr, "local")?; + } + let ip_val = b.ins().iconst( + pointer_type, + i64::try_from(exit.exit_ip) + .map_err(|_| VmError::JitNative("SSA exit ip out of range".to_string()))?, + ); + b.ins() + .store(MemFlags::new(), ip_val, vm_ptr, offsets.vm_ip); + let status = if halted { + b.ins().iconst(types::I32, STATUS_HALTED as i64) + } else if allow_link_handoff { + let helper_ptr = + iconst_ptr_from_addr(b, pointer_type, deopt_addrs.resume_linked_trace)?; + let call = + b.ins() + .call_indirect(deopt_refs.resume_linked_trace_ref, helper_ptr, &[vm_ptr]); + b.inst_results(call)[0] + } else { + b.ins().iconst(types::I32, STATUS_TRACE_EXIT as i64) + }; + jump_with_status(b, exit_block, status); + return Ok(()); + } + let stack_ptr = ssa_alloc_value_buffer(b, pointer_type, exit.stack.len(), layout.value.size)?; for (stack_index, materialization) in exit.stack.iter().enumerate() { let dst_addr = ssa_value_buffer_slot_addr( From 7e6e7dd920bd73dcd159a30151253b275b8f9391 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:11:57 +0800 Subject: [PATCH 08/19] perf(jit): dispatch linked traces from the outer loop --- src/vm/jit/native/lower.rs | 4 +++- src/vm/jit/runtime.rs | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 278e0696..5cd3e64e 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -191,7 +191,9 @@ fn try_compile_ssa_trace( sparse_restore_exit: restore_sparse_exit_state_entry_address(), resume_linked_trace: resume_linked_trace_entry_address(), }; - let allow_exit_link_handoff = !trace.has_yielding_call; + // The outer native dispatch loop links trace exits directly. Re-entering it through the + // native bridge adds a redundant depth check to every subsequent linked trace. + let allow_exit_link_handoff = false; let mut value_reprs = HashMap::new(); for block in &ssa.blocks { diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index a5866abe..10b70334 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -511,6 +511,7 @@ impl Vm { self.jit.compiled_trace_for_entry(self.ip, self.stack.len()) && next_trace_id != current_trace_id { + self.record_jit_link_handoff(); current_trace_id = next_trace_id; if let Err(err) = self.ensure_native_trace( current_trace_id, @@ -554,6 +555,7 @@ impl Vm { if let Some(next_trace_id) = next_trace_id && next_trace_id != current_trace_id { + self.record_jit_link_handoff(); current_trace_id = next_trace_id; if let Err(err) = self.ensure_native_trace( current_trace_id, From cf05a12146f4dd2405c01e409cf7e33ec0767107 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:15:39 +0800 Subject: [PATCH 09/19] perf(jit): move owned exit values into locals --- src/vm/jit/native/lower.rs | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 5cd3e64e..0fe653db 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -2764,6 +2764,7 @@ fn lower_ssa_exit_block( layout, offsets, entry_stack_depth, + owned_value_temps, helper_refs: deopt_refs, helper_addrs: deopt_addrs, .. @@ -2790,22 +2791,31 @@ fn lower_ssa_exit_block( deopt_addrs, }; - let inline_scalar_restore = entry_stack_depth == 0 + let mut moved_owned_values = BTreeSet::new(); + let inline_owned_restore = entry_stack_depth == 0 && exit.stack.is_empty() && exit .locals .iter() .zip(&exit.dirty_locals) .all(|(materialization, dirty)| { - !*dirty - || matches!( - materialization, - SsaMaterialization::BoxInt(_) - | SsaMaterialization::BoxBool(_) - | SsaMaterialization::BoxFloat(_) - ) + if !*dirty { + return true; + } + match materialization { + SsaMaterialization::BoxInt(_) + | SsaMaterialization::BoxBool(_) + | SsaMaterialization::BoxFloat(_) => true, + SsaMaterialization::Value(value) => { + owned_value_temps + .slots + .contains_key(&SsaTempValueSlotKey::Output(*value)) + && moved_owned_values.insert(*value) + } + SsaMaterialization::BoxHeapPtr { .. } => false, + } }); - if inline_scalar_restore { + if inline_owned_restore { let vm_locals_ptr = b .ins() .load(pointer_type, MemFlags::new(), vm_ptr, offsets.locals_ptr); @@ -2825,7 +2835,15 @@ fn lower_ssa_exit_block( ); let dst_addr = ssa_value_addr(b, pointer_type, vm_locals_ptr, index, layout.value.size); clear_owned_value_temp_slot(b, pointer_type, deopt_refs, deopt_addrs, dst_addr)?; - ssa_materialize_slot(b, materialize_ctx, materialization, dst_addr, "local")?; + if let SsaMaterialization::Value(value) = materialization { + let src = *exit_values.get(value).ok_or_else(|| { + VmError::JitNative("SSA exit tagged local value missing".to_string()) + })?; + ssa_copy_value_bytes(b, src, dst_addr, layout.value.size); + ssa_store_tag(b, layout.value, src, layout.value.null_tag); + } else { + ssa_materialize_slot(b, materialize_ctx, materialization, dst_addr, "local")?; + } } let ip_val = b.ins().iconst( pointer_type, From f636bf841be2e49c89bf79e4f3453cd8793ecaa1 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:19:08 +0800 Subject: [PATCH 10/19] perf(jit): initialize owned temps inline --- src/vm/jit/native/lower.rs | 59 +++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 0fe653db..659bb154 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -162,7 +162,6 @@ fn try_compile_ssa_trace( let deopt_refs = SsaDeoptHelperRefs { clone_value_ref: b.import_signature(clone_value_sig), non_yielding_host_call_ref: b.import_signature(non_yielding_host_call_sig), - init_null_slot_ref: b.import_signature(value_slot_sig.clone()), clear_value_slot_ref: b.import_signature(value_slot_sig), box_heap_value_ref: b.import_signature(box_heap_value_sig), map_has_ref: b.import_signature(map_has_sig), @@ -178,7 +177,6 @@ fn try_compile_ssa_trace( let deopt_addrs = SsaDeoptHelperAddrs { clone_value: clone_value_to_slot_entry_address(), non_yielding_host_call: non_yielding_host_call_entry_address(), - init_null_slot: init_null_value_slot_entry_address(), clear_value_slot: clear_value_slot_entry_address(), box_heap_value: write_heap_value_to_slot_entry_address(), map_has: map_has_entry_address(), @@ -305,14 +303,7 @@ fn try_compile_ssa_trace( ) })?, )?; - init_owned_value_temps( - &mut b, - exit_block, - pointer_type, - deopt_refs, - deopt_addrs, - &owned_value_temps, - )?; + init_owned_value_temps(&mut b, pointer_type, layout.value, &owned_value_temps)?; let entry_args = ssa_block_args(entry_args); b.ins().jump(entry_handle, &entry_args); @@ -377,6 +368,7 @@ fn try_compile_ssa_trace( &mut b, exit_block, pointer_type, + layout.value, deopt_refs, deopt_addrs, &owned_value_temps, @@ -428,7 +420,6 @@ struct SsaExitLowering { struct SsaDeoptHelperRefs { clone_value_ref: cranelift_codegen::ir::SigRef, non_yielding_host_call_ref: cranelift_codegen::ir::SigRef, - init_null_slot_ref: cranelift_codegen::ir::SigRef, clear_value_slot_ref: cranelift_codegen::ir::SigRef, box_heap_value_ref: cranelift_codegen::ir::SigRef, map_has_ref: cranelift_codegen::ir::SigRef, @@ -446,7 +437,6 @@ struct SsaDeoptHelperRefs { struct SsaDeoptHelperAddrs { clone_value: usize, non_yielding_host_call: usize, - init_null_slot: usize, clear_value_slot: usize, box_heap_value: usize, map_has: usize, @@ -771,22 +761,13 @@ fn ssa_create_value_stack_slot(b: &mut FunctionBuilder, value_size: i32) -> VmRe fn init_owned_value_temps( b: &mut FunctionBuilder, - exit_block: Block, pointer_type: cranelift_codegen::ir::Type, - helper_refs: SsaDeoptHelperRefs, - helper_addrs: SsaDeoptHelperAddrs, + value_layout: crate::vm::native::ValueLayout, temps: &SsaOwnedValueTemps, ) -> VmResult<()> { - let _ = exit_block; for slot in &temps.ordered { let addr = b.ins().stack_addr(pointer_type, *slot, 0); - ssa_call_infallible_helper( - b, - pointer_type, - helper_refs.init_null_slot_ref, - helper_addrs.init_null_slot, - &[addr], - )?; + ssa_store_tag(b, value_layout, addr, value_layout.null_tag); } Ok(()) } @@ -795,6 +776,7 @@ fn clear_owned_value_temps( b: &mut FunctionBuilder, exit_block: Block, pointer_type: cranelift_codegen::ir::Type, + value_layout: crate::vm::native::ValueLayout, helper_refs: SsaDeoptHelperRefs, helper_addrs: SsaDeoptHelperAddrs, temps: &SsaOwnedValueTemps, @@ -802,12 +784,13 @@ fn clear_owned_value_temps( let _ = exit_block; for slot in &temps.ordered { let addr = b.ins().stack_addr(pointer_type, *slot, 0); - ssa_call_infallible_helper( + clear_value_slot_if_heap( b, pointer_type, - helper_refs.clear_value_slot_ref, - helper_addrs.clear_value_slot, - &[addr], + value_layout, + helper_refs, + helper_addrs, + addr, )?; } Ok(()) @@ -842,6 +825,28 @@ fn clear_owned_value_temp_slot( ) } +fn clear_value_slot_if_heap( + b: &mut FunctionBuilder, + pointer_type: cranelift_codegen::ir::Type, + layout: crate::vm::native::ValueLayout, + helper_refs: SsaDeoptHelperRefs, + helper_addrs: SsaDeoptHelperAddrs, + addr: cranelift_codegen::ir::Value, +) -> VmResult<()> { + let tag = ssa_load_tag_i32(b, layout, addr); + let scalar = ssa_is_scalar_tag(b, layout, tag); + let done = b.create_block(); + let clear = b.create_block(); + b.ins().brif(scalar, done, &[], clear, &[]); + + b.switch_to_block(clear); + clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, addr)?; + b.ins().jump(done, &[]); + + b.switch_to_block(done); + Ok(()) +} + fn ssa_interrupt_charge_blocks(ssa: &SsaTrace) -> BTreeSet { let mut blocks = BTreeSet::new(); for block in &ssa.blocks { From b824f03a1f5d27403e7d79d2dfc2c9dd535c7e89 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:27:36 +0800 Subject: [PATCH 11/19] perf(jit): fast-path compiled trace handoffs --- src/vm/jit/runtime.rs | 74 +++++++++++++++++++++++++++++++++---------- src/vm/jit/trace.rs | 2 +- 2 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index 10b70334..a9ee8e0f 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -498,8 +498,8 @@ impl Vm { } let (mut entry, mut root_ip, mut terminal, _, mut has_yielding_call) = self.native_trace_state(current_trace_id)?; + native::clear_bridge_error(); loop { - native::clear_bridge_error(); let status = unsafe { entry(self as *mut Vm) }; self.native_trace_exec_count = self.native_trace_exec_count.saturating_add(1); self.jit.mark_trace_executed(current_trace_id); @@ -513,19 +513,26 @@ impl Vm { { self.record_jit_link_handoff(); current_trace_id = next_trace_id; - if let Err(err) = self.ensure_native_trace( + if let Some(state) = self.cached_native_trace_state( current_trace_id, native::NativeCompileProfile::Jit, ) { - if should_fallback_to_interpreter(&err) { - self.record_jit_helper_fallback(); - self.jit.block_trace(current_trace_id); - return Ok(ExecOutcome::Continue); + (entry, root_ip, terminal, _, has_yielding_call) = state; + } else { + if let Err(err) = self.ensure_native_trace( + current_trace_id, + native::NativeCompileProfile::Jit, + ) { + if should_fallback_to_interpreter(&err) { + self.record_jit_helper_fallback(); + self.jit.block_trace(current_trace_id); + return Ok(ExecOutcome::Continue); + } + return Err(err); } - return Err(err); + (entry, root_ip, terminal, _, has_yielding_call) = + self.native_trace_state(current_trace_id)?; } - (entry, root_ip, terminal, _, has_yielding_call) = - self.native_trace_state(current_trace_id)?; continue; } return Ok(ExecOutcome::Continue); @@ -557,19 +564,26 @@ impl Vm { { self.record_jit_link_handoff(); current_trace_id = next_trace_id; - if let Err(err) = self.ensure_native_trace( + if let Some(state) = self.cached_native_trace_state( current_trace_id, native::NativeCompileProfile::Jit, ) { - if should_fallback_to_interpreter(&err) { - self.record_jit_helper_fallback(); - self.jit.block_trace(current_trace_id); - return Ok(ExecOutcome::Continue); + (entry, root_ip, terminal, _, has_yielding_call) = state; + } else { + if let Err(err) = self.ensure_native_trace( + current_trace_id, + native::NativeCompileProfile::Jit, + ) { + if should_fallback_to_interpreter(&err) { + self.record_jit_helper_fallback(); + self.jit.block_trace(current_trace_id); + return Ok(ExecOutcome::Continue); + } + return Err(err); } - return Err(err); + (entry, root_ip, terminal, _, has_yielding_call) = + self.native_trace_state(current_trace_id)?; } - (entry, root_ip, terminal, _, has_yielding_call) = - self.native_trace_state(current_trace_id)?; continue; } } @@ -662,6 +676,32 @@ impl Vm { )) } + #[cfg(any( + all( + target_arch = "x86_64", + any(target_os = "windows", all(unix, not(target_os = "macos"))) + ), + all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")) + ))] + #[inline(always)] + fn cached_native_trace_state( + &self, + trace_id: usize, + compile_profile: native::NativeCompileProfile, + ) -> Option<(NativeTraceEntry, usize, JitTraceTerminal, bool, bool)> { + let native = self.native_traces.get(trace_id)?.as_ref()?; + (native.interrupt_settings == self.active_native_interrupt_settings() + && compile_profile_satisfies(native.compile_profile, compile_profile) + && native.drop_contract_events_enabled == self.drop_contract_events_enabled) + .then_some(( + native.entry, + native.root_ip, + native.terminal, + native.has_call, + native.has_yielding_call, + )) + } + #[cfg(any( all( target_arch = "x86_64", diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 3ca42a55..ccecf207 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -37,7 +37,7 @@ fn native_jit_supported() -> bool { && (cfg!(target_os = "linux") || cfg!(target_os = "macos"))) } -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum JitTraceTerminal { LoopBack, BranchExit, From 680d6afbb52e083c749b6935a779daff3bde3675 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:37:59 +0800 Subject: [PATCH 12/19] perf(jit): recognize array length set as push --- src/vm/jit/ir.rs | 8 ++++++++ src/vm/jit/recorder.rs | 36 +++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index 9e374385..9c88275c 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -735,6 +735,14 @@ impl SsaTraceBuilder { self.trace.entry } + pub(crate) fn defining_inst(&self, value: SsaValueId) -> Option<&SsaInst> { + self.trace + .blocks + .iter() + .flat_map(|block| &block.insts) + .find(|inst| inst.output.is_some_and(|output| output.id == value)) + } + pub(crate) fn create_block(&mut self) -> SsaBlockId { let id = SsaBlockId::new(self.trace.blocks.len() as u32); self.trace.blocks.push(SsaBlock { diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 75ec511b..ac8b424b 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -3255,23 +3255,37 @@ fn emit_specialized_builtin_call( actual: array.info.repr, }); } + let is_append = matches!( + builder.defining_inst(index.value.id).map(|inst| &inst.kind), + Some(SsaInstKind::ArrayLen { array: array_ptr }) + if matches!( + builder.defining_inst(*array_ptr).map(|inst| &inst.kind), + Some(SsaInstKind::UnboxHeapPtr { + input, + tag: ValueType::Array, + }) if *input == array.value.id + ) + ); + let kind = if is_append { + SsaInstKind::ArrayPush { + array: array.value.id, + value: value.value.id, + } + } else { + SsaInstKind::ArraySet { + array: array.value.id, + index: index.value.id, + value: value.value.id, + } + }; let out = builder - .append_value_inst( - block, - ip, - SsaValueRepr::Tagged, - SsaInstKind::ArraySet { - array: array.value.id, - index: index.value.id, - value: value.value.id, - }, - ) + .append_value_inst(block, ip, SsaValueRepr::Tagged, kind) .map(|value| SymbolicValue { value, info: ValueInfo::tagged_typed(ValueType::Array), }) .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; - Ok(("array_set", out)) + Ok((if is_append { "array_push" } else { "array_set" }, out)) } SpecializedBuiltinKind::ArrayPush => { let value = frame.pop()?; From 93261c2966983bd0247160dab3f561e009b520ec Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:51:01 +0800 Subject: [PATCH 13/19] perf(jit): reject non-loop entries before blocked lookup --- src/vm/jit/trace.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index ccecf207..2dc4c0d8 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -249,8 +249,8 @@ impl TraceJitEngine { if let Some(trace_id) = self.compiled_trace_for_key(key) { return Some(trace_id); } - if (!self.blocked_entries.is_empty() && self.blocked_entries.contains(&key)) - || !self.is_loop_header(program, ip) + if !self.is_loop_header(program, ip) + || (!self.blocked_entries.is_empty() && self.blocked_entries.contains(&key)) { return None; } From 1bdcf9f80ef35eaf8cf782fee59c578efc96d3db Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 02:58:53 +0800 Subject: [PATCH 14/19] perf(jit): borrow array reads into host calls --- src/vm/jit/ir.rs | 2 +- src/vm/jit/native/lower.rs | 248 +++++++++++++++++++++++++++++++------ 2 files changed, 214 insertions(+), 36 deletions(-) diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index 9c88275c..5fb78eae 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -350,7 +350,7 @@ pub(crate) enum SsaInstKind { } impl SsaInstKind { - fn inputs(&self) -> Vec { + pub(crate) fn inputs(&self) -> Vec { match self { Self::Constant(_) => Vec::new(), Self::HostCall { args, .. } => args.clone(), diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 659bb154..819b92ff 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -4,8 +4,8 @@ use super::super::runtime::resume_linked_trace_entry_address; use super::{NativeCompileProfile, TraceLoweringKind}; use crate::vm::jit::deopt::exit_inputs; use crate::vm::jit::ir::{ - SsaBranchTarget, SsaExitId, SsaInstKind, SsaMaterialization, SsaTerminator, SsaTrace, - SsaValueId, SsaValueRepr, + SsaBlockId, SsaBranchTarget, SsaExitId, SsaInstKind, SsaMaterialization, SsaTerminator, + SsaTrace, SsaValueId, SsaValueRepr, }; use crate::vm::native::{ ExecutableBuffer, HeapIntrinsicAddrs, HeapIntrinsicRefs, NativeInterruptMode, @@ -204,7 +204,9 @@ fn try_compile_ssa_trace( } } } - let owned_value_temps = allocate_owned_value_temps(&mut b, ssa, layout.value.size)?; + let borrowed_array_gets = borrowed_array_get_outputs(ssa); + let owned_value_temps = + allocate_owned_value_temps(&mut b, ssa, layout.value.size, &borrowed_array_gets)?; let mut block_handles = HashMap::new(); for block in &ssa.blocks { @@ -260,6 +262,7 @@ fn try_compile_ssa_trace( helper_refs: deopt_refs, helper_addrs: deopt_addrs, owned_value_temps: &owned_value_temps, + borrowed_array_gets: &borrowed_array_gets, value_reprs: &value_reprs, tagged_constant_addrs: &tagged_constant_addrs, }; @@ -495,6 +498,7 @@ struct SsaLowerCtx<'a> { helper_refs: SsaDeoptHelperRefs, helper_addrs: SsaDeoptHelperAddrs, owned_value_temps: &'a SsaOwnedValueTemps, + borrowed_array_gets: &'a BTreeSet, value_reprs: &'a HashMap, tagged_constant_addrs: &'a HashMap, } @@ -670,6 +674,7 @@ fn allocate_owned_value_temps( b: &mut FunctionBuilder, ssa: &SsaTrace, value_size: i32, + borrowed_array_gets: &BTreeSet, ) -> VmResult { let mut ordered = Vec::new(); let mut slots = HashMap::new(); @@ -678,7 +683,9 @@ fn allocate_owned_value_temps( let Some(output) = inst.output else { continue; }; - if ssa_inst_requires_owned_value_slot(&inst.kind) { + if ssa_inst_requires_owned_value_slot(&inst.kind) + && !borrowed_array_gets.contains(&output.id) + { let slot = ssa_create_value_stack_slot(b, value_size)?; ordered.push(slot); slots.insert(SsaTempValueSlotKey::Output(output.id), slot); @@ -724,6 +731,82 @@ fn allocate_owned_value_temps( Ok(SsaOwnedValueTemps { ordered, slots }) } +fn borrowed_array_get_outputs(ssa: &SsaTrace) -> BTreeSet { + let mut instruction_uses: HashMap> = HashMap::new(); + let mut non_instruction_uses = BTreeSet::new(); + + for block in &ssa.blocks { + for (index, inst) in block.insts.iter().enumerate() { + let host_call = matches!(inst.kind, SsaInstKind::HostCall { .. }); + for input in inst.kind.inputs() { + instruction_uses + .entry(input) + .or_default() + .push((block.id, index, host_call)); + } + } + let Some(terminator) = &block.terminator else { + continue; + }; + match terminator { + SsaTerminator::Jump { args, .. } => non_instruction_uses.extend(args.iter().copied()), + SsaTerminator::BranchBool { + condition, + if_true, + if_false, + } => { + non_instruction_uses.insert(*condition); + for target in [if_true, if_false] { + if let SsaBranchTarget::Block { args, .. } = target { + non_instruction_uses.extend(args.iter().copied()); + } + } + } + SsaTerminator::Exit { .. } | SsaTerminator::Return { .. } => {} + } + } + for exit in &ssa.exits { + non_instruction_uses.extend(exit_inputs(exit)); + } + + let mut borrowed = BTreeSet::new(); + for block in &ssa.blocks { + for (definition_index, inst) in block.insts.iter().enumerate() { + let Some(output) = inst.output else { + continue; + }; + if !matches!(inst.kind, SsaInstKind::ArrayGet { .. }) + || non_instruction_uses.contains(&output.id) + { + continue; + } + let Some([(use_block, use_index, true)]) = + instruction_uses.get(&output.id).map(Vec::as_slice) + else { + continue; + }; + if *use_block != block.id || *use_index <= definition_index { + continue; + } + let borrow_stays_valid = + block.insts[definition_index + 1..*use_index] + .iter() + .all(|between| { + !matches!( + between.kind, + SsaInstKind::ArraySet { .. } + | SsaInstKind::ArrayPush { .. } + | SsaInstKind::HostCall { .. } + ) + }); + if borrow_stays_valid { + borrowed.insert(output.id); + } + } + } + borrowed +} + fn ssa_inst_requires_owned_value_slot(kind: &SsaInstKind) -> bool { matches!( kind, @@ -989,6 +1072,7 @@ fn lower_ssa_inst( helper_refs, helper_addrs, owned_value_temps, + borrowed_array_gets, value_reprs, tagged_constant_addrs, .. @@ -1940,37 +2024,42 @@ fn lower_ssa_inst( layout.stack_vec.ptr_offset, ); let element_addr = ssa_value_addr(b, pointer_type, data_ptr, index, layout.value.size); - let out = owned_value_temp_slot_addr( - b, - pointer_type, - owned_value_temps, - SsaTempValueSlotKey::Output(output.id), - )?; - let tag = ssa_load_tag_i32(b, layout.value, element_addr); - let scalar = ssa_is_scalar_tag(b, layout.value, tag); - let fast = b.create_block(); - let slow = b.create_block(); - let clone_done = b.create_block(); - b.ins().brif(scalar, fast, &[], slow, &[]); - - b.switch_to_block(fast); - clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; - ssa_copy_value_bytes(b, element_addr, out, layout.value.size); - b.ins().jump(clone_done, &[]); - - b.switch_to_block(slow); - clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; - ssa_call_status_helper( - b, - exit_block, - pointer_type, - helper_refs.clone_value_ref, - helper_addrs.clone_value, - &[out, element_addr], - )?; - b.ins().jump(clone_done, &[]); - - b.switch_to_block(clone_done); + let out = if borrowed_array_gets.contains(&output.id) { + element_addr + } else { + let out = owned_value_temp_slot_addr( + b, + pointer_type, + owned_value_temps, + SsaTempValueSlotKey::Output(output.id), + )?; + let tag = ssa_load_tag_i32(b, layout.value, element_addr); + let scalar = ssa_is_scalar_tag(b, layout.value, tag); + let fast = b.create_block(); + let slow = b.create_block(); + let clone_done = b.create_block(); + b.ins().brif(scalar, fast, &[], slow, &[]); + + b.switch_to_block(fast); + clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; + ssa_copy_value_bytes(b, element_addr, out, layout.value.size); + b.ins().jump(clone_done, &[]); + + b.switch_to_block(slow); + clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; + ssa_call_status_helper( + b, + exit_block, + pointer_type, + helper_refs.clone_value_ref, + helper_addrs.clone_value, + &[out, element_addr], + )?; + b.ins().jump(clone_done, &[]); + + b.switch_to_block(clone_done); + out + }; b.ins().jump(done, &[]); b.switch_to_block(fail); @@ -4040,3 +4129,92 @@ fn native_isa(profile: NativeCompileProfile) -> VmResult { }); cached.clone().map_err(VmError::JitNative) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ValueType; + use crate::vm::jit::ir::SsaTraceBuilder; + + fn array_get_host_call_trace( + mutate_before_call: bool, + materialize_get_on_exit: bool, + ) -> (SsaTrace, SsaValueId) { + let mut builder = SsaTraceBuilder::new(0, 0); + let entry = builder.entry(); + let array = builder + .append_param(entry, SsaValueRepr::HeapPtr(ValueType::Array), "array") + .unwrap(); + let index = builder + .append_param(entry, SsaValueRepr::I64, "index") + .unwrap(); + let mutation_value = builder + .append_param(entry, SsaValueRepr::Tagged, "mutation_value") + .unwrap(); + let get = builder + .append_value_inst( + entry, + 1, + SsaValueRepr::Tagged, + SsaInstKind::ArrayGet { + array: array.id, + index: index.id, + }, + ) + .unwrap(); + if mutate_before_call { + builder + .append_value_inst( + entry, + 2, + SsaValueRepr::Tagged, + SsaInstKind::ArraySet { + array: array.id, + index: index.id, + value: mutation_value.id, + }, + ) + .unwrap(); + } + let call = builder + .append_value_inst( + entry, + 3, + SsaValueRepr::Tagged, + SsaInstKind::HostCall { + import: 0, + args: vec![get.id], + }, + ) + .unwrap(); + let stack_value = if materialize_get_on_exit { get } else { call }; + let exit = builder.add_exit( + 4, + vec![SsaMaterialization::Value(stack_value.id)], + Vec::new(), + Vec::new(), + ); + builder + .set_terminator(entry, SsaTerminator::Return { exit }) + .unwrap(); + (builder.finish(), get.id) + } + + #[test] + fn borrows_single_use_array_get_for_immediate_host_call() { + let (trace, get) = array_get_host_call_trace(false, false); + assert_eq!(borrowed_array_get_outputs(&trace), BTreeSet::from([get])); + } + + #[test] + fn does_not_borrow_array_get_across_array_mutation() { + let (trace, _) = array_get_host_call_trace(true, false); + assert!(borrowed_array_get_outputs(&trace).is_empty()); + } + + #[test] + fn does_not_borrow_array_get_that_escapes_to_an_exit() { + let (trace, _) = array_get_host_call_trace(false, true); + assert!(borrowed_array_get_outputs(&trace).is_empty()); + } +} From 439612097053519da4c2f1c236a5ea16ea615f18 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 03:11:45 +0800 Subject: [PATCH 15/19] perf(jit): specialize collection mutation bridges --- src/builtins/runtime/core.rs | 9 ++- src/vm/jit/native/lower.rs | 104 +++++++++++------------- src/vm/native/bridge.rs | 149 ++++++++++++++++++++++++++++++++++- src/vm/native/codegen.rs | 28 +++++++ src/vm/native/mod.rs | 37 ++++----- 5 files changed, 247 insertions(+), 80 deletions(-) diff --git a/src/builtins/runtime/core.rs b/src/builtins/runtime/core.rs index 7bbd502f..8fb1ff7e 100644 --- a/src/builtins/runtime/core.rs +++ b/src/builtins/runtime/core.rs @@ -215,7 +215,10 @@ pub(super) fn builtin_array_push_typed_impl( items } -fn builtin_array_push_shared_impl(mut items: SharedArray, value: AnyValue) -> SharedArray { +pub(crate) fn builtin_array_push_shared_impl( + mut items: SharedArray, + value: AnyValue, +) -> SharedArray { Arc::make_mut(&mut items).push(value); items } @@ -632,7 +635,7 @@ pub(super) fn builtin_set_array_impl( Ok(items) } -fn builtin_set_array_shared_impl( +pub(crate) fn builtin_set_array_shared_impl( mut items: SharedArray, index: i64, value: AnyValue, @@ -673,7 +676,7 @@ pub(super) fn builtin_set_map_impl( entries } -fn builtin_set_map_shared_impl( +pub(crate) fn builtin_set_map_shared_impl( mut entries: SharedMap, key: AnyValue, value: AnyValue, diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 819b92ff..af489ff3 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -11,22 +11,22 @@ use crate::vm::native::{ ExecutableBuffer, HeapIntrinsicAddrs, HeapIntrinsicRefs, NativeInterruptMode, NativeInterruptSettings, NativeStackLayout, STATUS_CONTINUE, STATUS_ERROR, STATUS_HALTED, STATUS_OUT_OF_FUEL, STATUS_TRACE_EXIT, alloc_buffer_signature, alloc_byte_buffer_entry_address, - alloc_value_buffer_entry_address, array_push_entry_address, box_heap_value_signature, - checked_add_i32, clear_value_slot_entry_address, clone_value_signature, - clone_value_to_slot_entry_address, collection_get_signature, collection_mutation_signature, - collection_predicate_signature, collection_set_entry_address, copy_bytes_entry_address, - copy_bytes_signature, detect_native_stack_layout, entry_signature, free_buffer_signature, - init_null_value_slot_entry_address, jump_with_status, map_get_entry_address, - map_has_entry_address, map_iter_next_entry_address, map_iter_next_signature, - map_iter_take_key_entry_address, map_iter_take_signature, map_iter_take_value_entry_address, - non_yielding_host_call_entry_address, non_yielding_host_call_signature, - pack_shared_signature, restore_sparse_exit_state_entry_address, - shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, - shared_string_from_buffer_entry_address, sparse_restore_exit_signature, - string_binary_transform_signature, string_contains_entry_address, string_contains_signature, - string_lower_ascii_entry_address, string_replace_literal_entry_address, - string_replace_signature, string_split_literal_entry_address, string_unary_transform_signature, - value_slot_signature, write_heap_value_to_slot_entry_address, zero_bytes_entry_address, + alloc_value_buffer_entry_address, array_push_entry_address, array_set_entry_address, + array_set_signature, box_heap_value_signature, checked_add_i32, clear_value_slot_entry_address, + clone_value_signature, clone_value_to_slot_entry_address, collection_get_signature, + collection_predicate_signature, copy_bytes_entry_address, copy_bytes_signature, + detect_native_stack_layout, entry_signature, free_buffer_signature, jump_with_status, + map_get_entry_address, map_has_entry_address, map_iter_next_entry_address, + map_iter_next_signature, map_iter_take_key_entry_address, map_iter_take_signature, + map_iter_take_value_entry_address, map_set_entry_address, map_set_signature, + non_yielding_host_call_entry_address, non_yielding_host_call_signature, pack_shared_signature, + restore_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, + shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, + sparse_restore_exit_signature, string_binary_transform_signature, + string_contains_entry_address, string_contains_signature, string_lower_ascii_entry_address, + string_replace_literal_entry_address, string_replace_signature, + string_split_literal_entry_address, string_unary_transform_signature, value_slot_signature, + write_heap_value_to_slot_entry_address, zero_bytes_entry_address, }; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::Ieee64; @@ -110,7 +110,8 @@ fn try_compile_ssa_trace( let map_iter_next_sig = map_iter_next_signature(pointer_type, call_conv); let map_iter_take_sig = map_iter_take_signature(pointer_type, call_conv); let array_push_sig = collection_get_signature(pointer_type, call_conv); - let collection_set_sig = collection_mutation_signature(pointer_type, call_conv); + let array_set_sig = array_set_signature(pointer_type, call_conv); + let map_set_sig = map_set_signature(pointer_type, call_conv); let sparse_restore_exit_sig = sparse_restore_exit_signature(pointer_type, call_conv); let resume_linked_trace_sig = entry_signature(pointer_type, call_conv); let string_contains_sig = string_contains_signature(pointer_type, call_conv); @@ -170,7 +171,8 @@ fn try_compile_ssa_trace( map_iter_take_key_ref: b.import_signature(map_iter_take_sig.clone()), map_iter_take_value_ref: b.import_signature(map_iter_take_sig), array_push_ref: b.import_signature(array_push_sig), - collection_set_ref: b.import_signature(collection_set_sig), + array_set_ref: b.import_signature(array_set_sig), + map_set_ref: b.import_signature(map_set_sig), sparse_restore_exit_ref: b.import_signature(sparse_restore_exit_sig), resume_linked_trace_ref: b.import_signature(resume_linked_trace_sig), }; @@ -185,7 +187,8 @@ fn try_compile_ssa_trace( map_iter_take_key: map_iter_take_key_entry_address(), map_iter_take_value: map_iter_take_value_entry_address(), array_push: array_push_entry_address(), - collection_set: collection_set_entry_address(), + array_set: array_set_entry_address(), + map_set: map_set_entry_address(), sparse_restore_exit: restore_sparse_exit_state_entry_address(), resume_linked_trace: resume_linked_trace_entry_address(), }; @@ -431,7 +434,8 @@ struct SsaDeoptHelperRefs { map_iter_take_key_ref: cranelift_codegen::ir::SigRef, map_iter_take_value_ref: cranelift_codegen::ir::SigRef, array_push_ref: cranelift_codegen::ir::SigRef, - collection_set_ref: cranelift_codegen::ir::SigRef, + array_set_ref: cranelift_codegen::ir::SigRef, + map_set_ref: cranelift_codegen::ir::SigRef, sparse_restore_exit_ref: cranelift_codegen::ir::SigRef, resume_linked_trace_ref: cranelift_codegen::ir::SigRef, } @@ -448,7 +452,8 @@ struct SsaDeoptHelperAddrs { map_iter_take_key: usize, map_iter_take_value: usize, array_push: usize, - collection_set: usize, + array_set: usize, + map_set: usize, sparse_restore_exit: usize, resume_linked_trace: usize, } @@ -714,17 +719,16 @@ fn allocate_owned_value_temps( ordered.push(slot); slots.insert(SsaTempValueSlotKey::MapKeyBox(output.id), slot); } - if matches!( - inst.kind, - SsaInstKind::ArraySet { .. } - | SsaInstKind::ArrayPush { .. } - | SsaInstKind::MapSet { .. } - ) { - for arg in 0..2 { - let slot = ssa_create_value_stack_slot(b, value_size)?; - ordered.push(slot); - slots.insert(SsaTempValueSlotKey::MutationArgBox(output.id, arg), slot); - } + let mutation_args: &[u8] = match inst.kind { + SsaInstKind::ArraySet { .. } => &[1], + SsaInstKind::ArrayPush { .. } => &[0], + SsaInstKind::MapSet { .. } => &[0, 1], + _ => &[], + }; + for arg in mutation_args { + let slot = ssa_create_value_stack_slot(b, value_size)?; + ordered.push(slot); + slots.insert(SsaTempValueSlotKey::MutationArgBox(output.id, *arg), slot); } } } @@ -2091,27 +2095,11 @@ fn lower_ssa_inst( owned_value_temps, SsaTempValueSlotKey::Output(output.id), )?; - let index_box = owned_value_temp_slot_addr( - b, - pointer_type, - owned_value_temps, - SsaTempValueSlotKey::MutationArgBox(output.id, 0), - )?; - let index_addr = ssa_ensure_boxed_value_addr( - b, - SsaBoxCtx { - exit_block, - pointer_type, - value_layout: layout.value, - helper_refs, - helper_addrs, - }, - Some(index_box), - *value_reprs.get(index).ok_or_else(|| { - VmError::JitNative("SSA array-set index representation missing".to_string()) - })?, - values[index], - )?; + if value_reprs.get(index) != Some(&SsaValueRepr::I64) { + return Err(VmError::JitNative( + "SSA array-set index must be lowered as i64".to_string(), + )); + } let value_box = owned_value_temp_slot_addr( b, pointer_type, @@ -2133,11 +2121,11 @@ fn lower_ssa_inst( })?, values[value], )?; - let helper_ptr = iconst_ptr_from_addr(b, pointer_type, helper_addrs.collection_set)?; + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, helper_addrs.array_set)?; let call = b.ins().call_indirect( - helper_refs.collection_set_ref, + helper_refs.array_set_ref, helper_ptr, - &[out, values[array], index_addr, value_addr], + &[out, values[array], values[index], value_addr], ); let status = b.inst_results(call)[0]; let fail = b.create_block(); @@ -2420,9 +2408,9 @@ fn lower_ssa_inst( })?, values[value], )?; - let helper_ptr = iconst_ptr_from_addr(b, pointer_type, helper_addrs.collection_set)?; + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, helper_addrs.map_set)?; let call = b.ins().call_indirect( - helper_refs.collection_set_ref, + helper_refs.map_set_ref, helper_ptr, &[out, values[map], key_addr, value_addr], ); diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index da94f4ce..3bb556f1 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -276,6 +276,14 @@ pub(crate) fn collection_set_entry_address() -> usize { pd_vm_native_collection_set as *const () as usize } +pub(crate) fn array_set_entry_address() -> usize { + pd_vm_native_array_set as *const () as usize +} + +pub(crate) fn map_set_entry_address() -> usize { + pd_vm_native_map_set as *const () as usize +} + pub(crate) fn array_push_entry_address() -> usize { pd_vm_native_array_push as *const () as usize } @@ -782,6 +790,76 @@ pub(crate) extern "C" fn pd_vm_native_collection_set( } } +pub(crate) extern "C" fn pd_vm_native_array_set( + dst: *mut Value, + array: *mut Value, + index: i64, + value: *const Value, +) -> i32 { + if dst.is_null() || array.is_null() || value.is_null() { + store_bridge_error(VmError::JitNative( + "native array-set helper received null pointer".to_string(), + )); + return STATUS_ERROR; + } + + let array = unsafe { std::ptr::replace(array, Value::Null) }; + let value = unsafe { (&*value).clone() }; + let result = match array { + Value::Array(values) => { + crate::builtins::runtime::core::builtin_set_array_shared_impl(values, index, value) + .map(Value::Array) + } + _ => Err(VmError::TypeMismatch("array")), + }; + match result { + Ok(result) => { + let previous = unsafe { std::ptr::replace(dst, result) }; + drop(previous); + STATUS_CONTINUE + } + Err(err) => { + store_bridge_error(err); + STATUS_ERROR + } + } +} + +pub(crate) extern "C" fn pd_vm_native_map_set( + dst: *mut Value, + map: *mut Value, + key: *const Value, + value: *const Value, +) -> i32 { + if dst.is_null() || map.is_null() || key.is_null() || value.is_null() { + store_bridge_error(VmError::JitNative( + "native map-set helper received null pointer".to_string(), + )); + return STATUS_ERROR; + } + + let map = unsafe { std::ptr::replace(map, Value::Null) }; + let key = unsafe { (&*key).clone() }; + let value = unsafe { (&*value).clone() }; + let result = match map { + Value::Map(entries) => Ok(Value::Map( + crate::builtins::runtime::core::builtin_set_map_shared_impl(entries, key, value), + )), + _ => Err(VmError::TypeMismatch("map")), + }; + match result { + Ok(result) => { + let previous = unsafe { std::ptr::replace(dst, result) }; + drop(previous); + STATUS_CONTINUE + } + Err(err) => { + store_bridge_error(err); + STATUS_ERROR + } + } +} + pub(crate) extern "C" fn pd_vm_native_array_push( dst: *mut Value, array: *mut Value, @@ -796,7 +874,13 @@ pub(crate) extern "C" fn pd_vm_native_array_push( let array = unsafe { std::ptr::replace(array, Value::Null) }; let value = unsafe { (&*value).clone() }; - match crate::builtins::runtime::core::builtin_array_push_owned(array, value) { + let result = match array { + Value::Array(values) => Ok(Value::Array( + crate::builtins::runtime::core::builtin_array_push_shared_impl(values, value), + )), + _ => Err(VmError::TypeMismatch("array")), + }; + match result { Ok(result) => { let previous = unsafe { std::ptr::replace(dst, result) }; drop(previous); @@ -1222,4 +1306,67 @@ mod tests { assert_eq!(Arc::strong_count(&replacement), 2); assert_eq!(vm.drop_contract_event_count(), 2); } + + #[test] + fn typed_array_set_detaches_shared_input() { + let shared = Arc::new(vec![Value::Int(10), Value::Int(20)]); + let alias = shared.clone(); + let mut input = Value::Array(shared); + let replacement = Value::Int(99); + let mut output = Value::Null; + + let status = pd_vm_native_array_set(&mut output, &mut input, 1, &replacement); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(input, Value::Null); + assert_eq!(alias.as_slice(), &[Value::Int(10), Value::Int(20)]); + let Value::Array(result) = output else { + panic!("expected array result"); + }; + assert_eq!(result.as_slice(), &[Value::Int(10), Value::Int(99)]); + assert!(!Arc::ptr_eq(&result, &alias)); + } + + #[test] + fn typed_array_push_detaches_shared_input() { + let shared = Arc::new(vec![Value::Int(10)]); + let alias = shared.clone(); + let mut input = Value::Array(shared); + let appended = Value::Int(20); + let mut output = Value::Null; + + let status = pd_vm_native_array_push(&mut output, &mut input, &appended); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(input, Value::Null); + assert_eq!(alias.as_slice(), &[Value::Int(10)]); + let Value::Array(result) = output else { + panic!("expected array result"); + }; + assert_eq!(result.as_slice(), &[Value::Int(10), Value::Int(20)]); + assert!(!Arc::ptr_eq(&result, &alias)); + } + + #[test] + fn typed_map_set_detaches_shared_input() { + let shared = Arc::new(VmMap::from(vec![(Value::Int(1), Value::Int(10))])); + let alias = shared.clone(); + let mut input = Value::Map(shared); + let key = Value::Int(2); + let value = Value::Int(20); + let mut output = Value::Null; + + let status = pd_vm_native_map_set(&mut output, &mut input, &key, &value); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(input, Value::Null); + assert_eq!(alias.get(&Value::Int(1)), Some(&Value::Int(10))); + assert_eq!(alias.get(&Value::Int(2)), None); + let Value::Map(result) = output else { + panic!("expected map result"); + }; + assert_eq!(result.get(&Value::Int(1)), Some(&Value::Int(10))); + assert_eq!(result.get(&Value::Int(2)), Some(&Value::Int(20))); + assert!(!Arc::ptr_eq(&result, &alias)); + } } diff --git a/src/vm/native/codegen.rs b/src/vm/native/codegen.rs index c06d18ee..258b40eb 100644 --- a/src/vm/native/codegen.rs +++ b/src/vm/native/codegen.rs @@ -213,6 +213,34 @@ pub(crate) fn collection_mutation_signature( sig } +#[cfg(feature = "cranelift-jit")] +pub(crate) fn map_set_signature( + pointer_type: cranelift_codegen::ir::Type, + call_conv: cranelift_codegen::isa::CallConv, +) -> Signature { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.returns.push(AbiParam::new(types::I32)); + sig +} + +#[cfg(feature = "cranelift-jit")] +pub(crate) fn array_set_signature( + pointer_type: cranelift_codegen::ir::Type, + call_conv: cranelift_codegen::isa::CallConv, +) -> Signature { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(types::I64)); + sig.params.push(AbiParam::new(pointer_type)); + sig.returns.push(AbiParam::new(types::I32)); + sig +} + #[cfg(feature = "cranelift-jit")] pub(crate) fn value_slot_signature( pointer_type: cranelift_codegen::ir::Type, diff --git a/src/vm/native/mod.rs b/src/vm/native/mod.rs index 7535c19a..51014d53 100644 --- a/src/vm/native/mod.rs +++ b/src/vm/native/mod.rs @@ -10,29 +10,30 @@ pub(crate) use bridge::{ STATUS_ERROR, STATUS_HALTED, STATUS_LINKED_CONTINUE, STATUS_OUT_OF_FUEL, STATUS_TRACE_EXIT, STATUS_WAITING, STATUS_YIELDED, alloc_byte_buffer_entry_address, alloc_value_buffer_entry_address, aot_call_boundary_interrupt_entry_address, - array_push_entry_address, clear_bridge_error, clear_value_slot_entry_address, - clone_value_to_slot_entry_address, collection_set_entry_address, copy_bytes_entry_address, - helper_entry_address, helper_entry_offset, init_null_value_slot_entry_address, - interrupt_helper_entry_address, interrupt_helper_entry_offset, map_get_entry_address, - map_has_entry_address, map_iter_next_entry_address, map_iter_take_key_entry_address, - map_iter_take_value_entry_address, non_yielding_host_call_entry_address, - restore_exit_state_entry_address, - restore_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, - shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, - store_bridge_error, string_contains_entry_address, string_lower_ascii_entry_address, - string_replace_literal_entry_address, string_split_literal_entry_address, take_bridge_error, - value_eq_entry_address, write_heap_value_to_slot_entry_address, zero_bytes_entry_address, + array_push_entry_address, array_set_entry_address, clear_bridge_error, + clear_value_slot_entry_address, clone_value_to_slot_entry_address, + collection_set_entry_address, copy_bytes_entry_address, helper_entry_address, + helper_entry_offset, init_null_value_slot_entry_address, interrupt_helper_entry_address, + interrupt_helper_entry_offset, map_get_entry_address, map_has_entry_address, + map_iter_next_entry_address, map_iter_take_key_entry_address, + map_iter_take_value_entry_address, map_set_entry_address, non_yielding_host_call_entry_address, + restore_exit_state_entry_address, restore_sparse_exit_state_entry_address, + shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, + shared_string_from_buffer_entry_address, store_bridge_error, string_contains_entry_address, + string_lower_ascii_entry_address, string_replace_literal_entry_address, + string_split_literal_entry_address, take_bridge_error, value_eq_entry_address, + write_heap_value_to_slot_entry_address, zero_bytes_entry_address, }; #[cfg(feature = "cranelift-jit")] pub(crate) use codegen::{ - alloc_buffer_signature, box_heap_value_signature, clone_value_signature, + alloc_buffer_signature, array_set_signature, box_heap_value_signature, clone_value_signature, collection_get_signature, collection_mutation_signature, collection_predicate_signature, copy_bytes_signature, entry_signature, free_buffer_signature, helper_signature, - jump_with_status, map_iter_next_signature, map_iter_take_signature, pack_shared_signature, - non_yielding_host_call_signature, - restore_exit_signature, sparse_restore_exit_signature, string_binary_transform_signature, - string_contains_signature, string_replace_signature, string_unary_transform_signature, - value_eq_signature, value_slot_signature, + jump_with_status, map_iter_next_signature, map_iter_take_signature, map_set_signature, + non_yielding_host_call_signature, pack_shared_signature, restore_exit_signature, + sparse_restore_exit_signature, string_binary_transform_signature, string_contains_signature, + string_replace_signature, string_unary_transform_signature, value_eq_signature, + value_slot_signature, }; pub(crate) use exec::{ExecutableBuffer, prepare_for_execution}; pub(crate) use layout::{ From 126ac00edef51e77402a87f8439b11e9a5ec681f Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 16 Jul 2026 03:15:34 +0800 Subject: [PATCH 16/19] perf(jit): borrow array reads into collection ops --- src/vm/jit/native/lower.rs | 117 ++++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 2 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index af489ff3..fc09e844 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -741,12 +741,22 @@ fn borrowed_array_get_outputs(ssa: &SsaTrace) -> BTreeSet { for block in &ssa.blocks { for (index, inst) in block.insts.iter().enumerate() { - let host_call = matches!(inst.kind, SsaInstKind::HostCall { .. }); for input in inst.kind.inputs() { + let borrows_input = match &inst.kind { + SsaInstKind::HostCall { .. } => true, + SsaInstKind::ArraySet { value, .. } | SsaInstKind::ArrayPush { value, .. } => { + *value == input + } + SsaInstKind::MapGet { key, .. } | SsaInstKind::MapHas { key, .. } => { + *key == input + } + SsaInstKind::MapSet { key, value, .. } => *key == input || *value == input, + _ => false, + }; instruction_uses .entry(input) .or_default() - .push((block.id, index, host_call)); + .push((block.id, index, borrows_input)); } } let Some(terminator) = &block.terminator else { @@ -4124,6 +4134,15 @@ mod tests { use crate::ValueType; use crate::vm::jit::ir::SsaTraceBuilder; + #[derive(Clone, Copy)] + enum BorrowingCollectionConsumer { + ArraySet, + ArrayPush, + MapGet, + MapHas, + MapSet, + } + fn array_get_host_call_trace( mutate_before_call: bool, materialize_get_on_exit: bool, @@ -4188,12 +4207,106 @@ mod tests { (builder.finish(), get.id) } + fn array_get_collection_consumer_trace( + consumer: BorrowingCollectionConsumer, + ) -> (SsaTrace, SsaValueId) { + let mut builder = SsaTraceBuilder::new(0, 0); + let entry = builder.entry(); + let array = builder + .append_param(entry, SsaValueRepr::HeapPtr(ValueType::Array), "array") + .unwrap(); + let index = builder + .append_param(entry, SsaValueRepr::I64, "index") + .unwrap(); + let map = builder + .append_param(entry, SsaValueRepr::HeapPtr(ValueType::Map), "map") + .unwrap(); + let map_key = builder + .append_param(entry, SsaValueRepr::Tagged, "map_key") + .unwrap(); + let get = builder + .append_value_inst( + entry, + 1, + SsaValueRepr::Tagged, + SsaInstKind::ArrayGet { + array: array.id, + index: index.id, + }, + ) + .unwrap(); + let (repr, kind) = match consumer { + BorrowingCollectionConsumer::ArraySet => ( + SsaValueRepr::Tagged, + SsaInstKind::ArraySet { + array: array.id, + index: index.id, + value: get.id, + }, + ), + BorrowingCollectionConsumer::ArrayPush => ( + SsaValueRepr::Tagged, + SsaInstKind::ArrayPush { + array: array.id, + value: get.id, + }, + ), + BorrowingCollectionConsumer::MapGet => ( + SsaValueRepr::Tagged, + SsaInstKind::MapGet { + map: map.id, + key: get.id, + }, + ), + BorrowingCollectionConsumer::MapHas => ( + SsaValueRepr::Bool, + SsaInstKind::MapHas { + map: map.id, + key: get.id, + }, + ), + BorrowingCollectionConsumer::MapSet => ( + SsaValueRepr::Tagged, + SsaInstKind::MapSet { + map: map.id, + key: map_key.id, + value: get.id, + }, + ), + }; + let result = builder.append_value_inst(entry, 2, repr, kind).unwrap(); + let exit = builder.add_exit( + 3, + vec![SsaMaterialization::Value(result.id)], + Vec::new(), + Vec::new(), + ); + builder + .set_terminator(entry, SsaTerminator::Return { exit }) + .unwrap(); + (builder.finish(), get.id) + } + #[test] fn borrows_single_use_array_get_for_immediate_host_call() { let (trace, get) = array_get_host_call_trace(false, false); assert_eq!(borrowed_array_get_outputs(&trace), BTreeSet::from([get])); } + #[test] + fn borrows_single_use_array_get_for_typed_collection_consumers() { + for consumer in [ + BorrowingCollectionConsumer::ArraySet, + BorrowingCollectionConsumer::ArrayPush, + BorrowingCollectionConsumer::MapGet, + BorrowingCollectionConsumer::MapHas, + BorrowingCollectionConsumer::MapSet, + ] { + let (trace, get) = array_get_collection_consumer_trace(consumer); + assert_eq!(borrowed_array_get_outputs(&trace), BTreeSet::from([get])); + } + } + #[test] fn does_not_borrow_array_get_across_array_mutation() { let (trace, _) = array_get_host_call_trace(true, false); From 481bb97001a95038b180d26d507fd9a395e41c17 Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 16 Jul 2026 12:27:03 +0800 Subject: [PATCH 17/19] fix(vm): enforce non-yielding host return contract --- README.md | 10 +++++++++ src/vm/host.rs | 41 +++++++++++++++++++++++++++++------- src/vm/native/bridge.rs | 17 ++------------- tests/vm/vm_runtime_tests.rs | 39 ++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index e85b3849..63ea18a3 100644 --- a/README.md +++ b/README.md @@ -457,6 +457,16 @@ The `call` opcode pops its arguments, dispatches to a builtin or bound host func | `Yield` | Retry next `run()` | **Rewound to `call` opcode** | **Args re-pushed** | | `Pending(op_id)` | Async result pending | Advanced past `call` | Empty (result injected via `complete_host_op`) | +**Non-yielding static host calls** — native-trace opt-in: + +- `Vm::bind_static_non_yielding_args_function` and + `Vm::register_static_non_yielding_args_function` let the trace JIT keep eligible host calls + inside native traces. +- The function must return exactly one value via `CallOutcome::Return(CallReturn::One(...))`. + Returning no value, `Halt`, `Yield`, or `Pending` is a contract violation and produces the same + `VmError::HostError` in interpreted and native execution. +- Use the ordinary static args APIs when the host function may suspend, halt, or return no value. + **`CallOutcome::Yield`** — cooperative "retry me later": - The VM re-pushes the original args onto the stack and rewinds `self.ip` to the start of the diff --git a/src/vm/host.rs b/src/vm/host.rs index 5a031dcc..77457755 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -439,6 +439,24 @@ pub(super) enum HostCallExecOutcome { Pending(HostOpId), } +pub(crate) fn require_non_yielding_host_value(outcome: CallOutcome) -> VmResult { + match outcome { + CallOutcome::Return(CallReturn::One(value)) => Ok(value), + CallOutcome::Return(CallReturn::None) => Err(VmError::HostError( + "non-yielding host function returned no value".to_string(), + )), + CallOutcome::Halt => Err(VmError::HostError( + "non-yielding host function returned halt".to_string(), + )), + CallOutcome::Yield => Err(VmError::HostError( + "non-yielding host function returned yield".to_string(), + )), + CallOutcome::Pending(_) => Err(VmError::HostError( + "non-yielding host function returned pending".to_string(), + )), + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct WaitingHostOp { pub(super) op_id: HostOpId, @@ -516,9 +534,9 @@ impl Vm { index } - /// Registers a static args-only host function that always returns synchronously. + /// Registers a static args-only host function that always returns one value synchronously. /// - /// Violating this contract by returning `Halt`, `Yield`, or `Pending` is a host error. + /// Returning no value, `Halt`, `Yield`, or `Pending` violates the contract and is a host error. pub fn register_static_non_yielding_args_function( &mut self, function: StaticHostArgsFunction, @@ -665,11 +683,11 @@ impl Vm { self.resolved_calls_dirty = true; } - /// Binds a static args-only host function that always returns synchronously. + /// Binds a static args-only host function that always returns one value synchronously. /// /// This is equivalent to [`Vm::bind_static_args_function`] except that the VM may keep - /// native JIT traces active across the call boundary. Returning `Halt`, `Yield`, or - /// `Pending` violates the contract and is reported as a host error. + /// native JIT traces active across the call boundary. Returning no value, `Halt`, `Yield`, + /// or `Pending` violates the contract and is reported as a host error. pub fn bind_static_non_yielding_args_function( &mut self, name: impl Into, @@ -1433,9 +1451,9 @@ impl Vm { .get_mut(resolved_index as usize) .ok_or(VmError::InvalidCall(resolved_index))?; match function { - VmHostFunction::ArgsDynamic(function) => function.call(args), - VmHostFunction::ArgsStatic(function) - | VmHostFunction::ArgsStaticNonYielding(function) => function(args), + VmHostFunction::ArgsDynamic(function) => (function.call(args), false), + VmHostFunction::ArgsStatic(function) => (function(args), false), + VmHostFunction::ArgsStaticNonYielding(function) => (function(args), true), VmHostFunction::Dynamic(_) | VmHostFunction::Static(_) | VmHostFunction::StackDynamic(_) @@ -1443,7 +1461,14 @@ impl Vm { } }; self.call_depth = self.call_depth.saturating_sub(1); + let (outcome, non_yielding) = outcome; let outcome = outcome?; + if non_yielding { + let value = require_non_yielding_host_value(outcome)?; + self.stack.truncate(arg_start); + self.stack.push(value); + return Ok(HostCallExecOutcome::Returned); + } match outcome { CallOutcome::Return(values) => { diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 3bb556f1..f2debc44 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -930,24 +930,11 @@ pub(crate) extern "C" fn pd_vm_native_non_yielding_host_call( vm.call_depth = vm.call_depth.saturating_add(1); let outcome = function(args); vm.call_depth = vm.call_depth.saturating_sub(1); - match outcome { - Ok(CallOutcome::Return(CallReturn::One(value))) => { + match outcome.and_then(crate::vm::host::require_non_yielding_host_value) { + Ok(value) => { unsafe { std::ptr::write(out, value) }; STATUS_CONTINUE } - Ok(CallOutcome::Return(CallReturn::None)) => { - store_bridge_error(VmError::HostError( - "non-yielding native host call returned no value".to_string(), - )); - STATUS_ERROR - } - Ok(CallOutcome::Halt | CallOutcome::Yield | CallOutcome::Pending(_)) => { - store_bridge_error(VmError::HostError( - "non-yielding native host call violated its synchronous return contract" - .to_string(), - )); - STATUS_ERROR - } Err(err) => { store_bridge_error(err); STATUS_ERROR diff --git a/tests/vm/vm_runtime_tests.rs b/tests/vm/vm_runtime_tests.rs index 79e0044b..9a838f95 100644 --- a/tests/vm/vm_runtime_tests.rs +++ b/tests/vm/vm_runtime_tests.rs @@ -3,6 +3,45 @@ mod common; use common::*; use vm::OpCode; +fn non_yielding_returns_none(_: &[Value]) -> Result { + Ok(CallOutcome::Return(vm::CallReturn::none())) +} + +fn non_yielding_halts(_: &[Value]) -> Result { + Ok(CallOutcome::Halt) +} + +fn non_yielding_yields(_: &[Value]) -> Result { + Ok(CallOutcome::Yield) +} + +fn non_yielding_pends(_: &[Value]) -> Result { + Ok(CallOutcome::Pending(7)) +} + +#[test] +fn non_yielding_args_contract_is_enforced_before_jit_compilation() { + let cases: [(&str, StaticHostArgsFunction); 4] = [ + ("no value", non_yielding_returns_none), + ("halt", non_yielding_halts), + ("yield", non_yielding_yields), + ("pending", non_yielding_pends), + ]; + + for (expected, function) in cases { + let compiled = compile_source("fn action() -> int; action();") + .expect("host call source should compile"); + let mut vm = Vm::new(compiled.program); + vm.bind_static_non_yielding_args_function("action", function); + + let err = vm.run().expect_err("contract violation should fail"); + assert!( + matches!(err, vm::VmError::HostError(ref detail) if detail.contains(expected)), + "unexpected error for {expected}: {err:?}" + ); + } +} + fn builtin_call_index_with_arity(source: &str, argc: u8) -> u16 { let compiled = compile_source(source).expect("compile should succeed"); let mut ip = 0usize; From 7ba306a624b9f97218fd152c76f399390694fdea Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 16 Jul 2026 12:40:24 +0800 Subject: [PATCH 18/19] fix(jit): isolate native bridge errors per thread --- src/vm/native/bridge.rs | 43 +++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index f2debc44..74927436 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -5,8 +5,8 @@ use crate::vm::{ CallOutcome, CallReturn, HostCallExecOutcome, NumericValue, Vm, VmError, VmHostFunction, VmResult, logical_shr_i64, }; +use std::cell::RefCell; use std::sync::Arc; -use std::sync::{Mutex, OnceLock}; pub(crate) const STATUS_CONTINUE: i32 = 0; pub(crate) const STATUS_HALTED: i32 = 1; @@ -72,29 +72,20 @@ impl NativeInterruptSettings { } } -static GENERIC_BRIDGE_ERROR: OnceLock>> = OnceLock::new(); - -fn generic_bridge_error_cell() -> &'static Mutex> { - GENERIC_BRIDGE_ERROR.get_or_init(|| Mutex::new(None)) +thread_local! { + static GENERIC_BRIDGE_ERROR: RefCell> = const { RefCell::new(None) }; } pub(crate) fn store_bridge_error(error: VmError) { - if let Ok(mut guard) = generic_bridge_error_cell().lock() { - *guard = Some(error); - } + GENERIC_BRIDGE_ERROR.with(|slot| *slot.borrow_mut() = Some(error)); } pub(crate) fn clear_bridge_error() { - if let Ok(mut guard) = generic_bridge_error_cell().lock() { - *guard = None; - } + GENERIC_BRIDGE_ERROR.with(|slot| *slot.borrow_mut() = None); } pub(crate) fn take_bridge_error() -> Option { - if let Ok(mut guard) = generic_bridge_error_cell().lock() { - return guard.take(); - } - None + GENERIC_BRIDGE_ERROR.with(|slot| slot.borrow_mut().take()) } fn arc_repr_word(value: &Arc) -> usize { @@ -1162,6 +1153,28 @@ mod tests { use super::*; use std::mem::ManuallyDrop; + #[test] + fn bridge_errors_are_isolated_between_threads() { + clear_bridge_error(); + store_bridge_error(VmError::HostError("main thread".to_string())); + + std::thread::spawn(|| { + assert!(take_bridge_error().is_none()); + store_bridge_error(VmError::HostError("worker thread".to_string())); + assert!(matches!( + take_bridge_error(), + Some(VmError::HostError(detail)) if detail == "worker thread" + )); + }) + .join() + .expect("worker should finish"); + + assert!(matches!( + take_bridge_error(), + Some(VmError::HostError(detail)) if detail == "main thread" + )); + } + #[test] fn sparse_exit_restore_accepts_null_buffers_for_zero_dirty_locals() { let preserved = Arc::new("preserved".to_string()); From 1a71f65a528d0057eb1ae0978ad5a1e94d576ae0 Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 16 Jul 2026 12:45:29 +0800 Subject: [PATCH 19/19] chore(jit): remove redundant terminal clones --- src/vm/jit/runtime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index a9ee8e0f..e4d75383 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -147,7 +147,7 @@ fn native_trace_cache_key( drop_contract_events_enabled, root_ip: trace.root_ip, entry_stack_depth: trace.entry_stack_depth, - terminal: trace.terminal.clone(), + terminal: trace.terminal, ssa_text: trace.ssa_text(), } } @@ -670,7 +670,7 @@ impl Vm { Ok(( native.entry, native.root_ip, - native.terminal.clone(), + native.terminal, native.has_call, native.has_yielding_call, ))