Skip to content

Commit e492fb1

Browse files
fffonionfffonion
committed
jit: lower hot allocation and utf8 builtins
1 parent 0976448 commit e492fb1

6 files changed

Lines changed: 353 additions & 3 deletions

File tree

‎src/vm/jit/ir.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,9 +188,13 @@ pub(crate) enum SsaInstKind {
188188
BytesFromArrayU8 {
189189
array: SsaValueId,
190190
},
191+
BytesToUtf8Ascii {
192+
bytes: SsaValueId,
193+
},
191194
BytesToArrayU8 {
192195
bytes: SsaValueId,
193196
},
197+
ArrayNew,
194198
ArrayLen {
195199
array: SsaValueId,
196200
},
@@ -428,7 +432,8 @@ impl SsaInstKind {
428432
Self::StringSplitLiteral { text, delimiter } => vec![*text, *delimiter],
429433
Self::StringConcat { lhs, rhs } | Self::BytesConcat { lhs, rhs } => vec![*lhs, *rhs],
430434
Self::BytesFromArrayU8 { array } => vec![*array],
431-
Self::BytesToArrayU8 { bytes } => vec![*bytes],
435+
Self::BytesToUtf8Ascii { bytes } | Self::BytesToArrayU8 { bytes } => vec![*bytes],
436+
Self::ArrayNew => Vec::new(),
432437
Self::ArrayGet { array, index } => vec![*array, *index],
433438
Self::ArrayHas { array, index } => vec![*array, *index],
434439
Self::ArraySet {
@@ -1067,7 +1072,9 @@ fn render_inst_kind(kind: &SsaInstKind) -> String {
10671072
SsaInstKind::StringConcat { lhs, rhs } => format!("string_concat {lhs}, {rhs}"),
10681073
SsaInstKind::BytesConcat { lhs, rhs } => format!("bytes_concat {lhs}, {rhs}"),
10691074
SsaInstKind::BytesFromArrayU8 { array } => format!("bytes_from_array_u8 {array}"),
1075+
SsaInstKind::BytesToUtf8Ascii { bytes } => format!("bytes_to_utf8_ascii {bytes}"),
10701076
SsaInstKind::BytesToArrayU8 { bytes } => format!("bytes_to_array_u8 {bytes}"),
1077+
SsaInstKind::ArrayNew => "array_new".to_string(),
10711078
SsaInstKind::ArrayLen { array } => format!("array_len {array}"),
10721079
SsaInstKind::ArrayGet { array, index } => format!("array_get {array}, {index}"),
10731080
SsaInstKind::ArrayHas { array, index } => format!("array_has {array}, {index}"),

‎src/vm/jit/native/lower.rs‎

Lines changed: 120 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1301,7 +1301,9 @@ fn ssa_trace_supported(ssa: &SsaTrace) -> bool {
13011301
| SsaInstKind::StringConcat { .. }
13021302
| SsaInstKind::BytesConcat { .. }
13031303
| SsaInstKind::BytesFromArrayU8 { .. }
1304+
| SsaInstKind::BytesToUtf8Ascii { .. }
13041305
| SsaInstKind::BytesToArrayU8 { .. }
1306+
| SsaInstKind::ArrayNew
13051307
| SsaInstKind::ArrayLen { .. }
13061308
| SsaInstKind::ArrayGet { .. }
13071309
| SsaInstKind::ArrayHas { .. }
@@ -1644,7 +1646,9 @@ fn ssa_inst_requires_owned_value_slot(kind: &SsaInstKind) -> bool {
16441646
| SsaInstKind::ToString { .. }
16451647
| SsaInstKind::StringSplitLiteral { .. }
16461648
| SsaInstKind::BytesFromArrayU8 { .. }
1649+
| SsaInstKind::BytesToUtf8Ascii { .. }
16471650
| SsaInstKind::BytesToArrayU8 { .. }
1651+
| SsaInstKind::ArrayNew
16481652
| SsaInstKind::StringConcat { .. }
16491653
| SsaInstKind::BytesConcat { .. }
16501654
| SsaInstKind::HostCall { .. }
@@ -2980,6 +2984,92 @@ fn lower_ssa_inst(
29802984
b.switch_to_block(cont);
29812985
out
29822986
}
2987+
SsaInstKind::BytesToUtf8Ascii { bytes } => {
2988+
let bytes = values[bytes];
2989+
let bytes_data = ssa_load_heap_data_ptr(b, layout.value, bytes);
2990+
let bytes_ptr = b.ins().load(
2991+
pointer_type,
2992+
MemFlags::new(),
2993+
bytes_data,
2994+
layout.stack_vec.ptr_offset,
2995+
);
2996+
let bytes_len = b.ins().load(
2997+
pointer_type,
2998+
MemFlags::new(),
2999+
bytes_data,
3000+
layout.stack_vec.len_offset,
3001+
);
3002+
let out = owned_value_temp_slot_addr(
3003+
b,
3004+
pointer_type,
3005+
owned_value_temps,
3006+
SsaTempValueSlotKey::Output(output.id),
3007+
)?;
3008+
let validate_loop = b.create_block();
3009+
let validate_step = b.create_block();
3010+
let finish = b.create_block();
3011+
let fail = b.create_block();
3012+
let cont = b.create_block();
3013+
b.append_block_param(validate_loop, pointer_type);
3014+
3015+
let zero = b.ins().iconst(pointer_type, 0);
3016+
b.ins().jump(validate_loop, &[BlockArg::Value(zero)]);
3017+
3018+
b.switch_to_block(validate_loop);
3019+
let index = b.block_params(validate_loop)[0];
3020+
let done = b
3021+
.ins()
3022+
.icmp(IntCC::UnsignedGreaterThanOrEqual, index, bytes_len);
3023+
b.ins().brif(done, finish, &[], validate_step, &[]);
3024+
3025+
b.switch_to_block(validate_step);
3026+
let byte_addr = b.ins().iadd(bytes_ptr, index);
3027+
let byte = b.ins().load(types::I8, MemFlags::new(), byte_addr, 0);
3028+
let is_ascii = b.ins().icmp_imm(IntCC::UnsignedLessThan, byte, 128);
3029+
let validate_next = b.create_block();
3030+
b.ins().brif(is_ascii, validate_next, &[], fail, &[]);
3031+
3032+
b.switch_to_block(validate_next);
3033+
let next_index = b.ins().iadd_imm(index, 1);
3034+
b.ins().jump(validate_loop, &[BlockArg::Value(next_index)]);
3035+
3036+
b.switch_to_block(finish);
3037+
let out_ptr = ssa_call_alloc_buffer(
3038+
b,
3039+
pointer_type,
3040+
heap_refs,
3041+
heap_addrs,
3042+
heap_addrs.alloc_byte_buffer,
3043+
bytes_len,
3044+
)?;
3045+
ssa_call_copy_bytes(
3046+
b,
3047+
pointer_type,
3048+
heap_refs,
3049+
heap_addrs,
3050+
out_ptr,
3051+
bytes_ptr,
3052+
bytes_len,
3053+
)?;
3054+
let out_raw = ssa_call_pack_shared(
3055+
b,
3056+
pointer_type,
3057+
heap_refs,
3058+
heap_addrs.pack_string,
3059+
out_ptr,
3060+
bytes_len,
3061+
bytes_len,
3062+
)?;
3063+
clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?;
3064+
ssa_store_heap_ptr_in_value(b, layout.value, out, layout.value.string_tag, out_raw);
3065+
b.ins().jump(cont, &[]);
3066+
3067+
b.switch_to_block(fail);
3068+
ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?;
3069+
3070+
b.switch_to_block(cont);
3071+
out
3072+
}
29833073
SsaInstKind::BytesToArrayU8 { bytes } => {
29843074
let bytes = values[bytes];
29853075
let bytes_data = ssa_load_heap_data_ptr(b, layout.value, bytes);
@@ -3067,6 +3157,35 @@ fn lower_ssa_inst(
30673157
b.switch_to_block(cont);
30683158
out
30693159
}
3160+
SsaInstKind::ArrayNew => {
3161+
let out = owned_value_temp_slot_addr(
3162+
b,
3163+
pointer_type,
3164+
owned_value_temps,
3165+
SsaTempValueSlotKey::Output(output.id),
3166+
)?;
3167+
let zero = b.ins().iconst(pointer_type, 0);
3168+
let out_ptr = ssa_call_alloc_buffer(
3169+
b,
3170+
pointer_type,
3171+
heap_refs,
3172+
heap_addrs,
3173+
heap_addrs.alloc_value_buffer,
3174+
zero,
3175+
)?;
3176+
let out_raw = ssa_call_pack_shared(
3177+
b,
3178+
pointer_type,
3179+
heap_refs,
3180+
heap_addrs.pack_array,
3181+
out_ptr,
3182+
zero,
3183+
zero,
3184+
)?;
3185+
clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?;
3186+
ssa_store_heap_ptr_in_value(b, layout.value, out, layout.value.array_tag, out_raw);
3187+
out
3188+
}
30703189
SsaInstKind::ArrayLen { array } => {
30713190
let array = values[array];
30723191
let vec_ptr = ssa_load_heap_data_ptr(b, layout.value, array);
@@ -4507,7 +4626,7 @@ fn lower_ssa_exit_block(
45074626
.contains_key(&SsaTempValueSlotKey::Output(*value))
45084627
&& moved_owned_values.insert(*value)
45094628
}
4510-
SsaMaterialization::BoxHeapPtr { .. } => true,
4629+
SsaMaterialization::BoxHeapPtr { .. } => false,
45114630
}
45124631
});
45134632
if inline_owned_restore {

‎src/vm/jit/recorder.rs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,7 +512,9 @@ enum SpecializedBuiltinKind {
512512
StringConcat,
513513
BytesConcat,
514514
BytesFromArrayU8,
515+
BytesToUtf8Ascii,
515516
BytesToArrayU8,
517+
ArrayNew,
516518
ArrayLen,
517519
ArrayGet,
518520
ArrayHas,
@@ -1225,6 +1227,19 @@ pub(crate) fn record_trace_with_local_count(
12251227
argc,
12261228
yields,
12271229
} => {
1230+
if builtin == Some(BuiltinFunction::ArrayNew) && argc == 0 {
1231+
let (name, out) = emit_specialized_builtin_call(
1232+
&mut builder,
1233+
current_block,
1234+
ip,
1235+
&mut frame,
1236+
SpecializedBuiltinKind::ArrayNew,
1237+
)?;
1238+
op_names.push(name.to_string());
1239+
frame.push(out);
1240+
has_useful_native_computation = true;
1241+
continue;
1242+
}
12281243
if let Some(builtin) = builtin
12291244
&& argc > 0
12301245
{
@@ -1477,6 +1492,13 @@ fn infer_loop_header_plan(
14771492
..
14781493
} => {
14791494
if argc == 0 {
1495+
if builtin == BuiltinFunction::ArrayNew {
1496+
let _ = analyze_specialized_builtin_call(
1497+
&mut frame,
1498+
SpecializedBuiltinKind::ArrayNew,
1499+
)?;
1500+
continue;
1501+
}
14801502
return Ok(None);
14811503
}
14821504
let args = call_arg_slice(&frame.stack, usize::from(argc))?;
@@ -2782,6 +2804,9 @@ fn select_specialized_builtin_kind(
27822804
BuiltinFunction::StringSplitLiteral => {
27832805
return Some(SpecializedBuiltinKind::StringSplitLiteral);
27842806
}
2807+
BuiltinFunction::BytesToUtf8 => {
2808+
return Some(SpecializedBuiltinKind::BytesToUtf8Ascii);
2809+
}
27852810
BuiltinFunction::MapIterNext => return Some(SpecializedBuiltinKind::MapIterNext),
27862811
BuiltinFunction::MapIterTakeKey => return Some(SpecializedBuiltinKind::MapIterTakeKey),
27872812
BuiltinFunction::MapIterTakeValue => {
@@ -2987,11 +3012,20 @@ fn analyze_specialized_builtin_call(
29873012
frame.push(ValueInfo::tagged_typed(ValueType::Bytes));
29883013
Ok("bytes_from_array_u8")
29893014
}
3015+
SpecializedBuiltinKind::BytesToUtf8Ascii => {
3016+
let _ = frame.pop()?;
3017+
frame.push(ValueInfo::tagged_typed(ValueType::String));
3018+
Ok("bytes_to_utf8_ascii")
3019+
}
29903020
SpecializedBuiltinKind::BytesToArrayU8 => {
29913021
let _ = frame.pop()?;
29923022
frame.push(ValueInfo::tagged_typed(ValueType::Array));
29933023
Ok("bytes_to_array_u8")
29943024
}
3025+
SpecializedBuiltinKind::ArrayNew => {
3026+
frame.push(ValueInfo::tagged_typed(ValueType::Array));
3027+
Ok("array_new")
3028+
}
29953029
SpecializedBuiltinKind::ArrayLen => {
29963030
let _ = frame.pop()?;
29973031
frame.push(ValueInfo::int(None));
@@ -3573,6 +3607,30 @@ fn emit_specialized_builtin_call(
35733607
.map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?;
35743608
Ok(("bytes_from_array_u8", out))
35753609
}
3610+
SpecializedBuiltinKind::BytesToUtf8Ascii => {
3611+
let bytes = ensure_heap_ptr(
3612+
builder,
3613+
block,
3614+
ip,
3615+
frame.pop()?,
3616+
HeapContainerKind::Bytes.value_type(),
3617+
)?;
3618+
let out = builder
3619+
.append_value_inst(
3620+
block,
3621+
ip,
3622+
SsaValueRepr::Tagged,
3623+
SsaInstKind::BytesToUtf8Ascii {
3624+
bytes: bytes.value.id,
3625+
},
3626+
)
3627+
.map(|value| SymbolicValue {
3628+
value,
3629+
info: ValueInfo::tagged_typed(ValueType::String),
3630+
})
3631+
.map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?;
3632+
Ok(("bytes_to_utf8_ascii", out))
3633+
}
35763634
SpecializedBuiltinKind::BytesToArrayU8 => {
35773635
let bytes = ensure_heap_ptr(
35783636
builder,
@@ -3597,6 +3655,16 @@ fn emit_specialized_builtin_call(
35973655
.map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?;
35983656
Ok(("bytes_to_array_u8", out))
35993657
}
3658+
SpecializedBuiltinKind::ArrayNew => {
3659+
let out = builder
3660+
.append_value_inst(block, ip, SsaValueRepr::Tagged, SsaInstKind::ArrayNew)
3661+
.map(|value| SymbolicValue {
3662+
value,
3663+
info: ValueInfo::tagged_typed(ValueType::Array),
3664+
})
3665+
.map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?;
3666+
Ok(("array_new", out))
3667+
}
36003668
SpecializedBuiltinKind::ArrayLen => {
36013669
let array = frame.pop()?;
36023670
let array = ensure_heap_ptr(

‎src/vm/jit/region.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ fn remap_inst_inputs(
205205
}};
206206
}
207207
match kind {
208-
SsaInstKind::Constant(_) => {}
208+
SsaInstKind::Constant(_) | SsaInstKind::ArrayNew => {}
209209
SsaInstKind::HostCall { args, .. } => {
210210
for arg in args {
211211
one!(arg);
@@ -227,6 +227,7 @@ fn remap_inst_inputs(
227227
| SsaInstKind::TypeOf { value: input }
228228
| SsaInstKind::ToString { value: input }
229229
| SsaInstKind::BytesFromArrayU8 { array: input }
230+
| SsaInstKind::BytesToUtf8Ascii { bytes: input }
230231
| SsaInstKind::BytesToArrayU8 { bytes: input }
231232
| SsaInstKind::MapIterNext { slot: input }
232233
| SsaInstKind::MapIterTakeKey { slot: input }

‎src/vm/jit/runtime.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,17 @@ pub(crate) fn resume_linked_trace_entry_address() -> usize {
231231
pd_vm_native_resume_linked_trace as *const () as usize
232232
}
233233

234+
#[cfg(not(any(
235+
all(
236+
target_arch = "x86_64",
237+
any(target_os = "windows", all(unix, not(target_os = "macos")))
238+
),
239+
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
240+
)))]
241+
pub(crate) fn resume_linked_trace_entry_address() -> usize {
242+
0
243+
}
244+
234245
#[cfg(any(
235246
all(
236247
target_arch = "x86_64",
@@ -549,6 +560,13 @@ impl Vm {
549560
self.publish_native_direct_slot(key.parent_trace_id, key.exit_id.raw(), child_trace_id)
550561
}
551562

563+
#[cfg(any(
564+
all(
565+
target_arch = "x86_64",
566+
any(target_os = "windows", all(unix, not(target_os = "macos")))
567+
),
568+
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
569+
))]
552570
fn publish_native_direct_slot(
553571
&mut self,
554572
parent_trace_id: usize,
@@ -590,6 +608,22 @@ impl Vm {
590608
Ok(())
591609
}
592610

611+
#[cfg(not(any(
612+
all(
613+
target_arch = "x86_64",
614+
any(target_os = "windows", all(unix, not(target_os = "macos")))
615+
),
616+
all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos"))
617+
)))]
618+
fn publish_native_direct_slot(
619+
&mut self,
620+
_parent_trace_id: usize,
621+
_slot_id: u32,
622+
_child_trace_id: usize,
623+
) -> VmResult<()> {
624+
Ok(())
625+
}
626+
593627
#[cfg(any(
594628
all(
595629
target_arch = "x86_64",

0 commit comments

Comments
 (0)