diff --git a/.github/workflows/zjit-macos.yml b/.github/workflows/zjit-macos.yml index feb8c462890303..e0cc2eff496ab7 100644 --- a/.github/workflows/zjit-macos.yml +++ b/.github/workflows/zjit-macos.yml @@ -98,7 +98,7 @@ jobs: rustup install ${{ matrix.rust_version }} --profile minimal rustup default ${{ matrix.rust_version }} - - uses: taiki-e/install-action@37f7c5781271959fb65b6b35224e28652ff2b63d # v2.87.0 + - uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index 9623059e035e18..4b57137ba7f42c 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -152,7 +152,7 @@ jobs: ruby-version: '3.1' bundler: none - - uses: taiki-e/install-action@37f7c5781271959fb65b6b35224e28652ff2b63d # v2.87.0 + - uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/box.c b/box.c index 559170c7a452ce..d58082e15bb37e 100644 --- a/box.c +++ b/box.c @@ -54,6 +54,7 @@ static bool tmp_dir_has_dirsep; /* process-private 0700 directory for box-local copies of extensions */ static char box_ext_tmp_dir[MAXPATHLEN]; +static rb_pid_t box_ext_tmp_dir_owner; static unsigned long box_ext_seq; #define BOX_TMP_PREFIX "_ruby_box_" @@ -574,6 +575,13 @@ system_tmpdir(void) static void ensure_box_ext_tmp_dir(void) { + /* A forked child inherits the parent's directory path; discard it so + * that each process creates its own directory and removes only that + * one at teardown. */ + if (box_ext_tmp_dir[0] && box_ext_tmp_dir_owner != getpid()) { + box_ext_tmp_dir[0] = '\0'; + box_ext_seq = 0; + } if (box_ext_tmp_dir[0]) return; int last_errno = 0; @@ -592,6 +600,7 @@ ensure_box_ext_tmp_dir(void) } if (mkdir(path, 0700) == 0) { strlcpy(box_ext_tmp_dir, path, sizeof(box_ext_tmp_dir)); + box_ext_tmp_dir_owner = getpid(); return; } last_errno = errno; @@ -852,7 +861,7 @@ rb_box_unload_local_extensions(void) ext = next; } #endif - if (box_ext_tmp_dir[0]) { + if (box_ext_tmp_dir[0] && box_ext_tmp_dir_owner == getpid()) { rmdir(box_ext_tmp_dir); box_ext_tmp_dir[0] = '\0'; } diff --git a/file.c b/file.c index 526c6efd338032..8f860f3dc10f74 100644 --- a/file.c +++ b/file.c @@ -2253,14 +2253,25 @@ rb_file_executable_real_p(VALUE obj, VALUE fname) /* * call-seq: - * File.file?(file) -> true or false - * - * Returns +true+ if the named +file+ exists and is a regular file. - * - * +file+ can be an IO object. + * File.file?(object) -> true or false + * + * Returns whether the given +object+, a string path or IO object, + * represents a filesystem entry that exists and is a regular file; + * see File.ftype: + * + * # Paths. + * File.file?('README.md') # => true + * File.file?('doc/') # => false + * File.file?('nosuch') # => false + * # IO objects. + * file = File.new('README.md') + * File.file?(file) # => true + * dir = Dir.new('doc/') + * File.file?(dir) # => false + * # Clean up. + * file.close + * dir.close * - * If the +file+ argument is a symbolic link, it will resolve the symbolic link - * and use the file referenced by the link. */ static VALUE @@ -7010,16 +7021,36 @@ rb_stat_ww(VALUE obj) } /* - * call-seq: - * stat.executable? -> true or false + * call-seq: + * executable? -> true or false * - * Returns true if stat is executable or if the - * operating system doesn't distinguish executable files from - * nonexecutable files. The tests are made using the effective owner of - * the process. + * Returns whether the filesystem entry represented by +self+ + * exists and is executable; + * raises Errno::ENOENT if the entry does not exist. + * + * On Windows, the entry is executable if its path has file extension + * +.bat+, +.cmd+, +.com+, or +.exe+: * - * File.stat("testfile").executable? #=> false + * File.stat('win32/rtname.cmd').executable? # => true + * File.stat('win32/file.c').executable? # => false * + * On other systems, the entry is executable if it has the execute/search + * permission for the effective user and group id of the current process; + * see {Permissions}[rdoc-ref:file/filesystem_modes.md@Permissions]. + * + * These examples use + * a {helper method}[rdoc-ref:file/filesystem_modes.md@Helper+Method], +mode+, + * that displays a mode both in octal digits and in characters: + * + * File.stat('.').executable? # => true + * mode('.') # => "040775 drwxrwxr-x" + * File.stat('bin/gem').executable? # => true + * mode('bin/gem') # => "100775 -rwxrwxr-x" + * File.stat('/etc/passwd').executable? # => false + * mode('/etc/passwd') # => "100644 -rw-r--r--" + * + * Note that some filesystem settings may cause this method to return +true+ + * even though the entry is not executable by the effective user/group. */ static VALUE @@ -7080,12 +7111,16 @@ rb_stat_X(VALUE obj) /* * call-seq: - * stat.file? -> true or false + * file? -> true or false + * + * Returns whether +self+ represents a filesystem entry that exists and is a regular file; + * see File::Stat.ftype: * - * Returns true if stat is a regular file (not - * a device file, pipe, socket, etc.). + * # Paths. + * File.stat('README.md').file? # => true + * File.stat('doc/').file? # => false + * File.stat('nosuch').file? # Raises Errno::ENOENT: No such file or directory. * - * File.stat("testfile").file? #=> true * */ diff --git a/pathname_builtin.rb b/pathname_builtin.rb index e538408ff62bc6..b5309a88f78555 100644 --- a/pathname_builtin.rb +++ b/pathname_builtin.rb @@ -1670,9 +1670,19 @@ def fnmatch(pattern, ...) File.fnmatch(pattern, @path, ...) end def fnmatch?(pattern, ...) File.fnmatch?(pattern, @path, ...) end # call-seq: - # pathname.ftype -> string + # ftype -> string # - # Returns the string type of the object at the path in +self+: + # Returns the string type of the object at the path in self, one of: + # + # - 'file'. + # - 'directory'. + # - 'characterSpecial'. + # - 'blockSpecial'. + # - 'fifo'. + # - 'link'. + # - 'socket'. + # + # Examples: # # Pathname('README.md').ftype # => "file" # Pathname('lib').ftype # => "directory" @@ -2280,12 +2290,33 @@ def empty? # call-seq: # executable? -> true or false # - # Returns whether the entry represented by `self` is executable; - # calls FileTest.executable? with argument `self.to_s`: + # Returns whether the entry represented by `self` exists and is executable. + # + # On Windows, the entry is executable if its path has file extension + # `.bat`, `.cmd`, `.com`, or `.exe`: + # + # ```ruby + # Pathname('bin/gem').executable? # => true + # mode('bin/gem') # => "100775 -rwxrwxr-x" + # Pathname('.').executable? # => true + # mode('.') # => "040775 drwxrwxr-x" + # Pathname('nosuch').executable? # => false + # ``` + # + # On other systems, the entry is executable if it has the execute/search + # permission for the effective user and group id of the current process; + # see {Permissions}[rdoc-ref:file/filesystem_modes.md@Permissions]. + # + # These examples use + # a {helper method}[rdoc-ref:file/filesystem_modes.md@Helper+Method], `mode`, + # that displays a mode both in octal digits and in characters: # # ```ruby - # Pathname('bin/gem').executable? # => true - # Pathname('README.md').executable? # => false + # Pathname('bin/gem').executable? # => true + # mode('bin/gem') # => "100775 -rwxrwxr-x" + # Pathname('.').executable? # => true + # mode('.') # => "040775 drwxrwxr-x" + # Pathname('nosuch').executable? # => false # ``` # def executable?() FileTest.executable?(@path) end @@ -2375,7 +2406,8 @@ def directory?() FileTest.directory?(@path) end # call-seq: # file? -> true or false # - # Returns whether the entry at the path in `self` exists and is a regular file: + # Returns whether the entry at the path in `self` exists and is a regular file; + # see #ftype: # # ```ruby # Pathname('README.md').file? # => true diff --git a/test/ruby/test_box.rb b/test/ruby/test_box.rb index 5491578e9fb303..b3ecd0c546394d 100644 --- a/test/ruby/test_box.rb +++ b/test/ruby/test_box.rb @@ -1301,6 +1301,18 @@ def test_loading_extension_from_deep_path_in_user_box end end + def test_extension_loading_survives_fork_in_user_box + omit "fork is not supported" unless Process.respond_to?(:fork) + + assert_ruby_status([ENV_ENABLE_BOX], "#{<<~"begin;"}\n#{<<~'end;'}") + begin; + Ruby::Box.new.require "digest/md5" + pid = fork {Ruby::Box.new.require "digest/sha2"} + raise "extension loading failed in the child" unless Process.wait2(pid)[1].success? + Ruby::Box.new.require "digest/sha2" + end; + end + def test_root_box_iclasses_should_be_boxable assert_separately([ENV_ENABLE_BOX], __FILE__, __LINE__, "#{<<~"begin;"}\n#{<<~'end;'}", ignore_stderr: true) begin; diff --git a/vm.c b/vm.c index 95a75af8ca31ea..67b44c100614bb 100644 --- a/vm.c +++ b/vm.c @@ -526,8 +526,10 @@ yjit_compile(rb_execution_context_t *ec) const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); - // Increment the ISEQ's call counter and trigger JIT compilation if not compiled - if (body->jit_entry == NULL) { + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled. + // Stop incrementing when not compiling (out of executable memory) so that + // ISEQs that failed to compile don't keep dirtying CoW pages after fork. + if (body->jit_entry == NULL && rb_yjit_compiling_p) { body->jit_entry_calls++; if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) { rb_yjit_compile_iseq(iseq, ec, false); @@ -546,7 +548,7 @@ zjit_compile(rb_execution_context_t *ec) const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); - if (body->jit_entry == NULL) { + if (body->jit_entry == NULL && rb_zjit_compiling_p) { body->jit_entry_calls++; // At profile-threshold, rewrite some of the YARV instructions @@ -607,7 +609,9 @@ jit_compile_exception(rb_execution_context_t *ec) struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); #if USE_ZJIT - if (body->jit_exception == NULL && rb_zjit_enabled_p) { + // rb_zjit_compiling_p is false until ZJIT is enabled, so no + // rb_zjit_enabled_p check is needed here. + if (body->jit_exception == NULL && rb_zjit_compiling_p) { body->jit_exception_calls++; // At profile-threshold, rewrite some of the YARV instructions @@ -624,8 +628,9 @@ jit_compile_exception(rb_execution_context_t *ec) #endif #if USE_YJIT - // Increment the ISEQ's call counter and trigger JIT compilation if not compiled - if (body->jit_exception == NULL && rb_yjit_enabled_p) { + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled. + // Like the ZJIT branch above, no rb_yjit_enabled_p check is needed here. + if (body->jit_exception == NULL && rb_yjit_compiling_p) { body->jit_exception_calls++; if (body->jit_exception_calls == rb_yjit_call_threshold) { rb_yjit_compile_iseq(iseq, ec, true); diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 3a0b7324046e59..ea4deb17727959 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1874,12 +1874,29 @@ vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, } } +// Fallback for YJIT. Prepare throw data and return it. VALUE rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj) { return vm_throw(ec, reg_cfp, throw_state, throwobj); } +// Fallback for ZJIT. Make a longjmp and unwind to the most recent vm_exec(). +VALUE +rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj) +{ + VALUE val = vm_throw(ec, reg_cfp, throw_state, throwobj); + + // vm_throw() has set ec->tag->state. On the longjmp path, vm_exec() reads the throw data from + // ec->errinfo instead of vm_exec_core()'s return value like THROW_EXCEPTION()'s path does, so + // we need to put it there instead. + enum ruby_tag_type state = ec->tag->state; + ec->errinfo = val; + EC_JUMP_TAG(ec, state); + + UNREACHABLE_RETURN(Qundef); +} + static inline void vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag) { diff --git a/yjit.h b/yjit.h index 481ee8b9b6350d..93acf4e60d1dd9 100644 --- a/yjit.h +++ b/yjit.h @@ -29,6 +29,7 @@ extern uint64_t rb_yjit_cold_threshold; extern uint64_t rb_yjit_live_iseq_count; extern uint64_t rb_yjit_iseq_alloc_count; extern bool rb_yjit_enabled_p; +extern bool rb_yjit_compiling_p; void rb_yjit_incr_counter(const char *counter_name); void rb_yjit_invalidate_all_method_lookup_assumptions(void); void rb_yjit_cme_invalidate(rb_callable_method_entry_t *cme); @@ -58,6 +59,7 @@ void rb_yjit_mark_all_executable(void); // In these builds, YJIT could never be turned on. Provide dummy implementations. #define rb_yjit_enabled_p false +#define rb_yjit_compiling_p false static inline void rb_yjit_incr_counter(const char *counter_name) {} static inline void rb_yjit_invalidate_all_method_lookup_assumptions(void) {} static inline void rb_yjit_cme_invalidate(rb_callable_method_entry_t *cme) {} diff --git a/yjit/src/yjit.rs b/yjit/src/yjit.rs index 380544ba33d981..33e4308b2999af 100644 --- a/yjit/src/yjit.rs +++ b/yjit/src/yjit.rs @@ -17,6 +17,15 @@ use crate::log::Log; #[no_mangle] pub static mut rb_yjit_enabled_p: bool = false; +/// Whether YJIT is compiling. Starts as false until YJIT is enabled, so the +/// interpreter doesn't need to check rb_yjit_enabled_p before it. Set back to +/// false when we run out of executable memory, in which case the interpreter +/// stops incrementing ISEQ call counters so that ISEQs that will never be +/// compiled stop dirtying CoW pages after fork. +#[allow(non_upper_case_globals)] +#[no_mangle] +pub static mut rb_yjit_compiling_p: bool = false; + // Time when YJIT was yjit was initialized (see yjit_init) pub static mut YJIT_INIT_TIME: Option = None; @@ -37,6 +46,14 @@ pub fn yjit_enabled_p() -> bool { unsafe { rb_yjit_enabled_p } } +/// Whether we have run out of executable memory. Used by the C code to +/// decide whether to keep incrementing ISEQ call counters. +pub fn out_of_memory_p() -> bool { + let cb = CodegenGlobals::get_inline_cb(); + let ocb = CodegenGlobals::get_outlined_cb(); + cb.has_dropped_bytes() || ocb.unwrap().has_dropped_bytes() +} + /// Register specialized codegen for builtin C method entries. /// Must be called at boot before ruby_init_prelude() since the prelude /// could redefine core methods (e.g. Kernel.prepend via bundler). @@ -77,7 +94,10 @@ fn yjit_init() { // YJIT enabled and initialized successfully assert!(unsafe{ !rb_yjit_enabled_p }); - unsafe { rb_yjit_enabled_p = true; } + unsafe { + rb_yjit_enabled_p = true; + rb_yjit_compiling_p = true; + } }); if let Err(_) = result { @@ -179,10 +199,11 @@ pub extern "C" fn rb_yjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exc let maybe_code_ptr = with_compile_time(|| { gen_entry_point(iseq, ec, jit_exception) }); - match maybe_code_ptr { - Some(ptr) => ptr, - None => std::ptr::null(), - } + // Stop compiling if we ran out of executable memory so that the + // interpreter stops incrementing ISEQ call counters. + unsafe { rb_yjit_compiling_p = !out_of_memory_p(); } + + maybe_code_ptr.unwrap_or(std::ptr::null()) } /// Free and recompile all existing JIT code diff --git a/zjit.h b/zjit.h index 7bda68202d4e96..f7d84e2a1157aa 100644 --- a/zjit.h +++ b/zjit.h @@ -107,6 +107,7 @@ ZJIT_STACK_MAP_BASE_PTR_STACK_SIZE(VALUE entry) } extern void *rb_zjit_entry; +extern bool rb_zjit_compiling_p; extern const zjit_jit_frame_t rb_zjit_c_frame; extern uint64_t rb_zjit_call_threshold; extern uint64_t rb_zjit_profile_threshold; @@ -170,6 +171,7 @@ CFP_ZJIT_FRAME(const rb_control_frame_t *cfp) } #else #define rb_zjit_entry 0 +#define rb_zjit_compiling_p false static inline void rb_zjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception) {} static inline void rb_zjit_profile_insn(uint32_t insn, rb_execution_context_t *ec) {} static inline void rb_zjit_profile_enable(const rb_iseq_t *iseq) {} diff --git a/zjit.rb b/zjit.rb index 2c6f5f4000d5be..3b52211eef3735 100644 --- a/zjit.rb +++ b/zjit.rb @@ -181,6 +181,8 @@ def stats_string :load_field_count, :store_field_count, + :throw_count, + :side_exit_size, :code_region_bytes, :side_exit_size_ratio, diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs index c172e6b6420684..d45e323253fbff 100644 --- a/zjit/src/asm/mod.rs +++ b/zjit/src/asm/mod.rs @@ -6,6 +6,7 @@ use std::ops::Range; use std::rc::Rc; use std::cell::RefCell; use std::mem; +use crate::state::rb_zjit_compiling_p; use crate::virtualmem::*; // Lots of manual vertical alignment in there that rustfmt doesn't handle well. @@ -218,6 +219,10 @@ impl CodeBlock { pub fn update_dropped_bytes(&mut self) { if self.mem_block.borrow().can_allocate() { self.dropped_bytes = false; + + // Memory is available again, so let the interpreter resume + // triggering compilation. + unsafe { rb_zjit_compiling_p = true; } } } diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 3b30e1bb2563de..f58f150a4a90bb 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -18,7 +18,7 @@ use crate::invariants::{ use crate::gc::append_gc_offsets; use crate::payload::{IseqCodePtrs, IseqStatus, IseqVersion, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; use crate::profile::reset_profiles_remaining; -use crate::state::ZJITState; +use crate::state::{rb_zjit_compiling_p, ZJITState}; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, trace_compile_phase, Counter, Counter::{compile_time_ns, exit_compile_error}}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; @@ -200,6 +200,14 @@ pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exc let cb = ZJITState::get_code_block(); let mut code_ptr = with_time_stat(compile_time_ns, || gen_iseq_entry_point(cb, iseq, jit_exception)); + // If this compile ran out of executable memory, stop compiling so + // that the interpreter stops incrementing ISEQ call counters. It is + // set back to true in update_dropped_bytes() if memory becomes + // available again. + if matches!(&code_ptr, Err(CompileError::OutOfMemory)) { + unsafe { rb_zjit_compiling_p = false; } + } + if let Err(err) = &code_ptr { // Assert that the ISEQ compiles if RubyVM::ZJIT.assert_compiles is enabled. // We assert only `jit_exception: false` cases until we support exception handlers. @@ -273,9 +281,26 @@ pub fn invalidate_iseq_version(cb: &mut CodeBlock, iseq: IseqPtr, version: &mut pub fn gen_iseq_call(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<(), CompileError> { trace_compile_phase("compile_stub", || { // Compile a function stub - let stub_ptr = gen_function_stub(cb, iseq_call.clone()).inspect_err(|err| { - debug!("{err:?}: gen_function_stub failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); - })?; + let stub_ptr = match iseq_call.stub_addr.get() { + // When gen_iseq_call() is called from invalidation and therefore this IseqCall has been + // compiled before, reuse the address of the previously-compiled stub. + Some(stub_ptr) => { + // gen_function_stub leaked one Rc reference into the stub's baked IseqCall pointer, + // and the previous function_stub_hit consumed it. Restore one leaked reference for + // the next stub hit's Rc::from_raw to reclaim. + unsafe { Rc::increment_strong_count(Rc::as_ptr(iseq_call)); } + stub_ptr + } + // When gen_iseq_call() is called from gen_iseq_body() and this IseqCall is compiled for + // the first time, generate a function stub and remember the address in the IseqCall. + None => { + let stub_ptr = gen_function_stub(cb, iseq_call.clone()).inspect_err(|err| { + debug!("{err:?}: gen_function_stub failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + iseq_call.stub_addr.set(Some(stub_ptr)); + stub_ptr + } + }; // Update the JIT-to-JIT call to call the stub let stub_addr = stub_ptr.raw_ptr(cb); @@ -804,7 +829,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::IsA { val, class } => gen_is_a(jit, asm, opnd!(val), opnd!(class)), &Insn::ArrayMax { ref elements, state } => gen_array_max(jit, asm, function, opnds!(elements), &function.frame_state(state)), &Insn::ArrayMin { ref elements, state } => gen_array_min(jit, asm, function, opnds!(elements), &function.frame_state(state)), - &Insn::Throw { state, .. } => no_output!(gen_throw(jit, asm, function, &function.frame_state(state))), + &Insn::Throw { throw_state, val, state } => no_output!(gen_throw(jit, asm, function, throw_state, opnd!(val), &function.frame_state(state))), &Insn::CondBranch { .. } | &Insn::Jump { .. } | Insn::Entries { .. } => unreachable!(), }; @@ -2761,10 +2786,25 @@ fn gen_return(asm: &mut Assembler, val: lir::Opnd) { asm.cret(C_RET_OPND); } -fn gen_throw(jit: &mut JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { - // TODO: Consider calling rb_vm_throw and propagating ec->tag->state to the interpreter. - // Also consider making it a jump on method inlining. - gen_side_exit(jit, asm, function, &SideExitReason::Throw, None, state); +fn gen_throw(jit: &mut JITState, asm: &mut Assembler, function: &Function, throw_state: u32, val: lir::Opnd, state: &FrameState) { + gen_incr_counter(asm, Counter::throw_count); + + // The interpreter pops the thrown value before calling vm_throw(), so keep it out of the cfp->sp we publish. + let state = state.with_stack_size(state.stack_size() - 1); // -1 for popped throw value + // rb_vm_throw() allocates with THROW_DATA_NEW() and may raise LocalJumpError, and the interpreter reads this + // frame's locals and stack while unwinding, so publish them in the same way as any other non-leaf fallback call. + gen_prepare_fallback_call(jit, asm, function, &state); + + asm_comment!(asm, "throw"); + unsafe extern "C" { + fn rb_zjit_throw(ec: EcPtr, cfp: CfpPtr, throw_state: usize, throwobj: VALUE) -> VALUE; + } + asm_ccall!(asm, rb_zjit_throw, EC, CFP, Opnd::UImm(throw_state.into()), val); + + // rb_zjit_throw() never returns. Trap in case it somehow does, and end the + // LIR block with an unreachable ret to give it a normal terminator. + asm.abort(); + asm.cret(C_RET_OPND); } /// Compile Fixnum + Fixnum @@ -4340,6 +4380,10 @@ pub struct IseqCall { /// Position where the call instruction ends (exclusive) end_addr: Cell>, + + /// Address of the function stub generated for this call, if already compiled. + // TODO(alan): Remove with removal of without_locals(), as this caching means `IseqCall` is never freed. + stub_addr: Cell>, } pub type IseqCallRef = Rc; @@ -4351,6 +4395,7 @@ impl IseqCall { iseq: Cell::new(iseq), start_addr: Cell::new(None), end_addr: Cell::new(None), + stub_addr: Cell::new(None), jit_entry_idx, argc, }; diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 79ff8dd539780a..0fd00630c1952a 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -6,7 +6,7 @@ use crate::backend::lir::Assembler; use crate::codegen::max_iseq_versions; use crate::cruby::*; use crate::hir::{Insn, iseq_to_hir}; -use crate::options::{get_option, rb_zjit_prepare_options, set_call_threshold, set_inline_threshold, set_max_versions}; +use crate::options::{get_option, rb_zjit_prepare_options, set_call_threshold, set_inline_threshold, set_max_versions, set_mem_bytes}; use crate::payload::IseqVersion; use crate::hir::tests::hir_build_tests::assert_contains_opcode; use crate::payload::*; @@ -829,6 +829,206 @@ fn test_yield_non_local_return() { assert_snapshot!(assert_compiles_allowing_exits("test"), @"42"); } +#[test] +fn test_throw_break_with_value_from_each() { + set_call_threshold(2); + eval(" + def test(a) = a.each { |x| break x * 10 if x == 3 } + test([1, 2, 3, 4]) + test([1, 2, 3, 4]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([1, 2, 3, 4])"), @"30"); +} + +#[test] +fn test_throw_no_break_returns_receiver() { + set_call_threshold(2); + eval(" + def test(a) = a.each { |x| break x if x == 99 } + test([1, 2]) + test([1, 2]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([1, 2])"), @"[1, 2]"); +} + +#[test] +fn test_throw_break_across_jit_to_jit_call() { + set_call_threshold(2); + eval(" + def inner = yield + def outer = inner { break 7 } + def test = outer + test + test + "); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"7"); +} + +#[test] +fn test_throw_break_three_frames_deep() { + set_call_threshold(2); + eval(" + def innermost(a) = a.each { |x| break x if x.even? } + def middle(a) = innermost(a) + def test(a) = middle(a) + test([1, 2, 3]) + test([1, 2, 3]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([1, 2, 3])"), @"2"); +} + +#[test] +fn test_throw_break_value_used_by_caller() { + set_call_threshold(2); + eval(" + def test(a) + v = a.each { |x| break x + 100 if x > 1 } + v.to_s + end + test([1, 2, 3]) + test([1, 2, 3]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([1, 2, 3])"), @r#""102""#); +} + +#[test] +fn test_throw_break_search_loop() { + set_call_threshold(2); + eval(" + def test(a) = a.each_with_index { |x, i| break i if x == :b } + test([:a, :b, :c]) + test([:a, :b, :c]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([:a, :b, :c])"), @"1"); +} + +#[test] +fn test_throw_break_runs_ensure() { + set_call_threshold(2); + eval(" + def test(a) + log = [] + r = a.each do |x| + begin + break x if x == 2 + ensure + log << x + end + end + [r, log] + end + test([1, 2, 3]) + test([1, 2, 3]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([1, 2, 3])"), @"[2, [1, 2]]"); +} + +#[test] +fn test_throw_return_from_proc() { + set_call_threshold(2); + eval(" + def test + p = proc { return 5 } + p.call + 99 + end + test + test + "); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_throw_return_from_lambda() { + set_call_threshold(2); + eval(" + def test + l = lambda { return 5 } + l.call + 1 + end + test + test + "); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"6"); +} + +#[test] +fn test_throw_orphan_break_raises_local_jump_error() { + set_call_threshold(2); + eval(" + def test + pr = proc { break 1 } + begin + pr.call + rescue LocalJumpError => e + e.class + end + end + test + test + "); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"LocalJumpError"); +} + +#[test] +fn test_throw_retry_in_rescue() { + set_call_threshold(2); + eval(" + def test + tries = 0 + begin + tries += 1 + raise 'boom' if tries < 3 + tries + rescue + retry + end + end + test + test + "); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"3"); +} + +#[test] +fn test_throw_next_with_ensure() { + set_call_threshold(2); + eval(" + def test(a) + a.map do |x| + begin + next x * 2 + ensure + nil + end + end + end + test([1, 2, 3]) + test([1, 2, 3]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([1, 2, 3])"), @"[2, 4, 6]"); +} + +#[test] +fn test_throw_break_inner_loop_repeatedly() { + set_call_threshold(2); + eval(" + def test(a) + sum = 0 + a.each do |x| + a.each do |y| + break if y > 2 + sum += x * y + end + end + sum + end + test([1, 2, 3]) + test([1, 2, 3]) + "); + assert_snapshot!(assert_compiles_allowing_exits("test([1, 2, 3])"), @"18"); +} + #[test] fn test_yield_autosplat() { // {|a, b|} auto-splats a single Array arg for yield (falls back). @@ -1105,6 +1305,41 @@ fn test_no_ep_escape_patch_point_after_send_does_not_repeat_send() { assert_snapshot!(assert_compiles_allowing_exits("[test, test, test]"), @"[1, 2, 3]"); } +#[test] +fn test_no_ep_escape_side_exit_restores_locals_while_oom() { + // A regression test for stub compilation failures on OOM. Functions patched by NoEPEscape + // is unsafe to enter (FrameState uses without_locals() and doesn't spill the entry state), + // so even under OOM, the re-stub after invalidation must succeed. + set_mem_bytes(2 * 1024 * 1024); + set_inline_threshold(0); + set_call_threshold(2); + assert_snapshot!(inspect(r#" + class Foo + def initialize = @perm = 7 + def callee(esc, local_to_spill = "spilled", perm = @perm) + binding if esc + local_to_spill + end + end + def kaller(foo, esc) = foo.callee(esc) + + foo = Foo.new + 300.times { kaller(foo, false) } # compile callee (with its NoEPEscape patch point) and the kaller->callee edge + + # Fill the code region so the re-stub after the EP escape fails with OutOfMemory. + 1000.times do |i| + body = (0...25).map { |k| "u#{k} = #{i} + #{k}; s += u#{k}" }.join("; ") + eval "def big#{i}(a = 1); s = 0; #{body}; s; end" + end + 1000.times { |i| 2.times { send(:"big#{i}") } } + + kaller(foo, true) # escape callee's EP; the kaller->callee re-stub OOMs + # Re-enter the patched callee. Each call must still return "spilled"; on the buggy + # build local_to_spill is read from a stale stack slot and comes back as junk. + 300.times.all? { kaller(foo, false) == "spilled" } + "#), @"true"); +} + #[test] fn test_send_without_block() { assert_snapshot!(inspect(" diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index dfa994c02c5dac..d4e6955bededdd 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -1264,7 +1264,7 @@ pub use manual_defs::*; pub mod test_utils { use std::{ptr::null, sync::Once}; - use crate::{options::{rb_zjit_call_threshold, rb_zjit_prepare_options, set_call_threshold, DEFAULT_CALL_THRESHOLD}, state::{rb_zjit_entry, ZJITState}}; + use crate::{options::{DEFAULT_CALL_THRESHOLD, rb_zjit_call_threshold, rb_zjit_prepare_options, set_call_threshold}, state::{ZJITState, rb_zjit_compiling_p, rb_zjit_entry}}; use super::*; @@ -1316,7 +1316,10 @@ pub mod test_utils { let zjit_entry = ZJITState::init(); // Enable zjit_* instructions - unsafe { rb_zjit_entry = zjit_entry; } + unsafe { + rb_zjit_entry = zjit_entry; + rb_zjit_compiling_p = true; + } } /// Make sure the Ruby VM is set up and run a given callback with rb_protect() diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index efe30cf05f66ff..afc0e10bc46c2d 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -650,7 +650,6 @@ pub enum SideExitReason { PatchPoint(Invariant), CalleeSideExit, Interrupt, - Throw, BlockParamProxyNotIseqOrIfunc, BlockParamProxyNotNil, BlockParamProxyNotProc, @@ -1266,6 +1265,7 @@ pub enum Insn { /// Control flow instructions Return { val: InsnId }, /// Non-local control flow. See the throw YARV instruction + /// TODO: Consider turning this into Insn::Jump when inlined. Throw { throw_state: u32, val: InsnId, state: InsnId }, /// Fixnum +, -, *, /, %, ==, !=, <, <=, >, >=, &, |, ^, << @@ -4112,13 +4112,13 @@ impl Function { /// - Result of [`Self::resolve_receiver_type_from_profile`] if we need to check profile data fn resolve_receiver_type(&self, recv: InsnId, recv_type: Type, state: InsnId) -> ReceiverTypeResolution { match self.resolve_receiver_type_from_profile(recv, state) { - ReceiverTypeResolution::NoProfile => { + resolution@(ReceiverTypeResolution::NoProfile|ReceiverTypeResolution::Megamorphic) => { // Use known type information as a fallback because it doesn't have shape // information (and we can generally eliminate duplicate guards). if let Some(class) = recv_type.runtime_exact_ruby_class() { ReceiverTypeResolution::StaticallyKnown { class } } else { - ReceiverTypeResolution::NoProfile + resolution } } resolution => resolution, @@ -6774,8 +6774,7 @@ impl Function { let mut necessary = InsnSet::with_capacity(self.insns.len()); // Now recursively traverse their data dependencies and mark those as necessary while let Some(insn_id) = worklist.pop_front() { - if necessary.get(insn_id) { continue; } - necessary.insert(insn_id); + if !necessary.insert(insn_id) { continue; } let insn_id = self.union_find.borrow().find_const(insn_id); self.insns[insn_id].for_each_operand(|operand| { worklist.push_back(self.union_find.borrow().find_const(operand)); @@ -10577,6 +10576,10 @@ fn compile_entry_block(fun: &mut Function, jit_entry_insns: &[u32], insn_idx_to_ let mut pc: Option = None; let &all_opts_passed_insn_idx = jit_entry_insns.last().unwrap(); + if get_option!(stats) { + fun.count_iseq_calls(entry_block); + } + // Check-and-jump for each missing optional PC let mut iter = jit_entry_insns.iter().peekable(); while let Some(&jit_entry_insn) = iter.next() { diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 5112f422c1467a..7d0c35244fdb83 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -5486,23 +5486,24 @@ mod hir_opt_tests { bb1(): EntryPoint interpreter v1:BasicObject = LoadSelf + IncrCounterPtr Jump bb3(v1) bb2(): EntryPoint JIT(0) - v4:BasicObject = LoadArg :self@0 + v5:BasicObject = LoadArg :self@0 IncrCounterPtr - Jump bb3(v4) - bb3(v7:BasicObject): + Jump bb3(v5) + bb3(v8:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v14:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v15:HashExact = HashDup v14 + v15:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v16:HashExact = HashDup v15 IncrCounter zjit_insn_count IncrCounter complex_arg_pass_caller_kw_splat - v18:BasicObject = Send v7, :foo, v15 # SendFallbackReason: Complex argument passing + v19:BasicObject = Send v8, :foo, v16 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v18 + Return v19 "); } @@ -5520,23 +5521,24 @@ mod hir_opt_tests { bb1(): EntryPoint interpreter v1:BasicObject = LoadSelf + IncrCounterPtr Jump bb3(v1) bb2(): EntryPoint JIT(0) - v4:BasicObject = LoadArg :self@0 + v5:BasicObject = LoadArg :self@0 IncrCounterPtr - Jump bb3(v4) - bb3(v7:BasicObject): + Jump bb3(v5) + bb3(v8:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v14:Fixnum[1] = Const Value(1) + v15:Fixnum[1] = Const Value(1) IncrCounter zjit_insn_count IncrCounter complex_arg_pass_keyword_to_positional_hash IncrCounter send_direct_fallback_context_send - v17:BasicObject = Send v7, :foo, v14 # SendFallbackReason: Complex argument passing + v18:BasicObject = Send v8, :foo, v15 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v17 + Return v18 "); } @@ -5554,23 +5556,24 @@ mod hir_opt_tests { bb1(): EntryPoint interpreter v1:BasicObject = LoadSelf + IncrCounterPtr Jump bb3(v1) bb2(): EntryPoint JIT(0) - v4:BasicObject = LoadArg :self@0 + v5:BasicObject = LoadArg :self@0 IncrCounterPtr - Jump bb3(v4) - bb3(v7:BasicObject): + Jump bb3(v5) + bb3(v8:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v14:Fixnum[1] = Const Value(1) + v15:Fixnum[1] = Const Value(1) IncrCounter zjit_insn_count IncrCounter complex_arg_pass_keyword_to_positional_hash IncrCounter send_direct_fallback_context_send - v17:BasicObject = Send v7, :foo, v14 # SendFallbackReason: Complex argument passing + v18:BasicObject = Send v8, :foo, v15 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v17 + Return v18 "); } @@ -5772,23 +5775,24 @@ mod hir_opt_tests { bb1(): EntryPoint interpreter v1:BasicObject = LoadSelf + IncrCounterPtr Jump bb3(v1) bb2(): EntryPoint JIT(0) - v4:BasicObject = LoadArg :self@0 + v5:BasicObject = LoadArg :self@0 IncrCounterPtr - Jump bb3(v4) - bb3(v7:BasicObject): + Jump bb3(v5) + bb3(v8:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v14:Fixnum[1] = Const Value(1) + v15:Fixnum[1] = Const Value(1) IncrCounter zjit_insn_count IncrCounter complex_arg_pass_param_kwrest IncrCounter send_direct_fallback_context_send - v17:BasicObject = Send v7, :foo, v14 # SendFallbackReason: Complex argument passing + v18:BasicObject = Send v8, :foo, v15 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v17 + Return v18 "); } @@ -5893,23 +5897,24 @@ mod hir_opt_tests { bb1(): EntryPoint interpreter v1:BasicObject = LoadSelf + IncrCounterPtr Jump bb3(v1) bb2(): EntryPoint JIT(0) - v4:BasicObject = LoadArg :self@0 + v5:BasicObject = LoadArg :self@0 IncrCounterPtr - Jump bb3(v4) - bb3(v7:BasicObject): + Jump bb3(v5) + bb3(v8:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v14:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v15:HashExact = HashDup v14 + v15:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v16:HashExact = HashDup v15 IncrCounter zjit_insn_count IncrCounter complex_arg_pass_caller_kw_splat - v18:BasicObject = Send v7, :foo, v15 # SendFallbackReason: Complex argument passing + v19:BasicObject = Send v8, :foo, v16 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v18 + Return v19 "); } @@ -5927,21 +5932,22 @@ mod hir_opt_tests { bb1(): EntryPoint interpreter v1:BasicObject = LoadSelf + IncrCounterPtr Jump bb3(v1) bb2(): EntryPoint JIT(0) - v4:BasicObject = LoadArg :self@0 + v5:BasicObject = LoadArg :self@0 IncrCounterPtr - Jump bb3(v4) - bb3(v7:BasicObject): + Jump bb3(v5) + bb3(v8:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter complex_arg_pass_param_kwrest IncrCounter send_direct_fallback_context_send - v14:BasicObject = Send v7, :foo # SendFallbackReason: Complex argument passing + v15:BasicObject = Send v8, :foo # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v14 + Return v15 "); } @@ -14694,25 +14700,26 @@ mod hir_opt_tests { v1:BasicObject = LoadSelf v2:CPtr = LoadSP v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr Jump bb3(v1, v3) bb2(): EntryPoint JIT(0) - v6:BasicObject = LoadArg :self@0 - v7:BasicObject = LoadArg :args@1 + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 IncrCounterPtr - Jump bb3(v6, v7) - bb3(v10:BasicObject, v11:BasicObject): + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v20:ArrayExact = ToArray v11 + v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count IncrCounter complex_arg_pass_caller_splat IncrCounter caller_splat_profile_monomorphic - v23:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v23 + Return v24 "); } @@ -14734,25 +14741,26 @@ mod hir_opt_tests { v1:BasicObject = LoadSelf v2:CPtr = LoadSP v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr Jump bb3(v1, v3) bb2(): EntryPoint JIT(0) - v6:BasicObject = LoadArg :self@0 - v7:BasicObject = LoadArg :args@1 + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 IncrCounterPtr - Jump bb3(v6, v7) - bb3(v10:BasicObject, v11:BasicObject): + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v20:ArrayExact = ToArray v11 + v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count IncrCounter complex_arg_pass_caller_splat IncrCounter caller_splat_profile_polymorphic - v23:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v23 + Return v24 "); } @@ -17933,20 +17941,21 @@ mod hir_opt_tests { bb1(): EntryPoint interpreter v1:HeapBasicObject = LoadSelf + IncrCounterPtr Jump bb3(v1) bb2(): EntryPoint JIT(0) - v4:HeapBasicObject = LoadArg :self@0 + v5:HeapBasicObject = LoadArg :self@0 IncrCounterPtr - Jump bb3(v4) - bb3(v7:HeapBasicObject): + Jump bb3(v5) + bb3(v8:HeapBasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter send_direct_fallback_context_super - v14:BasicObject = InvokeSuper v7, 0x1000 # SendFallbackReason: Argument count does not match parameter count + v15:BasicObject = InvokeSuper v8, 0x1000 # SendFallbackReason: Argument count does not match parameter count IncrCounter zjit_insn_count CheckInterrupts - Return v14 + Return v15 "); } @@ -18107,6 +18116,46 @@ mod hir_opt_tests { "); } + #[test] + fn test_specialize_inlined_megamorphic_receiver() { + set_call_threshold(6); + eval(" + def klass_eq(klass) = klass == Integer + + def test = klass_eq(String) + + # 5 distinct receiver classes at the == site: one more than the profile's + # 4 buckets, so the distribution is megamorphic. + klass_eq(Integer); klass_eq(Array); klass_eq(Hash); klass_eq(Symbol); klass_eq(Float) + 6.times { test } + "); + assert_snapshot!(hir_string("test"), @" + fn test@:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint StableConstantNames(0x1000, String) + v12:ClassSubclass[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Object@0x1010, klass_eq@0x1018, cme:0x1020) + v21:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile + PushInlineFrame :klass_eq, v21 (0x1048), num_args=1 + PatchPoint StableConstantNames(0x1068, Integer) + v31:ClassSubclass[Integer@0x1070] = Const Value(VALUE(0x1070)) + PatchPoint MethodRedefined(Class@0x1078, ==@0x1080, cme:0x1088) + v45:CBool = IsBitEqual v12, v31 + v46:BoolExact = BoxBool v45 + CheckInterrupts + PopInlineFrame + Return v46 + "); + } + #[test] fn specialize_polymorphic_send_preserves_argument_profiles() { // Each arm of a polymorphic dispatch must still see the profiled types of diff --git a/zjit/src/options.rs b/zjit/src/options.rs index dc696dcdb85cc8..3529bbc8c143e7 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -666,6 +666,13 @@ pub fn set_inline_threshold(inline_threshold: InlineThreshold) { unsafe { OPTIONS.as_mut().unwrap().inline_threshold = inline_threshold; } } +/// Set --zjit-mem-size for testing. It's used to force OOM in tests. +#[cfg(test)] +pub fn set_mem_bytes(mem_bytes: usize) { + rb_zjit_prepare_options(); + unsafe { OPTIONS.as_mut().unwrap().mem_bytes = mem_bytes; } +} + /// Enable --zjit-stats for testing #[cfg(test)] pub fn enable_zjit_stats() { diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 096b0871d26457..38cb44576aa4c1 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -143,7 +143,7 @@ pub fn num_arguments_on_stack(cd: *const rb_call_data) -> usize { (unsafe { vm_ci_argc(ci) }) as usize + has_blockarg as usize } -const DISTRIBUTION_SIZE: usize = 4; +const DISTRIBUTION_SIZE: usize = 8; pub type TypeDistribution = Distribution; diff --git a/zjit/src/state.rs b/zjit/src/state.rs index 9889997d1e4838..4fda1e792db5a6 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -20,6 +20,15 @@ use std::ptr::null; #[unsafe(no_mangle)] pub static mut rb_zjit_entry: *const u8 = null(); +/// Whether ZJIT is compiling. Starts as false until ZJIT is enabled, so the +/// interpreter doesn't need to check rb_zjit_enabled_p before it. Set back to +/// false when we run out of executable memory, in which case the interpreter +/// stops incrementing ISEQ call counters so that ISEQs that will never be +/// compiled stop dirtying CoW pages after fork. +#[allow(non_upper_case_globals)] +#[unsafe(no_mangle)] +pub static mut rb_zjit_compiling_p: bool = false; + /// Like rb_zjit_enabled_p, but for Rust code. pub fn zjit_enabled_p() -> bool { unsafe { rb_zjit_entry != null() } @@ -410,7 +419,10 @@ fn zjit_enable() { // ZJIT enabled and initialized successfully assert!(unsafe{ rb_zjit_entry == null() }); - unsafe { rb_zjit_entry = zjit_entry; } + unsafe { + rb_zjit_entry = zjit_entry; + rb_zjit_compiling_p = true; + } }); if result.is_err() { diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index aae92a2e84bed2..c4476db2d9cc40 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -230,7 +230,6 @@ make_counters! { exit_patchpoint_root_box_only, exit_callee_side_exit, exit_interrupt, - exit_throw, exit_stackoverflow, exit_block_param_proxy_not_iseq_or_ifunc, exit_block_param_proxy_not_nil, @@ -457,6 +456,9 @@ make_counters! { // TODO(max): Implement // vm_reify_stack_count, + // The number of throw instructions executed in JIT code + throw_count, + // The number of times we ran a dynamic check guard_type_count, guard_shape_count, @@ -627,7 +629,6 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { GuardSuperMethodEntry => exit_guard_super_method_entry, CalleeSideExit => exit_callee_side_exit, Interrupt => exit_interrupt, - Throw => exit_throw, StackOverflow => exit_stackoverflow, BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, BlockParamProxyNotNil => exit_block_param_proxy_not_nil,