diff --git a/bootstraptest/test_ractor.rb b/bootstraptest/test_ractor.rb index ed80dd08628e9d..6c3a45148c23e1 100644 --- a/bootstraptest/test_ractor.rb +++ b/bootstraptest/test_ractor.rb @@ -2370,3 +2370,24 @@ def call_test(obj) :ok end RUBY + +assert_equal 'ok', <<~'RUBY' + begin + 100.times do |i| + Ractor.new(i) do |j| + 1000.times do |i| + "#{j}-#{i}" + end + end + pid = fork do + GC.verify_internal_consistency + end + _, status = Process.waitpid2 pid + raise unless status.success? + end + + :ok + rescue NotImplementedError + :ok + end +RUBY diff --git a/cont.c b/cont.c index d167685f13bec3..c49256c977a5b3 100644 --- a/cont.c +++ b/cont.c @@ -2005,10 +2005,9 @@ fiber_alloc(VALUE klass) } static rb_serial_t -next_fiber_serial(void) +next_fiber_serial(rb_ractor_t *cr) { - static rbimpl_atomic_uint64_t fiber_serial = 1; - return (rb_serial_t)ATOMIC_U64_FETCH_ADD(fiber_serial, 1); + return cr->next_fiber_serial++; } static rb_fiber_t* @@ -2027,7 +2026,7 @@ fiber_t_alloc(VALUE fiber_value, unsigned int blocking) fiber->cont.type = FIBER_CONTEXT; fiber->blocking = blocking; fiber->killed = 0; - fiber->serial = next_fiber_serial(); + fiber->serial = next_fiber_serial(th->ractor); cont_init(&fiber->cont, th); fiber->cont.saved_ec.fiber_ptr = fiber; @@ -2580,7 +2579,7 @@ rb_threadptr_root_fiber_setup(rb_thread_t *th) fiber->cont.saved_ec.thread_ptr = th; fiber->blocking = 1; fiber->killed = 0; - fiber->serial = next_fiber_serial(); + fiber->serial = next_fiber_serial(th->ractor); fiber_status_set(fiber, FIBER_RESUMED); /* skip CREATED */ th->ec = &fiber->cont.saved_ec; cont_init_jit_cont(&fiber->cont); diff --git a/gc/default/default.c b/gc/default/default.c index b6a9dfa1f57d9d..9a2efab562402b 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -622,6 +622,8 @@ typedef struct rb_objspace { rb_postponed_job_handle_t finalize_deferred_pjob; unsigned long live_ractor_cache_count; + + int fork_vm_lock_lev; } rb_objspace_t; #ifndef HEAP_PAGE_ALIGN_LOG @@ -9324,12 +9326,20 @@ gc_malloc_allocations(VALUE self) void rb_gc_impl_before_fork(void *objspace_ptr) { - /* no-op */ + rb_objspace_t *objspace = objspace_ptr; + + objspace->fork_vm_lock_lev = RB_GC_VM_LOCK(); + rb_gc_vm_barrier(); } void rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) { + rb_objspace_t *objspace = objspace_ptr; + + RB_GC_VM_UNLOCK(objspace->fork_vm_lock_lev); + objspace->fork_vm_lock_lev = 0; + if (pid == 0) { /* child process */ rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL); } diff --git a/insns.def b/insns.def index 3d13f4cb646d39..7df36726157455 100644 --- a/insns.def +++ b/insns.def @@ -224,6 +224,7 @@ setinstancevariable (VALUE val) () // attr bool leaf = false; /* has rb_check_frozen() */ +// attr bool zjit_profile = true; { vm_setinstancevariable(GET_ISEQ(), GET_SELF(), id, val, ic); } diff --git a/ractor.c b/ractor.c index 48fbf7cfb94404..3fc507128c1939 100644 --- a/ractor.c +++ b/ractor.c @@ -418,6 +418,7 @@ ractor_alloc(VALUE klass) VALUE rv = TypedData_Make_Struct(klass, rb_ractor_t, &ractor_data_type, r); FL_SET_RAW(rv, RUBY_FL_SHAREABLE); r->pub.self = rv; + r->next_fiber_serial = 1; VM_ASSERT(ractor_status_p(r, ractor_created)); return rv; } @@ -435,6 +436,7 @@ rb_ractor_main_alloc(void) r->name = Qnil; r->pub.self = Qnil; r->newobj_cache = rb_gc_ractor_cache_alloc(r); + r->next_fiber_serial = 1; ruby_single_main_ractor = r; return r; diff --git a/ractor_core.h b/ractor_core.h index 130ccb11189fa6..81374c0769f6ca 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -88,6 +88,8 @@ struct rb_ractor_struct { // ractor local data + rb_serial_t next_fiber_serial; + st_table *local_storage; struct rb_id_table *idkey_local_storage; VALUE local_storage_store_lock; diff --git a/ruby_atomic.h b/ruby_atomic.h index b60c56cfd89048..cbcfe682ceddb9 100644 --- a/ruby_atomic.h +++ b/ruby_atomic.h @@ -70,27 +70,4 @@ rbimpl_atomic_u64_set_relaxed(volatile rbimpl_atomic_uint64_t *address, uint64_t } #define ATOMIC_U64_SET_RELAXED(var, val) rbimpl_atomic_u64_set_relaxed(&(var), val) -static inline uint64_t -rbimpl_atomic_u64_fetch_add(volatile rbimpl_atomic_uint64_t *ptr, uint64_t val) -{ -#if defined(HAVE_GCC_ATOMIC_BUILTINS_64) - return __atomic_fetch_add(ptr, val, __ATOMIC_SEQ_CST); -#elif defined(_WIN32) - return InterlockedExchangeAdd64((volatile LONG64 *)ptr, val); -#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)) - return atomic_add_64_nv(ptr, val) - val; -#elif defined(HAVE_STDATOMIC_H) - return atomic_fetch_add_explicit((_Atomic uint64_t *)ptr, val, memory_order_seq_cst); -#else - // Fallback using mutex for platforms without 64-bit atomics - static rb_nativethread_mutex_t lock = RB_NATIVETHREAD_LOCK_INIT; - rb_native_mutex_lock(&lock); - uint64_t old = *ptr; - *ptr = old + val; - rb_native_mutex_unlock(&lock); - return old; -#endif -} -#define ATOMIC_U64_FETCH_ADD(var, val) rbimpl_atomic_u64_fetch_add(&(var), val) - #endif diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index bac17f4a6d4452..fe082504b82302 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -101,6 +101,7 @@ fn main() { .allowlist_function("rb_shape_get_iv_index") .allowlist_function("rb_shape_transition_add_ivar_no_warnings") .allowlist_var("rb_invalid_shape_id") + .allowlist_type("shape_id_fl_type") .allowlist_var("VM_KW_SPECIFIED_BITS_MAX") .allowlist_var("SHAPE_ID_NUM_BITS") .allowlist_function("rb_obj_is_kind_of") diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 83919e5369e5d9..1a60e2a7fe2a06 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -263,6 +263,10 @@ impl ShapeId { pub fn is_too_complex(self) -> bool { unsafe { rb_jit_shape_too_complex_p(self.0) } } + + pub fn is_frozen(self) -> bool { + (self.0 & SHAPE_ID_FL_FROZEN) != 0 + } } // Given an ISEQ pointer, convert PC to insn_idx diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index fe2055d4cc0486..aaecfa2f89769a 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1462,6 +1462,13 @@ pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; pub type vm_frame_env_flags = u32; pub type attr_index_t = u16; pub type shape_id_t = u32; +pub const SHAPE_ID_HEAP_INDEX_MASK: shape_id_fl_type = 29360128; +pub const SHAPE_ID_FL_FROZEN: shape_id_fl_type = 33554432; +pub const SHAPE_ID_FL_HAS_OBJECT_ID: shape_id_fl_type = 67108864; +pub const SHAPE_ID_FL_TOO_COMPLEX: shape_id_fl_type = 134217728; +pub const SHAPE_ID_FL_NON_CANONICAL_MASK: shape_id_fl_type = 100663296; +pub const SHAPE_ID_FLAGS_MASK: shape_id_fl_type = 264241152; +pub type shape_id_fl_type = u32; #[repr(C)] pub struct rb_cvar_class_tbl_entry { pub index: u32, @@ -1744,35 +1751,36 @@ pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 215; pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 216; pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 217; pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 218; -pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 219; -pub const YARVINSN_zjit_send: ruby_vminsn_type = 220; -pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 221; -pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 222; -pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 223; -pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 224; -pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 225; -pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 226; -pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 227; -pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 228; -pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 229; -pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 230; -pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 231; -pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 232; -pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 233; -pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 234; -pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 235; -pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 236; -pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 237; -pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 238; -pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 239; -pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 240; -pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 241; -pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 242; -pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 243; -pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 244; -pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 245; -pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 246; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 219; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 220; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 221; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 222; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 223; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 224; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 247; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 248; pub type ruby_vminsn_type = u32; pub type rb_iseq_callback = ::std::option::Option< unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index dd7bc953b9219e..b7149ad14c66fe 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -2873,6 +2873,50 @@ impl Function { }; self.make_equal_to(insn_id, replacement); } + Insn::SetIvar { self_val, id, val, state, ic: _ } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_too_complex() { + // too-complex shapes can't use index access + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_frozen() { + // Can't set ivars on frozen objects + self.push_insn_id(block, insn_id); continue; + } + let mut ivar_index: u16 = 0; + if !unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // TODO(max): Shape transition + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: types::HeapBasicObject, state }); + let self_val = self.push_insn(block, Insn::GuardShape { val: self_val, shape: recv_type.shape(), state }); + // Current shape contains this ivar + let (ivar_storage, offset) = if recv_type.flags().is_embedded() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + (self_val, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: ID!(_as_heap), offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + (as_heap, offset) + }; + let replacement = self.push_insn(block, Insn::StoreField { recv: ivar_storage, id, offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv: self_val, val }); + self.make_equal_to(insn_id, replacement); + } _ => { self.push_insn_id(block, insn_id); } } } @@ -4906,6 +4950,8 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { // profiled cfp->self. if opcode == YARVINSN_getinstancevariable || opcode == YARVINSN_trace_getinstancevariable { profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_setinstancevariable || opcode == YARVINSN_trace_setinstancevariable { + profiles.profile_self(&exit_state, self_param); } else if opcode == YARVINSN_definedivar || opcode == YARVINSN_trace_definedivar { profiles.profile_self(&exit_state, self_param); } else if opcode == YARVINSN_invokeblock || opcode == YARVINSN_trace_invokeblock { diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 866d0ec06dcf23..c460942fc13e42 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -3479,6 +3479,151 @@ mod hir_opt_tests { "); } + #[test] + fn test_specialize_monomorphic_setivar_already_in_shape() { + eval(" + @foo = 4 + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v19:HeapBasicObject = GuardType v6, HeapBasicObject + v20:HeapBasicObject = GuardShape v19, 0x1000 + StoreField v20, :@foo@0x1001, v10 + WriteBarrier v20, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_monomorphic_setivar_with_shape_transition() { + eval(" + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_setivar_with_t_data() { + eval(" + class C < Range + def test = @a = 5 + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_setivar() { + set_call_threshold(3); + eval(" + class C + def test = @a = 5 + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_setivar() { + eval(r#" + class C + def test = @a = 5 + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + #[test] fn test_elide_freeze_with_frozen_hash() { eval(" diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 10afdf2cc6475c..e7efbcad34a678 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -82,6 +82,7 @@ fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) { YARVINSN_opt_aset => profile_operands(profiler, profile, 3), YARVINSN_opt_not => profile_operands(profiler, profile, 1), YARVINSN_getinstancevariable => profile_self(profiler, profile), + YARVINSN_setinstancevariable => profile_self(profiler, profile), YARVINSN_definedivar => profile_self(profiler, profile), YARVINSN_opt_regexpmatch2 => profile_operands(profiler, profile, 2), YARVINSN_objtostring => profile_operands(profiler, profile, 1),