[pull] master from ruby:master

gc/mmtk/mmtk.c

@@ -330,6 +330,11 @@ rb_mmtk_call_obj_free(MMTk_ObjectReference object)
     }
 
     rb_gc_obj_free(objspace, obj);
+
+    // TODO: uncomment this when done debugging
+// #ifdef MMTK_DEBUG
+    memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj));
+// #endif
 }
 
 static size_t

lib/prism/lex_compat.rb

@@ -198,58 +198,6 @@ def deconstruct_keys(keys)
       "__END__": :on___end__
     }.freeze
 
-    # Pretty much a 1:1 copy of Ripper::Lexer::State. We list all the available states
-    # to reimplement to_s without using Ripper.
-    class State
-      # Ripper-internal bitflags.
-      ALL = %i[
-        BEG END ENDARG ENDFN ARG CMDARG MID FNAME DOT CLASS LABEL LABELED FITEM
-      ].map.with_index.to_h { |name, i| [2 ** i, name] }
-      ALL[0] = :NONE
-      ALL.freeze
-      ALL.each { |value, name| const_set(name, value) }
-
-      # :stopdoc:
-
-      attr_reader :to_int, :to_s
-
-      def initialize(i)
-        @to_int = i
-        @to_s = state_name(i)
-        freeze
-      end
-
-      def [](index)
-        case index
-        when 0, :to_int
-          @to_int
-        when 1, :to_s
-          @to_s
-        else
-          nil
-        end
-      end
-
-      alias to_i to_int
-      alias inspect to_s
-      def pretty_print(q) q.text(to_s) end
-      def ==(i) super or to_int == i end
-      def &(i) self.class.new(to_int & i) end
-      def |(i) self.class.new(to_int | i) end
-      def allbits?(i) to_int.allbits?(i) end
-      def anybits?(i) to_int.anybits?(i) end
-      def nobits?(i) to_int.nobits?(i) end
-
-      # :startdoc:
-
-      private
-
-      # Convert the state flags into the format exposed by ripper.
-      def state_name(bits)
-        ALL.filter_map { |flag, name| name if bits & flag != 0  }.join("|")
-      end
-    end
-
     # When we produce tokens, we produce the same arrays that Ripper does.
     # However, we add a couple of convenience methods onto them to make them a
     # little easier to work with. We delegate all other methods to the array.
@@ -300,8 +248,8 @@ def ==(other) # :nodoc:
     class IdentToken < Token
       def ==(other) # :nodoc:
         (self[0...-1] == other[0...-1]) && (
-          (other[3] == State::LABEL | State::END) ||
-          (other[3] & (State::ARG | State::CMDARG) != 0)
+          (other[3] == Translation::Ripper::EXPR_LABEL | Translation::Ripper::EXPR_END) ||
+          (other[3] & (Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_CMDARG) != 0)
         )
       end
     end
@@ -312,8 +260,8 @@ class IgnoredNewlineToken < Token
       def ==(other) # :nodoc:
         return false unless self[0...-1] == other[0...-1]
 
-        if self[3] == State::ARG | State::LABELED
-          other[3] & State::ARG | State::LABELED != 0
+        if self[3] == Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED
+          other[3] & Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED != 0
         else
           self[3] == other[3]
         end
@@ -331,8 +279,8 @@ def ==(other) # :nodoc:
     class ParamToken < Token
       def ==(other) # :nodoc:
         (self[0...-1] == other[0...-1]) && (
-          (other[3] == State::END) ||
-          (other[3] == State::END | State::LABEL)
+          (other[3] == Translation::Ripper::EXPR_END) ||
+          (other[3] == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL)
         )
       end
     end
@@ -727,7 +675,7 @@ def result
 
         event = RIPPER.fetch(token.type)
         value = token.value
-        lex_state = State.new(lex_state)
+        lex_state = Translation::Ripper::Lexer::State.new(lex_state)
 
         token =
           case event
@@ -741,7 +689,7 @@ def result
             last_heredoc_end = token.location.end_offset
             IgnoreStateToken.new([[lineno, column], event, value, lex_state])
           when :on_ident
-            if lex_state == State::END
+            if lex_state == Translation::Ripper::EXPR_END
               # If we have an identifier that follows a method name like:
               #
               #     def foo bar
@@ -751,7 +699,7 @@ def result
               # yet. We do this more accurately, so we need to allow comparing
               # against both END and END|LABEL.
               ParamToken.new([[lineno, column], event, value, lex_state])
-            elsif lex_state == State::END | State::LABEL
+            elsif lex_state == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL
               # In the event that we're comparing identifiers, we're going to
               # allow a little divergence. Ripper doesn't account for local
               # variables introduced through named captures in regexes, and we
@@ -791,7 +739,7 @@ def result
                   counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
                 end
 
-                State.new(result_value[current_index][1])
+                Translation::Ripper::Lexer::State.new(result_value[current_index][1])
               else
                 previous_state
               end

lib/prism/prism.gemspec

@@ -104,6 +104,7 @@ Gem::Specification.new do |spec|
     "lib/prism/translation/parser/compiler.rb",
     "lib/prism/translation/parser/lexer.rb",
     "lib/prism/translation/ripper.rb",
+    "lib/prism/translation/ripper/lexer.rb",
     "lib/prism/translation/ripper/sexp.rb",
     "lib/prism/translation/ripper/shim.rb",
     "lib/prism/translation/ruby_parser.rb",

lib/prism/translation/ripper.rb

@@ -424,9 +424,34 @@ def self.sexp_raw(src, filename = "-", lineno = 1, raise_errors: false)
         end
       end
 
+      autoload :Lexer, "prism/translation/ripper/lexer"
       autoload :SexpBuilder, "prism/translation/ripper/sexp"
       autoload :SexpBuilderPP, "prism/translation/ripper/sexp"
 
+      # :stopdoc:
+      # This is not part of the public API but used by some gems.
+
+      # Ripper-internal bitflags.
+      LEX_STATE_NAMES = %i[
+        BEG END ENDARG ENDFN ARG CMDARG MID FNAME DOT CLASS LABEL LABELED FITEM
+      ].map.with_index.to_h { |name, i| [2 ** i, name] }.freeze
+      private_constant :LEX_STATE_NAMES
+
+      LEX_STATE_NAMES.each do |value, key|
+        const_set("EXPR_#{key}", value)
+      end
+      EXPR_NONE = 0
+      EXPR_VALUE = EXPR_BEG
+      EXPR_BEG_ANY = EXPR_BEG | EXPR_MID | EXPR_CLASS
+      EXPR_ARG_ANY = EXPR_ARG | EXPR_CMDARG
+      EXPR_END_ANY = EXPR_END | EXPR_ENDARG | EXPR_ENDFN
+
+      def self.lex_state_name(state)
+        LEX_STATE_NAMES.filter_map { |flag, name| name if state & flag != 0  }.join("|")
+      end
+
+      # :startdoc:
+
       # The source that is being parsed.
       attr_reader :source
 

lib/prism/translation/ripper/lexer.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+require_relative "../ripper"
+
+module Prism
+  module Translation
+    class Ripper
+      class Lexer # :nodoc:
+        # :stopdoc:
+        class State
+
+          attr_reader :to_int, :to_s
+
+          def initialize(i)
+            @to_int = i
+            @to_s = Ripper.lex_state_name(i)
+            freeze
+          end
+
+          def [](index)
+            case index
+            when 0, :to_int
+              @to_int
+            when 1, :to_s
+              @to_s
+            else
+              nil
+            end
+          end
+
+          alias to_i to_int
+          alias inspect to_s
+          def pretty_print(q) q.text(to_s) end
+          def ==(i) super or to_int == i end
+          def &(i) self.class.new(to_int & i) end
+          def |(i) self.class.new(to_int | i) end
+          def allbits?(i) to_int.allbits?(i) end
+          def anybits?(i) to_int.anybits?(i) end
+          def nobits?(i) to_int.nobits?(i) end
+        end
+        # :startdoc:
+      end
+    end
+  end
+end

test/prism/ruby/ripper_test.rb

@@ -65,13 +65,12 @@ class RipperTest < TestCase
 
     # Check that the hardcoded values don't change without us noticing.
     def test_internals
-      actual = LexCompat::State::ALL
-      expected = Ripper.constants.select { |name| name.start_with?("EXPR_") }
-      expected -= %i[EXPR_VALUE EXPR_BEG_ANY EXPR_ARG_ANY EXPR_END_ANY]
+      actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
+      expected = Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
 
-      assert_equal(expected.size, actual.size)
-      expected.each do |const_name|
-        assert_equal(const_name.to_s.delete_prefix("EXPR_").to_sym, actual[Ripper.const_get(const_name)])
+      assert_equal(expected, actual)
+      expected.zip(actual).each do |ripper, prism|
+        assert_equal(Ripper.const_get(ripper), Translation::Ripper.const_get(prism))
       end
     end
 

test/ruby/test_zjit.rb

@@ -383,6 +383,43 @@ def f(a)
     }, call_threshold: 2
   end
 
+  def test_kwargs_with_exit_and_local_invalidation
+    assert_compiles ':ok', %q{
+      def a(b:, c:)
+        if c == :b
+          return -> {}
+        end
+        Class # invalidate locals
+
+        raise "c is :b!" if c == :b
+      end
+
+      def test
+        # note opposite order of kwargs
+        a(c: :c, b: :b)
+      end
+
+      4.times { test }
+      :ok
+    }, call_threshold: 2
+  end
+
+  def test_kwargs_with_max_direct_send_arg_count
+    # Ensure that we only reorder the args when we _can_ use direct send (< 6 args).
+    assert_compiles '[[1, 2, 3, 4, 5, 6, 7, 8]]', %q{
+      def kwargs(five, six, a:, b:, c:, d:, e:, f:)
+        [a, b, c, d, five, six, e, f]
+      end
+
+      5.times.flat_map do
+        [
+          kwargs(5, 6, d: 4, c: 3, a: 1, b: 2, e: 7, f: 8),
+          kwargs(5, 6, d: 4, c: 3, b: 2, a: 1, e: 7, f: 8)
+        ]
+      end.uniq
+    }, call_threshold: 2
+  end
+
   def test_setlocal_on_eval
     assert_compiles '1', %q{
       @b = binding

zjit/src/codegen.rs

@@ -401,9 +401,6 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio
         &Insn::Send { cd, blockiseq, state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason),
         &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason),
         &Insn::SendWithoutBlock { cd, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason),
-        // Give up SendWithoutBlockDirect for 6+ args since asm.ccall() doesn't support it.
-        Insn::SendWithoutBlockDirect { cd, state, args, .. } if args.len() + 1 > C_ARG_OPNDS.len() => // +1 for self
-            gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), SendFallbackReason::TooManyArgsForLir),
         Insn::SendWithoutBlockDirect { cme, iseq, recv, args, state, .. } => gen_send_without_block_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state)),
         &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason),
         &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason),
@@ -439,6 +436,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio
             gen_fixnum_rshift(asm, opnd!(left), shift_amount)
         }
         &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)),
+        &Insn::FixnumAref { recv, index } => gen_fixnum_aref(asm, opnd!(recv), opnd!(index)),
         Insn::IsNil { val } => gen_isnil(asm, opnd!(val)),
         &Insn::IsMethodCfunc { val, cd, cfunc, state: _ } => gen_is_method_cfunc(jit, asm, opnd!(val), cd, cfunc),
         &Insn::IsBitEqual { left, right } => gen_is_bit_equal(asm, opnd!(left), opnd!(right)),
@@ -1911,6 +1909,10 @@ fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, righ
     asm_ccall!(asm, rb_fix_mod_fix, left, right)
 }
 
+fn gen_fixnum_aref(asm: &mut Assembler, recv: lir::Opnd, index: lir::Opnd) -> lir::Opnd {
+    asm_ccall!(asm, rb_fix_aref, recv, index)
+}
+
 // Compile val == nil
 fn gen_isnil(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd {
     asm.cmp(val, Qnil.into());

zjit/src/cruby_methods.rs

@@ -219,9 +219,8 @@ pub fn init() -> Annotations {
     annotate!(rb_cString, "ascii_only?", types::BoolExact, no_gc, leaf);
     annotate!(rb_cModule, "name", types::StringExact.union(types::NilClass), no_gc, leaf, elidable);
     annotate!(rb_cModule, "===", inline_module_eqq, types::BoolExact, no_gc, leaf);
-    annotate!(rb_cArray, "length", types::Fixnum, no_gc, leaf, elidable);
-    annotate!(rb_cArray, "size", types::Fixnum, no_gc, leaf, elidable);
-    annotate!(rb_cArray, "empty?", types::BoolExact, no_gc, leaf, elidable);
+    annotate!(rb_cArray, "length", inline_array_length, types::Fixnum, no_gc, leaf, elidable);
+    annotate!(rb_cArray, "empty?", inline_array_empty_p, types::BoolExact, no_gc, leaf, elidable);
     annotate!(rb_cArray, "reverse", types::ArrayExact, leaf, elidable);
     annotate!(rb_cArray, "join", types::StringExact);
     annotate!(rb_cArray, "[]", inline_array_aref);
@@ -256,6 +255,7 @@ pub fn init() -> Annotations {
     annotate!(rb_cInteger, "<=", inline_integer_le);
     annotate!(rb_cInteger, "<<", inline_integer_lshift);
     annotate!(rb_cInteger, ">>", inline_integer_rshift);
+    annotate!(rb_cInteger, "[]", inline_integer_aref);
     annotate!(rb_cInteger, "to_s", types::StringExact);
     annotate!(rb_cString, "to_s", inline_string_to_s, types::StringExact);
     let thread_singleton = unsafe { rb_singleton_class(rb_cThread) };
@@ -537,6 +537,30 @@ fn inline_module_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::In
     None
 }
 
+fn inline_array_length(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> {
+    let &[] = args else { return None; };
+    if fun.likely_a(recv, types::Array, state) {
+        let recv = fun.coerce_to(block, recv, types::Array, state);
+        let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv });
+        let result = fun.push_insn(block, hir::Insn::BoxFixnum { val: length_cint, state });
+        return Some(result);
+    }
+    None
+}
+
+fn inline_array_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> {
+    let &[] = args else { return None; };
+    if fun.likely_a(recv, types::Array, state) {
+        let recv = fun.coerce_to(block, recv, types::Array, state);
+        let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv });
+        let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) });
+        let result_c = fun.push_insn(block, hir::Insn::IsBitEqual { left: length_cint, right: zero });
+        let result = fun.push_insn(block, hir::Insn::BoxBool { val: result_c });
+        return Some(result);
+    }
+    None
+}
+
 fn inline_integer_succ(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> {
     if !args.is_empty() { return None; }
     if fun.likely_a(recv, types::Fixnum, state) {
@@ -656,6 +680,17 @@ fn inline_integer_rshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir
     try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumRShift { left, right }, BOP_GTGT, recv, other, state)
 }
 
+fn inline_integer_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> {
+    let &[index] = args else { return None; };
+    if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(index, types::Fixnum, state) {
+        let recv = fun.coerce_to(block, recv, types::Fixnum, state);
+        let index = fun.coerce_to(block, index, types::Fixnum, state);
+        let result = fun.push_insn(block, hir::Insn::FixnumAref { recv, index });
+        return Some(result);
+    }
+    None
+}
+
 fn inline_basic_object_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> {
     let &[other] = args else { return None; };
     let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other });