ruby
diff --git a/‎insns.def‎
Lines changed: 1 addition & 0 deletions b/‎insns.def‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎yjit/src/cruby_bindings.inc.rs‎
Lines changed: 29 additions & 28 deletions b/‎yjit/src/cruby_bindings.inc.rs‎
Lines changed: 29 additions & 28 deletions
diff --git a/‎zjit/src/codegen.rs‎
Lines changed: 56 additions & 8 deletions b/‎zjit/src/codegen.rs‎
Lines changed: 56 additions & 8 deletions
diff --git a/‎zjit/src/codegen_tests.rs‎
Lines changed: 165 additions & 0 deletions b/‎zjit/src/codegen_tests.rs‎
Lines changed: 165 additions & 0 deletions
diff --git a/‎zjit/src/cruby.rs‎
Lines changed: 5 additions & 0 deletions b/‎zjit/src/cruby.rs‎
Lines changed: 5 additions & 0 deletions
@@ -876,6 +876,7 @@ sendforward
 (CALL_DATA cd, ISEQ blockiseq)
 (...)
 (VALUE val)
+// attr bool zjit_profile = true;
 // attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci);
 // attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci);
 {
 
@@ -1110,6 +1110,12 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio
         Insn::Comment { .. } => return Ok(()), // comment instruction, no code generation
         &Insn::Const { val: Const::Value(val) } => gen_const_value(val),
         &Insn::Const { val: Const::CPtr(val) } => gen_const_cptr(val),
+        // The `...` local of an inlined forwardable frame is just the caller's callinfo
+        // pointer. `gen_push_inline_frame` already stored the same value in the frame slot.
+        // Materialized into a register rather than left as an immediate because `getlocal ...`
+        // puts it on the modeled VM stack, and a JITFrame stack map can only reconstruct a slot
+        // from a register or an immediate `VALUE` -- which a callinfo pointer is not.
+        &Insn::ForwardingCallInfo { ci, .. } => asm.load(gen_const_cptr(ci as *const u8)),
         &Insn::Const { val: Const::CInt64(val) } => gen_const_long(val),
         &Insn::Const { val: Const::CUInt16(val) } => gen_const_uint16(val),
         &Insn::Const { val: Const::CUInt32(val) } => gen_const_uint32(val),
@@ -1165,8 +1171,11 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio
                 *kw_bits, *jit_entry_idx, &function.frame_state(*state), *block, block_arg,
             )
         }
-        Insn::PushInlineFrame { cme, iseq, recv, num_args, blockiseq, state, .. } => {
-            no_output!(gen_push_inline_frame(jit, asm, function, *cme, *iseq, opnd!(recv), *num_args, &function.frame_state(*state), *blockiseq))
+        Insn::PushInlineFrame { cme, iseq, recv, num_args, blockiseq, forwarded, state, .. } => {
+            let forwarded = forwarded.as_ref().map(|forwarded| {
+                (forwarded.ci, forwarded.args.iter().map(|&arg| opnd!(arg)).collect::<Vec<_>>())
+            });
+            no_output!(gen_push_inline_frame(jit, asm, function, *cme, *iseq, opnd!(recv), *num_args, &function.frame_state(*state), *blockiseq, forwarded))
         },
         Insn::PopInlineFrame { iseq, argc, state } => {
             no_output!(gen_pop_inline_frame(asm, *iseq, *argc, &function.frame_state(*state)))
@@ -2282,8 +2291,20 @@ fn gen_push_inline_frame(
     num_args: u16,
     state: &FrameState,
     blockiseq: Option<IseqPtr>,
+    forwarded: Option<(*const rb_callinfo, Vec<Opnd>)>,
 ) {
-    let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize();
+    // A `def foo(...)` callee keeps the caller's arguments in the frame instead of binding them
+    // to parameters, so the frame is `argc` slots taller than the local table, exactly as
+    // `vm_call_iseq_forwardable` makes it. See `gen_send_iseq_direct`, which builds the same
+    // frame for the out-of-line call.
+    let forwarded_argc = forwarded.as_ref().map_or(0, |(_, args)| args.len());
+    debug_assert_eq!(
+        forwarded.is_some(),
+        unsafe { rb_get_iseq_flags_forwardable(iseq) },
+        "forwarded arguments are set exactly for a forwardable callee",
+    );
+    debug_assert!(forwarded.is_none() || forwarded_argc == num_args.to_usize());
+    let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize() + forwarded_argc;
     let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize();
     gen_stack_overflow_check(jit, asm, function, state, stack_growth);
 
@@ -2325,9 +2346,28 @@ fn gen_push_inline_frame(
         frame_type,
         specval,
         write_block_code: iseq_may_write_block_code(iseq),
-        forwarded_argc: None, // `can_inline` rejects forwardable callees
+        forwarded_argc: forwarded.is_some().then_some(forwarded_argc),
     });
 
+    // A forwardable callee reads its arguments out of the frame rather than out of parameter
+    // locals, so they have to be in memory: a side exit lands the interpreter on a `sendforward`
+    // whose `vm_adjust_stack_forwarding` copies them back out from below the frame, the GC scans
+    // them there once this frame's `cfp->sp` is published above them, and a `getlocal ...` reads
+    // the callinfo from the slot directly above them. This mirrors the same block in
+    // `gen_send_iseq_direct`; the inlined body reads none of it, because the `...` local is an
+    // `Insn::ForwardingCallInfo` constant and the arguments are the caller's own SSA values.
+    if let Some((ci, args)) = forwarded {
+        asm_comment!(asm, "copy forwarded arguments and callinfo to inlined callee frame");
+        let locals_base = state.stack().len() - args.len();
+        for (idx, &arg) in args.iter().enumerate() {
+            asm.store(Opnd::mem(64, SP, ((locals_base + idx) * SIZEOF_VALUE) as i32), arg);
+        }
+        asm.store(
+            Opnd::mem(64, SP, ((locals_base + forwarded_argc) * SIZEOF_VALUE) as i32),
+            Opnd::const_ptr(ci as *const u8),
+        );
+    }
+
     // Publish the inlined callee's entry JITFrame before the inlined body runs.
     // Frame walking functions such as rb_profile_frames can inspect the new
     // CFP between this frame push and the first inlined gen_write_jit_frame, so
@@ -2392,7 +2432,10 @@ fn gen_pop_inline_frame(
     argc: usize,
     state: &FrameState,
 ) {
-    let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize();
+    // Undo `gen_push_inline_frame`'s SP math, including the extension a forwardable callee's
+    // frame was grown by.
+    let forwarded_argc = if unsafe { rb_get_iseq_flags_forwardable(iseq) } { argc } else { 0 };
+    let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize() + forwarded_argc;
     let sp_offset = (state.stack().len() + local_size - argc + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE;
 
     asm_comment!(asm, "restore caller SP after inline");
@@ -4273,6 +4316,9 @@ fn gen_prepare_fallback_call(jit: &JITState, asm: &mut Assembler, function: &Fun
 /// on the native stack are known.
 fn build_stack_map(jit: &JITState, function: &Function, state: &FrameState) -> Vec<StackMapEntry> {
     let mut stack = Vec::new();
+    // Owned rather than borrowed: `Function::frame_state` resolves the state's operands through
+    // the union-find, which `jit.get_opnd` below needs. `frame_state_ref` would hand back the
+    // stored state with stale operand ids.
     let mut current_state = state.clone();
     loop {
         stack.extend(current_state.stack().rev().copied().map(|insn_id| {
@@ -4287,17 +4333,19 @@ fn build_stack_map(jit: &JITState, function: &Function, state: &FrameState) -> V
         let Some(caller) = current_state.caller() else {
             break;
         };
-        stack.push(StackMapEntry::Skip(inline_frame_stack_gap(current_state.iseq)));
+        stack.push(StackMapEntry::Skip(inline_frame_stack_gap(current_state.iseq, current_state.extra_locals)));
         current_state = function.frame_state(caller);
     }
     stack
 }
 
-fn inline_frame_stack_gap(iseq: IseqPtr) -> usize {
+fn inline_frame_stack_gap(iseq: IseqPtr, extra_locals: u16) -> usize {
     // The extra slot is for the callee's receiver below its local table.
     // We currently never map out the stack for `invokeblock`, which doesn't
     // put a receiver on cfp->sp stack.
-    1 + unsafe { get_iseq_body_local_table_size(iseq) }.to_usize() + VM_ENV_DATA_SIZE.to_usize()
+    // `extra_locals` is the frame extension a `def foo(...)` callee's arguments live in, which
+    // makes the frame that much taller. See `FrameState::extra_locals`.
+    1 + unsafe { get_iseq_body_local_table_size(iseq) }.to_usize() + extra_locals as usize + VM_ENV_DATA_SIZE.to_usize()
 }
 
 /// Prepare for calling a C function that may call an arbitrary method.
 
@@ -819,6 +819,171 @@ fn test_zsuper_to_forwardable_callee() {
     "#), @r#"[["base", 3, 4]]"#);
 }
 
+#[test]
+fn test_inlined_forwarder_positional_args() {
+    // The forwarder is inlined into its caller, so the `bar(...)` inside it sees the caller's
+    // callinfo at compile time and becomes a direct call to `target`.
+    assert_snapshot!(with_inlining(|| assert_inlines("
+        def target(a, b) = a - b
+        def fwd(...) = target(...)
+        def entry = fwd(7, 2)
+        200.times { entry }
+        entry
+    ")), @"5");
+}
+
+#[test]
+fn test_inlined_forwarder_keyword_args() {
+    // `vm_caller_setup_fwd_args` gives the merged callinfo the *caller's* keyword table, so the
+    // expanded call has to bind the trailing arguments as keywords, not as positionals.
+    assert_snapshot!(with_inlining(|| assert_inlines(r#"
+        def target(a, b:, c: 3) = [a, b, c]
+        def fwd(...) = target(...)
+        def entry = [fwd(1, b: 2), fwd(1, c: 9, b: 2)]
+        200.times { entry }
+        entry
+    "#)), @"[[1, 2, 3], [1, 2, 9]]");
+}
+
+#[test]
+fn test_inlined_forwarder_site_writes_its_own_args() {
+    // `bar(x, ...)`: the merged argument list is the site's own arguments followed by the
+    // caller's, in that order.
+    assert_snapshot!(with_inlining(|| assert_inlines("
+        def target(a, b, c) = [a, b, c]
+        def fwd(x, ...) = target(x, ...)
+        def entry = fwd(1, 2, 3)
+        200.times { entry }
+        entry
+    ")), @"[1, 2, 3]");
+}
+
+#[test]
+fn test_inlined_forwarder_carrying_a_literal_block() {
+    // `bh = VM_ENV_BLOCK_HANDLER(GET_LEP())`: the forwarded call gets the forwarder frame's own
+    // block handler, which the expanded call reads back out of the frame's EP. Re-deriving the
+    // literal block instead would capture the wrong frame, since the block belongs to `entry`.
+    assert_snapshot!(with_inlining(|| assert_inlines("
+        def target(x) = yield(x)
+        def fwd(...) = target(...)
+        def entry = fwd(4) { |v| v * 2 }
+        200.times { entry }
+        entry
+    ")), @"8");
+}
+
+#[test]
+fn test_inlined_forwarder_block_present_on_some_calls_only() {
+    // A `&blk` handed to the forwarder may be a Proc on one call and nothing on the next, so the
+    // handler the expanded call passes on is only known at run time. `block_given?` in the target
+    // has to see each call for what it was.
+    assert_snapshot!(with_inlining(|| assert_inlines_allowing_exits(r#"
+        def target(x) = [x, block_given? ? yield(x) : :none]
+        def fwd(...) = target(...)
+        def entry(i, &b) = fwd(i, &b)
+        200.times { |i| i.even? ? entry(i) { |v| v } : entry(i) }
+        [entry(1) { |v| v * 2 }, entry(1)]
+    "#)), @"[[1, 2], [1, :none]]");
+}
+
+#[test]
+fn test_inlined_forwarder_argument_error() {
+    // The argument check belongs to the target, which the inlined forwarder now calls directly.
+    assert_snapshot!(with_inlining(|| assert_inlines_allowing_exits(r#"
+        def target(a, b) = a + b
+        def fwd(...) = target(...)
+        def entry = (fwd(1) rescue $!.message)
+        200.times { entry }
+        entry
+    "#)), @r#""wrong number of arguments (given 1, expected 2)""#);
+}
+
+#[test]
+fn test_inlined_forwarder_side_exit_resumes_the_sendforward() {
+    // A guard inside the inlined forwarder exits to the `sendforward` instruction, and the
+    // interpreter's `vm_adjust_stack_forwarding` rebuilds the argument list by reading below the
+    // frame at `lep - (local_table_size + argc + 2)`. That only works because the inlined frame
+    // push copied the arguments into those slots and put the callinfo above them, the way
+    // `vm_call_iseq_forwardable` does.
+    assert_snapshot!(with_inlining(|| assert_inlines_allowing_exits(r#"
+        class A; def m(a, b, c) = [:a, a, b, c]; end
+        class B; def m(a, b, c) = [:b, a, b, c]; end
+        class Fwd
+          def initialize(t) = @t = t
+          def m(...) = @t.m(...)
+        end
+        fa = Fwd.new(A.new)
+        fb = Fwd.new(B.new)
+        # Warm up on A alone so the expanded call guards on A.
+        200.times { fa.m(1, 2, 3) }
+        # B fails that guard mid-forwarder.
+        [fa.m(1, 2, 3), fb.m(4, 5, 6)]
+    "#)), @"[[:a, 1, 2, 3], [:b, 4, 5, 6]]");
+}
+
+#[test]
+fn test_inlined_forwarder_chained_forwarding_falls_back() {
+    // The inner target is itself a `def bar(...)`, whose `...` local has to receive a real
+    // callinfo. No `rb_callinfo` describes the merged call, so the site keeps its `sendforward`.
+    assert_snapshot!(with_inlining(|| assert_inlines("
+        def target(a, b:) = [a, b]
+        def inner(...) = target(...)
+        def outer(...) = inner(...)
+        def entry = outer(1, b: 2)
+        200.times { entry }
+        entry
+    ")), @"[1, 2]");
+}
+
+#[test]
+fn test_inlined_forwarder_ruby2_keywords() {
+    // A `ruby2_keywords` frame splats into the forwarder, which keeps the call site off the
+    // direct send entirely; the flagged Hash still has to reach the target as keywords.
+    assert_snapshot!(with_inlining(|| assert_inlines_allowing_exits("
+        def target(*a, **k) = [a, k]
+        def fwd(...) = target(...)
+        ruby2_keywords def r2k(*a) = fwd(*a)
+        def entry = r2k(1, k: 2)
+        200.times { entry }
+        entry
+    ")), @"[[1], {k: 2}]");
+}
+
+#[test]
+fn test_inlined_forwarder_super_is_unaffected() {
+    // `super` out of a forwardable frame goes through `invokesuperforward`, which
+    // `vm_search_super_method` rebuilds the callinfo for at run time. Inlining the frame must
+    // not disturb it.
+    assert_snapshot!(with_inlining(|| assert_inlines_allowing_exits(r#"
+        class Base
+          def run(*a, **k) = ["base", a, k]
+        end
+        class Child < Base
+          def run(...) = super
+        end
+        c = Child.new
+        def call_it(c) = c.run(1, k: 2)
+        200.times { call_it(c) }
+        call_it(c)
+    "#)), @r#"["base", [1], {k: 2}]"#);
+}
+
+#[test]
+fn test_inlined_forwarder_with_extra_locals() {
+    // The `...` local is local 0 and the frame extension sits below the whole local table, so a
+    // forwarder with locals of its own still finds its arguments where the interpreter left them.
+    assert_snapshot!(with_inlining(|| assert_inlines("
+        def target(a) = a * 2
+        def fwd(...)
+          extra = 10
+          extra + target(...)
+        end
+        def entry = fwd(3)
+        200.times { entry }
+        entry
+    ")), @"16");
+}
+
 #[test]
 fn test_setlocal_on_eval() {
     assert_snapshot!(inspect("
 
@@ -1083,6 +1083,11 @@ pub fn ruby_call_method_name(cd: *const rb_call_data) -> String {
     mid.contents_lossy().to_string()
 }
 
+/// The method name a callinfo names, for HIR printing.
+pub fn ruby_call_info_method_name(ci: *const rb_callinfo) -> String {
+    unsafe { rb_vm_ci_mid(ci) }.contents_lossy().to_string()
+}
+
 /// A location in Rust code for integrating with debugging facilities defined in C.
 /// Use the [src_loc!] macro to crate an instance.
 pub struct SourceLocation {
Original file line number	Diff line number	Diff line change
`@@ -876,6 +876,7 @@ sendforward`
`876`	`876`	`(CALL_DATA cd, ISEQ blockiseq)`
`877`	`877`	`(...)`
`878`	`878`	`(VALUE val)`
	`879`	`+// attr bool zjit_profile = true;`
`879`	`880`	`// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci);`
`880`	`881`	`// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci);`
`881`	`882`	`{`