Linux premium180.web-hosting.com 4.18.0-553.54.1.lve.el8.x86_64 #1 SMP Wed Jun 4 13:01:13 UTC 2025 x86_64
LiteSpeed
: 162.0.209.168 | : 216.73.216.187
Cant Read [ /etc/named.conf ]
8.3.30
nortrmdp
www.github.com/MadExploits
Terminal
AUTO ROOT
Adminer
Backdoor Destroyer
Linux Exploit
Lock Shell
Lock File
Create User
CREATE RDP
PHP Mailer
BACKCONNECT
UNLOCK SHELL
HASH IDENTIFIER
CPANEL RESET
CREATE WP USER
BLACK DEFEND!
README
+ Create Folder
+ Create File
/
opt /
alt /
ruby34 /
share /
ruby /
ruby_vm /
rjit /
[ HOME SHELL ]
Name
Size
Permission
Action
assembler.rb
32.81
KB
-rw-r--r--
block.rb
568
B
-rw-r--r--
branch_stub.rb
879
B
-rw-r--r--
c_pointer.rb
10.68
KB
-rw-r--r--
c_type.rb
2.66
KB
-rw-r--r--
code_block.rb
2.28
KB
-rw-r--r--
compiler.rb
16.72
KB
-rw-r--r--
context.rb
11.14
KB
-rw-r--r--
entry_stub.rb
226
B
-rw-r--r--
exit_compiler.rb
4.83
KB
-rw-r--r--
hooks.rb
949
B
-rw-r--r--
insn_compiler.rb
204.81
KB
-rw-r--r--
instruction.rb
38.66
KB
-rw-r--r--
invariants.rb
4.72
KB
-rw-r--r--
jit_state.rb
1.96
KB
-rw-r--r--
stats.rb
7.4
KB
-rw-r--r--
type.rb
5.91
KB
-rw-r--r--
Delete
Unzip
Zip
${this.title}
Close
Code Editor : insn_compiler.rb
# frozen_string_literal: true module RubyVM::RJIT class InsnCompiler # struct rb_calling_info. Storing flags instead of ci. CallingInfo = Struct.new(:argc, :flags, :kwarg, :ci_addr, :send_shift, :block_handler) do def kw_splat = flags & C::VM_CALL_KW_SPLAT != 0 end # @param ocb [CodeBlock] # @param exit_compiler [RubyVM::RJIT::ExitCompiler] def initialize(cb, ocb, exit_compiler) @ocb = ocb @exit_compiler = exit_compiler @cfunc_codegen_table = {} register_cfunc_codegen_funcs end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] # @param insn `RubyVM::RJIT::Instruction` def compile(jit, ctx, asm, insn) asm.incr_counter(:rjit_insns_count) stack = ctx.stack_size.times.map do |stack_idx| ctx.get_opnd_type(StackOpnd[ctx.stack_size - stack_idx - 1]).type end locals = jit.iseq.body.local_table_size.times.map do |local_idx| (ctx.local_types[local_idx] || Type::Unknown).type end insn_idx = format('%04d', (jit.pc.to_i - jit.iseq.body.iseq_encoded.to_i) / C.VALUE.size) asm.comment("Insn: #{insn_idx} #{insn.name} (stack: [#{stack.join(', ')}], locals: [#{locals.join(', ')}])") # 83/102 case insn.name when :nop then nop(jit, ctx, asm) when :getlocal then getlocal(jit, ctx, asm) when :setlocal then setlocal(jit, ctx, asm) when :getblockparam then getblockparam(jit, ctx, asm) # setblockparam when :getblockparamproxy then getblockparamproxy(jit, ctx, asm) when :getspecial then getspecial(jit, ctx, asm) # setspecial when :getinstancevariable then getinstancevariable(jit, ctx, asm) when :setinstancevariable then setinstancevariable(jit, ctx, asm) when :getclassvariable then getclassvariable(jit, ctx, asm) when :setclassvariable then setclassvariable(jit, ctx, asm) when :opt_getconstant_path then opt_getconstant_path(jit, ctx, asm) when :getconstant then getconstant(jit, ctx, asm) # setconstant when :getglobal then getglobal(jit, ctx, asm) # setglobal when :putnil then putnil(jit, ctx, asm) when :putself then putself(jit, ctx, asm) when :putobject then putobject(jit, ctx, asm) when :putspecialobject then putspecialobject(jit, ctx, asm) when :putstring then putstring(jit, ctx, asm) when :putchilledstring then putchilledstring(jit, ctx, asm) when :concatstrings then concatstrings(jit, ctx, asm) when :anytostring then anytostring(jit, ctx, asm) when :toregexp then toregexp(jit, ctx, asm) when :intern then intern(jit, ctx, asm) when :newarray then newarray(jit, ctx, asm) when :duparray then duparray(jit, ctx, asm) # duphash when :expandarray then expandarray(jit, ctx, asm) when :concatarray then concatarray(jit, ctx, asm) when :splatarray then splatarray(jit, ctx, asm) when :newhash then newhash(jit, ctx, asm) when :newrange then newrange(jit, ctx, asm) when :pop then pop(jit, ctx, asm) when :dup then dup(jit, ctx, asm) when :dupn then dupn(jit, ctx, asm) when :swap then swap(jit, ctx, asm) # opt_reverse when :topn then topn(jit, ctx, asm) when :setn then setn(jit, ctx, asm) when :adjuststack then adjuststack(jit, ctx, asm) when :defined then defined(jit, ctx, asm) when :definedivar then definedivar(jit, ctx, asm) # checkmatch when :checkkeyword then checkkeyword(jit, ctx, asm) # checktype # defineclass # definemethod # definesmethod when :send then send(jit, ctx, asm) when :opt_send_without_block then opt_send_without_block(jit, ctx, asm) when :objtostring then objtostring(jit, ctx, asm) when :opt_str_freeze then opt_str_freeze(jit, ctx, asm) when :opt_ary_freeze then opt_ary_freeze(jit, ctx, asm) when :opt_hash_freeze then opt_hash_freeze(jit, ctx, asm) when :opt_nil_p then opt_nil_p(jit, ctx, asm) # opt_str_uminus when :opt_newarray_send then opt_newarray_send(jit, ctx, asm) when :invokesuper then invokesuper(jit, ctx, asm) when :invokeblock then invokeblock(jit, ctx, asm) when :leave then leave(jit, ctx, asm) when :throw then throw(jit, ctx, asm) when :jump then jump(jit, ctx, asm) when :branchif then branchif(jit, ctx, asm) when :branchunless then branchunless(jit, ctx, asm) when :branchnil then branchnil(jit, ctx, asm) # once when :opt_case_dispatch then opt_case_dispatch(jit, ctx, asm) when :opt_plus then opt_plus(jit, ctx, asm) when :opt_minus then opt_minus(jit, ctx, asm) when :opt_mult then opt_mult(jit, ctx, asm) when :opt_div then opt_div(jit, ctx, asm) when :opt_mod then opt_mod(jit, ctx, asm) when :opt_eq then opt_eq(jit, ctx, asm) when :opt_neq then opt_neq(jit, ctx, asm) when :opt_lt then opt_lt(jit, ctx, asm) when :opt_le then opt_le(jit, ctx, asm) when :opt_gt then opt_gt(jit, ctx, asm) when :opt_ge then opt_ge(jit, ctx, asm) when :opt_ltlt then opt_ltlt(jit, ctx, asm) when :opt_and then opt_and(jit, ctx, asm) when :opt_or then opt_or(jit, ctx, asm) when :opt_aref then opt_aref(jit, ctx, asm) when :opt_aset then opt_aset(jit, ctx, asm) # opt_aset_with # opt_aref_with when :opt_length then opt_length(jit, ctx, asm) when :opt_size then opt_size(jit, ctx, asm) when :opt_empty_p then opt_empty_p(jit, ctx, asm) when :opt_succ then opt_succ(jit, ctx, asm) when :opt_not then opt_not(jit, ctx, asm) when :opt_regexpmatch2 then opt_regexpmatch2(jit, ctx, asm) # invokebuiltin when :opt_invokebuiltin_delegate then opt_invokebuiltin_delegate(jit, ctx, asm) when :opt_invokebuiltin_delegate_leave then opt_invokebuiltin_delegate_leave(jit, ctx, asm) when :getlocal_WC_0 then getlocal_WC_0(jit, ctx, asm) when :getlocal_WC_1 then getlocal_WC_1(jit, ctx, asm) when :setlocal_WC_0 then setlocal_WC_0(jit, ctx, asm) when :setlocal_WC_1 then setlocal_WC_1(jit, ctx, asm) when :putobject_INT2FIX_0_ then putobject_INT2FIX_0_(jit, ctx, asm) when :putobject_INT2FIX_1_ then putobject_INT2FIX_1_(jit, ctx, asm) else CantCompile end end private # # Insns # # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def nop(jit, ctx, asm) # Do nothing KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getlocal(jit, ctx, asm) idx = jit.operand(0) level = jit.operand(1) jit_getlocal_generic(jit, ctx, asm, idx:, level:) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getlocal_WC_0(jit, ctx, asm) idx = jit.operand(0) jit_getlocal_generic(jit, ctx, asm, idx:, level: 0) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getlocal_WC_1(jit, ctx, asm) idx = jit.operand(0) jit_getlocal_generic(jit, ctx, asm, idx:, level: 1) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def setlocal(jit, ctx, asm) idx = jit.operand(0) level = jit.operand(1) jit_setlocal_generic(jit, ctx, asm, idx:, level:) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def setlocal_WC_0(jit, ctx, asm) idx = jit.operand(0) jit_setlocal_generic(jit, ctx, asm, idx:, level: 0) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def setlocal_WC_1(jit, ctx, asm) idx = jit.operand(0) jit_setlocal_generic(jit, ctx, asm, idx:, level: 1) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getblockparam(jit, ctx, asm) # EP level level = jit.operand(1) # Save the PC and SP because we might allocate jit_prepare_routine_call(jit, ctx, asm) # A mirror of the interpreter code. Checking for the case # where it's pushing rb_block_param_proxy. side_exit = side_exit(jit, ctx) # Load environment pointer EP from CFP ep_reg = :rax jit_get_ep(asm, level, reg: ep_reg) # Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero # FIXME: This is testing bits in the same place that the WB check is testing. # We should combine these at some point asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) # If the frame flag has been modified, then the actual proc value is # already in the EP and we should just use the value. frame_flag_modified = asm.new_label('frame_flag_modified') asm.jnz(frame_flag_modified) # This instruction writes the block handler to the EP. If we need to # fire a write barrier for the write, then exit (we'll let the # interpreter handle it so it can fire the write barrier). # flags & VM_ENV_FLAG_WB_REQUIRED asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_ENV_FLAG_WB_REQUIRED) # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 asm.jnz(side_exit) # Convert the block handler in to a proc # call rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler) asm.mov(C_ARGS[0], EC) # The block handler for the current frame # note, VM_ASSERT(VM_ENV_LOCAL_P(ep)) asm.mov(C_ARGS[1], [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) asm.call(C.rb_vm_bh_to_procval) # Load environment pointer EP from CFP (again) ep_reg = :rcx jit_get_ep(asm, level, reg: ep_reg) # Write the value at the environment pointer idx = jit.operand(0) offs = -(C.VALUE.size * idx) asm.mov([ep_reg, offs], C_RET); # Set the frame modified flag asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS]) # flag_check asm.or(:rax, C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) # modified_flag asm.mov([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], :rax) asm.write_label(frame_flag_modified) # Push the proc on the stack stack_ret = ctx.stack_push(Type::Unknown) ep_reg = :rax jit_get_ep(asm, level, reg: ep_reg) asm.mov(:rax, [ep_reg, offs]) asm.mov(stack_ret, :rax) KeepCompiling end # setblockparam # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getblockparamproxy(jit, ctx, asm) # To get block_handler unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end starting_context = ctx.dup # make a copy for use with jit_chain_guard # A mirror of the interpreter code. Checking for the case # where it's pushing rb_block_param_proxy. side_exit = side_exit(jit, ctx) # EP level level = jit.operand(1) # Peek at the block handler so we can check whether it's nil comptime_handler = jit.peek_at_block_handler(level) # When a block handler is present, it should always be a GC-guarded # pointer (VM_BH_ISEQ_BLOCK_P) if comptime_handler != 0 && comptime_handler & 0x3 != 0x1 asm.incr_counter(:getblockpp_not_gc_guarded) return CantCompile end # Load environment pointer EP from CFP ep_reg = :rax jit_get_ep(asm, level, reg: ep_reg) # Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) asm.jnz(counted_exit(side_exit, :getblockpp_block_param_modified)) # Load the block handler for the current frame # note, VM_ASSERT(VM_ENV_LOCAL_P(ep)) block_handler = :rax asm.mov(block_handler, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # Specialize compilation for the case where no block handler is present if comptime_handler == 0 # Bail if there is a block handler asm.cmp(block_handler, 0) jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_block_handler_none)) putobject(jit, ctx, asm, val: Qnil) else # Block handler is a tagged pointer. Look at the tag. 0x03 is from VM_BH_ISEQ_BLOCK_P(). asm.and(block_handler, 0x3) # Bail unless VM_BH_ISEQ_BLOCK_P(bh). This also checks for null. asm.cmp(block_handler, 0x1) jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_not_iseq_block)) # Push rb_block_param_proxy. It's a root, so no need to use jit_mov_gc_ptr. top = ctx.stack_push(Type::BlockParamProxy) asm.mov(:rax, C.rb_block_param_proxy) asm.mov(top, :rax) end jump_to_next_insn(jit, ctx, asm) EndBlock end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getspecial(jit, ctx, asm) # This takes two arguments, key and type # key is only used when type == 0 # A non-zero type determines which type of backref to fetch #rb_num_t key = jit.jit_get_arg(0); rtype = jit.operand(1) if rtype == 0 # not yet implemented return CantCompile; elsif rtype & 0x01 != 0 # Fetch a "special" backref based on a char encoded by shifting by 1 # Can raise if matchdata uninitialized jit_prepare_routine_call(jit, ctx, asm) # call rb_backref_get() asm.comment('rb_backref_get') asm.call(C.rb_backref_get) asm.mov(C_ARGS[0], C_RET) # backref case [rtype >> 1].pack('c') in ?& asm.comment("rb_reg_last_match") asm.call(C.rb_reg_last_match) in ?` asm.comment("rb_reg_match_pre") asm.call(C.rb_reg_match_pre) in ?' asm.comment("rb_reg_match_post") asm.call(C.rb_reg_match_post) in ?+ asm.comment("rb_reg_match_last") asm.call(C.rb_reg_match_last) end stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) KeepCompiling else # Fetch the N-th match from the last backref based on type shifted by 1 # Can raise if matchdata uninitialized jit_prepare_routine_call(jit, ctx, asm) # call rb_backref_get() asm.comment('rb_backref_get') asm.call(C.rb_backref_get) # rb_reg_nth_match((int)(type >> 1), backref); asm.comment('rb_reg_nth_match') asm.mov(C_ARGS[0], rtype >> 1) asm.mov(C_ARGS[1], C_RET) # backref asm.call(C.rb_reg_nth_match) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) KeepCompiling end end # setspecial # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getinstancevariable(jit, ctx, asm) # Specialize on a compile-time receiver, and split a block for chain guards unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end id = jit.operand(0) comptime_obj = jit.peek_at_self jit_getivar(jit, ctx, asm, comptime_obj, id, nil, SelfOpnd) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def setinstancevariable(jit, ctx, asm) starting_context = ctx.dup # make a copy for use with jit_chain_guard # Defer compilation so we can specialize on a runtime `self` unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end ivar_name = jit.operand(0) comptime_receiver = jit.peek_at_self # If the comptime receiver is frozen, writing an IV will raise an exception # and we don't want to JIT code to deal with that situation. if C.rb_obj_frozen_p(comptime_receiver) asm.incr_counter(:setivar_frozen) return CantCompile end # Check if the comptime receiver is a T_OBJECT receiver_t_object = C::BUILTIN_TYPE(comptime_receiver) == C::T_OBJECT # If the receiver isn't a T_OBJECT, or uses a custom allocator, # then just write out the IV write as a function call. # too-complex shapes can't use index access, so we use rb_ivar_get for them too. if !receiver_t_object || shape_too_complex?(comptime_receiver) || ctx.chain_depth >= 10 asm.comment('call rb_vm_setinstancevariable') ic = jit.operand(1) # The function could raise exceptions. # Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm) # Get the operands from the stack val_opnd = ctx.stack_pop(1) # Call rb_vm_setinstancevariable(iseq, obj, id, val, ic); asm.mov(:rdi, jit.iseq.to_i) asm.mov(:rsi, [CFP, C.rb_control_frame_t.offsetof(:self)]) asm.mov(:rdx, ivar_name) asm.mov(:rcx, val_opnd) asm.mov(:r8, ic) asm.call(C.rb_vm_setinstancevariable) else # Get the iv index shape_id = C.rb_shape_get_shape_id(comptime_receiver) ivar_index = C.rb_shape_get_iv_index(shape_id, ivar_name) # Get the receiver asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # Generate a side exit side_exit = side_exit(jit, ctx) # Upgrade type guard_object_is_heap(jit, ctx, asm, :rax, SelfOpnd, :setivar_not_heap) asm.comment('guard shape') asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id) megamorphic_side_exit = counted_exit(side_exit, :setivar_megamorphic) jit_chain_guard(:jne, jit, starting_context, asm, megamorphic_side_exit) # If we don't have an instance variable index, then we need to # transition out of the current shape. if ivar_index.nil? shape = C.rb_shape_get_shape_by_id(shape_id) current_capacity = shape.capacity dest_shape = C.rb_shape_get_next_no_warnings(shape, comptime_receiver, ivar_name) new_shape_id = C.rb_shape_id(dest_shape) if new_shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID asm.incr_counter(:setivar_too_complex) return CantCompile end ivar_index = shape.next_iv_index # If the new shape has a different capacity, we need to # reallocate the object. needs_extension = dest_shape.capacity != shape.capacity if needs_extension # Generate the C call so that runtime code will increase # the capacity and set the buffer. asm.mov(C_ARGS[0], :rax) asm.mov(C_ARGS[1], current_capacity) asm.mov(C_ARGS[2], dest_shape.capacity) asm.call(C.rb_ensure_iv_list_size) # Load the receiver again after the function call asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) end write_val = ctx.stack_pop(1) jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, needs_extension) # Store the new shape asm.comment('write shape') asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv asm.mov(DwordPtr[:rax, C.rb_shape_id_offset], new_shape_id) else # If the iv index already exists, then we don't need to # transition to a new shape. The reason is because we find # the iv index by searching up the shape tree. If we've # made the transition already, then there's no reason to # update the shape on the object. Just set the IV. write_val = ctx.stack_pop(1) jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, false) end skip_wb = asm.new_label('skip_wb') # If the value we're writing is an immediate, we don't need to WB asm.test(write_val, C::RUBY_IMMEDIATE_MASK) asm.jnz(skip_wb) # If the value we're writing is nil or false, we don't need to WB asm.cmp(write_val, Qnil) asm.jbe(skip_wb) asm.comment('write barrier') asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv asm.mov(C_ARGS[1], write_val) asm.call(C.rb_gc_writebarrier) asm.write_label(skip_wb) end KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getclassvariable(jit, ctx, asm) # rb_vm_getclassvariable can raise exceptions. jit_prepare_routine_call(jit, ctx, asm) asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)]) asm.mov(C_ARGS[1], CFP) asm.mov(C_ARGS[2], jit.operand(0)) asm.mov(C_ARGS[3], jit.operand(1)) asm.call(C.rb_vm_getclassvariable) top = ctx.stack_push(Type::Unknown) asm.mov(top, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def setclassvariable(jit, ctx, asm) # rb_vm_setclassvariable can raise exceptions. jit_prepare_routine_call(jit, ctx, asm) asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)]) asm.mov(C_ARGS[1], CFP) asm.mov(C_ARGS[2], jit.operand(0)) asm.mov(C_ARGS[3], ctx.stack_pop(1)) asm.mov(C_ARGS[4], jit.operand(1)) asm.call(C.rb_vm_setclassvariable) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_getconstant_path(jit, ctx, asm) # Cut the block for invalidation unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end ic = C.iseq_inline_constant_cache.new(jit.operand(0)) idlist = ic.segments # Make sure there is an exit for this block as the interpreter might want # to invalidate this block from rb_rjit_constant_ic_update(). # For now, we always take an entry exit even if it was a side exit. Invariants.ensure_block_entry_exit(jit, cause: 'opt_getconstant_path') # See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update(). ice = ic.entry if ice.nil? # In this case, leave a block that unconditionally side exits # for the interpreter to invalidate. asm.incr_counter(:optgetconst_not_cached) return CantCompile end if ice.ic_cref # with cref # Cache is keyed on a certain lexical scope. Use the interpreter's cache. side_exit = side_exit(jit, ctx) # Call function to verify the cache. It doesn't allocate or call methods. asm.mov(C_ARGS[0], ic.to_i) asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:ep)]) asm.call(C.rb_vm_ic_hit_p) # Check the result. SysV only specifies one byte for _Bool return values, # so it's important we only check one bit to ignore the higher bits in the register. asm.test(C_RET, 1) asm.jz(counted_exit(side_exit, :optgetconst_cache_miss)) asm.mov(:rax, ic.to_i) # inline_cache asm.mov(:rax, [:rax, C.iseq_inline_constant_cache.offsetof(:entry)]) # ic_entry asm.mov(:rax, [:rax, C.iseq_inline_constant_cache_entry.offsetof(:value)]) # ic_entry_val # Push ic->entry->value stack_top = ctx.stack_push(Type::Unknown) asm.mov(stack_top, :rax) else # without cref # TODO: implement this # Optimize for single ractor mode. # if !assume_single_ractor_mode(jit, ocb) # return CantCompile # end # Invalidate output code on any constant writes associated with # constants referenced within the current block. Invariants.assume_stable_constant_names(jit, idlist) putobject(jit, ctx, asm, val: ice.value) end jump_to_next_insn(jit, ctx, asm) EndBlock end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getconstant(jit, ctx, asm) id = jit.operand(0) # vm_get_ev_const can raise exceptions. jit_prepare_routine_call(jit, ctx, asm) allow_nil_opnd = ctx.stack_pop(1) klass_opnd = ctx.stack_pop(1) asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], klass_opnd) asm.mov(C_ARGS[2], id) asm.mov(C_ARGS[3], allow_nil_opnd) asm.call(C.rb_vm_get_ev_const) top = ctx.stack_push(Type::Unknown) asm.mov(top, C_RET) KeepCompiling end # setconstant # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def getglobal(jit, ctx, asm) gid = jit.operand(0) # Save the PC and SP because we might make a Ruby call for warning jit_prepare_routine_call(jit, ctx, asm) asm.mov(C_ARGS[0], gid) asm.call(C.rb_gvar_get) top = ctx.stack_push(Type::Unknown) asm.mov(top, C_RET) KeepCompiling end # setglobal # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putnil(jit, ctx, asm) putobject(jit, ctx, asm, val: Qnil) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putself(jit, ctx, asm) stack_top = ctx.stack_push_self asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) asm.mov(stack_top, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putobject(jit, ctx, asm, val: jit.operand(0)) # Push it to the stack val_type = Type.from(C.to_ruby(val)) stack_top = ctx.stack_push(val_type) if asm.imm32?(val) asm.mov(stack_top, val) else # 64-bit immediates can't be directly written to memory asm.mov(:rax, val) asm.mov(stack_top, :rax) end KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putspecialobject(jit, ctx, asm) object_type = jit.operand(0) if object_type == C::VM_SPECIAL_OBJECT_VMCORE stack_top = ctx.stack_push(Type::UnknownHeap) asm.mov(:rax, C.rb_mRubyVMFrozenCore) asm.mov(stack_top, :rax) KeepCompiling else # TODO: implement for VM_SPECIAL_OBJECT_CBASE and # VM_SPECIAL_OBJECT_CONST_BASE CantCompile end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putstring(jit, ctx, asm) put_val = jit.operand(0, ruby: true) # Save the PC and SP because the callee will allocate jit_prepare_routine_call(jit, ctx, asm) asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], to_value(put_val)) asm.mov(C_ARGS[2], 0) asm.call(C.rb_ec_str_resurrect) stack_top = ctx.stack_push(Type::TString) asm.mov(stack_top, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putchilledstring(jit, ctx, asm) put_val = jit.operand(0, ruby: true) # Save the PC and SP because the callee will allocate jit_prepare_routine_call(jit, ctx, asm) asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], to_value(put_val)) asm.mov(C_ARGS[2], 1) asm.call(C.rb_ec_str_resurrect) stack_top = ctx.stack_push(Type::TString) asm.mov(stack_top, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def concatstrings(jit, ctx, asm) n = jit.operand(0) # Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm) asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * n)) # call rb_str_concat_literals(size_t n, const VALUE *strings); asm.mov(C_ARGS[0], n) asm.mov(C_ARGS[1], :rax) asm.call(C.rb_str_concat_literals) ctx.stack_pop(n) stack_ret = ctx.stack_push(Type::TString) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def anytostring(jit, ctx, asm) # Save the PC and SP since we might call #to_s jit_prepare_routine_call(jit, ctx, asm) str = ctx.stack_pop(1) val = ctx.stack_pop(1) asm.mov(C_ARGS[0], str) asm.mov(C_ARGS[1], val) asm.call(C.rb_obj_as_string_result) # Push the return value stack_ret = ctx.stack_push(Type::TString) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def toregexp(jit, ctx, asm) opt = jit.operand(0, signed: true) cnt = jit.operand(1) # Save the PC and SP because this allocates an object and could # raise an exception. jit_prepare_routine_call(jit, ctx, asm) asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * cnt)) # values_ptr ctx.stack_pop(cnt) asm.mov(C_ARGS[0], 0) asm.mov(C_ARGS[1], cnt) asm.mov(C_ARGS[2], :rax) # values_ptr asm.call(C.rb_ary_tmp_new_from_values) # Save the array so we can clear it later asm.push(C_RET) asm.push(C_RET) # Alignment asm.mov(C_ARGS[0], C_RET) asm.mov(C_ARGS[1], opt) asm.call(C.rb_reg_new_ary) # The actual regex is in RAX now. Pop the temp array from # rb_ary_tmp_new_from_values into C arg regs so we can clear it asm.pop(:rcx) # Alignment asm.pop(:rcx) # ary # The value we want to push on the stack is in RAX right now stack_ret = ctx.stack_push(Type::UnknownHeap) asm.mov(stack_ret, C_RET) # Clear the temp array. asm.mov(C_ARGS[0], :rcx) # ary asm.call(C.rb_ary_clear) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def intern(jit, ctx, asm) # Save the PC and SP because we might allocate jit_prepare_routine_call(jit, ctx, asm); str = ctx.stack_pop(1) asm.mov(C_ARGS[0], str) asm.call(C.rb_str_intern) # Push the return value stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def newarray(jit, ctx, asm) n = jit.operand(0) # Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm) # If n is 0, then elts is never going to be read, so we can just pass null if n == 0 values_ptr = 0 else asm.comment('load pointer to array elts') offset_magnitude = C.VALUE.size * n values_opnd = ctx.sp_opnd(-(offset_magnitude)) asm.lea(:rax, values_opnd) values_ptr = :rax end # call rb_ec_ary_new_from_values(struct rb_execution_context_struct *ec, long n, const VALUE *elts); asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], n) asm.mov(C_ARGS[2], values_ptr) asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def duparray(jit, ctx, asm) ary = jit.operand(0) # Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm) # call rb_ary_resurrect(VALUE ary); asm.comment('call rb_ary_resurrect') asm.mov(C_ARGS[0], ary) asm.call(C.rb_ary_resurrect) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling end # duphash # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def expandarray(jit, ctx, asm) # Both arguments are rb_num_t which is unsigned num = jit.operand(0) flag = jit.operand(1) # If this instruction has the splat flag, then bail out. if flag & 0x01 != 0 asm.incr_counter(:expandarray_splat) return CantCompile end # If this instruction has the postarg flag, then bail out. if flag & 0x02 != 0 asm.incr_counter(:expandarray_postarg) return CantCompile end side_exit = side_exit(jit, ctx) array_opnd = ctx.stack_opnd(0) array_stack_opnd = StackOpnd[0] # num is the number of requested values. If there aren't enough in the # array then we're going to push on nils. if ctx.get_opnd_type(array_stack_opnd) == Type::Nil ctx.stack_pop(1) # pop after using the type info # special case for a, b = nil pattern # push N nils onto the stack num.times do push_opnd = ctx.stack_push(Type::Nil) asm.mov(push_opnd, Qnil) end return KeepCompiling end # Move the array from the stack and check that it's an array. asm.mov(:rax, array_opnd) guard_object_is_array(jit, ctx, asm, :rax, :rcx, array_stack_opnd, :expandarray_not_array) ctx.stack_pop(1) # pop after using the type info # If we don't actually want any values, then just return. if num == 0 return KeepCompiling end jit_array_len(asm, :rax, :rcx) # Only handle the case where the number of values in the array is greater # than or equal to the number of values requested. asm.cmp(:rcx, num) asm.jl(counted_exit(side_exit, :expandarray_rhs_too_small)) # Conditionally load the address of the heap array into REG1. # (struct RArray *)(obj)->as.heap.ptr #asm.mov(:rax, array_opnd) asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)]) asm.test(:rcx, C::RARRAY_EMBED_FLAG); asm.mov(:rcx, [:rax, C.RArray.offsetof(:as, :heap, :ptr)]) # Load the address of the embedded array into REG1. # (struct RArray *)(obj)->as.ary asm.lea(:rax, [:rax, C.RArray.offsetof(:as, :ary)]) asm.cmovnz(:rcx, :rax) # Loop backward through the array and push each element onto the stack. (num - 1).downto(0).each do |i| top = ctx.stack_push(Type::Unknown) asm.mov(:rax, [:rcx, i * C.VALUE.size]) asm.mov(top, :rax) end KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def concatarray(jit, ctx, asm) # Save the PC and SP because the callee may allocate # Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm) # Get the operands from the stack ary2st_opnd = ctx.stack_pop(1) ary1_opnd = ctx.stack_pop(1) # Call rb_vm_concat_array(ary1, ary2st) asm.mov(C_ARGS[0], ary1_opnd) asm.mov(C_ARGS[1], ary2st_opnd) asm.call(C.rb_vm_concat_array) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def splatarray(jit, ctx, asm) flag = jit.operand(0) # Save the PC and SP because the callee may allocate # Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm) # Get the operands from the stack ary_opnd = ctx.stack_pop(1) # Call rb_vm_splat_array(flag, ary) asm.mov(C_ARGS[0], flag) asm.mov(C_ARGS[1], ary_opnd) asm.call(C.rb_vm_splat_array) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def newhash(jit, ctx, asm) num = jit.operand(0) # Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm) if num != 0 # val = rb_hash_new_with_size(num / 2); asm.mov(C_ARGS[0], num / 2) asm.call(C.rb_hash_new_with_size) # Save the allocated hash as we want to push it after insertion asm.push(C_RET) asm.push(C_RET) # x86 alignment # Get a pointer to the values to insert into the hash asm.lea(:rcx, ctx.stack_opnd(num - 1)) # rb_hash_bulk_insert(num, STACK_ADDR_FROM_TOP(num), val); asm.mov(C_ARGS[0], num) asm.mov(C_ARGS[1], :rcx) asm.mov(C_ARGS[2], C_RET) asm.call(C.rb_hash_bulk_insert) asm.pop(:rax) asm.pop(:rax) ctx.stack_pop(num) stack_ret = ctx.stack_push(Type::Hash) asm.mov(stack_ret, :rax) else # val = rb_hash_new(); asm.call(C.rb_hash_new) stack_ret = ctx.stack_push(Type::Hash) asm.mov(stack_ret, C_RET) end KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def newrange(jit, ctx, asm) flag = jit.operand(0) # rb_range_new() allocates and can raise jit_prepare_routine_call(jit, ctx, asm) # val = rb_range_new(low, high, (int)flag); asm.mov(C_ARGS[0], ctx.stack_opnd(1)) asm.mov(C_ARGS[1], ctx.stack_opnd(0)) asm.mov(C_ARGS[2], flag) asm.call(C.rb_range_new) ctx.stack_pop(2) stack_ret = ctx.stack_push(Type::UnknownHeap) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def pop(jit, ctx, asm) ctx.stack_pop KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def dup(jit, ctx, asm) dup_val = ctx.stack_opnd(0) mapping, tmp_type = ctx.get_opnd_mapping(StackOpnd[0]) loc0 = ctx.stack_push_mapping([mapping, tmp_type]) asm.mov(:rax, dup_val) asm.mov(loc0, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def dupn(jit, ctx, asm) n = jit.operand(0) # In practice, seems to be only used for n==2 if n != 2 return CantCompile end opnd1 = ctx.stack_opnd(1) opnd0 = ctx.stack_opnd(0) mapping1 = ctx.get_opnd_mapping(StackOpnd[1]) mapping0 = ctx.get_opnd_mapping(StackOpnd[0]) dst1 = ctx.stack_push_mapping(mapping1) asm.mov(:rax, opnd1) asm.mov(dst1, :rax) dst0 = ctx.stack_push_mapping(mapping0) asm.mov(:rax, opnd0) asm.mov(dst0, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def swap(jit, ctx, asm) stack_swap(jit, ctx, asm, 0, 1) KeepCompiling end # opt_reverse # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def topn(jit, ctx, asm) n = jit.operand(0) top_n_val = ctx.stack_opnd(n) mapping = ctx.get_opnd_mapping(StackOpnd[n]) loc0 = ctx.stack_push_mapping(mapping) asm.mov(:rax, top_n_val) asm.mov(loc0, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def setn(jit, ctx, asm) n = jit.operand(0) top_val = ctx.stack_pop(0) dst_opnd = ctx.stack_opnd(n) asm.mov(:rax, top_val) asm.mov(dst_opnd, :rax) mapping = ctx.get_opnd_mapping(StackOpnd[0]) ctx.set_opnd_mapping(StackOpnd[n], mapping) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def adjuststack(jit, ctx, asm) n = jit.operand(0) ctx.stack_pop(n) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def defined(jit, ctx, asm) op_type = jit.operand(0) obj = jit.operand(1, ruby: true) pushval = jit.operand(2, ruby: true) # Save the PC and SP because the callee may allocate # Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm) # Get the operands from the stack v_opnd = ctx.stack_pop(1) # Call vm_defined(ec, reg_cfp, op_type, obj, v) asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], CFP) asm.mov(C_ARGS[2], op_type) asm.mov(C_ARGS[3], to_value(obj)) asm.mov(C_ARGS[4], v_opnd) asm.call(C.rb_vm_defined) asm.test(C_RET, 255) asm.mov(:rax, Qnil) asm.mov(:rcx, to_value(pushval)) asm.cmovnz(:rax, :rcx) # Push the return value onto the stack out_type = if C::SPECIAL_CONST_P(pushval) Type::UnknownImm else Type::Unknown end stack_ret = ctx.stack_push(out_type) asm.mov(stack_ret, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def definedivar(jit, ctx, asm) # Defer compilation so we can specialize base on a runtime receiver unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end ivar_name = jit.operand(0) # Value that will be pushed on the stack if the ivar is defined. In practice this is always the # string "instance-variable". If the ivar is not defined, nil will be pushed instead. pushval = jit.operand(2, ruby: true) # Get the receiver recv = :rcx asm.mov(recv, [CFP, C.rb_control_frame_t.offsetof(:self)]) # Specialize base on compile time values comptime_receiver = jit.peek_at_self if shape_too_complex?(comptime_receiver) # Fall back to calling rb_ivar_defined # Save the PC and SP because the callee may allocate # Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax # Call rb_ivar_defined(recv, ivar_name) asm.mov(C_ARGS[0], recv) asm.mov(C_ARGS[1], ivar_name) asm.call(C.rb_ivar_defined) # if (rb_ivar_defined(recv, ivar_name)) { # val = pushval; # } asm.test(C_RET, 255) asm.mov(:rax, Qnil) asm.mov(:rcx, to_value(pushval)) asm.cmovnz(:rax, :rcx) # Push the return value onto the stack out_type = C::SPECIAL_CONST_P(pushval) ? Type::UnknownImm : Type::Unknown stack_ret = ctx.stack_push(out_type) asm.mov(stack_ret, :rax) return KeepCompiling end shape_id = C.rb_shape_get_shape_id(comptime_receiver) ivar_exists = C.rb_shape_get_iv_index(shape_id, ivar_name) side_exit = side_exit(jit, ctx) # Guard heap object (recv_opnd must be used before stack_pop) guard_object_is_heap(jit, ctx, asm, recv, SelfOpnd) shape_opnd = DwordPtr[recv, C.rb_shape_id_offset] asm.comment('guard shape') asm.cmp(shape_opnd, shape_id) jit_chain_guard(:jne, jit, ctx, asm, side_exit) result = ivar_exists ? C.to_value(pushval) : Qnil putobject(jit, ctx, asm, val: result) # Jump to next instruction. This allows guard chains to share the same successor. jump_to_next_insn(jit, ctx, asm) return EndBlock end # checkmatch # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def checkkeyword(jit, ctx, asm) # When a keyword is unspecified past index 32, a hash will be used # instead. This can only happen in iseqs taking more than 32 keywords. if jit.iseq.body.param.keyword.num >= 32 return CantCompile end # The EP offset to the undefined bits local bits_offset = jit.operand(0) # The index of the keyword we want to check index = jit.operand(1, signed: true) # Load environment pointer EP ep_reg = :rax jit_get_ep(asm, 0, reg: ep_reg) # VALUE kw_bits = *(ep - bits) bits_opnd = [ep_reg, C.VALUE.size * -bits_offset] # unsigned int b = (unsigned int)FIX2ULONG(kw_bits); # if ((b & (0x01 << idx))) { # # We can skip the FIX2ULONG conversion by shifting the bit we test bit_test = 0x01 << (index + 1) asm.test(bits_opnd, bit_test) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.cmovz(:rax, :rcx) stack_ret = ctx.stack_push(Type::UnknownImm) asm.mov(stack_ret, :rax) KeepCompiling end # checktype # defineclass # definemethod # definesmethod # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def send(jit, ctx, asm) # Specialize on a compile-time receiver, and split a block for chain guards unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end cd = C.rb_call_data.new(jit.operand(0)) blockiseq = jit.operand(1) # calling->ci mid = C.vm_ci_mid(cd.ci) calling = build_calling(ci: cd.ci, block_handler: blockiseq) if calling.flags & C::VM_CALL_FORWARDING != 0 return CantCompile end # vm_sendish cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling) if cme == CantCompile return CantCompile end jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_send_without_block(jit, ctx, asm, cd: C.rb_call_data.new(jit.operand(0))) # Specialize on a compile-time receiver, and split a block for chain guards unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end # calling->ci mid = C.vm_ci_mid(cd.ci) calling = build_calling(ci: cd.ci, block_handler: C::VM_BLOCK_HANDLER_NONE) # vm_sendish cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling) if cme == CantCompile return CantCompile end jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def objtostring(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end recv = ctx.stack_opnd(0) comptime_recv = jit.peek_at_stack(0) if C.RB_TYPE_P(comptime_recv, C::RUBY_T_STRING) side_exit = side_exit(jit, ctx) jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[0], comptime_recv, side_exit) # No work needed. The string value is already on the top of the stack. KeepCompiling else cd = C.rb_call_data.new(jit.operand(0)) opt_send_without_block(jit, ctx, asm, cd:) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_ary_freeze(jit, ctx, asm) unless Invariants.assume_bop_not_redefined(jit, C::ARRAY_REDEFINED_OP_FLAG, C::BOP_FREEZE) return CantCompile; end ary = jit.operand(0, ruby: true) # Push the return value onto the stack stack_ret = ctx.stack_push(Type::CArray) asm.mov(:rax, to_value(ary)) asm.mov(stack_ret, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_hash_freeze(jit, ctx, asm) unless Invariants.assume_bop_not_redefined(jit, C::HASH_REDEFINED_OP_FLAG, C::BOP_FREEZE) return CantCompile; end hash = jit.operand(0, ruby: true) # Push the return value onto the stack stack_ret = ctx.stack_push(Type::CHash) asm.mov(:rax, to_value(hash)) asm.mov(stack_ret, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_str_freeze(jit, ctx, asm) unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_FREEZE) return CantCompile; end str = jit.operand(0, ruby: true) # Push the return value onto the stack stack_ret = ctx.stack_push(Type::CString) asm.mov(:rax, to_value(str)) asm.mov(stack_ret, :rax) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_nil_p(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # opt_str_uminus # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_newarray_send(jit, ctx, asm) type = C.ID2SYM jit.operand(1) case type when :min then opt_newarray_min(jit, ctx, asm) when :max then opt_newarray_max(jit, ctx, asm) when :hash then opt_newarray_hash(jit, ctx, asm) else return CantCompile end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_newarray_min(jit, ctx, asm) num = jit.operand(0) # Save the PC and SP because we may allocate jit_prepare_routine_call(jit, ctx, asm) offset_magnitude = C.VALUE.size * num values_opnd = ctx.sp_opnd(-offset_magnitude) asm.lea(:rax, values_opnd) asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], num) asm.mov(C_ARGS[2], :rax) asm.call(C.rb_vm_opt_newarray_min) ctx.stack_pop(num) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_newarray_max(jit, ctx, asm) num = jit.operand(0) # Save the PC and SP because we may allocate jit_prepare_routine_call(jit, ctx, asm) offset_magnitude = C.VALUE.size * num values_opnd = ctx.sp_opnd(-offset_magnitude) asm.lea(:rax, values_opnd) asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], num) asm.mov(C_ARGS[2], :rax) asm.call(C.rb_vm_opt_newarray_max) ctx.stack_pop(num) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_newarray_hash(jit, ctx, asm) num = jit.operand(0) # Save the PC and SP because we may allocate jit_prepare_routine_call(jit, ctx, asm) offset_magnitude = C.VALUE.size * num values_opnd = ctx.sp_opnd(-offset_magnitude) asm.lea(:rax, values_opnd) asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], num) asm.mov(C_ARGS[2], :rax) asm.call(C.rb_vm_opt_newarray_hash) ctx.stack_pop(num) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def invokesuper(jit, ctx, asm) cd = C.rb_call_data.new(jit.operand(0)) block = jit.operand(1) # Defer compilation so we can specialize on class of receiver unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end me = C.rb_vm_frame_method_entry(jit.cfp) if me.nil? return CantCompile end # FIXME: We should track and invalidate this block when this cme is invalidated current_defined_class = me.defined_class mid = me.def.original_id if me.to_i != C.rb_callable_method_entry(current_defined_class, me.called_id).to_i # Though we likely could generate this call, as we are only concerned # with the method entry remaining valid, assume_method_lookup_stable # below requires that the method lookup matches as well return CantCompile end # vm_search_normal_superclass rbasic_klass = C.to_ruby(C.RBasic.new(C.to_value(current_defined_class)).klass) if C::BUILTIN_TYPE(current_defined_class) == C::RUBY_T_ICLASS && C::BUILTIN_TYPE(rbasic_klass) == C::RUBY_T_MODULE && \ C::FL_TEST_RAW(rbasic_klass, C::RMODULE_IS_REFINEMENT) return CantCompile end comptime_superclass = C.rb_class_get_superclass(C.RCLASS_ORIGIN(current_defined_class)) ci = cd.ci argc = C.vm_ci_argc(ci) ci_flags = C.vm_ci_flag(ci) # Don't JIT calls that aren't simple # Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block. if ci_flags & C::VM_CALL_KWARG != 0 asm.incr_counter(:send_keywords) return CantCompile end if ci_flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_kw_splat) return CantCompile end if ci_flags & C::VM_CALL_ARGS_BLOCKARG != 0 asm.incr_counter(:send_block_arg) return CantCompile end # Ensure we haven't rebound this method onto an incompatible class. # In the interpreter we try to avoid making this check by performing some # cheaper calculations first, but since we specialize on the method entry # and so only have to do this once at compile time this is fine to always # check and side exit. comptime_recv = jit.peek_at_stack(argc) unless C.obj_is_kind_of(comptime_recv, current_defined_class) return CantCompile end # Do method lookup cme = C.rb_callable_method_entry(comptime_superclass, mid) if cme.nil? return CantCompile end # Check that we'll be able to write this method dispatch before generating checks cme_def_type = cme.def.type if cme_def_type != C::VM_METHOD_TYPE_ISEQ && cme_def_type != C::VM_METHOD_TYPE_CFUNC # others unimplemented return CantCompile end asm.comment('guard known me') lep_opnd = :rax jit_get_lep(jit, asm, reg: lep_opnd) ep_me_opnd = [lep_opnd, C.VALUE.size * C::VM_ENV_DATA_INDEX_ME_CREF] asm.mov(:rcx, me.to_i) asm.cmp(ep_me_opnd, :rcx) asm.jne(counted_exit(side_exit(jit, ctx), :invokesuper_me_changed)) if block == C::VM_BLOCK_HANDLER_NONE # Guard no block passed # rb_vm_frame_block_handler(GET_EC()->cfp) == VM_BLOCK_HANDLER_NONE # note, we assume VM_ASSERT(VM_ENV_LOCAL_P(ep)) # # TODO: this could properly forward the current block handler, but # would require changes to gen_send_* asm.comment('guard no block given') ep_specval_opnd = [lep_opnd, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL] asm.cmp(ep_specval_opnd, C::VM_BLOCK_HANDLER_NONE) asm.jne(counted_exit(side_exit(jit, ctx), :invokesuper_block)) end # We need to assume that both our current method entry and the super # method entry we invoke remain stable Invariants.assume_method_lookup_stable(jit, me) Invariants.assume_method_lookup_stable(jit, cme) # Method calls may corrupt types ctx.clear_local_types calling = build_calling(ci:, block_handler: block) case cme_def_type in C::VM_METHOD_TYPE_ISEQ iseq = def_iseq_ptr(cme.def) frame_type = C::VM_FRAME_MAGIC_METHOD | C::VM_ENV_FLAG_LOCAL jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type:) in C::VM_METHOD_TYPE_CFUNC jit_call_cfunc(jit, ctx, asm, cme, calling) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def invokeblock(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end # Get call info cd = C.rb_call_data.new(jit.operand(0)) calling = build_calling(ci: cd.ci, block_handler: :captured) # Get block_handler cfp = jit.cfp lep = C.rb_vm_ep_local_ep(cfp.ep) comptime_handler = lep[C::VM_ENV_DATA_INDEX_SPECVAL] # Handle each block_handler type if comptime_handler == C::VM_BLOCK_HANDLER_NONE # no block given asm.incr_counter(:invokeblock_none) CantCompile elsif comptime_handler & 0x3 == 0x1 # VM_BH_ISEQ_BLOCK_P asm.comment('get local EP') ep_reg = :rax jit_get_lep(jit, asm, reg: ep_reg) asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd asm.comment('guard block_handler type') side_exit = side_exit(jit, ctx) asm.mov(:rcx, :rax) asm.and(:rcx, 0x3) # block_handler is a tagged pointer asm.cmp(:rcx, 0x1) # VM_BH_ISEQ_BLOCK_P tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed) jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit) comptime_captured = C.rb_captured_block.new(comptime_handler & ~0x3) comptime_iseq = comptime_captured.code.iseq asm.comment('guard known ISEQ') asm.and(:rax, ~0x3) # captured asm.mov(:rax, [:rax, C.VALUE.size * 2]) # captured->iseq asm.mov(:rcx, comptime_iseq.to_i) asm.cmp(:rax, :rcx) block_changed_exit = counted_exit(side_exit, :invokeblock_iseq_block_changed) jit_chain_guard(:jne, jit, ctx, asm, block_changed_exit) jit_call_iseq(jit, ctx, asm, nil, calling, comptime_iseq, frame_type: C::VM_FRAME_MAGIC_BLOCK) elsif comptime_handler & 0x3 == 0x3 # VM_BH_IFUNC_P # We aren't handling CALLER_SETUP_ARG and CALLER_REMOVE_EMPTY_KW_SPLAT yet. if calling.flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:invokeblock_ifunc_args_splat) return CantCompile end if calling.flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:invokeblock_ifunc_kw_splat) return CantCompile end asm.comment('get local EP') jit_get_lep(jit, asm, reg: :rax) asm.mov(:rcx, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd asm.comment('guard block_handler type'); side_exit = side_exit(jit, ctx) asm.mov(:rax, :rcx) # block_handler_opnd asm.and(:rax, 0x3) # tag_opnd: block_handler is a tagged pointer asm.cmp(:rax, 0x3) # VM_BH_IFUNC_P tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed) jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit) # The cfunc may not be leaf jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax asm.comment('call ifunc') asm.and(:rcx, ~0x3) # captured_opnd asm.lea(:rax, ctx.sp_opnd(-calling.argc * C.VALUE.size)) # argv asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], :rcx) # captured_opnd asm.mov(C_ARGS[2], calling.argc) asm.mov(C_ARGS[3], :rax) # argv asm.call(C.rb_vm_yield_with_cfunc) ctx.stack_pop(calling.argc) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) # cfunc calls may corrupt types ctx.clear_local_types # Share the successor with other chains jump_to_next_insn(jit, ctx, asm) EndBlock elsif symbol?(comptime_handler) asm.incr_counter(:invokeblock_symbol) CantCompile else # Proc asm.incr_counter(:invokeblock_proc) CantCompile end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def leave(jit, ctx, asm) assert_equal(ctx.stack_size, 1) jit_check_ints(jit, ctx, asm) asm.comment('pop stack frame') asm.lea(:rax, [CFP, C.rb_control_frame_t.size]) asm.mov(CFP, :rax) asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], :rax) # Return a value (for compile_leave_exit) ret_opnd = ctx.stack_pop asm.mov(:rax, ret_opnd) # Set caller's SP and push a value to its stack (for JIT) asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # Note: SP is in the position after popping a receiver and arguments asm.mov([SP], :rax) # Jump to cfp->jit_return asm.jmp([CFP, -C.rb_control_frame_t.size + C.rb_control_frame_t.offsetof(:jit_return)]) EndBlock end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def throw(jit, ctx, asm) throw_state = jit.operand(0) asm.mov(:rcx, ctx.stack_pop(1)) # throwobj # THROW_DATA_NEW allocates. Save SP for GC and PC for allocation tracing as # well as handling the catch table. However, not using jit_prepare_routine_call # since we don't need a patch point for this implementation. jit_save_pc(jit, asm) # clobbers rax jit_save_sp(ctx, asm) # rb_vm_throw verifies it's a valid throw, sets ec->tag->state, and returns throw # data, which is throwobj or a vm_throw_data wrapping it. When ec->tag->state is # set, JIT code callers will handle the throw with vm_exec_handle_exception. asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], CFP) asm.mov(C_ARGS[2], throw_state) # asm.mov(C_ARGS[3], :rcx) # same reg asm.call(C.rb_vm_throw) asm.comment('exit from throw') asm.pop(SP) asm.pop(EC) asm.pop(CFP) # return C_RET as C_RET asm.ret EndBlock end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jump(jit, ctx, asm) # Check for interrupts, but only on backward branches that may create loops jump_offset = jit.operand(0, signed: true) if jump_offset < 0 jit_check_ints(jit, ctx, asm) end pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) jit_direct_jump(jit.iseq, pc, ctx, asm) EndBlock end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def branchif(jit, ctx, asm) # Check for interrupts, but only on backward branches that may create loops jump_offset = jit.operand(0, signed: true) if jump_offset < 0 jit_check_ints(jit, ctx, asm) end # Get the branch target instruction offsets next_pc = jit.pc + C.VALUE.size * jit.insn.len jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) val_type = ctx.get_opnd_type(StackOpnd[0]) val_opnd = ctx.stack_pop(1) if (result = val_type.known_truthy) != nil target_pc = result ? jump_pc : next_pc jit_direct_jump(jit.iseq, target_pc, ctx, asm) else # This `test` sets ZF only for Qnil and Qfalse, which let jz jump. asm.test(val_opnd, ~Qnil) # Set stubs branch_stub = BranchStub.new( iseq: jit.iseq, shape: Default, target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.target1.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) @ocb.write(ocb_asm) end # Jump to target0 on jnz branch_stub.compile = compile_branchif(branch_stub) branch_stub.compile.call(asm) end EndBlock end def compile_branchif(branch_stub) # Proc escapes arguments in memory proc do |branch_asm| branch_asm.comment("branchif #{branch_stub.shape}") branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.jnz(branch_stub.target0.address) branch_asm.jmp(branch_stub.target1.address) in Next0 branch_asm.jz(branch_stub.target1.address) in Next1 branch_asm.jnz(branch_stub.target0.address) end end end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def branchunless(jit, ctx, asm) # Check for interrupts, but only on backward branches that may create loops jump_offset = jit.operand(0, signed: true) if jump_offset < 0 jit_check_ints(jit, ctx, asm) end # Get the branch target instruction offsets next_pc = jit.pc + C.VALUE.size * jit.insn.len jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) val_type = ctx.get_opnd_type(StackOpnd[0]) val_opnd = ctx.stack_pop(1) if (result = val_type.known_truthy) != nil target_pc = result ? next_pc : jump_pc jit_direct_jump(jit.iseq, target_pc, ctx, asm) else # This `test` sets ZF only for Qnil and Qfalse, which let jz jump. asm.test(val_opnd, ~Qnil) # Set stubs branch_stub = BranchStub.new( iseq: jit.iseq, shape: Default, target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.target1.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) @ocb.write(ocb_asm) end # Jump to target0 on jz branch_stub.compile = compile_branchunless(branch_stub) branch_stub.compile.call(asm) end EndBlock end def compile_branchunless(branch_stub) # Proc escapes arguments in memory proc do |branch_asm| branch_asm.comment("branchunless #{branch_stub.shape}") branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.jz(branch_stub.target0.address) branch_asm.jmp(branch_stub.target1.address) in Next0 branch_asm.jnz(branch_stub.target1.address) in Next1 branch_asm.jz(branch_stub.target0.address) end end end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def branchnil(jit, ctx, asm) # Check for interrupts, but only on backward branches that may create loops jump_offset = jit.operand(0, signed: true) if jump_offset < 0 jit_check_ints(jit, ctx, asm) end # Get the branch target instruction offsets next_pc = jit.pc + C.VALUE.size * jit.insn.len jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) val_type = ctx.get_opnd_type(StackOpnd[0]) val_opnd = ctx.stack_pop(1) if (result = val_type.known_nil) != nil target_pc = result ? jump_pc : next_pc jit_direct_jump(jit.iseq, target_pc, ctx, asm) else asm.cmp(val_opnd, Qnil) # Set stubs branch_stub = BranchStub.new( iseq: jit.iseq, shape: Default, target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.target1.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) @ocb.write(ocb_asm) end # Jump to target0 on je branch_stub.compile = compile_branchnil(branch_stub) branch_stub.compile.call(asm) end EndBlock end def compile_branchnil(branch_stub) # Proc escapes arguments in memory proc do |branch_asm| branch_asm.comment("branchnil #{branch_stub.shape}") branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.je(branch_stub.target0.address) branch_asm.jmp(branch_stub.target1.address) in Next0 branch_asm.jne(branch_stub.target1.address) in Next1 branch_asm.je(branch_stub.target0.address) end end end end # once # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_case_dispatch(jit, ctx, asm) # Normally this instruction would lookup the key in a hash and jump to an # offset based on that. # Instead we can take the fallback case and continue with the next # instruction. # We'd hope that our jitted code will be sufficiently fast without the # hash lookup, at least for small hashes, but it's worth revisiting this # assumption in the future. unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end starting_context = ctx.dup case_hash = jit.operand(0, ruby: true) else_offset = jit.operand(1) # Try to reorder case/else branches so that ones that are actually used come first. # Supporting only Fixnum for now so that the implementation can be an equality check. key_opnd = ctx.stack_pop(1) comptime_key = jit.peek_at_stack(0) # Check that all cases are fixnums to avoid having to register BOP assumptions on # all the types that case hashes support. This spends compile time to save memory. if fixnum?(comptime_key) && comptime_key <= 2**32 && C.rb_hash_keys(case_hash).all? { |key| fixnum?(key) } unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQQ) return CantCompile end # Check if the key is the same value asm.cmp(key_opnd, to_value(comptime_key)) side_exit = side_exit(jit, starting_context) jit_chain_guard(:jne, jit, starting_context, asm, side_exit) # Get the offset for the compile-time key offset = C.rb_hash_stlike_lookup(case_hash, comptime_key) # NOTE: If we hit the else branch with various values, it could negatively impact the performance. jump_offset = offset || else_offset # Jump to the offset of case or else target_pc = jit.pc + (jit.insn.len + jump_offset) * C.VALUE.size jit_direct_jump(jit.iseq, target_pc, ctx, asm) EndBlock else KeepCompiling # continue with === branches end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_plus(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end comptime_recv = jit.peek_at_stack(1) comptime_obj = jit.peek_at_stack(0) if fixnum?(comptime_recv) && fixnum?(comptime_obj) unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_PLUS) return CantCompile end # Check that both operands are fixnums guard_two_fixnums(jit, ctx, asm) obj_opnd = ctx.stack_pop recv_opnd = ctx.stack_pop asm.mov(:rax, recv_opnd) asm.sub(:rax, 1) # untag asm.mov(:rcx, obj_opnd) asm.add(:rax, :rcx) asm.jo(side_exit(jit, ctx)) dst_opnd = ctx.stack_push(Type::Fixnum) asm.mov(dst_opnd, :rax) KeepCompiling else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_minus(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end comptime_recv = jit.peek_at_stack(1) comptime_obj = jit.peek_at_stack(0) if fixnum?(comptime_recv) && fixnum?(comptime_obj) unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MINUS) return CantCompile end # Check that both operands are fixnums guard_two_fixnums(jit, ctx, asm) obj_opnd = ctx.stack_pop recv_opnd = ctx.stack_pop asm.mov(:rax, recv_opnd) asm.mov(:rcx, obj_opnd) asm.sub(:rax, :rcx) asm.jo(side_exit(jit, ctx)) asm.add(:rax, 1) # re-tag dst_opnd = ctx.stack_push(Type::Fixnum) asm.mov(dst_opnd, :rax) KeepCompiling else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_mult(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_div(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_mod(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end if two_fixnums_on_stack?(jit) unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MOD) return CantCompile end # Check that both operands are fixnums guard_two_fixnums(jit, ctx, asm) # Get the operands and destination from the stack arg1 = ctx.stack_pop(1) arg0 = ctx.stack_pop(1) # Check for arg0 % 0 asm.cmp(arg1, 0) asm.je(side_exit(jit, ctx)) # Call rb_fix_mod_fix(VALUE recv, VALUE obj) asm.mov(C_ARGS[0], arg0) asm.mov(C_ARGS[1], arg1) asm.call(C.rb_fix_mod_fix) # Push the return value onto the stack stack_ret = ctx.stack_push(Type::Fixnum) asm.mov(stack_ret, C_RET) KeepCompiling else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_eq(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end if jit_equality_specialized(jit, ctx, asm, true) jump_to_next_insn(jit, ctx, asm) EndBlock else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_neq(jit, ctx, asm) # opt_neq is passed two rb_call_data as arguments: # first for ==, second for != neq_cd = C.rb_call_data.new(jit.operand(1)) opt_send_without_block(jit, ctx, asm, cd: neq_cd) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_lt(jit, ctx, asm) jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovl, bop: C::BOP_LT) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_le(jit, ctx, asm) jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovle, bop: C::BOP_LE) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_gt(jit, ctx, asm) jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovg, bop: C::BOP_GT) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_ge(jit, ctx, asm) jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovge, bop: C::BOP_GE) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_ltlt(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_and(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end if two_fixnums_on_stack?(jit) unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_AND) return CantCompile end # Check that both operands are fixnums guard_two_fixnums(jit, ctx, asm) # Get the operands and destination from the stack arg1 = ctx.stack_pop(1) arg0 = ctx.stack_pop(1) asm.comment('bitwise and') asm.mov(:rax, arg0) asm.and(:rax, arg1) # Push the return value onto the stack dst = ctx.stack_push(Type::Fixnum) asm.mov(dst, :rax) KeepCompiling else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_or(jit, ctx, asm) unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end if two_fixnums_on_stack?(jit) unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_OR) return CantCompile end # Check that both operands are fixnums guard_two_fixnums(jit, ctx, asm) # Get the operands and destination from the stack asm.comment('bitwise or') arg1 = ctx.stack_pop(1) arg0 = ctx.stack_pop(1) # Do the bitwise or arg0 | arg1 asm.mov(:rax, arg0) asm.or(:rax, arg1) # Push the return value onto the stack dst = ctx.stack_push(Type::Fixnum) asm.mov(dst, :rax) KeepCompiling else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_aref(jit, ctx, asm) cd = C.rb_call_data.new(jit.operand(0)) argc = C.vm_ci_argc(cd.ci) if argc != 1 asm.incr_counter(:optaref_argc_not_one) return CantCompile end unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end comptime_recv = jit.peek_at_stack(1) comptime_obj = jit.peek_at_stack(0) side_exit = side_exit(jit, ctx) if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_obj) unless Invariants.assume_bop_not_redefined(jit, C::ARRAY_REDEFINED_OP_FLAG, C::BOP_AREF) return CantCompile end idx_opnd = ctx.stack_opnd(0) recv_opnd = ctx.stack_opnd(1) not_array_exit = counted_exit(side_exit, :optaref_recv_not_array) jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, StackOpnd[1], comptime_recv, not_array_exit) # Bail if idx is not a FIXNUM asm.mov(:rax, idx_opnd) asm.test(:rax, C::RUBY_FIXNUM_FLAG) asm.jz(counted_exit(side_exit, :optaref_arg_not_fixnum)) # Call VALUE rb_ary_entry_internal(VALUE ary, long offset). # It never raises or allocates, so we don't need to write to cfp->pc. asm.sar(:rax, 1) # Convert fixnum to int asm.mov(C_ARGS[0], recv_opnd) asm.mov(C_ARGS[1], :rax) asm.call(C.rb_ary_entry_internal) # Pop the argument and the receiver ctx.stack_pop(2) # Push the return value onto the stack stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) # Let guard chains share the same successor jump_to_next_insn(jit, ctx, asm) EndBlock elsif C.rb_class_of(comptime_recv) == Hash unless Invariants.assume_bop_not_redefined(jit, C::HASH_REDEFINED_OP_FLAG, C::BOP_AREF) return CantCompile end recv_opnd = ctx.stack_opnd(1) # Guard that the receiver is a Hash not_hash_exit = counted_exit(side_exit, :optaref_recv_not_hash) jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, StackOpnd[1], comptime_recv, not_hash_exit) # Prepare to call rb_hash_aref(). It might call #hash on the key. jit_prepare_routine_call(jit, ctx, asm) asm.comment('call rb_hash_aref') key_opnd = ctx.stack_opnd(0) recv_opnd = ctx.stack_opnd(1) asm.mov(:rdi, recv_opnd) asm.mov(:rsi, key_opnd) asm.call(C.rb_hash_aref) # Pop the key and the receiver ctx.stack_pop(2) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) # Let guard chains share the same successor jump_to_next_insn(jit, ctx, asm) EndBlock else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_aset(jit, ctx, asm) # Defer compilation so we can specialize on a runtime `self` unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end comptime_recv = jit.peek_at_stack(2) comptime_key = jit.peek_at_stack(1) # Get the operands from the stack recv = ctx.stack_opnd(2) key = ctx.stack_opnd(1) _val = ctx.stack_opnd(0) if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_key) side_exit = side_exit(jit, ctx) # Guard receiver is an Array jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[2], comptime_recv, side_exit) # Guard key is a fixnum jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_key), key, StackOpnd[1], comptime_key, side_exit) # We might allocate or raise jit_prepare_routine_call(jit, ctx, asm) asm.comment('call rb_ary_store') recv = ctx.stack_opnd(2) key = ctx.stack_opnd(1) val = ctx.stack_opnd(0) asm.mov(:rax, key) asm.sar(:rax, 1) # FIX2LONG(key) asm.mov(C_ARGS[0], recv) asm.mov(C_ARGS[1], :rax) asm.mov(C_ARGS[2], val) asm.call(C.rb_ary_store) # rb_ary_store returns void # stored value should still be on stack val = ctx.stack_opnd(0) # Push the return value onto the stack ctx.stack_pop(3) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(:rax, val) asm.mov(stack_ret, :rax) jump_to_next_insn(jit, ctx, asm) EndBlock elsif C.rb_class_of(comptime_recv) == Hash side_exit = side_exit(jit, ctx) # Guard receiver is a Hash jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[2], comptime_recv, side_exit) # We might allocate or raise jit_prepare_routine_call(jit, ctx, asm) # Call rb_hash_aset recv = ctx.stack_opnd(2) key = ctx.stack_opnd(1) val = ctx.stack_opnd(0) asm.mov(C_ARGS[0], recv) asm.mov(C_ARGS[1], key) asm.mov(C_ARGS[2], val) asm.call(C.rb_hash_aset) # Push the return value onto the stack ctx.stack_pop(3) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) jump_to_next_insn(jit, ctx, asm) EndBlock else opt_send_without_block(jit, ctx, asm) end end # opt_aset_with # opt_aref_with # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_length(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_size(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_empty_p(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_succ(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_not(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_regexpmatch2(jit, ctx, asm) opt_send_without_block(jit, ctx, asm) end # invokebuiltin # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_invokebuiltin_delegate(jit, ctx, asm) bf = C.rb_builtin_function.new(jit.operand(0)) bf_argc = bf.argc start_index = jit.operand(1) # ec, self, and arguments if bf_argc + 2 > C_ARGS.size return CantCompile end # If the calls don't allocate, do they need up to date PC, SP? jit_prepare_routine_call(jit, ctx, asm) # Call the builtin func (ec, recv, arg1, arg2, ...) asm.comment('call builtin func') asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:self)]) # Copy arguments from locals if bf_argc > 0 # Load environment pointer EP from CFP asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:ep)]) bf_argc.times do |i| table_size = jit.iseq.body.local_table_size offs = -table_size - C::VM_ENV_DATA_SIZE + 1 + start_index + i asm.mov(C_ARGS[2 + i], [:rax, offs * C.VALUE.size]) end end asm.call(bf.func_ptr) # Push the return value stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) KeepCompiling end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def opt_invokebuiltin_delegate_leave(jit, ctx, asm) opt_invokebuiltin_delegate(jit, ctx, asm) # opt_invokebuiltin_delegate is always followed by leave insn end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putobject_INT2FIX_0_(jit, ctx, asm) putobject(jit, ctx, asm, val: C.to_value(0)) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def putobject_INT2FIX_1_(jit, ctx, asm) putobject(jit, ctx, asm, val: C.to_value(1)) end # # C func # # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_true(jit, ctx, asm, argc, _known_recv_class) return false if argc != 0 asm.comment('nil? == true') ctx.stack_pop(1) stack_ret = ctx.stack_push(Type::True) asm.mov(stack_ret, Qtrue) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_false(jit, ctx, asm, argc, _known_recv_class) return false if argc != 0 asm.comment('nil? == false') ctx.stack_pop(1) stack_ret = ctx.stack_push(Type::False) asm.mov(stack_ret, Qfalse) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_kernel_is_a(jit, ctx, asm, argc, known_recv_class) if argc != 1 return false end # If this is a super call we might not know the class if known_recv_class.nil? return false end # Important note: The output code will simply `return true/false`. # Correctness follows from: # - `known_recv_class` implies there is a guard scheduled before here # for a particular `CLASS_OF(lhs)`. # - We guard that rhs is identical to the compile-time sample # - In general, for any two Class instances A, B, `A < B` does not change at runtime. # Class#superclass is stable. sample_rhs = jit.peek_at_stack(0) sample_lhs = jit.peek_at_stack(1) # We are not allowing module here because the module hierarchy can change at runtime. if C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS) return false end sample_is_a = C.obj_is_kind_of(sample_lhs, sample_rhs) side_exit = side_exit(jit, ctx) asm.comment('Kernel#is_a?') asm.mov(:rax, to_value(sample_rhs)) asm.cmp(ctx.stack_opnd(0), :rax) asm.jne(counted_exit(side_exit, :send_is_a_class_mismatch)) ctx.stack_pop(2) if sample_is_a stack_ret = ctx.stack_push(Type::True) asm.mov(stack_ret, Qtrue) else stack_ret = ctx.stack_push(Type::False) asm.mov(stack_ret, Qfalse) end return true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_kernel_instance_of(jit, ctx, asm, argc, known_recv_class) if argc != 1 return false end # If this is a super call we might not know the class if known_recv_class.nil? return false end # Important note: The output code will simply `return true/false`. # Correctness follows from: # - `known_recv_class` implies there is a guard scheduled before here # for a particular `CLASS_OF(lhs)`. # - We guard that rhs is identical to the compile-time sample # - For a particular `CLASS_OF(lhs)`, `rb_obj_class(lhs)` does not change. # (because for any singleton class `s`, `s.superclass.equal?(s.attached_object.class)`) sample_rhs = jit.peek_at_stack(0) sample_lhs = jit.peek_at_stack(1) # Filters out cases where the C implementation raises unless C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS) || C.RB_TYPE_P(sample_rhs, C::RUBY_T_MODULE) return false end # We need to grab the class here to deal with singleton classes. # Instance of grabs the "real class" of the object rather than the # singleton class. sample_lhs_real_class = C.rb_obj_class(sample_lhs) sample_instance_of = (sample_lhs_real_class == sample_rhs) side_exit = side_exit(jit, ctx) asm.comment('Kernel#instance_of?') asm.mov(:rax, to_value(sample_rhs)) asm.cmp(ctx.stack_opnd(0), :rax) asm.jne(counted_exit(side_exit, :send_instance_of_class_mismatch)) ctx.stack_pop(2) if sample_instance_of stack_ret = ctx.stack_push(Type::True) asm.mov(stack_ret, Qtrue) else stack_ret = ctx.stack_push(Type::False) asm.mov(stack_ret, Qfalse) end return true; end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_obj_not(jit, ctx, asm, argc, _known_recv_class) return false if argc != 0 recv_type = ctx.get_opnd_type(StackOpnd[0]) case recv_type.known_truthy in false asm.comment('rb_obj_not(nil_or_false)') ctx.stack_pop(1) out_opnd = ctx.stack_push(Type::True) asm.mov(out_opnd, Qtrue) in true # Note: recv_type != Type::Nil && recv_type != Type::False. asm.comment('rb_obj_not(truthy)') ctx.stack_pop(1) out_opnd = ctx.stack_push(Type::False) asm.mov(out_opnd, Qfalse) in nil asm.comment('rb_obj_not') recv = ctx.stack_pop # This `test` sets ZF only for Qnil and Qfalse, which let cmovz set. asm.test(recv, ~Qnil) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.cmovz(:rax, :rcx) stack_ret = ctx.stack_push(Type::UnknownImm) asm.mov(stack_ret, :rax) end true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_obj_equal(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 asm.comment('equal?') obj1 = ctx.stack_pop(1) obj2 = ctx.stack_pop(1) asm.mov(:rax, obj1) asm.mov(:rcx, obj2) asm.cmp(:rax, :rcx) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.cmove(:rax, :rcx) stack_ret = ctx.stack_push(Type::UnknownImm) asm.mov(stack_ret, :rax) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_obj_not_equal(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 jit_equality_specialized(jit, ctx, asm, false) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_mod_eqq(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 asm.comment('Module#===') # By being here, we know that the receiver is a T_MODULE or a T_CLASS, because Module#=== can # only live on these objects. With that, we can call rb_obj_is_kind_of() without # jit_prepare_routine_call() or a control frame push because it can't raise, allocate, or call # Ruby methods with these inputs. # Note the difference in approach from Kernel#is_a? because we don't get a free guard for the # right hand side. lhs = ctx.stack_opnd(1) # the module rhs = ctx.stack_opnd(0) asm.mov(C_ARGS[0], rhs); asm.mov(C_ARGS[1], lhs); asm.call(C.rb_obj_is_kind_of) # Return the result ctx.stack_pop(2) stack_ret = ctx.stack_push(Type::UnknownImm) asm.mov(stack_ret, C_RET) return true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_int_equal(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 return false unless two_fixnums_on_stack?(jit) guard_two_fixnums(jit, ctx, asm) # Compare the arguments asm.comment('rb_int_equal') arg1 = ctx.stack_pop(1) arg0 = ctx.stack_pop(1) asm.mov(:rax, arg1) asm.cmp(arg0, :rax) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.cmove(:rax, :rcx) stack_ret = ctx.stack_push(Type::UnknownImm) asm.mov(stack_ret, :rax) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_int_mul(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 return false unless two_fixnums_on_stack?(jit) guard_two_fixnums(jit, ctx, asm) asm.comment('rb_int_mul') y_opnd = ctx.stack_pop x_opnd = ctx.stack_pop asm.mov(C_ARGS[0], x_opnd) asm.mov(C_ARGS[1], y_opnd) asm.call(C.rb_fix_mul_fix) ret_opnd = ctx.stack_push(Type::Unknown) asm.mov(ret_opnd, C_RET) true end def jit_rb_int_div(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 return false unless two_fixnums_on_stack?(jit) guard_two_fixnums(jit, ctx, asm) asm.comment('rb_int_div') y_opnd = ctx.stack_pop x_opnd = ctx.stack_pop asm.mov(:rax, y_opnd) asm.cmp(:rax, C.to_value(0)) asm.je(side_exit(jit, ctx)) asm.mov(C_ARGS[0], x_opnd) asm.mov(C_ARGS[1], :rax) asm.call(C.rb_fix_div_fix) ret_opnd = ctx.stack_push(Type::Unknown) asm.mov(ret_opnd, C_RET) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_int_aref(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 return false unless two_fixnums_on_stack?(jit) guard_two_fixnums(jit, ctx, asm) asm.comment('rb_int_aref') y_opnd = ctx.stack_pop x_opnd = ctx.stack_pop asm.mov(C_ARGS[0], x_opnd) asm.mov(C_ARGS[1], y_opnd) asm.call(C.rb_fix_aref) ret_opnd = ctx.stack_push(Type::UnknownImm) asm.mov(ret_opnd, C_RET) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_str_empty_p(jit, ctx, asm, argc, known_recv_class) recv_opnd = ctx.stack_pop(1) out_opnd = ctx.stack_push(Type::UnknownImm) asm.comment('get string length') asm.mov(:rax, recv_opnd) str_len_opnd = [:rax, C.RString.offsetof(:len)] asm.cmp(str_len_opnd, 0) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.cmove(:rax, :rcx) asm.mov(out_opnd, :rax) return true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_str_to_s(jit, ctx, asm, argc, known_recv_class) return false if argc != 0 if known_recv_class == String asm.comment('to_s on plain string') # The method returns the receiver, which is already on the stack. # No stack movement. return true end false end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_str_bytesize(jit, ctx, asm, argc, known_recv_class) asm.comment('String#bytesize') recv = ctx.stack_pop(1) asm.mov(C_ARGS[0], recv) asm.call(C.rb_str_bytesize) out_opnd = ctx.stack_push(Type::Fixnum) asm.mov(out_opnd, C_RET) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_str_concat(jit, ctx, asm, argc, known_recv_class) # The << operator can accept integer codepoints for characters # as the argument. We only specially optimise string arguments. # If the peeked-at compile time argument is something other than # a string, assume it won't be a string later either. comptime_arg = jit.peek_at_stack(0) unless C.RB_TYPE_P(comptime_arg, C::RUBY_T_STRING) return false end # Guard that the concat argument is a string asm.mov(:rax, ctx.stack_opnd(0)) guard_object_is_string(jit, ctx, asm, :rax, :rcx, StackOpnd[0]) # Guard buffers from GC since rb_str_buf_append may allocate. During the VM lock on GC, # other Ractors may trigger global invalidation, so we need ctx.clear_local_types. # PC is used on errors like Encoding::CompatibilityError raised by rb_str_buf_append. jit_prepare_routine_call(jit, ctx, asm) concat_arg = ctx.stack_pop(1) recv = ctx.stack_pop(1) # Test if string encodings differ. If different, use rb_str_append. If the same, # use rb_yjit_str_simple_append, which calls rb_str_cat. asm.comment('<< on strings') # Take receiver's object flags XOR arg's flags. If any # string-encoding flags are different between the two, # the encodings don't match. recv_reg = :rax asm.mov(recv_reg, recv) concat_arg_reg = :rcx asm.mov(concat_arg_reg, concat_arg) asm.mov(recv_reg, [recv_reg, C.RBasic.offsetof(:flags)]) asm.mov(concat_arg_reg, [concat_arg_reg, C.RBasic.offsetof(:flags)]) asm.xor(recv_reg, concat_arg_reg) asm.test(recv_reg, C::RUBY_ENCODING_MASK) # Push once, use the resulting operand in both branches below. stack_ret = ctx.stack_push(Type::TString) enc_mismatch = asm.new_label('enc_mismatch') asm.jnz(enc_mismatch) # If encodings match, call the simple append function and jump to return asm.mov(C_ARGS[0], recv) asm.mov(C_ARGS[1], concat_arg) asm.call(C.rjit_str_simple_append) ret_label = asm.new_label('func_return') asm.mov(stack_ret, C_RET) asm.jmp(ret_label) # If encodings are different, use a slower encoding-aware concatenate asm.write_label(enc_mismatch) asm.mov(C_ARGS[0], recv) asm.mov(C_ARGS[1], concat_arg) asm.call(C.rb_str_buf_append) asm.mov(stack_ret, C_RET) # Drop through to return asm.write_label(ret_label) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_str_uplus(jit, ctx, asm, argc, _known_recv_class) if argc != 0 return false end # We allocate when we dup the string jit_prepare_routine_call(jit, ctx, asm) asm.comment('Unary plus on string') asm.mov(:rax, ctx.stack_pop(1)) # recv_opnd asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)]) # flags_opnd asm.test(:rcx, C::RUBY_FL_FREEZE) ret_label = asm.new_label('stack_ret') # String#+@ can only exist on T_STRING stack_ret = ctx.stack_push(Type::TString) # If the string isn't frozen, we just return it. asm.mov(stack_ret, :rax) # recv_opnd asm.jz(ret_label) # Str is frozen - duplicate it asm.mov(C_ARGS[0], :rax) # recv_opnd asm.call(C.rb_str_dup) asm.mov(stack_ret, C_RET) asm.write_label(ret_label) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_str_getbyte(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 asm.comment('rb_str_getbyte') index_opnd = ctx.stack_pop str_opnd = ctx.stack_pop asm.mov(C_ARGS[0], str_opnd) asm.mov(C_ARGS[1], index_opnd) asm.call(C.rb_str_getbyte) ret_opnd = ctx.stack_push(Type::Fixnum) asm.mov(ret_opnd, C_RET) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_ary_empty_p(jit, ctx, asm, argc, _known_recv_class) array_reg = :rax asm.mov(array_reg, ctx.stack_pop(1)) jit_array_len(asm, array_reg, :rcx) asm.test(:rcx, :rcx) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.cmovz(:rax, :rcx) out_opnd = ctx.stack_push(Type::UnknownImm) asm.mov(out_opnd, :rax) return true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_ary_push(jit, ctx, asm, argc, _known_recv_class) return false if argc != 1 asm.comment('rb_ary_push') jit_prepare_routine_call(jit, ctx, asm) item_opnd = ctx.stack_pop ary_opnd = ctx.stack_pop asm.mov(C_ARGS[0], ary_opnd) asm.mov(C_ARGS[1], item_opnd) asm.call(C.rb_ary_push) ret_opnd = ctx.stack_push(Type::TArray) asm.mov(ret_opnd, C_RET) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_obj_respond_to(jit, ctx, asm, argc, known_recv_class) # respond_to(:sym) or respond_to(:sym, true) if argc != 1 && argc != 2 return false end if known_recv_class.nil? return false end recv_class = known_recv_class # Get the method_id from compile time. We will later add a guard against it. mid_sym = jit.peek_at_stack(argc - 1) unless static_symbol?(mid_sym) return false end mid = C.rb_sym2id(mid_sym) # This represents the value of the "include_all" argument and whether it's known allow_priv = if argc == 1 # Default is false false else # Get value from type information (may or may not be known) ctx.get_opnd_type(StackOpnd[0]).known_truthy end target_cme = C.rb_callable_method_entry_or_negative(recv_class, mid) # Should never be null, as in that case we will be returned a "negative CME" assert_equal(false, target_cme.nil?) cme_def_type = C.UNDEFINED_METHOD_ENTRY_P(target_cme) ? C::VM_METHOD_TYPE_UNDEF : target_cme.def.type if cme_def_type == C::VM_METHOD_TYPE_REFINED return false end visibility = if cme_def_type == C::VM_METHOD_TYPE_UNDEF C::METHOD_VISI_UNDEF else C.METHOD_ENTRY_VISI(target_cme) end result = case [visibility, allow_priv] in C::METHOD_VISI_UNDEF, _ then Qfalse # No method => false in C::METHOD_VISI_PUBLIC, _ then Qtrue # Public method => true regardless of include_all in _, true then Qtrue # include_all => always true else return false # not public and include_all not known, can't compile end if result != Qtrue # Only if respond_to_missing? hasn't been overridden # In the future, we might want to jit the call to respond_to_missing? unless Invariants.assume_method_basic_definition(jit, recv_class, C.idRespond_to_missing) return false end end # Invalidate this block if method lookup changes for the method being queried. This works # both for the case where a method does or does not exist, as for the latter we asked for a # "negative CME" earlier. Invariants.assume_method_lookup_stable(jit, target_cme) # Generate a side exit side_exit = side_exit(jit, ctx) if argc == 2 # pop include_all argument (we only use its type info) ctx.stack_pop(1) end sym_opnd = ctx.stack_pop(1) _recv_opnd = ctx.stack_pop(1) # This is necessary because we have no guarantee that sym_opnd is a constant asm.comment('guard known mid') asm.mov(:rax, to_value(mid_sym)) asm.cmp(sym_opnd, :rax) asm.jne(side_exit) putobject(jit, ctx, asm, val: result) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_f_block_given_p(jit, ctx, asm, argc, _known_recv_class) asm.comment('block_given?') # Same as rb_vm_frame_block_handler jit_get_lep(jit, asm, reg: :rax) asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler ctx.stack_pop(1) out_opnd = ctx.stack_push(Type::UnknownImm) # Return `block_handler != VM_BLOCK_HANDLER_NONE` asm.cmp(:rax, C::VM_BLOCK_HANDLER_NONE) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.cmovne(:rax, :rcx) # block_given asm.mov(out_opnd, :rax) true end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_thread_s_current(jit, ctx, asm, argc, _known_recv_class) return false if argc != 0 asm.comment('Thread.current') ctx.stack_pop(1) # ec->thread_ptr asm.mov(:rax, [EC, C.rb_execution_context_t.offsetof(:thread_ptr)]) # thread->self asm.mov(:rax, [:rax, C.rb_thread_struct.offsetof(:self)]) stack_ret = ctx.stack_push(Type::UnknownHeap) asm.mov(stack_ret, :rax) true end # # Helpers # def register_cfunc_codegen_funcs # Specialization for C methods. See register_cfunc_method for details. register_cfunc_method(BasicObject, :!, :jit_rb_obj_not) register_cfunc_method(NilClass, :nil?, :jit_rb_true) register_cfunc_method(Kernel, :nil?, :jit_rb_false) register_cfunc_method(Kernel, :is_a?, :jit_rb_kernel_is_a) register_cfunc_method(Kernel, :kind_of?, :jit_rb_kernel_is_a) register_cfunc_method(Kernel, :instance_of?, :jit_rb_kernel_instance_of) register_cfunc_method(BasicObject, :==, :jit_rb_obj_equal) register_cfunc_method(BasicObject, :equal?, :jit_rb_obj_equal) register_cfunc_method(BasicObject, :!=, :jit_rb_obj_not_equal) register_cfunc_method(Kernel, :eql?, :jit_rb_obj_equal) register_cfunc_method(Module, :==, :jit_rb_obj_equal) register_cfunc_method(Module, :===, :jit_rb_mod_eqq) register_cfunc_method(Symbol, :==, :jit_rb_obj_equal) register_cfunc_method(Symbol, :===, :jit_rb_obj_equal) register_cfunc_method(Integer, :==, :jit_rb_int_equal) register_cfunc_method(Integer, :===, :jit_rb_int_equal) # rb_str_to_s() methods in string.c register_cfunc_method(String, :empty?, :jit_rb_str_empty_p) register_cfunc_method(String, :to_s, :jit_rb_str_to_s) register_cfunc_method(String, :to_str, :jit_rb_str_to_s) register_cfunc_method(String, :bytesize, :jit_rb_str_bytesize) register_cfunc_method(String, :<<, :jit_rb_str_concat) register_cfunc_method(String, :+@, :jit_rb_str_uplus) # rb_ary_empty_p() method in array.c register_cfunc_method(Array, :empty?, :jit_rb_ary_empty_p) register_cfunc_method(Kernel, :respond_to?, :jit_obj_respond_to) register_cfunc_method(Kernel, :block_given?, :jit_rb_f_block_given_p) # Thread.current register_cfunc_method(C.rb_singleton_class(Thread), :current, :jit_thread_s_current) #--- register_cfunc_method(Array, :<<, :jit_rb_ary_push) register_cfunc_method(Integer, :*, :jit_rb_int_mul) register_cfunc_method(Integer, :/, :jit_rb_int_div) register_cfunc_method(Integer, :[], :jit_rb_int_aref) register_cfunc_method(String, :getbyte, :jit_rb_str_getbyte) end def register_cfunc_method(klass, mid_sym, func) mid = C.rb_intern(mid_sym.to_s) me = C.rb_method_entry_at(klass, mid) assert_equal(false, me.nil?) # Only cfuncs are supported method_serial = me.def.method_serial @cfunc_codegen_table[method_serial] = method(func) end def lookup_cfunc_codegen(cme_def) @cfunc_codegen_table[cme_def.method_serial] end def stack_swap(_jit, ctx, asm, offset0, offset1) stack0_mem = ctx.stack_opnd(offset0) stack1_mem = ctx.stack_opnd(offset1) mapping0 = ctx.get_opnd_mapping(StackOpnd[offset0]) mapping1 = ctx.get_opnd_mapping(StackOpnd[offset1]) asm.mov(:rax, stack0_mem) asm.mov(:rcx, stack1_mem) asm.mov(stack0_mem, :rcx) asm.mov(stack1_mem, :rax) ctx.set_opnd_mapping(StackOpnd[offset0], mapping1) ctx.set_opnd_mapping(StackOpnd[offset1], mapping0) end def jit_getlocal_generic(jit, ctx, asm, idx:, level:) # Load environment pointer EP (level 0) from CFP ep_reg = :rax jit_get_ep(asm, level, reg: ep_reg) # Load the local from the block # val = *(vm_get_ep(GET_EP(), level) - idx); asm.mov(:rax, [ep_reg, -idx * C.VALUE.size]) # Write the local at SP stack_top = if level == 0 local_idx = ep_offset_to_local_idx(jit.iseq, idx) ctx.stack_push_local(local_idx) else ctx.stack_push(Type::Unknown) end asm.mov(stack_top, :rax) KeepCompiling end def jit_setlocal_generic(jit, ctx, asm, idx:, level:) value_type = ctx.get_opnd_type(StackOpnd[0]) # Load environment pointer EP at level ep_reg = :rax jit_get_ep(asm, level, reg: ep_reg) # Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers # only affect heap objects being written. If we know an immediate value is being written we # can skip this check. unless value_type.imm? # flags & VM_ENV_FLAG_WB_REQUIRED flags_opnd = [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS] asm.test(flags_opnd, C::VM_ENV_FLAG_WB_REQUIRED) # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 asm.jnz(side_exit(jit, ctx)) end if level == 0 local_idx = ep_offset_to_local_idx(jit.iseq, idx) ctx.set_local_type(local_idx, value_type) end # Pop the value to write from the stack stack_top = ctx.stack_pop(1) # Write the value at the environment pointer asm.mov(:rcx, stack_top) asm.mov([ep_reg, -(C.VALUE.size * idx)], :rcx) KeepCompiling end # Compute the index of a local variable from its slot index def ep_offset_to_local_idx(iseq, ep_offset) # Layout illustration # This is an array of VALUE # | VM_ENV_DATA_SIZE | # v v # low addr <+-------+-------+-------+-------+------------------+ # |local 0|local 1| ... |local n| .... | # +-------+-------+-------+-------+------------------+ # ^ ^ ^ ^ # +-------+---local_table_size----+ cfp->ep--+ # | | # +------------------ep_offset---------------+ # # See usages of local_var_name() from iseq.c for similar calculation. # Equivalent of iseq->body->local_table_size local_table_size = iseq.body.local_table_size op = ep_offset - C::VM_ENV_DATA_SIZE local_idx = local_table_size - op - 1 assert_equal(true, local_idx >= 0 && local_idx < local_table_size) local_idx end # Compute the index of a local variable from its slot index def slot_to_local_idx(iseq, slot_idx) # Layout illustration # This is an array of VALUE # | VM_ENV_DATA_SIZE | # v v # low addr <+-------+-------+-------+-------+------------------+ # |local 0|local 1| ... |local n| .... | # +-------+-------+-------+-------+------------------+ # ^ ^ ^ ^ # +-------+---local_table_size----+ cfp->ep--+ # | | # +------------------slot_idx----------------+ # # See usages of local_var_name() from iseq.c for similar calculation. local_table_size = iseq.body.local_table_size op = slot_idx - C::VM_ENV_DATA_SIZE local_table_size - op - 1 end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def guard_object_is_heap(jit, ctx, asm, object, object_opnd, counter = nil) object_type = ctx.get_opnd_type(object_opnd) if object_type.heap? return end side_exit = side_exit(jit, ctx) side_exit = counted_exit(side_exit, counter) if counter asm.comment('guard object is heap') # Test that the object is not an immediate asm.test(object, C::RUBY_IMMEDIATE_MASK) asm.jnz(side_exit) # Test that the object is not false asm.cmp(object, Qfalse) asm.je(side_exit) if object_type.diff(Type::UnknownHeap) != TypeDiff::Incompatible ctx.upgrade_opnd_type(object_opnd, Type::UnknownHeap) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def guard_object_is_array(jit, ctx, asm, object_reg, flags_reg, object_opnd, counter = nil) object_type = ctx.get_opnd_type(object_opnd) if object_type.array? return end guard_object_is_heap(jit, ctx, asm, object_reg, object_opnd, counter) side_exit = side_exit(jit, ctx) side_exit = counted_exit(side_exit, counter) if counter asm.comment('guard object is array') # Pull out the type mask asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)]) asm.and(flags_reg, C::RUBY_T_MASK) # Compare the result with T_ARRAY asm.cmp(flags_reg, C::RUBY_T_ARRAY) asm.jne(side_exit) if object_type.diff(Type::TArray) != TypeDiff::Incompatible ctx.upgrade_opnd_type(object_opnd, Type::TArray) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def guard_object_is_string(jit, ctx, asm, object_reg, flags_reg, object_opnd, counter = nil) object_type = ctx.get_opnd_type(object_opnd) if object_type.string? return end guard_object_is_heap(jit, ctx, asm, object_reg, object_opnd, counter) side_exit = side_exit(jit, ctx) side_exit = counted_exit(side_exit, counter) if counter asm.comment('guard object is string') # Pull out the type mask asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)]) asm.and(flags_reg, C::RUBY_T_MASK) # Compare the result with T_STRING asm.cmp(flags_reg, C::RUBY_T_STRING) asm.jne(side_exit) if object_type.diff(Type::TString) != TypeDiff::Incompatible ctx.upgrade_opnd_type(object_opnd, Type::TString) end end # clobbers object_reg def guard_object_is_not_ruby2_keyword_hash(asm, object_reg, flags_reg, side_exit) asm.comment('guard object is not ruby2 keyword hash') not_ruby2_keyword = asm.new_label('not_ruby2_keyword') asm.test(object_reg, C::RUBY_IMMEDIATE_MASK) asm.jnz(not_ruby2_keyword) asm.cmp(object_reg, Qfalse) asm.je(not_ruby2_keyword) asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)]) type_reg = object_reg asm.mov(type_reg, flags_reg) asm.and(type_reg, C::RUBY_T_MASK) asm.cmp(type_reg, C::RUBY_T_HASH) asm.jne(not_ruby2_keyword) asm.test(flags_reg, C::RHASH_PASS_AS_KEYWORDS) asm.jnz(side_exit) asm.write_label(not_ruby2_keyword) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_chain_guard(opcode, jit, ctx, asm, side_exit, limit: 20) opcode => :je | :jne | :jnz | :jz if ctx.chain_depth < limit deeper = ctx.dup deeper.chain_depth += 1 branch_stub = BranchStub.new( iseq: jit.iseq, shape: Default, target0: BranchTarget.new(ctx: deeper, pc: jit.pc), ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(deeper, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.compile = compile_jit_chain_guard(branch_stub, opcode:) branch_stub.compile.call(asm) else asm.public_send(opcode, side_exit) end end def compile_jit_chain_guard(branch_stub, opcode:) # Proc escapes arguments in memory proc do |branch_asm| # Not using `asm.comment` here since it's usually put before cmp/test before this. branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.public_send(opcode, branch_stub.target0.address) end end end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_guard_known_klass(jit, ctx, asm, known_klass, obj_opnd, insn_opnd, comptime_obj, side_exit, limit: 10) # Only memory operand is supported for now assert_equal(true, obj_opnd.is_a?(Array)) known_klass = C.to_value(known_klass) val_type = ctx.get_opnd_type(insn_opnd) if val_type.known_class == known_klass # We already know from type information that this is a match return end # Touching this as Ruby could crash for FrozenCore if known_klass == C.rb_cNilClass assert(!val_type.heap?) assert(val_type.unknown?) asm.comment('guard object is nil') asm.cmp(obj_opnd, Qnil) jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) ctx.upgrade_opnd_type(insn_opnd, Type::Nil) elsif known_klass == C.rb_cTrueClass assert(!val_type.heap?) assert(val_type.unknown?) asm.comment('guard object is true') asm.cmp(obj_opnd, Qtrue) jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) ctx.upgrade_opnd_type(insn_opnd, Type::True) elsif known_klass == C.rb_cFalseClass assert(!val_type.heap?) assert(val_type.unknown?) asm.comment('guard object is false') asm.cmp(obj_opnd, Qfalse) jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) ctx.upgrade_opnd_type(insn_opnd, Type::False) elsif known_klass == C.rb_cInteger && fixnum?(comptime_obj) # We will guard fixnum and bignum as though they were separate classes # BIGNUM can be handled by the general else case below assert(val_type.unknown?) asm.comment('guard object is fixnum') asm.test(obj_opnd, C::RUBY_FIXNUM_FLAG) jit_chain_guard(:jz, jit, ctx, asm, side_exit, limit:) ctx.upgrade_opnd_type(insn_opnd, Type::Fixnum) elsif known_klass == C.rb_cSymbol && static_symbol?(comptime_obj) assert(!val_type.heap?) # We will guard STATIC vs DYNAMIC as though they were separate classes # DYNAMIC symbols can be handled by the general else case below if val_type != Type::ImmSymbol || !val_type.imm? assert(val_type.unknown?) asm.comment('guard object is static symbol') assert_equal(8, C::RUBY_SPECIAL_SHIFT) asm.cmp(BytePtr[*obj_opnd], C::RUBY_SYMBOL_FLAG) jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) ctx.upgrade_opnd_type(insn_opnd, Type::ImmSymbol) end elsif known_klass == C.rb_cFloat && flonum?(comptime_obj) assert(!val_type.heap?) if val_type != Type::Flonum || !val_type.imm? assert(val_type.unknown?) # We will guard flonum vs heap float as though they were separate classes asm.comment('guard object is flonum') asm.mov(:rax, obj_opnd) asm.and(:rax, C::RUBY_FLONUM_MASK) asm.cmp(:rax, C::RUBY_FLONUM_FLAG) jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) ctx.upgrade_opnd_type(insn_opnd, Type::Flonum) end elsif C.RCLASS_SINGLETON_P(known_klass) && comptime_obj == C.rb_class_attached_object(known_klass) # Singleton classes are attached to one specific object, so we can # avoid one memory access (and potentially the is_heap check) by # looking for the expected object directly. # Note that in case the sample instance has a singleton class that # doesn't attach to the sample instance, it means the sample instance # has an empty singleton class that hasn't been materialized yet. In # this case, comparing against the sample instance doesn't guarantee # that its singleton class is empty, so we can't avoid the memory # access. As an example, `Object.new.singleton_class` is an object in # this situation. asm.comment('guard known object with singleton class') asm.mov(:rax, to_value(comptime_obj)) asm.cmp(obj_opnd, :rax) jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) elsif val_type == Type::CString && known_klass == C.rb_cString # guard elided because the context says we've already checked assert_equal(C.to_value(C.rb_class_of(comptime_obj)), C.rb_cString) else assert(!val_type.imm?) # Load memory to a register asm.mov(:rax, obj_opnd) obj_opnd = :rax # Check that the receiver is a heap object # Note: if we get here, the class doesn't have immediate instances. unless val_type.heap? asm.comment('guard not immediate') asm.test(obj_opnd, C::RUBY_IMMEDIATE_MASK) jit_chain_guard(:jnz, jit, ctx, asm, side_exit, limit:) asm.cmp(obj_opnd, Qfalse) jit_chain_guard(:je, jit, ctx, asm, side_exit, limit:) end # Bail if receiver class is different from known_klass klass_opnd = [obj_opnd, C.RBasic.offsetof(:klass)] asm.comment("guard known class #{known_klass}") asm.mov(:rcx, known_klass) asm.cmp(klass_opnd, :rcx) jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) if known_klass == C.rb_cString # Upgrading to Type::CString here is incorrect. # The guard we put only checks RBASIC_CLASS(obj), # which adding a singleton class can change. We # additionally need to know the string is frozen # to claim Type::CString. ctx.upgrade_opnd_type(insn_opnd, Type::TString) elsif known_klass == C.rb_cArray ctx.upgrade_opnd_type(insn_opnd, Type::TArray) end end end # @param jit [RubyVM::RJIT::JITState] def two_fixnums_on_stack?(jit) comptime_recv = jit.peek_at_stack(1) comptime_arg = jit.peek_at_stack(0) return fixnum?(comptime_recv) && fixnum?(comptime_arg) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def guard_two_fixnums(jit, ctx, asm) # Get stack operands without popping them arg1 = ctx.stack_opnd(0) arg0 = ctx.stack_opnd(1) # Get the stack operand types arg1_type = ctx.get_opnd_type(StackOpnd[0]) arg0_type = ctx.get_opnd_type(StackOpnd[1]) if arg0_type.heap? || arg1_type.heap? asm.comment('arg is heap object') asm.jmp(side_exit(jit, ctx)) return end if arg0_type != Type::Fixnum && arg0_type.specific? asm.comment('arg0 not fixnum') asm.jmp(side_exit(jit, ctx)) return end if arg1_type != Type::Fixnum && arg1_type.specific? asm.comment('arg1 not fixnum') asm.jmp(side_exit(jit, ctx)) return end assert(!arg0_type.heap?) assert(!arg1_type.heap?) assert(arg0_type == Type::Fixnum || arg0_type.unknown?) assert(arg1_type == Type::Fixnum || arg1_type.unknown?) # If not fixnums at run-time, fall back if arg0_type != Type::Fixnum asm.comment('guard arg0 fixnum') asm.test(arg0, C::RUBY_FIXNUM_FLAG) jit_chain_guard(:jz, jit, ctx, asm, side_exit(jit, ctx)) end if arg1_type != Type::Fixnum asm.comment('guard arg1 fixnum') asm.test(arg1, C::RUBY_FIXNUM_FLAG) jit_chain_guard(:jz, jit, ctx, asm, side_exit(jit, ctx)) end # Set stack types in context ctx.upgrade_opnd_type(StackOpnd[0], Type::Fixnum) ctx.upgrade_opnd_type(StackOpnd[1], Type::Fixnum) end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_fixnum_cmp(jit, ctx, asm, opcode:, bop:) opcode => :cmovl | :cmovle | :cmovg | :cmovge unless jit.at_current_insn? defer_compilation(jit, ctx, asm) return EndBlock end comptime_recv = jit.peek_at_stack(1) comptime_obj = jit.peek_at_stack(0) if fixnum?(comptime_recv) && fixnum?(comptime_obj) unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, bop) return CantCompile end # Check that both operands are fixnums guard_two_fixnums(jit, ctx, asm) obj_opnd = ctx.stack_pop recv_opnd = ctx.stack_pop asm.mov(:rax, obj_opnd) asm.cmp(recv_opnd, :rax) asm.mov(:rax, Qfalse) asm.mov(:rcx, Qtrue) asm.public_send(opcode, :rax, :rcx) dst_opnd = ctx.stack_push(Type::UnknownImm) asm.mov(dst_opnd, :rax) KeepCompiling else opt_send_without_block(jit, ctx, asm) end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_equality_specialized(jit, ctx, asm, gen_eq) # Create a side-exit to fall back to the interpreter side_exit = side_exit(jit, ctx) a_opnd = ctx.stack_opnd(1) b_opnd = ctx.stack_opnd(0) comptime_a = jit.peek_at_stack(1) comptime_b = jit.peek_at_stack(0) if two_fixnums_on_stack?(jit) unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQ) return false end guard_two_fixnums(jit, ctx, asm) asm.comment('check fixnum equality') asm.mov(:rax, a_opnd) asm.mov(:rcx, b_opnd) asm.cmp(:rax, :rcx) asm.mov(:rax, gen_eq ? Qfalse : Qtrue) asm.mov(:rcx, gen_eq ? Qtrue : Qfalse) asm.cmove(:rax, :rcx) # Push the output on the stack ctx.stack_pop(2) dst = ctx.stack_push(Type::UnknownImm) asm.mov(dst, :rax) true elsif C.rb_class_of(comptime_a) == String && C.rb_class_of(comptime_b) == String unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_EQ) # if overridden, emit the generic version return false end # Guard that a is a String jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_a), a_opnd, StackOpnd[1], comptime_a, side_exit) equal_label = asm.new_label(:equal) ret_label = asm.new_label(:ret) # If they are equal by identity, return true asm.mov(:rax, a_opnd) asm.mov(:rcx, b_opnd) asm.cmp(:rax, :rcx) asm.je(equal_label) # Otherwise guard that b is a T_STRING (from type info) or String (from runtime guard) btype = ctx.get_opnd_type(StackOpnd[0]) unless btype.string? # Note: any T_STRING is valid here, but we check for a ::String for simplicity # To pass a mutable static variable (rb_cString) requires an unsafe block jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_b), b_opnd, StackOpnd[0], comptime_b, side_exit) end asm.comment('call rb_str_eql_internal') asm.mov(C_ARGS[0], a_opnd) asm.mov(C_ARGS[1], b_opnd) asm.call(gen_eq ? C.rb_str_eql_internal : C.rjit_str_neq_internal) # Push the output on the stack ctx.stack_pop(2) dst = ctx.stack_push(Type::UnknownImm) asm.mov(dst, C_RET) asm.jmp(ret_label) asm.write_label(equal_label) asm.mov(dst, gen_eq ? Qtrue : Qfalse) asm.write_label(ret_label) true else false end end # NOTE: This clobbers :rax # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_prepare_routine_call(jit, ctx, asm) jit.record_boundary_patch_point = true jit_save_pc(jit, asm) jit_save_sp(ctx, asm) # In case the routine calls Ruby methods, it can set local variables # through Kernel#binding and other means. ctx.clear_local_types end # NOTE: This clobbers :rax # @param jit [RubyVM::RJIT::JITState] # @param asm [RubyVM::RJIT::Assembler] def jit_save_pc(jit, asm, comment: 'save PC to CFP') next_pc = jit.pc + jit.insn.len * C.VALUE.size # Use the next one for backtrace and side exits asm.comment(comment) asm.mov(:rax, next_pc) asm.mov([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax) end # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_save_sp(ctx, asm) if ctx.sp_offset != 0 asm.comment('save SP to CFP') asm.lea(SP, ctx.sp_opnd) asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) ctx.sp_offset = 0 end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jump_to_next_insn(jit, ctx, asm) reset_depth = ctx.dup reset_depth.chain_depth = 0 next_pc = jit.pc + jit.insn.len * C.VALUE.size # We are at the end of the current instruction. Record the boundary. if jit.record_boundary_patch_point exit_pos = Assembler.new.then do |ocb_asm| @exit_compiler.compile_side_exit(next_pc, ctx, ocb_asm) @ocb.write(ocb_asm) end Invariants.record_global_inval_patch(asm, exit_pos) jit.record_boundary_patch_point = false end jit_direct_jump(jit.iseq, next_pc, reset_depth, asm, comment: 'jump_to_next_insn') end # rb_vm_check_ints # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_check_ints(jit, ctx, asm) asm.comment('RUBY_VM_CHECK_INTS(ec)') asm.mov(:eax, DwordPtr[EC, C.rb_execution_context_t.offsetof(:interrupt_flag)]) asm.test(:eax, :eax) asm.jnz(side_exit(jit, ctx)) end # See get_lvar_level in compile.c def get_lvar_level(iseq) level = 0 while iseq.to_i != iseq.body.local_iseq.to_i level += 1 iseq = iseq.body.parent_iseq end return level end # GET_LEP # @param jit [RubyVM::RJIT::JITState] # @param asm [RubyVM::RJIT::Assembler] def jit_get_lep(jit, asm, reg:) level = get_lvar_level(jit.iseq) jit_get_ep(asm, level, reg:) end # vm_get_ep # @param asm [RubyVM::RJIT::Assembler] def jit_get_ep(asm, level, reg:) asm.mov(reg, [CFP, C.rb_control_frame_t.offsetof(:ep)]) level.times do # GET_PREV_EP: ep[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03 asm.mov(reg, [reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) asm.and(reg, ~0x03) end end # vm_getivar # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_getivar(jit, ctx, asm, comptime_obj, ivar_id, obj_opnd, obj_yarv_opnd) side_exit = side_exit(jit, ctx) starting_ctx = ctx.dup # copy for jit_chain_guard # Guard not special const if C::SPECIAL_CONST_P(comptime_obj) asm.incr_counter(:getivar_special_const) return CantCompile end case C::BUILTIN_TYPE(comptime_obj) when C::T_OBJECT # This is the only supported case for now (ROBJECT_IVPTR) else # General case. Call rb_ivar_get(). # VALUE rb_ivar_get(VALUE obj, ID id) asm.comment('call rb_ivar_get()') asm.mov(C_ARGS[0], obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)]) asm.mov(C_ARGS[1], ivar_id) # The function could raise exceptions. jit_prepare_routine_call(jit, ctx, asm) # clobbers obj_opnd and :rax asm.call(C.rb_ivar_get) if obj_opnd # attr_reader ctx.stack_pop end # Push the ivar on the stack out_opnd = ctx.stack_push(Type::Unknown) asm.mov(out_opnd, C_RET) # Jump to next instruction. This allows guard chains to share the same successor. jump_to_next_insn(jit, ctx, asm) return EndBlock end asm.mov(:rax, obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)]) guard_object_is_heap(jit, ctx, asm, :rax, obj_yarv_opnd, :getivar_not_heap) shape_id = C.rb_shape_get_shape_id(comptime_obj) if shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID asm.incr_counter(:getivar_too_complex) return CantCompile end asm.comment('guard shape') asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id) jit_chain_guard(:jne, jit, starting_ctx, asm, counted_exit(side_exit, :getivar_megamorphic)) if obj_opnd ctx.stack_pop # pop receiver for attr_reader end index = C.rb_shape_get_iv_index(shape_id, ivar_id) # If there is no IVAR index, then the ivar was undefined # when we entered the compiler. That means we can just return # nil for this shape + iv name if index.nil? stack_opnd = ctx.stack_push(Type::Nil) val_opnd = Qnil else asm.comment('ROBJECT_IVPTR') if C::FL_TEST_RAW(comptime_obj, C::ROBJECT_EMBED) # Access embedded array asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :ary) + (index * C.VALUE.size)]) else # Pull out an ivar table on heap asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :heap, :ivptr)]) # Read the table asm.mov(:rax, [:rax, index * C.VALUE.size]) end stack_opnd = ctx.stack_push(Type::Unknown) val_opnd = :rax end asm.mov(stack_opnd, val_opnd) # Let guard chains share the same successor jump_to_next_insn(jit, ctx, asm) EndBlock end def jit_write_iv(asm, comptime_receiver, recv_reg, temp_reg, ivar_index, set_value, needs_extension) # Compile time self is embedded and the ivar index lands within the object embed_test_result = C::FL_TEST_RAW(comptime_receiver, C::ROBJECT_EMBED) && !needs_extension if embed_test_result # Find the IV offset offs = C.RObject.offsetof(:as, :ary) + ivar_index * C.VALUE.size # Write the IV asm.comment('write IV') asm.mov(temp_reg, set_value) asm.mov([recv_reg, offs], temp_reg) else # Compile time value is *not* embedded. # Get a pointer to the extended table asm.mov(recv_reg, [recv_reg, C.RObject.offsetof(:as, :heap, :ivptr)]) # Write the ivar in to the extended table asm.comment("write IV"); asm.mov(temp_reg, set_value) asm.mov([recv_reg, C.VALUE.size * ivar_index], temp_reg) end end # vm_caller_setup_arg_block: Handle VM_CALL_ARGS_BLOCKARG cases. # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def guard_block_arg(jit, ctx, asm, calling) if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 block_arg_type = ctx.get_opnd_type(StackOpnd[0]) case block_arg_type in Type::Nil calling.block_handler = C::VM_BLOCK_HANDLER_NONE in Type::BlockParamProxy calling.block_handler = C.rb_block_param_proxy else asm.incr_counter(:send_block_arg) return CantCompile end end end # vm_search_method # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_search_method(jit, ctx, asm, mid, calling) assert_equal(true, jit.at_current_insn?) # Generate a side exit side_exit = side_exit(jit, ctx) # kw_splat is not supported yet if calling.flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_kw_splat) return CantCompile end # Get a compile-time receiver and its class recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet recv_idx += calling.send_shift comptime_recv = jit.peek_at_stack(recv_idx) comptime_recv_klass = C.rb_class_of(comptime_recv) # Guard the receiver class (part of vm_search_method_fastpath) recv_opnd = ctx.stack_opnd(recv_idx) megamorphic_exit = counted_exit(side_exit, :send_klass_megamorphic) jit_guard_known_klass(jit, ctx, asm, comptime_recv_klass, recv_opnd, StackOpnd[recv_idx], comptime_recv, megamorphic_exit) # Do method lookup (vm_cc_cme(cc) != NULL) cme = C.rb_callable_method_entry(comptime_recv_klass, mid) if cme.nil? asm.incr_counter(:send_missing_cme) return CantCompile # We don't support vm_call_method_name end # Invalidate on redefinition (part of vm_search_method_fastpath) Invariants.assume_method_lookup_stable(jit, cme) return cme, comptime_recv_klass end # vm_call_general # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_general(jit, ctx, asm, mid, calling, cme, known_recv_class) jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) end # vm_call_method # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] # @param send_shift [Integer] The number of shifts needed for VM_CALL_OPT_SEND def jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) # The main check of vm_call_method before vm_call_method_each_type case C::METHOD_ENTRY_VISI(cme) in C::METHOD_VISI_PUBLIC # You can always call public methods in C::METHOD_VISI_PRIVATE # Allow only callsites without a receiver if calling.flags & C::VM_CALL_FCALL == 0 asm.incr_counter(:send_private) return CantCompile end in C::METHOD_VISI_PROTECTED # If the method call is an FCALL, it is always valid if calling.flags & C::VM_CALL_FCALL == 0 # otherwise we need an ancestry check to ensure the receiver is valid to be called as protected jit_protected_callee_ancestry_guard(asm, cme, side_exit(jit, ctx)) end end # Get a compile-time receiver recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet recv_idx += calling.send_shift comptime_recv = jit.peek_at_stack(recv_idx) recv_opnd = ctx.stack_opnd(recv_idx) jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) end # Generate ancestry guard for protected callee. # Calls to protected callees only go through when self.is_a?(klass_that_defines_the_callee). def jit_protected_callee_ancestry_guard(asm, cme, side_exit) # See vm_call_method(). def_class = cme.defined_class # Note: PC isn't written to current control frame as rb_is_kind_of() shouldn't raise. # VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass); asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)]) asm.mov(C_ARGS[1], to_value(def_class)) asm.call(C.rb_obj_is_kind_of) asm.test(C_RET, C_RET) asm.jz(counted_exit(side_exit, :send_protected_check_failed)) end # vm_call_method_each_type # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) case cme.def.type in C::VM_METHOD_TYPE_ISEQ iseq = def_iseq_ptr(cme.def) jit_call_iseq(jit, ctx, asm, cme, calling, iseq) in C::VM_METHOD_TYPE_NOTIMPLEMENTED asm.incr_counter(:send_notimplemented) return CantCompile in C::VM_METHOD_TYPE_CFUNC jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class:) in C::VM_METHOD_TYPE_ATTRSET jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) in C::VM_METHOD_TYPE_IVAR jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) in C::VM_METHOD_TYPE_MISSING asm.incr_counter(:send_missing) return CantCompile in C::VM_METHOD_TYPE_BMETHOD jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) in C::VM_METHOD_TYPE_ALIAS jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) in C::VM_METHOD_TYPE_OPTIMIZED jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class) in C::VM_METHOD_TYPE_UNDEF asm.incr_counter(:send_undef) return CantCompile in C::VM_METHOD_TYPE_ZSUPER asm.incr_counter(:send_zsuper) return CantCompile in C::VM_METHOD_TYPE_REFINED asm.incr_counter(:send_refined) return CantCompile end end # vm_call_iseq_setup # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type: nil, prev_ep: nil) argc = calling.argc flags = calling.flags send_shift = calling.send_shift # When you have keyword arguments, there is an extra object that gets # placed on the stack the represents a bitmap of the keywords that were not # specified at the call site. We need to keep track of the fact that this # value is present on the stack in order to properly set up the callee's # stack pointer. doing_kw_call = iseq.body.param.flags.has_kw supplying_kws = flags & C::VM_CALL_KWARG != 0 if flags & C::VM_CALL_TAILCALL != 0 # We can't handle tailcalls asm.incr_counter(:send_tailcall) return CantCompile end # No support for callees with these parameters yet as they require allocation # or complex handling. if iseq.body.param.flags.has_post asm.incr_counter(:send_iseq_has_opt) return CantCompile end if iseq.body.param.flags.has_kwrest asm.incr_counter(:send_iseq_has_kwrest) return CantCompile end # In order to handle backwards compatibility between ruby 3 and 2 # ruby2_keywords was introduced. It is called only on methods # with splat and changes they way they handle them. # We are just going to not compile these. # https://www.rubydoc.info/stdlib/core/Proc:ruby2_keywords if iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_iseq_ruby2_keywords) return CantCompile end iseq_has_rest = iseq.body.param.flags.has_rest if iseq_has_rest && calling.block_handler == :captured asm.incr_counter(:send_iseq_has_rest_and_captured) return CantCompile end if iseq_has_rest && iseq.body.param.flags.has_kw && supplying_kws asm.incr_counter(:send_iseq_has_rest_and_kw_supplied) return CantCompile end # If we have keyword arguments being passed to a callee that only takes # positionals, then we need to allocate a hash. For now we're going to # call that too complex and bail. if supplying_kws && !iseq.body.param.flags.has_kw asm.incr_counter(:send_iseq_has_no_kw) return CantCompile end # If we have a method accepting no kwargs (**nil), exit if we have passed # it any kwargs. if supplying_kws && iseq.body.param.flags.accepts_no_kwarg asm.incr_counter(:send_iseq_accepts_no_kwarg) return CantCompile end # For computing number of locals to set up for the callee num_params = iseq.body.param.size # Block parameter handling. This mirrors setup_parameters_complex(). if iseq.body.param.flags.has_block if iseq.body.local_iseq.to_i == iseq.to_i num_params -= 1 else # In this case (param.flags.has_block && local_iseq != iseq), # the block argument is setup as a local variable and requires # materialization (allocation). Bail. asm.incr_counter(:send_iseq_materialized_block) return CantCompile end end if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0 # zsuper methods are super calls without any arguments. # They are also marked as splat, but don't actually have an array # they pull arguments from, instead we need to change to call # a different method with the current stack. asm.incr_counter(:send_iseq_zsuper) return CantCompile end start_pc_offset = 0 required_num = iseq.body.param.lead_num # This struct represents the metadata about the caller-specified # keyword arguments. kw_arg = calling.kwarg kw_arg_num = if kw_arg.nil? 0 else kw_arg.keyword_len end # Arity handling and optional parameter setup opts_filled = argc - required_num - kw_arg_num opt_num = iseq.body.param.opt_num opts_missing = opt_num - opts_filled if doing_kw_call && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_iseq_splat_with_kw) return CantCompile end if flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_iseq_kw_splat) return CantCompile end if iseq_has_rest && opt_num != 0 asm.incr_counter(:send_iseq_has_rest_and_optional) return CantCompile end if opts_filled < 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 # Too few arguments and no splat to make up for it asm.incr_counter(:send_iseq_arity_error) return CantCompile end if opts_filled > opt_num && !iseq_has_rest # Too many arguments and no place to put them (i.e. rest arg) asm.incr_counter(:send_iseq_arity_error) return CantCompile end block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0 # Guard block_arg_type if guard_block_arg(jit, ctx, asm, calling) == CantCompile return CantCompile end # If we have unfilled optional arguments and keyword arguments then we # would need to adjust the arguments location to account for that. # For now we aren't handling this case. if doing_kw_call && opts_missing > 0 asm.incr_counter(:send_iseq_missing_optional_kw) return CantCompile end # We will handle splat case later if opt_num > 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 num_params -= opts_missing start_pc_offset = iseq.body.param.opt_table[opts_filled] end if doing_kw_call # Here we're calling a method with keyword arguments and specifying # keyword arguments at this call site. # This struct represents the metadata about the callee-specified # keyword parameters. keyword = iseq.body.param.keyword keyword_num = keyword.num keyword_required_num = keyword.required_num required_kwargs_filled = 0 if keyword_num > 30 # We have so many keywords that (1 << num) encoded as a FIXNUM # (which shifts it left one more) no longer fits inside a 32-bit # immediate. asm.incr_counter(:send_iseq_too_many_kwargs) return CantCompile end # Check that the kwargs being passed are valid if supplying_kws # This is the list of keyword arguments that the callee specified # in its initial declaration. # SAFETY: see compile.c for sizing of this slice. callee_kwargs = keyword_num.times.map { |i| keyword.table[i] } # Here we're going to build up a list of the IDs that correspond to # the caller-specified keyword arguments. If they're not in the # same order as the order specified in the callee declaration, then # we're going to need to generate some code to swap values around # on the stack. caller_kwargs = [] kw_arg.keyword_len.times do |kwarg_idx| sym = C.to_ruby(kw_arg[:keywords][kwarg_idx]) caller_kwargs << C.rb_sym2id(sym) end # First, we're going to be sure that the names of every # caller-specified keyword argument correspond to a name in the # list of callee-specified keyword parameters. caller_kwargs.each do |caller_kwarg| search_result = callee_kwargs.map.with_index.find { |kwarg, _| kwarg == caller_kwarg } case search_result in nil # If the keyword was never found, then we know we have a # mismatch in the names of the keyword arguments, so we need to # bail. asm.incr_counter(:send_iseq_kwargs_mismatch) return CantCompile in _, callee_idx if callee_idx < keyword_required_num # Keep a count to ensure all required kwargs are specified required_kwargs_filled += 1 else end end end assert_equal(true, required_kwargs_filled <= keyword_required_num) if required_kwargs_filled != keyword_required_num asm.incr_counter(:send_iseq_kwargs_mismatch) return CantCompile end end # Check if we need the arg0 splat handling of vm_callee_setup_block_arg arg_setup_block = (calling.block_handler == :captured) # arg_setup_type: arg_setup_block (invokeblock) block_arg0_splat = arg_setup_block && argc == 1 && (iseq.body.param.flags.has_lead || opt_num > 1) && !iseq.body.param.flags.ambiguous_param0 if block_arg0_splat # If block_arg0_splat, we still need side exits after splat, but # doing push_splat_args here disallows it. So bail out. if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest asm.incr_counter(:invokeblock_iseq_arg0_args_splat) return CantCompile end # The block_arg0_splat implementation is for the rb_simple_iseq_p case, # but doing_kw_call means it's not a simple ISEQ. if doing_kw_call asm.incr_counter(:invokeblock_iseq_arg0_has_kw) return CantCompile end # The block_arg0_splat implementation cannot deal with optional parameters. # This is a setup_parameters_complex() situation and interacts with the # starting position of the callee. if opt_num > 1 asm.incr_counter(:invokeblock_iseq_arg0_optional) return CantCompile end end if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest array = jit.peek_at_stack(block_arg ? 1 : 0) splat_array_length = if array.nil? 0 else array.length end if opt_num == 0 && required_num != splat_array_length + argc - 1 asm.incr_counter(:send_iseq_splat_arity_error) return CantCompile end end # Don't compile forwardable iseqs if iseq.body.param.flags.forwardable return CantCompile end # We will not have CantCompile from here. if block_arg ctx.stack_pop(1) end if calling.block_handler == C::VM_BLOCK_HANDLER_NONE && iseq.body.builtin_attrs & C::BUILTIN_ATTR_LEAF != 0 if jit_leaf_builtin_func(jit, ctx, asm, flags, iseq) return KeepCompiling end end # Number of locals that are not parameters num_locals = iseq.body.local_table_size - num_params # Stack overflow check # Note that vm_push_frame checks it against a decremented cfp, hence the multiply by 2. # #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) asm.comment('stack overflow check') locals_offs = C.VALUE.size * (num_locals + iseq.body.stack_max) + 2 * C.rb_control_frame_t.size asm.lea(:rax, ctx.sp_opnd(locals_offs)) asm.cmp(CFP, :rax) asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow)) # push_splat_args does stack manipulation so we can no longer side exit if splat_array_length remaining_opt = (opt_num + required_num) - (splat_array_length + (argc - 1)) if opt_num > 0 # We are going to jump to the correct offset based on how many optional # params are remaining. offset = opt_num - remaining_opt start_pc_offset = iseq.body.param.opt_table[offset] end # We are going to assume that the splat fills # all the remaining arguments. In the generated code # we test if this is true and if not side exit. argc = argc - 1 + splat_array_length + remaining_opt push_splat_args(splat_array_length, jit, ctx, asm) remaining_opt.times do # We need to push nil for the optional arguments stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, Qnil) end end # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end if iseq_has_rest # We are going to allocate so setting pc and sp. jit_save_pc(jit, asm) # clobbers rax jit_save_sp(ctx, asm) if flags & C::VM_CALL_ARGS_SPLAT != 0 non_rest_arg_count = argc - 1 # We start by dupping the array because someone else might have # a reference to it. array = ctx.stack_pop(1) asm.mov(C_ARGS[0], array) asm.call(C.rb_ary_dup) array = C_RET if non_rest_arg_count > required_num # If we have more arguments than required, we need to prepend # the items from the stack onto the array. diff = (non_rest_arg_count - required_num) # diff is >0 so no need to worry about null pointer asm.comment('load pointer to array elements') offset_magnitude = C.VALUE.size * diff values_opnd = ctx.sp_opnd(-offset_magnitude) values_ptr = :rcx asm.lea(values_ptr, values_opnd) asm.comment('prepend stack values to rest array') asm.mov(C_ARGS[0], diff) asm.mov(C_ARGS[1], values_ptr) asm.mov(C_ARGS[2], array) asm.call(C.rb_ary_unshift_m) ctx.stack_pop(diff) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) # We now should have the required arguments # and an array of all the rest arguments argc = required_num + 1 elsif non_rest_arg_count < required_num # If we have fewer arguments than required, we need to take some # from the array and move them to the stack. diff = (required_num - non_rest_arg_count) # This moves the arguments onto the stack. But it doesn't modify the array. move_rest_args_to_stack(array, diff, jit, ctx, asm) # We will now slice the array to give us a new array of the correct size asm.mov(C_ARGS[0], array) asm.mov(C_ARGS[1], diff) asm.call(C.rjit_rb_ary_subseq_length) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) # We now should have the required arguments # and an array of all the rest arguments argc = required_num + 1 else # The arguments are equal so we can just push to the stack assert_equal(non_rest_arg_count, required_num) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, array) end else assert_equal(true, argc >= required_num) n = (argc - required_num) argc = required_num + 1 # If n is 0, then elts is never going to be read, so we can just pass null if n == 0 values_ptr = 0 else asm.comment('load pointer to array elements') offset_magnitude = C.VALUE.size * n values_opnd = ctx.sp_opnd(-offset_magnitude) values_ptr = :rcx asm.lea(values_ptr, values_opnd) end asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], n) asm.mov(C_ARGS[2], values_ptr) asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) end end if doing_kw_call # Here we're calling a method with keyword arguments and specifying # keyword arguments at this call site. # Number of positional arguments the callee expects before the first # keyword argument args_before_kw = required_num + opt_num # This struct represents the metadata about the caller-specified # keyword arguments. ci_kwarg = calling.kwarg caller_keyword_len = if ci_kwarg.nil? 0 else ci_kwarg.keyword_len end # This struct represents the metadata about the callee-specified # keyword parameters. keyword = iseq.body.param.keyword asm.comment('keyword args') # This is the list of keyword arguments that the callee specified # in its initial declaration. callee_kwargs = keyword.table total_kwargs = keyword.num # Here we're going to build up a list of the IDs that correspond to # the caller-specified keyword arguments. If they're not in the # same order as the order specified in the callee declaration, then # we're going to need to generate some code to swap values around # on the stack. caller_kwargs = [] caller_keyword_len.times do |kwarg_idx| sym = C.to_ruby(ci_kwarg[:keywords][kwarg_idx]) caller_kwargs << C.rb_sym2id(sym) end kwarg_idx = caller_keyword_len unspecified_bits = 0 keyword_required_num = keyword.required_num (keyword_required_num...total_kwargs).each do |callee_idx| already_passed = false callee_kwarg = callee_kwargs[callee_idx] caller_keyword_len.times do |caller_idx| if caller_kwargs[caller_idx] == callee_kwarg already_passed = true break end end unless already_passed # Reserve space on the stack for each default value we'll be # filling in (which is done in the next loop). Also increments # argc so that the callee's SP is recorded correctly. argc += 1 default_arg = ctx.stack_push(Type::Unknown) # callee_idx - keyword->required_num is used in a couple of places below. req_num = keyword.required_num extra_args = callee_idx - req_num # VALUE default_value = keyword->default_values[callee_idx - keyword->required_num]; default_value = keyword.default_values[extra_args] if default_value == Qundef # Qundef means that this value is not constant and must be # recalculated at runtime, so we record it in unspecified_bits # (Qnil is then used as a placeholder instead of Qundef). unspecified_bits |= 0x01 << extra_args default_value = Qnil end asm.mov(:rax, default_value) asm.mov(default_arg, :rax) caller_kwargs[kwarg_idx] = callee_kwarg kwarg_idx += 1 end end assert_equal(kwarg_idx, total_kwargs) # Next, we're going to loop through every keyword that was # specified by the caller and make sure that it's in the correct # place. If it's not we're going to swap it around with another one. total_kwargs.times do |kwarg_idx| callee_kwarg = callee_kwargs[kwarg_idx] # If the argument is already in the right order, then we don't # need to generate any code since the expected value is already # in the right place on the stack. if callee_kwarg == caller_kwargs[kwarg_idx] next end # In this case the argument is not in the right place, so we # need to find its position where it _should_ be and swap with # that location. ((kwarg_idx + 1)...total_kwargs).each do |swap_idx| if callee_kwarg == caller_kwargs[swap_idx] # First we're going to generate the code that is going # to perform the actual swapping at runtime. offset0 = argc - 1 - swap_idx - args_before_kw offset1 = argc - 1 - kwarg_idx - args_before_kw stack_swap(jit, ctx, asm, offset0, offset1) # Next we're going to do some bookkeeping on our end so # that we know the order that the arguments are # actually in now. caller_kwargs[kwarg_idx], caller_kwargs[swap_idx] = caller_kwargs[swap_idx], caller_kwargs[kwarg_idx] break end end end # Keyword arguments cause a special extra local variable to be # pushed onto the stack that represents the parameters that weren't # explicitly given a value and have a non-constant default. asm.mov(ctx.stack_opnd(-1), C.to_value(unspecified_bits)) end # Same as vm_callee_setup_block_arg_arg0_check and vm_callee_setup_block_arg_arg0_splat # on vm_callee_setup_block_arg for arg_setup_block. This is done after CALLER_SETUP_ARG # and CALLER_REMOVE_EMPTY_KW_SPLAT, so this implementation is put here. This may need # side exits, so you still need to allow side exits here if block_arg0_splat is true. # Note that you can't have side exits after this arg0 splat. if block_arg0_splat asm.incr_counter(:send_iseq_block_arg0_splat) return CantCompile end # Create a context for the callee callee_ctx = Context.new # Set the argument types in the callee's context argc.times do |arg_idx| stack_offs = argc - arg_idx - 1 arg_type = ctx.get_opnd_type(StackOpnd[stack_offs]) callee_ctx.set_local_type(arg_idx, arg_type) end recv_type = if calling.block_handler == :captured Type::Unknown # we don't track the type information of captured->self for now else ctx.get_opnd_type(StackOpnd[argc]) end callee_ctx.upgrade_opnd_type(SelfOpnd, recv_type) # Setup the new frame frame_type ||= C::VM_FRAME_MAGIC_METHOD | C::VM_ENV_FLAG_LOCAL jit_push_frame( jit, ctx, asm, cme, flags, argc, frame_type, calling.block_handler, iseq: iseq, local_size: num_locals, stack_max: iseq.body.stack_max, prev_ep:, doing_kw_call:, ) # Directly jump to the entry point of the callee pc = (iseq.body.iseq_encoded + start_pc_offset).to_i jit_direct_jump(iseq, pc, callee_ctx, asm) EndBlock end def jit_leaf_builtin_func(jit, ctx, asm, flags, iseq) builtin_func = builtin_function(iseq) if builtin_func.nil? return false end # this is a .send call not currently supported for builtins if flags & C::VM_CALL_OPT_SEND != 0 return false end builtin_argc = builtin_func.argc if builtin_argc + 1 >= C_ARGS.size return false end asm.comment('inlined leaf builtin') # The callee may allocate, e.g. Integer#abs on a Bignum. # Save SP for GC, save PC for allocation tracing, and prepare # for global invalidation after GC's VM lock contention. jit_prepare_routine_call(jit, ctx, asm) # Call the builtin func (ec, recv, arg1, arg2, ...) asm.mov(C_ARGS[0], EC) # Copy self and arguments (0..builtin_argc).each do |i| stack_opnd = ctx.stack_opnd(builtin_argc - i) asm.mov(C_ARGS[i + 1], stack_opnd) end ctx.stack_pop(builtin_argc + 1) asm.call(builtin_func.func_ptr) # Push the return value stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) return true end # vm_call_cfunc # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class: nil) argc = calling.argc flags = calling.flags cfunc = cme.def.body.cfunc cfunc_argc = cfunc.argc # If the function expects a Ruby array of arguments if cfunc_argc < 0 && cfunc_argc != -1 asm.incr_counter(:send_cfunc_ruby_array_varg) return CantCompile end # We aren't handling a vararg cfuncs with splat currently. if flags & C::VM_CALL_ARGS_SPLAT != 0 && cfunc_argc == -1 asm.incr_counter(:send_args_splat_cfunc_var_args) return CantCompile end if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0 # zsuper methods are super calls without any arguments. # They are also marked as splat, but don't actually have an array # they pull arguments from, instead we need to change to call # a different method with the current stack. asm.incr_counter(:send_args_splat_cfunc_zuper) return CantCompile; end # In order to handle backwards compatibility between ruby 3 and 2 # ruby2_keywords was introduced. It is called only on methods # with splat and changes they way they handle them. # We are just going to not compile these. # https://docs.ruby-lang.org/en/3.2/Module.html#method-i-ruby2_keywords if jit.iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_args_splat_cfunc_ruby2_keywords) return CantCompile; end kw_arg = calling.kwarg kw_arg_num = if kw_arg.nil? 0 else kw_arg.keyword_len end if kw_arg_num != 0 && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_cfunc_splat_with_kw) return CantCompile end if c_method_tracing_currently_enabled? # Don't JIT if tracing c_call or c_return asm.incr_counter(:send_cfunc_tracing) return CantCompile end # Delegate to codegen for C methods if we have it. if kw_arg.nil? && flags & C::VM_CALL_OPT_SEND == 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 && (cfunc_argc == -1 || argc == cfunc_argc) known_cfunc_codegen = lookup_cfunc_codegen(cme.def) if known_cfunc_codegen&.call(jit, ctx, asm, argc, known_recv_class) # cfunc codegen generated code. Terminate the block so # there isn't multiple calls in the same block. jump_to_next_insn(jit, ctx, asm) return EndBlock end end # Check for interrupts jit_check_ints(jit, ctx, asm) # Stack overflow check # #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) # REG_CFP <= REG_SP + 4 * SIZEOF_VALUE + sizeof(rb_control_frame_t) asm.comment('stack overflow check') asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * 4 + 2 * C.rb_control_frame_t.size)) asm.cmp(CFP, :rax) asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow)) # Number of args which will be passed through to the callee # This is adjusted by the kwargs being combined into a hash. passed_argc = if kw_arg.nil? argc else argc - kw_arg_num + 1 end # If the argument count doesn't match if cfunc_argc >= 0 && cfunc_argc != passed_argc && flags & C::VM_CALL_ARGS_SPLAT == 0 asm.incr_counter(:send_cfunc_argc_mismatch) return CantCompile end # Don't JIT functions that need C stack arguments for now if cfunc_argc >= 0 && passed_argc + 1 > C_ARGS.size asm.incr_counter(:send_cfunc_toomany_args) return CantCompile end block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0 # Guard block_arg_type if guard_block_arg(jit, ctx, asm, calling) == CantCompile return CantCompile end if block_arg ctx.stack_pop(1) end # push_splat_args does stack manipulation so we can no longer side exit if flags & C::VM_CALL_ARGS_SPLAT != 0 assert_equal(true, cfunc_argc >= 0) required_args = cfunc_argc - (argc - 1) # + 1 because we pass self if required_args + 1 >= C_ARGS.size asm.incr_counter(:send_cfunc_toomany_args) return CantCompile end # We are going to assume that the splat fills # all the remaining arguments. So the number of args # should just equal the number of args the cfunc takes. # In the generated code we test if this is true # and if not side exit. argc = cfunc_argc passed_argc = argc push_splat_args(required_args, jit, ctx, asm) end # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift: calling.send_shift) end # Points to the receiver operand on the stack # Store incremented PC into current control frame in case callee raises. jit_save_pc(jit, asm) # Increment the stack pointer by 3 (in the callee) # sp += 3 frame_type = C::VM_FRAME_MAGIC_CFUNC | C::VM_FRAME_FLAG_CFRAME | C::VM_ENV_FLAG_LOCAL if kw_arg frame_type |= C::VM_FRAME_FLAG_CFRAME_KW end jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, calling.block_handler) if kw_arg # Build a hash from all kwargs passed asm.comment('build_kwhash') imemo_ci = calling.ci_addr # we assume all callinfos with kwargs are on the GC heap assert_equal(true, C.imemo_type_p(imemo_ci, C.imemo_callinfo)) asm.mov(C_ARGS[0], imemo_ci) asm.lea(C_ARGS[1], ctx.sp_opnd(0)) asm.call(C.rjit_build_kwhash) # Replace the stack location at the start of kwargs with the new hash stack_opnd = ctx.stack_opnd(argc - passed_argc) asm.mov(stack_opnd, C_RET) end # Copy SP because REG_SP will get overwritten sp = :rax asm.lea(sp, ctx.sp_opnd(0)) # Pop the C function arguments from the stack (in the caller) ctx.stack_pop(argc + 1) # Write interpreter SP into CFP. # Needed in case the callee yields to the block. jit_save_sp(ctx, asm) # Non-variadic method case cfunc_argc in (0..) # Non-variadic method # Copy the arguments from the stack to the C argument registers # self is the 0th argument and is at index argc from the stack top (0..passed_argc).each do |i| asm.mov(C_ARGS[i], [sp, -(argc + 1 - i) * C.VALUE.size]) end in -1 # Variadic method: rb_f_puts(int argc, VALUE *argv, VALUE recv) # The method gets a pointer to the first argument # rb_f_puts(int argc, VALUE *argv, VALUE recv) asm.mov(C_ARGS[0], passed_argc) asm.lea(C_ARGS[1], [sp, -argc * C.VALUE.size]) # argv asm.mov(C_ARGS[2], [sp, -(argc + 1) * C.VALUE.size]) # recv end # Call the C function # VALUE ret = (cfunc->func)(recv, argv[0], argv[1]); # cfunc comes from compile-time cme->def, which we assume to be stable. # Invalidation logic is in yjit_method_lookup_change() asm.comment('call C function') asm.mov(:rax, cfunc.func) asm.call(:rax) # TODO: use rel32 if close enough # Record code position for TracePoint patching. See full_cfunc_return(). Invariants.record_global_inval_patch(asm, full_cfunc_return) # Push the return value on the Ruby stack stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) # Pop the stack frame (ec->cfp++) # Instead of recalculating, we can reuse the previous CFP, which is stored in a callee-saved # register asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], CFP) # cfunc calls may corrupt types ctx.clear_local_types # Note: the return block of jit_call_iseq has ctx->sp_offset == 1 # which allows for sharing the same successor. # Jump (fall through) to the call continuation block # We do this to end the current block after the call assert_equal(1, ctx.sp_offset) jump_to_next_insn(jit, ctx, asm) EndBlock end # vm_call_attrset # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) argc = calling.argc flags = calling.flags send_shift = calling.send_shift if flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_attrset_splat) return CantCompile end if flags & C::VM_CALL_KWARG != 0 asm.incr_counter(:send_attrset_kwarg) return CantCompile elsif argc != 1 || !C.RB_TYPE_P(comptime_recv, C::RUBY_T_OBJECT) asm.incr_counter(:send_attrset_method) return CantCompile elsif c_method_tracing_currently_enabled? # Can't generate code for firing c_call and c_return events # See :attr-tracing: asm.incr_counter(:send_c_tracingg) return CantCompile elsif flags & C::VM_CALL_ARGS_BLOCKARG != 0 asm.incr_counter(:send_block_arg) return CantCompile end ivar_name = cme.def.body.attr.id # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end # Save the PC and SP because the callee may allocate # Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm) # Get the operands from the stack val_opnd = ctx.stack_pop(1) recv_opnd = ctx.stack_pop(1) # Call rb_vm_set_ivar_id with the receiver, the ivar name, and the value asm.mov(C_ARGS[0], recv_opnd) asm.mov(C_ARGS[1], ivar_name) asm.mov(C_ARGS[2], val_opnd) asm.call(C.rb_vm_set_ivar_id) out_opnd = ctx.stack_push(Type::Unknown) asm.mov(out_opnd, C_RET) KeepCompiling end # vm_call_ivar (+ part of vm_call_method_each_type) # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) argc = calling.argc flags = calling.flags if flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_ivar_splat) return CantCompile end if argc != 0 asm.incr_counter(:send_arity) return CantCompile end # We don't support handle_opt_send_shift_stack for this yet. if flags & C::VM_CALL_OPT_SEND != 0 asm.incr_counter(:send_ivar_opt_send) return CantCompile end ivar_id = cme.def.body.attr.id # Not handling block_handler if flags & C::VM_CALL_ARGS_BLOCKARG != 0 asm.incr_counter(:send_block_arg) return CantCompile end jit_getivar(jit, ctx, asm, comptime_recv, ivar_id, recv_opnd, StackOpnd[0]) end # vm_call_bmethod # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) proc_addr = cme.def.body.bmethod.proc proc_t = C.rb_yjit_get_proc_ptr(proc_addr) proc_block = proc_t.block if proc_block.type != C.block_type_iseq asm.incr_counter(:send_bmethod_not_iseq) return CantCompile end capture = proc_block.as.captured iseq = capture.code.iseq # TODO: implement this # Optimize for single ractor mode and avoid runtime check for # "defined with an un-shareable Proc in a different Ractor" # if !assume_single_ractor_mode(jit, ocb) # return CantCompile; # end # Passing a block to a block needs logic different from passing # a block to a method and sometimes requires allocation. Bail for now. if calling.block_handler != C::VM_BLOCK_HANDLER_NONE asm.incr_counter(:send_bmethod_blockarg) return CantCompile end jit_call_iseq( jit, ctx, asm, cme, calling, iseq, frame_type: C::VM_FRAME_MAGIC_BLOCK | C::VM_FRAME_FLAG_BMETHOD | C::VM_FRAME_FLAG_LAMBDA, prev_ep: capture.ep, ) end # vm_call_alias # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) cme = C.rb_aliased_callable_method_entry(cme) jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) end # vm_call_optimized # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class) if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 # Not working yet asm.incr_counter(:send_block_arg) return CantCompile end case cme.def.body.optimized.type in C::OPTIMIZED_METHOD_TYPE_SEND jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class) in C::OPTIMIZED_METHOD_TYPE_CALL jit_call_opt_call(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift) in C::OPTIMIZED_METHOD_TYPE_BLOCK_CALL asm.incr_counter(:send_optimized_block_call) return CantCompile in C::OPTIMIZED_METHOD_TYPE_STRUCT_AREF jit_call_opt_struct_aref(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift) in C::OPTIMIZED_METHOD_TYPE_STRUCT_ASET asm.incr_counter(:send_optimized_struct_aset) return CantCompile end end # vm_call_opt_send # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class) if jit_caller_setup_arg(jit, ctx, asm, calling.flags) == CantCompile return CantCompile end if calling.argc == 0 asm.incr_counter(:send_optimized_send_no_args) return CantCompile end calling.argc -= 1 # We aren't handling `send(:send, ...)` yet. This might work, but not tested yet. if calling.send_shift > 0 asm.incr_counter(:send_optimized_send_send) return CantCompile end # Lazily handle stack shift in handle_opt_send_shift_stack calling.send_shift += 1 jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, C::VM_CALL_FCALL) end # vm_call_opt_call # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_opt_call(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:) if block_handler != C::VM_BLOCK_HANDLER_NONE asm.incr_counter(:send_optimized_call_block) return CantCompile end if flags & C::VM_CALL_KWARG != 0 asm.incr_counter(:send_optimized_call_kwarg) return CantCompile end if flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_optimized_call_splat) return CantCompile end # TODO: implement this # Optimize for single ractor mode and avoid runtime check for # "defined with an un-shareable Proc in a different Ractor" # if !assume_single_ractor_mode(jit, ocb) # return CantCompile # end # If this is a .send call we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end # About to reset the SP, need to load this here recv_idx = argc # blockarg is not supported. send_shift is already handled. asm.mov(:rcx, ctx.stack_opnd(recv_idx)) # recv # Save the PC and SP because the callee can make Ruby calls jit_prepare_routine_call(jit, ctx, asm) # NOTE: clobbers rax asm.lea(:rax, ctx.sp_opnd(0)) # sp kw_splat = flags & C::VM_CALL_KW_SPLAT asm.mov(C_ARGS[0], :rcx) asm.mov(C_ARGS[1], EC) asm.mov(C_ARGS[2], argc) asm.lea(C_ARGS[3], [:rax, -argc * C.VALUE.size]) # stack_argument_pointer. NOTE: C_ARGS[3] is rcx asm.mov(C_ARGS[4], kw_splat) asm.mov(C_ARGS[5], C::VM_BLOCK_HANDLER_NONE) asm.call(C.rjit_optimized_call) ctx.stack_pop(argc + 1) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) return KeepCompiling end # vm_call_opt_struct_aref # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_opt_struct_aref(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:) if argc != 0 asm.incr_counter(:send_optimized_struct_aref_error) return CantCompile end if c_method_tracing_currently_enabled? # Don't JIT if tracing c_call or c_return asm.incr_counter(:send_cfunc_tracing) return CantCompile end off = cme.def.body.optimized.index recv_idx = argc # blockarg is not supported recv_idx += send_shift comptime_recv = jit.peek_at_stack(recv_idx) # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end # All structs from the same Struct class should have the same # length. So if our comptime_recv is embedded all runtime # structs of the same class should be as well, and the same is # true of the converse. embedded = C::FL_TEST_RAW(comptime_recv, C::RSTRUCT_EMBED_LEN_MASK) asm.comment('struct aref') asm.mov(:rax, ctx.stack_pop(1)) # recv if embedded asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :ary) + (C.VALUE.size * off)]) else asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :heap, :ptr)]) asm.mov(:rax, [:rax, C.VALUE.size * off]) end ret = ctx.stack_push(Type::Unknown) asm.mov(ret, :rax) jump_to_next_insn(jit, ctx, asm) EndBlock end # vm_call_opt_send (lazy part) # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) # We don't support `send(:send, ...)` for now. assert_equal(1, send_shift) asm.comment('shift stack') (0...argc).reverse_each do |i| opnd = ctx.stack_opnd(i) opnd2 = ctx.stack_opnd(i + 1) asm.mov(:rax, opnd) asm.mov(opnd2, :rax) end ctx.shift_stack(argc) end # vm_call_symbol # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, flags) flags |= C::VM_CALL_OPT_SEND | (calling.kw_splat ? C::VM_CALL_KW_SPLAT : 0) comptime_symbol = jit.peek_at_stack(calling.argc) if comptime_symbol.class != String && !static_symbol?(comptime_symbol) asm.incr_counter(:send_optimized_send_not_sym_or_str) return CantCompile end mid = C.get_symbol_id(comptime_symbol) if mid == 0 asm.incr_counter(:send_optimized_send_null_mid) return CantCompile end asm.comment("Guard #{comptime_symbol.inspect} is on stack") class_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_class_changed) jit_guard_known_klass( jit, ctx, asm, C.rb_class_of(comptime_symbol), ctx.stack_opnd(calling.argc), StackOpnd[calling.argc], comptime_symbol, class_changed_exit, ) asm.mov(C_ARGS[0], ctx.stack_opnd(calling.argc)) asm.call(C.rb_get_symbol_id) asm.cmp(C_RET, mid) id_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_id_changed) jit_chain_guard(:jne, jit, ctx, asm, id_changed_exit) # rb_callable_method_entry_with_refinements calling.flags = flags cme, _ = jit_search_method(jit, ctx, asm, mid, calling) if cme == CantCompile return CantCompile end if flags & C::VM_CALL_FCALL != 0 return jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) end raise NotImplementedError # unreachable for now end # vm_push_frame # # Frame structure: # | args | locals | cme/cref | block_handler/prev EP | frame type (EP here) | stack bottom (SP here) # # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, block_handler, iseq: nil, local_size: 0, stack_max: 0, prev_ep: nil, doing_kw_call: nil) # Save caller SP and PC before pushing a callee frame for backtrace and side exits asm.comment('save SP to caller CFP') recv_idx = argc # blockarg is already popped recv_idx += (block_handler == :captured) ? 0 : 1 # receiver is not on stack when captured->self is used if iseq # Skip setting this to SP register. This cfp->sp will be copied to SP on leave insn. asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * -recv_idx)) # Pop receiver and arguments to prepare for side exits asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], :rax) else asm.lea(SP, ctx.sp_opnd(C.VALUE.size * -recv_idx)) asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) ctx.sp_offset = recv_idx end jit_save_pc(jit, asm, comment: 'save PC to caller CFP') sp_offset = ctx.sp_offset + 3 + local_size + (doing_kw_call ? 1 : 0) # callee_sp local_size.times do |i| asm.comment('set local variables') if i == 0 local_index = sp_offset + i - local_size - 3 asm.mov([SP, C.VALUE.size * local_index], Qnil) end asm.comment('set up EP with managing data') ep_offset = sp_offset - 1 # ep[-2]: cref_or_me asm.mov(:rax, cme.to_i) asm.mov([SP, C.VALUE.size * (ep_offset - 2)], :rax) # ep[-1]: block handler or prev env ptr (specval) if prev_ep asm.mov(:rax, prev_ep.to_i | 1) # tagged prev ep asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) elsif block_handler == :captured # Set captured->ep, saving captured in :rcx for captured->self ep_reg = :rcx jit_get_lep(jit, asm, reg: ep_reg) asm.mov(:rcx, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler asm.and(:rcx, ~0x3) # captured asm.mov(:rax, [:rcx, C.VALUE.size]) # captured->ep asm.or(:rax, 0x1) # GC_GUARDED_PTR asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) elsif block_handler == C::VM_BLOCK_HANDLER_NONE asm.mov([SP, C.VALUE.size * (ep_offset - 1)], C::VM_BLOCK_HANDLER_NONE) elsif block_handler == C.rb_block_param_proxy # vm_caller_setup_arg_block: block_code == rb_block_param_proxy jit_get_lep(jit, asm, reg: :rax) # VM_CF_BLOCK_HANDLER: VM_CF_LEP asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # VM_CF_BLOCK_HANDLER: VM_ENV_BLOCK_HANDLER asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax) # reg_cfp->block_code = handler asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) # return handler; else # assume blockiseq asm.mov(:rax, block_handler) asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax) asm.lea(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # VM_CFP_TO_CAPTURED_BLOCK asm.or(:rax, 1) # VM_BH_FROM_ISEQ_BLOCK asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) end # ep[-0]: ENV_FLAGS asm.mov([SP, C.VALUE.size * (ep_offset - 0)], frame_type) asm.comment('set up new frame') cfp_offset = -C.rb_control_frame_t.size # callee CFP # For ISEQ, JIT code will set it as needed. However, C func needs 0 there for svar frame detection. if iseq.nil? asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:pc)], 0) end asm.mov(:rax, iseq.to_i) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:iseq)], :rax) if block_handler == :captured asm.mov(:rax, [:rcx]) # captured->self else self_index = ctx.sp_offset - (1 + argc) # blockarg has been popped asm.mov(:rax, [SP, C.VALUE.size * self_index]) end asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:self)], :rax) asm.lea(:rax, [SP, C.VALUE.size * ep_offset]) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:ep)], :rax) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:block_code)], 0) # Update SP register only for ISEQ calls. SP-relative operations should be done above this. sp_reg = iseq ? SP : :rax asm.lea(sp_reg, [SP, C.VALUE.size * sp_offset]) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:sp)], sp_reg) # cfp->jit_return is used only for ISEQs if iseq # The callee might change locals through Kernel#binding and other means. ctx.clear_local_types # Stub cfp->jit_return return_ctx = ctx.dup return_ctx.stack_pop(argc + ((block_handler == :captured) ? 0 : 1)) # Pop args and receiver. blockarg has been popped return_ctx.stack_push(Type::Unknown) # push callee's return value return_ctx.sp_offset = 1 # SP is in the position after popping a receiver and arguments return_ctx.chain_depth = 0 branch_stub = BranchStub.new( iseq: jit.iseq, shape: Default, target0: BranchTarget.new(ctx: return_ctx, pc: jit.pc + jit.insn.len * C.VALUE.size), ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(return_ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.compile = compile_jit_return(branch_stub, cfp_offset:) branch_stub.compile.call(asm) end asm.comment('switch to callee CFP') # Update CFP register only for ISEQ calls cfp_reg = iseq ? CFP : :rax asm.lea(cfp_reg, [CFP, cfp_offset]) asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], cfp_reg) end def compile_jit_return(branch_stub, cfp_offset:) # Proc escapes arguments in memory proc do |branch_asm| branch_asm.comment('set jit_return to callee CFP') branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.mov(:rax, branch_stub.target0.address) branch_asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:jit_return)], :rax) end end end end # CALLER_SETUP_ARG: Return CantCompile if not supported # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_caller_setup_arg(jit, ctx, asm, flags) if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_args_splat_kw_splat) return CantCompile elsif flags & C::VM_CALL_ARGS_SPLAT != 0 # splat is not supported in this path asm.incr_counter(:send_args_splat) return CantCompile elsif flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_args_kw_splat) return CantCompile elsif flags & C::VM_CALL_KWARG != 0 asm.incr_counter(:send_kwarg) return CantCompile end end # Pushes arguments from an array to the stack. Differs from push splat because # the array can have items left over. # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def move_rest_args_to_stack(array, num_args, jit, ctx, asm) side_exit = side_exit(jit, ctx) asm.comment('move_rest_args_to_stack') # array is :rax array_len_opnd = :rcx jit_array_len(asm, array, array_len_opnd) asm.comment('Side exit if length is less than required') asm.cmp(array_len_opnd, num_args) asm.jl(counted_exit(side_exit, :send_iseq_has_rest_and_splat_not_equal)) asm.comment('Push arguments from array') # Load the address of the embedded array # (struct RArray *)(obj)->as.ary array_reg = array # Conditionally load the address of the heap array # (struct RArray *)(obj)->as.heap.ptr flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) heap_ptr_opnd = [array_reg, C.RArray.offsetof(:as, :heap, :ptr)] # Load the address of the embedded array # (struct RArray *)(obj)->as.ary ary_opnd = :rdx # NOTE: array :rax is used after move_rest_args_to_stack too asm.lea(:rcx, [array_reg, C.RArray.offsetof(:as, :ary)]) asm.mov(ary_opnd, heap_ptr_opnd) asm.cmovnz(ary_opnd, :rcx) num_args.times do |i| top = ctx.stack_push(Type::Unknown) asm.mov(:rcx, [ary_opnd, i * C.VALUE.size]) asm.mov(top, :rcx) end end # vm_caller_setup_arg_splat (+ CALLER_SETUP_ARG): # Pushes arguments from an array to the stack that are passed with a splat (i.e. *args). # It optimistically compiles to a static size that is the exact number of arguments needed for the function. # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def push_splat_args(required_args, jit, ctx, asm) side_exit = side_exit(jit, ctx) asm.comment('push_splat_args') array_opnd = ctx.stack_opnd(0) array_stack_opnd = StackOpnd[0] array_reg = :rax asm.mov(array_reg, array_opnd) guard_object_is_array(jit, ctx, asm, array_reg, :rcx, array_stack_opnd, :send_args_splat_not_array) array_len_opnd = :rcx jit_array_len(asm, array_reg, array_len_opnd) asm.comment('Side exit if length is not equal to remaining args') asm.cmp(array_len_opnd, required_args) asm.jne(counted_exit(side_exit, :send_args_splat_length_not_equal)) asm.comment('Check last argument is not ruby2keyword hash') ary_opnd = :rcx jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg last_array_value = :rax asm.mov(last_array_value, [ary_opnd, (required_args - 1) * C.VALUE.size]) ruby2_exit = counted_exit(side_exit, :send_args_splat_ruby2_hash); guard_object_is_not_ruby2_keyword_hash(asm, last_array_value, :rcx, ruby2_exit) # clobbers :rax asm.comment('Push arguments from array') array_opnd = ctx.stack_pop(1) if required_args > 0 # Load the address of the embedded array # (struct RArray *)(obj)->as.ary array_reg = :rax asm.mov(array_reg, array_opnd) # Conditionally load the address of the heap array # (struct RArray *)(obj)->as.heap.ptr flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) heap_ptr_opnd = [array_reg, C.RArray.offsetof(:as, :heap, :ptr)] # Load the address of the embedded array # (struct RArray *)(obj)->as.ary asm.lea(:rcx, [array_reg, C.RArray.offsetof(:as, :ary)]) asm.mov(:rax, heap_ptr_opnd) asm.cmovnz(:rax, :rcx) ary_opnd = :rax (0...required_args).each do |i| top = ctx.stack_push(Type::Unknown) asm.mov(:rcx, [ary_opnd, i * C.VALUE.size]) asm.mov(top, :rcx) end asm.comment('end push_each') end end # Generate RARRAY_LEN. For array_opnd, use Opnd::Reg to reduce memory access, # and use Opnd::Mem to save registers. def jit_array_len(asm, array_reg, len_reg) asm.comment('get array length for embedded or heap') # Pull out the embed flag to check if it's an embedded array. asm.mov(len_reg, [array_reg, C.RBasic.offsetof(:flags)]) # Get the length of the array asm.and(len_reg, C::RARRAY_EMBED_LEN_MASK) asm.sar(len_reg, C::RARRAY_EMBED_LEN_SHIFT) # Conditionally move the length of the heap array asm.test([array_reg, C.RBasic.offsetof(:flags)], C::RARRAY_EMBED_FLAG) # Select the array length value asm.cmovz(len_reg, [array_reg, C.RArray.offsetof(:as, :heap, :len)]) end # Generate RARRAY_CONST_PTR (part of RARRAY_AREF) def jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg asm.comment('get array pointer for embedded or heap') flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) # Load the address of the embedded array # (struct RArray *)(obj)->as.ary asm.mov(ary_opnd, [array_reg, C.RArray.offsetof(:as, :heap, :ptr)]) asm.lea(array_reg, [array_reg, C.RArray.offsetof(:as, :ary)]) # clobbers array_reg asm.cmovnz(ary_opnd, array_reg) end def assert(cond) assert_equal(cond, true) end def assert_equal(left, right) if left != right raise "'#{left.inspect}' was not '#{right.inspect}'" end end def fixnum?(obj) (C.to_value(obj) & C::RUBY_FIXNUM_FLAG) == C::RUBY_FIXNUM_FLAG end def flonum?(obj) (C.to_value(obj) & C::RUBY_FLONUM_MASK) == C::RUBY_FLONUM_FLAG end def symbol?(obj) static_symbol?(obj) || dynamic_symbol?(obj) end def static_symbol?(obj) (C.to_value(obj) & 0xff) == C::RUBY_SYMBOL_FLAG end def dynamic_symbol?(obj) return false if C::SPECIAL_CONST_P(obj) C.RB_TYPE_P(obj, C::RUBY_T_SYMBOL) end def shape_too_complex?(obj) C.rb_shape_get_shape_id(obj) == C::OBJ_TOO_COMPLEX_SHAPE_ID end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def defer_compilation(jit, ctx, asm) # Make a stub to compile the current insn if ctx.chain_depth != 0 raise "double defer!" end ctx.chain_depth += 1 jit_direct_jump(jit.iseq, jit.pc, ctx, asm, comment: 'defer_compilation') end def jit_direct_jump(iseq, pc, ctx, asm, comment: 'jit_direct_jump') branch_stub = BranchStub.new( iseq:, shape: Default, target0: BranchTarget.new(ctx:, pc:), ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.compile = compile_jit_direct_jump(branch_stub, comment:) branch_stub.compile.call(asm) end def compile_jit_direct_jump(branch_stub, comment:) # Proc escapes arguments in memory proc do |branch_asm| branch_asm.comment(comment) branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.jmp(branch_stub.target0.address) in Next0 # Just write the block without a jump end end end end # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] def side_exit(jit, ctx) # We use the latest ctx.sp_offset to generate a side exit to tolerate sp_offset changes by jit_save_sp. # However, we want to simulate an old stack_size when we take a side exit. We do that by adjusting the # sp_offset because gen_outlined_exit uses ctx.sp_offset to move SP. ctx = ctx.with_stack_size(jit.stack_size_for_pc) jit.side_exit_for_pc[ctx.sp_offset] ||= Assembler.new.then do |asm| @exit_compiler.compile_side_exit(jit.pc, ctx, asm) @ocb.write(asm) end end def counted_exit(side_exit, name) asm = Assembler.new asm.incr_counter(name) asm.jmp(side_exit) @ocb.write(asm) end def def_iseq_ptr(cme_def) C.rb_iseq_check(cme_def.body.iseq.iseqptr) end def to_value(obj) GC_REFS << obj C.to_value(obj) end def full_cfunc_return @full_cfunc_return ||= Assembler.new.then do |asm| @exit_compiler.compile_full_cfunc_return(asm) @ocb.write(asm) end end def c_method_tracing_currently_enabled? C.rb_rjit_global_events & (C::RUBY_EVENT_C_CALL | C::RUBY_EVENT_C_RETURN) != 0 end # Return a builtin function if a given iseq consists of only that builtin function def builtin_function(iseq) opt_invokebuiltin_delegate_leave = INSNS.values.find { |i| i.name == :opt_invokebuiltin_delegate_leave } leave = INSNS.values.find { |i| i.name == :leave } if iseq.body.iseq_size == opt_invokebuiltin_delegate_leave.len + leave.len && C.rb_vm_insn_decode(iseq.body.iseq_encoded[0]) == opt_invokebuiltin_delegate_leave.bin && C.rb_vm_insn_decode(iseq.body.iseq_encoded[opt_invokebuiltin_delegate_leave.len]) == leave.bin C.rb_builtin_function.new(iseq.body.iseq_encoded[1]) end end def build_calling(ci:, block_handler:) CallingInfo.new( argc: C.vm_ci_argc(ci), flags: C.vm_ci_flag(ci), kwarg: C.vm_ci_kwarg(ci), ci_addr: ci.to_i, send_shift: 0, block_handler:, ) end end end
Close