From f5bbb0480ef1ea03d7011af128369d10ca726d72 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 15 Sep 2020 15:57:57 +0300 Subject: [PATCH] Tracing JIT for INIT_DYNAMIC_CALL (closure only) --- ext/opcache/jit/zend_jit.c | 1 + ext/opcache/jit/zend_jit_internal.h | 3 +- ext/opcache/jit/zend_jit_trace.c | 55 +++++--- ext/opcache/jit/zend_jit_vm_helpers.c | 11 +- ext/opcache/jit/zend_jit_x86.dasc | 194 ++++++++++++++++++++++++-- 5 files changed, 227 insertions(+), 37 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index cfc391f0e8..b5c6c20e8d 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -22,6 +22,7 @@ #include "Zend/zend_vm.h" #include "Zend/zend_exceptions.h" #include "Zend/zend_constants.h" +#include "Zend/zend_closures.h" #include "Zend/zend_ini.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 18a3096204..0bbdb949a8 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -211,6 +211,7 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_EXIT_FREE_OP1 (1<<5) #define ZEND_JIT_EXIT_FREE_OP2 (1<<6) #define ZEND_JIT_EXIT_PACKED_GUARD (1<<7) +#define ZEND_JIT_EXIT_DYNAMIC_CALL (1<<8) /* exit because of polymorphic INTI_DYNAMIC_CALL call */ typedef union _zend_op_trace_info { zend_op dummy; /* the size of this structure must be the same as zend_op */ @@ -458,7 +459,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HAN int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline); int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); -zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphc); +zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphc); static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, zend_bool *exit_if_true) { diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 253ae68a7f..15bc71a691 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -1361,13 +1361,15 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin level = 0; for (;;p++) { if (p->op == ZEND_JIT_TRACE_VM) { - uint8_t orig_op1_type, op1_type, op2_type, op3_type; + uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type; +// zend_class_entry *op1_ce = NULL; + zend_class_entry *op2_ce = NULL; // TODO: range inference ??? opline = p->opline; op1_type = orig_op1_type = p->op1_type; - op2_type = p->op2_type; + op2_type = orig_op2_type = p->op2_type; op3_type = p->op3_type; if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { op1_type = IS_UNKNOWN; @@ -1383,11 +1385,11 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { - // TODO: support for recorded classes ??? +// op1_ce = (zend_class_entry*)(p+1)->ce; p++; } if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { - // TODO: support for recorded classes ??? + op2_ce = (zend_class_entry*)(p+1)->ce; p++; } @@ -1701,6 +1703,11 @@ propagate_arg: } ADD_OP1_TRACE_GUARD(); break; + case ZEND_INIT_DYNAMIC_CALL: + if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) { + ADD_OP2_TRACE_GUARD(); + } + break; default: break; } @@ -3361,8 +3368,10 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par uint8_t op2_type = p->op2_type; uint8_t op3_type = p->op3_type; uint8_t orig_op1_type = op1_type; + uint8_t orig_op2_type = op2_type; zend_bool op1_indirect; zend_class_entry *op1_ce = NULL; + zend_class_entry *op2_ce = NULL; opline = p->opline; if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { @@ -3383,7 +3392,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par p++; } if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { - // TODO: support for recorded classes ??? + op2_ce = (zend_class_entry*)(p+1)->ce; p++; } @@ -4970,8 +4979,17 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } goto done; - case ZEND_INIT_METHOD_CALL: case ZEND_INIT_DYNAMIC_CALL: + if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) { + op2_info = OP2_INFO(); + CHECK_OP2_TRACE_TYPE(); + if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1)) { + goto jit_failure; + } + goto done; + } + /* break missing intentionally */ + case ZEND_INIT_METHOD_CALL: if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) { goto jit_failure; } @@ -6058,7 +6076,7 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) { fprintf(stderr, "/CALL"); } - if (t->exit_info[i].flags & ZEND_JIT_EXIT_POLYMORPHISM) { + if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_DYNAMIC_CALL)) { fprintf(stderr, "/POLY"); } if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) { @@ -6398,7 +6416,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 int ret = 0; uint32_t trace_num; zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; - zend_bool is_megamorphic = 0; + uint32_t is_megamorphic = 0; uint32_t polymorphism = 0; trace_num = ZEND_JIT_TRACE_NUM; @@ -6427,15 +6445,18 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 goto abort; } - if (EX(call) - && JIT_G(max_polymorphic_calls) > 0 - && (zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)) { - if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) { - is_megamorphic = 1; - } else if (!zend_jit_traces[parent_num].polymorphism) { - polymorphism = 1; - } else if (exit_num == 0) { - polymorphism = zend_jit_traces[parent_num].polymorphism + 1; + if (JIT_G(max_polymorphic_calls) > 0) { + if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_DYNAMIC_CALL) + || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM) + && EX(call))) { + if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) { + is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags & + (ZEND_JIT_EXIT_DYNAMIC_CALL | ZEND_JIT_EXIT_POLYMORPHISM); + } else if (!zend_jit_traces[parent_num].polymorphism) { + polymorphism = 1; + } else if (exit_num == 0) { + polymorphism = zend_jit_traces[parent_num].polymorphism + 1; + } } } diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 88a4eeb369..6cb8df08d9 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -474,7 +474,7 @@ static int zend_jit_trace_bad_loop_exit(const zend_op *opline) return 0; } -static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level, uint32_t init_level, uint32_t *call_level) +static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t *megamorphic, uint32_t level, uint32_t init_level, uint32_t *call_level) { zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR; @@ -507,7 +507,7 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend } func = (zend_function*)jit_extension->op_array; } - if (is_megamorphic + if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM /* TODO: use more accurate check ??? */ && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC) || func->common.scope)) { @@ -522,7 +522,7 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend return idx; } -static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level) +static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t *megamorphic, uint32_t level) { uint32_t call_level = 0; @@ -570,7 +570,7 @@ static int zend_jit_trace_call_level(const zend_execute_data *call) * */ -zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphic) +zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphic) { #ifdef HAVE_GCC_GLOBAL_REGS @@ -929,6 +929,9 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, if (JIT_G(max_polymorphic_calls) == 0 && zend_jit_may_be_polymorphic_call(opline - 1)) { func = NULL; + } else if (is_megamorphic == ZEND_JIT_EXIT_DYNAMIC_CALL + && trace_buffer[1].opline == opline - 1) { + func = NULL; } call_level = zend_jit_trace_call_level(EX(call)); ZEND_ASSERT(ret_level + level + call_level < 32); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ae3763622e..d4e40c431f 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8468,7 +8468,16 @@ static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1 return 1; } -static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func) +/* copy of hidden zend_closure */ +typedef struct _zend_closure { + zend_object std; + zend_function func; + zval this_ptr; + zend_class_entry *called_scope; + zif_handler orig_internal_handler; +} zend_closure; + +static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func, zend_bool is_closure) { uint32_t used_stack; @@ -8478,15 +8487,26 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); | // if (EXPECTED(ZEND_USER_CODE(func->type))) { - | test byte [r0 + offsetof(zend_function, type)], 1 - | mov FCARG1a, used_stack - | jnz >1 + if (!is_closure) { + | test byte [r0 + offsetof(zend_function, type)], 1 + | mov FCARG1a, used_stack + | jnz >1 + } else { + | mov FCARG1a, used_stack + } | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); | mov edx, opline->extended_value - | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] - | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] - | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] - | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] + if (!is_closure) { + | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] + | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] + | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] + | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] + } else { + | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] + | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] + | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] + | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] + } | shl edx, 5 |.if X64 | movsxd r2, edx @@ -8522,7 +8542,6 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); |.cold_code |1: - | SET_EX_OPLINE opline, r0 if (func) { | mov FCARG1d, used_stack } @@ -8531,9 +8550,15 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen #else if (func && func->type == ZEND_INTERNAL_FUNCTION) { #endif + | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_int_extend_stack_helper, r0 } else { - | mov FCARG2a, r0 + if (!is_closure) { + | mov FCARG2a, r0 + } else { + | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] + } + | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_extend_stack_helper, r0 } | mov RX, r0 @@ -8549,20 +8574,59 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); | // ZEND_SET_CALL_INFO(call, 0, call_info); | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) - | // call->func = func; #ifdef _WIN32 if (0) { #else if (func && func->type == ZEND_INTERNAL_FUNCTION) { #endif + | // call->func = func; |1: | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 } else { - | mov aword EX:RX->func, r0 + if (!is_closure) { + | // call->func = func; + | mov aword EX:RX->func, r0 + } else { + | // call->func = &closure->func; + | lea r1, aword [r0 + offsetof(zend_closure, func)] + | mov aword EX:RX->func, r1 + } + |1: + } + if (!is_closure) { + | // Z_CE(call->This) = called_scope; + | mov aword EX:RX->This.value.ptr, 0 + } else { + if (opline->op2_type == IS_CV) { + | // GC_ADDREF(closure); + | add dword [r0], 1 + } + | // object_or_called_scope = closure->called_scope; + | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] + | mov aword EX:RX->This.value.ptr, r1 + | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | + | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); + | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] + | and edx, ZEND_ACC_FAKE_CLOSURE + | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) + | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { + | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF + | jz >1 + | // call_info |= ZEND_CALL_HAS_THIS; + | or edx, ZEND_CALL_HAS_THIS + | // object_or_called_scope = Z_OBJ(closure->this_ptr); + | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] + |1: + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | or dword EX:RX->This.u1.type_info, edx + | // Z_PTR(call->This) = object_or_called_scope; + | mov aword EX:RX->This.value.ptr, r1 + | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 + | jnz >1 + | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] + | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 |1: } - | // Z_CE(call->This) = called_scope; - | mov aword EX:RX->This.value.ptr, 0 | // ZEND_CALL_NUM_ARGS(call) = num_args; | mov dword EX:RX->This.u2.num_args, opline->extended_value return 1; @@ -8949,7 +9013,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t |3: } - if (!zend_jit_push_call_frame(Dst, opline, func)) { + if (!zend_jit_push_call_frame(Dst, opline, func, 0)) { return 0; } @@ -8965,6 +9029,106 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t return 1; } +static int zend_jit_init_closure_call(dasm_State **Dst, + const zend_op *opline, + uint32_t b, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + int call_level, + zend_jit_trace_rec *trace) +{ + zend_function *func = NULL; + zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + + | GET_ZVAL_PTR r0, op2_addr + + if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure + && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + |.if X64 + || if (!IS_SIGNED_32BIT(zend_ce_closure)) { + | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) + | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a + || } else { + | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure + || } + |.else + | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure + |.endif + | jne &exit_addr + if (ssa->var_info && ssa_op->op2_use >= 0) { + ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; + ssa->var_info[ssa_op->op2_use].is_instanceof = 0; + } + } + + if (trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL + && trace->func + && trace->func->type == ZEND_USER_FUNCTION) { + const zend_op *opcodes; + int32_t exit_point; + const void *exit_addr; + + func = (zend_function*)trace->func; + opcodes = func->op_array.opcodes; + exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_DYNAMIC_CALL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + | .if X64 + || if (!IS_SIGNED_32BIT(opcodes)) { + | mov64 FCARG1a, ((ptrdiff_t)opcodes) + | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a + || } else { + | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes + || } + | .else + | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes + | .endif + | jne &exit_addr + } + + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + + if (!zend_jit_push_call_frame(Dst, opline, func, 1)) { + return 0; + } + + if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, trace)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + + if (trace + && trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + if (!zend_jit_set_valid_ip(Dst, opline + 1)) { + return 0; + } + } + + return 1; +} + static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) { uint32_t num_args = 0; -- 2.40.0