From: Dmitry Stogov Date: Thu, 14 May 2020 11:21:46 +0000 (+0300) Subject: Tracing JIT support for delayed call chain X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=161ee110bf6a0abb673cdf7429868be62236207d;p=php Tracing JIT support for delayed call chain --- diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 09c997b5b1..bcc1fa863c 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -112,6 +112,7 @@ static const void *zend_jit_loop_counter_handler = NULL; static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa); static void ZEND_FASTCALL zend_runtime_jit(void); +static int zend_jit_trace_op_len(const zend_op *opline); static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline, zend_jit_trace_rec *trace); static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const zend_op *to_opline, zend_jit_trace_rec *trace, uint32_t flags); static const void *zend_jit_trace_get_exit_addr(uint32_t n); @@ -2460,7 +2461,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op case ZEND_INIT_FCALL: case ZEND_INIT_FCALL_BY_NAME: case ZEND_INIT_NS_FCALL_BY_NAME: - if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, call_level, NULL)) { + if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 4d20aac6a5..b139c4071e 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -225,6 +225,7 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_EXIT_JITED (1<<0) #define ZEND_JIT_EXIT_BLACKLISTED (1<<1) #define ZEND_JIT_EXIT_TO_VM (1<<2) /* exit to VM without attempt to create a side trace */ +#define ZEND_JIT_EXIT_RESTORE_CALL (1<<3) /* deoptimizer should restore EX(call) chain */ typedef union _zend_op_trace_info { zend_op dummy; /* the size of this structure must be the same as zend_op */ diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index d3d586b739..cb766b907f 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -136,6 +136,10 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *from_opline, const uint32_t stack_size; zend_jit_trace_stack *stack = NULL; + if (delayed_call_chain) { + assert(to_opline != NULL); /* CALL and IP share the same register */ + flags |= ZEND_JIT_EXIT_RESTORE_CALL; + } if (JIT_G(current_frame)) { op_array = &JIT_G(current_frame)->func->op_array; stack_size = op_array->last_var + op_array->T; @@ -2582,16 +2586,24 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } - // TODO: Merge two loops implementing parallel move ??? - for (i = 0; i < parent_vars_count; i++) { - if (STACK_REG(parent_stack, i) != ZREG_NONE) { - if (ra && ra[i] && ra[i]->reg == STACK_REG(parent_stack, i)) { - /* register already loaded by parent trace */ - SET_STACK_REG(stack, i, ra[i]->reg); - } else if (!zend_jit_store_var(&dasm_state, ssa->var_info[i].type, i, STACK_REG(parent_stack, i))) { - goto jit_failure; + if (parent_trace) { + /* Deoptimization */ + + // TODO: Merge this loop with the following LOAD loop to implement parallel move ??? + for (i = 0; i < parent_vars_count; i++) { + if (STACK_REG(parent_stack, i) != ZREG_NONE) { + if (ra && ra[i] && ra[i]->reg == STACK_REG(parent_stack, i)) { + /* register already loaded by parent trace */ + SET_STACK_REG(stack, i, ra[i]->reg); + } else if (!zend_jit_store_var(&dasm_state, ssa->var_info[i].type, i, STACK_REG(parent_stack, i))) { + goto jit_failure; + } } } + + if (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + zend_jit_save_call_chain(&dasm_state, -1); + } } if (ra @@ -3069,7 +3081,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par case ZEND_INIT_FCALL: case ZEND_INIT_FCALL_BY_NAME: case ZEND_INIT_NS_FCALL_BY_NAME: - if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, op_array_ssa, frame->call_level, p + 1)) { + if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1)) { goto jit_failure; } goto done; @@ -4094,10 +4106,11 @@ jit_cleanup: static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) { const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; + uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags; uint32_t stack_size; zend_jit_trace_stack *stack; - if (opline) { + if (opline || (flags & ZEND_JIT_EXIT_RESTORE_CALL)) { return 1; } @@ -4133,6 +4146,9 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n zend_jit_align_func(&dasm_state); /* Deoptimization */ + if (zend_jit_traces[trace_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + zend_jit_save_call_chain(&dasm_state, -1); + } stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; for (i = 0; i < stack_size; i++) { @@ -4535,6 +4551,9 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) { fprintf(stderr, "/VM"); } + if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + fprintf(stderr, "/CALL"); + } for (j = 0; j < stack_size; j++) { zend_uchar type = STACK_TYPE(stack, j); if (type != IS_UNKNOWN) { @@ -4956,6 +4975,12 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf uint32_t stack_size = t->exit_info[exit_num].stack_size; zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset; + if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + zend_execute_data *call = (zend_execute_data *)regs->r[ZREG_RX]; + call->prev_execute_data = EX(call); + EX(call) = call; + } + for (i = 0; i < stack_size; i++) { if (STACK_REG(stack, i) != ZREG_NONE) { if (STACK_TYPE(stack, i) == IS_LONG) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 01d8752f67..4af3481c51 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2328,6 +2328,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | // Save CPU registers |.if X64 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ + | mov aword [r4+15*8], r15 | mov aword [r4+11*8], r11 | mov aword [r4+10*8], r10 | mov aword [r4+9*8], r9 @@ -2357,6 +2358,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | movsd qword [r4+16*8+0*8], xmm0 |.else | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ + | mov aword [r4+7*4], edi | mov aword [r4+2*4], edx | mov aword [r4+1*4], ecx | mov aword [r4+0*4], eax @@ -7673,18 +7675,83 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con return 1; } -static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline) +static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, zend_jit_trace_rec *trace) { int skip; + if (trace) { + zend_jit_trace_rec *p = trace; + + ssa_op++; + while (1) { + if (p->op == ZEND_JIT_TRACE_VM) { + switch (p->opline->opcode) { + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + case ZEND_INIT_FCALL: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_NEW: + case ZEND_INIT_USER_CALL: + case ZEND_FAST_CALL: + case ZEND_JMP: + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_ASSERT_CHECK: + case ZEND_CATCH: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + return 1; + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + return 0; + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + /* skip */ + break; + default: + if (zend_may_throw(opline, ssa_op, op_array, ssa)) { + return 1; + } + } + ssa_op += zend_jit_trace_op_len(opline); + } else if (p->op == ZEND_JIT_TRACE_ENTER || + p->op == ZEND_JIT_TRACE_BACK || + p->op == ZEND_JIT_TRACE_END) { + return 1; + } + p++; + } + } + if (!call_info) { const zend_op *end = op_array->opcodes + op_array->last; opline++; + ssa_op++; skip = 1; while (opline != end) { if (!skip) { - zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; if (zend_may_throw(opline, ssa_op, op_array, ssa)) { return 1; } @@ -7740,6 +7807,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons return 0; } opline++; + ssa_op++; } return 1; @@ -7752,6 +7820,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons } opline++; + ssa_op++; skip = 1; while (opline != end) { if (skip) { @@ -7772,12 +7841,12 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons return 1; } } else { - zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; if (zend_may_throw(opline, ssa_op, op_array, ssa)) { return 1; } } opline++; + ssa_op++; } return 0; @@ -7847,7 +7916,7 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, const zend_op *opline, co return 1; } -static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, int call_level, zend_jit_trace_rec *trace) +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; @@ -7984,7 +8053,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - if (trace || zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) { + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, trace)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; }