From: Dmitry Stogov Date: Fri, 6 Nov 2020 09:09:56 +0000 (+0300) Subject: Move stack overflow checks out of the loops X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=98e4f9466d979c2cc6ea42193ec038f5d08915c8;p=php Move stack overflow checks out of the loops --- diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 18cab65613..eea6b5c7ba 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2723,7 +2723,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op case ZEND_INIT_FCALL: case ZEND_INIT_FCALL_BY_NAME: case ZEND_INIT_NS_FCALL_BY_NAME: - if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL)) { + if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 1)) { goto jit_failure; } goto done; @@ -3296,7 +3296,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, op1_info, op1_addr, ce, ce_is_instanceof, 0, NULL, - NULL)) { + NULL, 1)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index fa81941d6c..22a5aa4100 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -392,6 +392,7 @@ struct _zend_jit_trace_stack_frame { const zend_op *call_opline; uint32_t call_level; uint32_t _info; + int used_stack; zend_jit_trace_stack stack[1]; }; diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 27362312e5..14826ee69b 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -987,6 +987,7 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u typedef struct _zend_tssa { zend_ssa ssa; const zend_op **tssa_opcodes; + int used_stack; } zend_tssa; static const zend_op _nop_opcode = {0}; @@ -1005,7 +1006,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin zend_jit_trace_stack *stack; uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; uint32_t optimization_level = 0; - int call_level, level, num_op_arrays; + int call_level, level, num_op_arrays, used_stack, max_used_stack; size_t frame_size, stack_top, stack_size, stack_bottom; zend_jit_op_array_trace_extension *jit_extension; zend_ssa *ssa; @@ -1172,6 +1173,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin tssa->cfg.blocks[0].successors_count = 0; tssa->cfg.blocks[0].predecessors_count = 0; } + ((zend_tssa*)tssa)->used_stack = -1; if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { return tssa; @@ -1440,11 +1442,18 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin top = zend_jit_trace_call_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); + frame->used_stack = 0; for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1); } memset(&return_value_info, 0, sizeof(return_value_info)); + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { + max_used_stack = used_stack = 0; + } else { + max_used_stack = used_stack = -1; + } + p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; idx = 0; level = 0; @@ -1817,6 +1826,11 @@ propagate_arg: ADD_OP2_TRACE_GUARD(); } break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_UNPACK: + case ZEND_CHECK_UNDEF_ARGS: + case ZEND_INCLUDE_OR_EVAL: + max_used_stack = used_stack = -1; default: break; } @@ -1961,6 +1975,7 @@ propagate_arg: /* Trace missed INIT_FCALL opcode */ call = top; TRACE_FRAME_INIT(call, op_array, 0, 0); + call->used_stack = 0; top = zend_jit_trace_call_frame(top, op_array); for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); @@ -2077,12 +2092,17 @@ propagate_arg: top = frame; if (frame->prev) { + if (used_stack > 0) { + used_stack -= frame->used_stack; + } frame = frame->prev; ZEND_ASSERT(&frame->func->op_array == op_array); } else { + max_used_stack = used_stack = -1; frame = zend_jit_trace_ret_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); + frame->used_stack = 0; for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1); } @@ -2092,6 +2112,7 @@ propagate_arg: call = top; TRACE_FRAME_INIT(call, p->func, 0, 0); call->prev = frame->call; + call->used_stack = 0; frame->call = call; top = zend_jit_trace_call_frame(top, p->op_array); if (p->func && p->func->type == ZEND_USER_FUNCTION) { @@ -2099,6 +2120,33 @@ propagate_arg: SET_STACK_INFO(call->stack, i, -1); } } + if (used_stack >= 0 + && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { + if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) { + max_used_stack = used_stack = -1; + } else { + const zend_op *opline = (p-1)->opline; + + switch (opline->opcode) { + case ZEND_INIT_FCALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_DYNAMIC_CALL: + //case ZEND_INIT_STATIC_METHOD_CALL: + //case ZEND_INIT_USER_CALL: + //case ZEND_NEW: + frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func); + used_stack += frame->used_stack; + if (used_stack > max_used_stack) { + max_used_stack = used_stack; + } + break; + default: + max_used_stack = used_stack = -1; + } + } + } } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { call = frame->call; if (call) { @@ -2110,6 +2158,8 @@ propagate_arg: } } + ((zend_tssa*)tssa)->used_stack = max_used_stack; + if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { @@ -3379,6 +3429,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par zend_uchar res_type = IS_UNKNOWN; const zend_op *opline, *orig_opline; const zend_ssa_op *ssa_op, *orig_ssa_op; + int used_stack; JIT_G(current_trace) = trace_buffer; @@ -3391,6 +3442,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; + used_stack = ((zend_tssa*)ssa)->used_stack; /* Register allocation */ if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) @@ -3446,6 +3498,12 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par int parent_vars_count = 0; zend_jit_trace_stack *parent_stack = NULL; + if (used_stack > 0) { + if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) { + goto jit_failure; + } + } + if (parent_trace) { parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size, op_array->last_var + op_array->T); @@ -4424,7 +4482,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par case ZEND_INIT_FCALL: case ZEND_INIT_FCALL_BY_NAME: case ZEND_INIT_NS_FCALL_BY_NAME: - if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1)) { + if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, used_stack < 0)) { goto jit_failure; } goto done; @@ -5403,7 +5461,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, op1_info, op1_addr, ce, ce_is_instanceof, delayed_fetch_this, op1_ce, - p + 1)) { + p + 1, used_stack < 0)) { goto jit_failure; } goto done; @@ -5413,7 +5471,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } op2_info = OP2_INFO(); CHECK_OP2_TRACE_TYPE(); - if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1)) { + if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, used_stack < 0)) { goto jit_failure; } goto done; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 312c858792..c90dd67457 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8604,7 +8604,25 @@ typedef struct _zend_closure { zif_handler orig_internal_handler; } zend_closure; -static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func, zend_bool is_closure, zend_bool use_this) +static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | // Check Stack Overflow + | MEM_OP2_2_ZTS mov, r1, aword, executor_globals, vm_stack_end, r0 + | MEM_OP2_2_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 + | cmp r1, used_stack + | jb &exit_addr + + return 1; +} + +static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func, zend_bool is_closure, zend_bool use_this, zend_bool stack_check) { uint32_t used_stack; @@ -8646,51 +8664,54 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX - | // Check Stack Overflow - | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2 - | sub r2, RX - if (func) { - | cmp r2, used_stack - } else { - | cmp r2, FCARG1a - } - - if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); - const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); - - if (!exit_addr) { - return 0; - } - | jb &exit_addr - } else { - | jb >1 - | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); - |.cold_code - |1: + if (stack_check) { + | // Check Stack Overflow + | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2 + | sub r2, RX if (func) { - | mov FCARG1d, used_stack + | cmp r2, used_stack + } else { + | cmp r2, FCARG1a } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | jb &exit_addr + } else { + | jb >1 + | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + |.cold_code + |1: + if (func) { + | mov FCARG1d, used_stack + } #ifdef _WIN32 - if (0) { + if (0) { #else - if (func && func->type == ZEND_INTERNAL_FUNCTION) { + if (func && func->type == ZEND_INTERNAL_FUNCTION) { #endif - | SET_EX_OPLINE opline, r0 - | EXT_CALL zend_jit_int_extend_stack_helper, r0 - } else { - if (!is_closure) { - | mov FCARG2a, r0 + | SET_EX_OPLINE opline, r0 + | EXT_CALL zend_jit_int_extend_stack_helper, r0 } else { - | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] + if (!is_closure) { + | mov FCARG2a, r0 + } else { + | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] + } + | SET_EX_OPLINE opline, r0 + | EXT_CALL zend_jit_extend_stack_helper, r0 } - | SET_EX_OPLINE opline, r0 - | EXT_CALL zend_jit_extend_stack_helper, r0 + | mov RX, r0 + | jmp >1 + |.code } - | mov RX, r0 - | jmp >1 - |.code } if (func) { @@ -9032,7 +9053,7 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zen return 1; } -static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace) +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, zend_bool stack_check) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; @@ -9165,7 +9186,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t |3: } - if (!zend_jit_push_call_frame(Dst, opline, func, 0, 0)) { + if (!zend_jit_push_call_frame(Dst, opline, func, 0, 0, stack_check)) { return 0; } @@ -9194,7 +9215,8 @@ static int zend_jit_init_method_call(dasm_State **Dst, zend_bool ce_is_instanceof, zend_bool use_this, zend_class_entry *trace_ce, - zend_jit_trace_rec *trace) + zend_jit_trace_rec *trace, + zend_bool stack_check) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; @@ -9404,7 +9426,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { - if (!zend_jit_push_call_frame(Dst, opline, func, 0, use_this)) { + if (!zend_jit_push_call_frame(Dst, opline, func, 0, use_this, stack_check)) { return 0; } } @@ -9433,7 +9455,8 @@ static int zend_jit_init_closure_call(dasm_State **Dst, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, - zend_jit_trace_rec *trace) + zend_jit_trace_rec *trace, + zend_bool stack_check) { zend_function *func = NULL; zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); @@ -9502,7 +9525,7 @@ static int zend_jit_init_closure_call(dasm_State **Dst, } } - if (!zend_jit_push_call_frame(Dst, opline, func, 1, 0)) { + if (!zend_jit_push_call_frame(Dst, opline, func, 1, 0, stack_check)) { return 0; }