Move stack overflow checks out of the loops

diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index 18cab6561355a7a91aa06bdbe5a63dbe5d703518..eea6b5c7ba972fa17d1af71cd820bd1d1512b8fd 100644 (file)
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -2723,7 +2723,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
                                        case ZEND_INIT_FCALL:
                                        case ZEND_INIT_FCALL_BY_NAME:
                                        case ZEND_INIT_NS_FCALL_BY_NAME:
-                                               if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL)) {
+                                               if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 1)) {
                                                        goto jit_failure;
                                                }
                                                goto done;
@@ -3296,7 +3296,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
                                                }
                                                if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
                                                                op1_info, op1_addr, ce, ce_is_instanceof, 0, NULL,
-                                                               NULL)) {
+                                                               NULL, 1)) {
                                                        goto jit_failure;
                                                }
                                                goto done;

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index fa81941d6ccf1f2c78f4bc2e90218ea4ee1056e4..22a5aa4100b257cb2289c69cb388102e9be70b31 100644 (file)
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -392,6 +392,7 @@ struct _zend_jit_trace_stack_frame {
        const zend_op              *call_opline;
        uint32_t                    call_level;
        uint32_t                    _info;
+       int                         used_stack;
        zend_jit_trace_stack        stack[1];
 };
 

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index 27362312e543cf8525105c474d745aefd988a0f2..14826ee69bbaa97600054635b30e4f3cf4ba2dc6 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -987,6 +987,7 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u
 typedef struct _zend_tssa {
        zend_ssa        ssa;
        const zend_op **tssa_opcodes;
+       int             used_stack;
 } zend_tssa;
 
 static const zend_op _nop_opcode = {0};
@@ -1005,7 +1006,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
        zend_jit_trace_stack *stack;
        uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
        uint32_t optimization_level = 0;
-       int call_level, level, num_op_arrays;
+       int call_level, level, num_op_arrays, used_stack, max_used_stack;
        size_t frame_size, stack_top, stack_size, stack_bottom;
        zend_jit_op_array_trace_extension *jit_extension;
        zend_ssa *ssa;
@@ -1172,6 +1173,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                tssa->cfg.blocks[0].successors_count = 0;
                tssa->cfg.blocks[0].predecessors_count = 0;
        }
+       ((zend_tssa*)tssa)->used_stack = -1;
 
        if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
                return tssa;
@@ -1440,11 +1442,18 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
        top = zend_jit_trace_call_frame(frame, op_array);
        TRACE_FRAME_INIT(frame, op_array, 0, 0);
        TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
+       frame->used_stack = 0;
        for (i = 0; i < op_array->last_var + op_array->T; i++) {
                SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
        }
        memset(&return_value_info, 0, sizeof(return_value_info));
 
+       if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
+               max_used_stack = used_stack = 0;
+       } else {
+               max_used_stack = used_stack = -1;
+       }
+
        p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
        idx = 0;
        level = 0;
@@ -1817,6 +1826,11 @@ propagate_arg:
                                                ADD_OP2_TRACE_GUARD();
                                        }
                                        break;
+                               case ZEND_SEND_ARRAY:
+                               case ZEND_SEND_UNPACK:
+                               case ZEND_CHECK_UNDEF_ARGS:
+                               case ZEND_INCLUDE_OR_EVAL:
+                                       max_used_stack = used_stack = -1;
                                default:
                                        break;
                        }
@@ -1961,6 +1975,7 @@ propagate_arg:
                                /* Trace missed INIT_FCALL opcode */
                                call = top;
                                TRACE_FRAME_INIT(call, op_array, 0, 0);
+                               call->used_stack = 0;
                                top = zend_jit_trace_call_frame(top, op_array);
                                for (i = 0; i < op_array->last_var + op_array->T; i++) {
                                        SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
@@ -2077,12 +2092,17 @@ propagate_arg:
 
                        top = frame;
                        if (frame->prev) {
+                               if (used_stack > 0) {
+                                       used_stack -= frame->used_stack;
+                               }
                                frame = frame->prev;
                                ZEND_ASSERT(&frame->func->op_array == op_array);
                        } else {
+                               max_used_stack = used_stack = -1;
                                frame = zend_jit_trace_ret_frame(frame, op_array);
                                TRACE_FRAME_INIT(frame, op_array, 0, 0);
                                TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
+                               frame->used_stack = 0;
                                for (i = 0; i < op_array->last_var + op_array->T; i++) {
                                        SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
                                }
@@ -2092,6 +2112,7 @@ propagate_arg:
                        call = top;
                        TRACE_FRAME_INIT(call, p->func, 0, 0);
                        call->prev = frame->call;
+                       call->used_stack = 0;
                        frame->call = call;
                        top = zend_jit_trace_call_frame(top, p->op_array);
                        if (p->func && p->func->type == ZEND_USER_FUNCTION) {
@@ -2099,6 +2120,33 @@ propagate_arg:
                                        SET_STACK_INFO(call->stack, i, -1);
                                }
                        }
+                       if (used_stack >= 0
+                        && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
+                               if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
+                                       max_used_stack = used_stack = -1;
+                               } else {
+                                       const zend_op *opline = (p-1)->opline;
+
+                                       switch (opline->opcode) {
+                                               case ZEND_INIT_FCALL:
+                                               case ZEND_INIT_FCALL_BY_NAME:
+                                               case ZEND_INIT_NS_FCALL_BY_NAME:
+                                               case ZEND_INIT_METHOD_CALL:
+                                               case ZEND_INIT_DYNAMIC_CALL:
+                                               //case ZEND_INIT_STATIC_METHOD_CALL:
+                                               //case ZEND_INIT_USER_CALL:
+                                               //case ZEND_NEW:
+                                                       frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
+                                                       used_stack += frame->used_stack;
+                                                       if (used_stack > max_used_stack) {
+                                                               max_used_stack = used_stack;
+                                                       }
+                                                       break;
+                                               default:
+                                                       max_used_stack = used_stack = -1;
+                                       }
+                               }
+                       }
                } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
                        call = frame->call;
                        if (call) {
@@ -2110,6 +2158,8 @@ propagate_arg:
                }
        }
 
+       ((zend_tssa*)tssa)->used_stack = max_used_stack;
+
        if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
         || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
         || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
@@ -3379,6 +3429,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
        zend_uchar res_type = IS_UNKNOWN;
        const zend_op *opline, *orig_opline;
        const zend_ssa_op *ssa_op, *orig_ssa_op;
+       int used_stack;
 
        JIT_G(current_trace) = trace_buffer;
 
@@ -3391,6 +3442,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
        }
 
        ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
+       used_stack = ((zend_tssa*)ssa)->used_stack;
 
        /* Register allocation */
        if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
@@ -3446,6 +3498,12 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                int parent_vars_count = 0;
                zend_jit_trace_stack *parent_stack = NULL;
 
+               if (used_stack > 0) {
+                       if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) {
+                               goto jit_failure;
+                       }
+               }
+
                if (parent_trace) {
                        parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
                                op_array->last_var + op_array->T);
@@ -4424,7 +4482,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                        case ZEND_INIT_FCALL:
                                        case ZEND_INIT_FCALL_BY_NAME:
                                        case ZEND_INIT_NS_FCALL_BY_NAME:
-                                               if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1)) {
+                                               if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, used_stack < 0)) {
                                                        goto jit_failure;
                                                }
                                                goto done;
@@ -5403,7 +5461,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                                                op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
                                                                op_array, ssa, ssa_op, frame->call_level,
                                                                op1_info, op1_addr, ce, ce_is_instanceof, delayed_fetch_this, op1_ce,
-                                                               p + 1)) {
+                                                               p + 1, used_stack < 0)) {
                                                        goto jit_failure;
                                                }
                                                goto done;
@@ -5413,7 +5471,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                                }
                                                op2_info = OP2_INFO();
                                                CHECK_OP2_TRACE_TYPE();
-                                               if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1)) {
+                                               if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, used_stack < 0)) {
                                                        goto jit_failure;
                                                }
                                                goto done;

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 312c8587926f42ce2096abfbde6b4601d3693ca7..c90dd6745738ff5725c122624ca0f9b2149cd0aa 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -8604,7 +8604,25 @@ typedef struct _zend_closure {
        zif_handler       orig_internal_handler;
 } zend_closure;
 
-static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func, zend_bool is_closure, zend_bool use_this)
+static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack)
+{
+       int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+       const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+       if (!exit_addr) {
+               return 0;
+       }
+
+       |       // Check Stack Overflow
+       |       MEM_OP2_2_ZTS mov, r1, aword, executor_globals, vm_stack_end, r0
+       |       MEM_OP2_2_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0
+       |       cmp r1, used_stack
+       |       jb &exit_addr
+
+       return 1;
+}
+
+static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_function *func, zend_bool is_closure, zend_bool use_this, zend_bool stack_check)
 {
        uint32_t used_stack;
 
@@ -8646,51 +8664,54 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zen
 
        |       // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
        |       MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX
-       |       // Check Stack Overflow
-       |       MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2
-       |       sub r2, RX
-       if (func) {
-               |       cmp r2, used_stack
-       } else {
-               |       cmp r2, FCARG1a
-       }
-
-       if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
-               int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
-               const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
-
-               if (!exit_addr) {
-                       return 0;
-               }
 
-               |       jb &exit_addr
-       } else {
-               |       jb >1
-               |       // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
-               |.cold_code
-               |1:
+       if (stack_check) {
+               |       // Check Stack Overflow
+               |       MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2
+               |       sub r2, RX
                if (func) {
-                       |       mov FCARG1d, used_stack
+                       |       cmp r2, used_stack
+               } else {
+                       |       cmp r2, FCARG1a
                }
+
+               if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+                       int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+                       const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+
+                       if (!exit_addr) {
+                               return 0;
+                       }
+
+                       |       jb &exit_addr
+               } else {
+                       |       jb >1
+                       |       // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
+                       |.cold_code
+                       |1:
+                       if (func) {
+                               |       mov FCARG1d, used_stack
+                       }
 #ifdef _WIN32
-               if (0) {
+                       if (0) {
 #else
-               if (func && func->type == ZEND_INTERNAL_FUNCTION) {
+                       if (func && func->type == ZEND_INTERNAL_FUNCTION) {
 #endif
-                       |       SET_EX_OPLINE opline, r0
-                       |       EXT_CALL zend_jit_int_extend_stack_helper, r0
-               } else {
-                       if (!is_closure) {
-                               |       mov FCARG2a, r0
+                               |       SET_EX_OPLINE opline, r0
+                               |       EXT_CALL zend_jit_int_extend_stack_helper, r0
                        } else {
-                               |       lea FCARG2a, aword [r0 + offsetof(zend_closure, func)]
+                               if (!is_closure) {
+                                       |       mov FCARG2a, r0
+                               } else {
+                                       |       lea FCARG2a, aword [r0 + offsetof(zend_closure, func)]
+                               }
+                               |       SET_EX_OPLINE opline, r0
+                               |       EXT_CALL zend_jit_extend_stack_helper, r0
                        }
-                       |       SET_EX_OPLINE opline, r0
-                       |       EXT_CALL zend_jit_extend_stack_helper, r0
+                       |       mov RX, r0
+                       |       jmp >1
+                       |.code
                }
-               |       mov RX, r0
-               |       jmp >1
-               |.code
        }
 
        if (func) {
@@ -9032,7 +9053,7 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zen
        return 1;
 }
 
-static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace)
+static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, zend_bool stack_check)
 {
        zend_func_info *info = ZEND_FUNC_INFO(op_array);
        zend_call_info *call_info = NULL;
@@ -9165,7 +9186,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t
                |3:
        }
 
-       if (!zend_jit_push_call_frame(Dst, opline, func, 0, 0)) {
+       if (!zend_jit_push_call_frame(Dst, opline, func, 0, 0, stack_check)) {
                return 0;
        }
 
@@ -9194,7 +9215,8 @@ static int zend_jit_init_method_call(dasm_State          **Dst,
                                      zend_bool             ce_is_instanceof,
                                      zend_bool             use_this,
                                      zend_class_entry     *trace_ce,
-                                     zend_jit_trace_rec   *trace)
+                                     zend_jit_trace_rec   *trace,
+                                     zend_bool             stack_check)
 {
        zend_func_info *info = ZEND_FUNC_INFO(op_array);
        zend_call_info *call_info = NULL;
@@ -9404,7 +9426,7 @@ static int zend_jit_init_method_call(dasm_State          **Dst,
        }
 
        if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
-               if (!zend_jit_push_call_frame(Dst, opline, func, 0, use_this)) {
+               if (!zend_jit_push_call_frame(Dst, opline, func, 0, use_this, stack_check)) {
                        return 0;
                }
        }
@@ -9433,7 +9455,8 @@ static int zend_jit_init_closure_call(dasm_State          **Dst,
                                       zend_ssa             *ssa,
                                       const zend_ssa_op    *ssa_op,
                                       int                   call_level,
-                                      zend_jit_trace_rec   *trace)
+                                      zend_jit_trace_rec   *trace,
+                                      zend_bool             stack_check)
 {
        zend_function *func = NULL;
        zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
@@ -9502,7 +9525,7 @@ static int zend_jit_init_closure_call(dasm_State          **Dst,
                }
        }
 
-       if (!zend_jit_push_call_frame(Dst, opline, func, 1, 0)) {
+       if (!zend_jit_push_call_frame(Dst, opline, func, 1, 0, stack_check)) {
                return 0;
        }