Improved tracing JIT for nested calls

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index a385463029df00fee31b6d4db5ae24a30edfb1b3..c787bbf8b114b8a677aec409f91a1fe9ae35176a 100644 (file)
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -363,6 +363,7 @@ struct _zend_jit_trace_stack_frame {
        zend_jit_trace_stack_frame *call;
        zend_jit_trace_stack_frame *prev;
        const zend_function        *func;
+       uint32_t                    call_level;
        uint32_t                    _info;
        zend_jit_trace_stack        stack[1];
 };
@@ -384,6 +385,7 @@ struct _zend_jit_trace_stack_frame {
                _frame->call = NULL; \
                _frame->prev = NULL; \
                _frame->func = (const zend_function*)_func; \
+               _frame->call_level = 0; \
                _frame->_info = (((uint32_t)(num_args)) << TRACE_FRAME_SHIFT_NUM_ARGS) & TRACE_FRAME_MASK_NUM_ARGS; \
                if (nested) { \
                        _frame->_info |= TRACE_FRAME_MASK_NESTED; \

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index 7efe94517f8889200c3a2a5cbbc5d30773f763a7..20696f5f5f48bc040bea00efd55f4141438cc1c2 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -1614,8 +1614,12 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
                                        v++;
                                }
                                if (return_value_info.type != 0) {
-                                       if ((p+1)->op == ZEND_JIT_TRACE_VM) {
-                                               const zend_op *opline = (p+1)->opline - 1;
+                                       zend_jit_trace_rec *q = p + 1;
+                                       while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
+                                               q++;
+                                       }
+                                       if (q->op == ZEND_JIT_TRACE_VM) {
+                                               const zend_op *opline = q->opline - 1;
                                                if (opline->result_type != IS_UNUSED) {
                                                        ssa_var_info[
                                                                p->first_ssa_var +
@@ -2429,7 +2433,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
        const zend_op_array *op_array;
        zend_ssa *ssa, *op_array_ssa;
        zend_jit_trace_rec *p;
-       int call_level = -1; // TODO: proper support for inlined functions ???
        zend_jit_op_array_trace_extension *jit_extension;
        int num_op_arrays = 0;
        zend_jit_trace_info *t;
@@ -2671,8 +2674,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                p++;
                        }
 
-#if 0
-                       // TODO: call level calculation doesn't work for traces ???
                        switch (opline->opcode) {
                                case ZEND_INIT_FCALL:
                                case ZEND_INIT_FCALL_BY_NAME:
@@ -2682,9 +2683,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                case ZEND_INIT_STATIC_METHOD_CALL:
                                case ZEND_INIT_USER_CALL:
                                case ZEND_NEW:
-                                       call_level++;
+                                       frame->call_level++;
                        }
-#endif
 
                        if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) {
                                switch (opline->opcode) {
@@ -3043,7 +3043,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                        case ZEND_INIT_FCALL:
                                        case ZEND_INIT_FCALL_BY_NAME:
                                        case ZEND_INIT_NS_FCALL_BY_NAME:
-                                               if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, op_array_ssa, call_level, p + 1)) {
+                                               if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, op_array_ssa, frame->call_level, p + 1)) {
                                                        goto jit_failure;
                                                }
                                                goto done;
@@ -3128,7 +3128,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                                        case ZEND_DO_ICALL:
                                        case ZEND_DO_FCALL_BY_NAME:
                                        case ZEND_DO_FCALL:
-                                               if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, call_level, -1, p + 1)) {
+                                               if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
                                                        goto jit_failure;
                                                }
                                                goto done;
@@ -3608,16 +3608,13 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                        }
 
 done:
-#if 0
-                       // TODO: call level calculation doesn't work for traces ???
                        switch (opline->opcode) {
                                case ZEND_DO_FCALL:
                                case ZEND_DO_ICALL:
                                case ZEND_DO_UCALL:
                                case ZEND_DO_FCALL_BY_NAME:
-                                       call_level--;
+                                       frame->call_level--;
                        }
-#endif
 
                        if (ra) {
                                zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
@@ -3946,6 +3943,7 @@ done:
                                if (!skip_guard && !zend_jit_init_fcall_guard(&dasm_state, NULL, p->func)) {
                                        goto jit_failure;
                                }
+                               frame->call_level++;
                        }
                } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
                        call = frame->call;

diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c
index b2faaeb1a09c0b2284fbc67d242794c0aef77637..7fcd14feca8c13cb18eeb358d3c76c000707ace6 100644 (file)
--- a/ext/opcache/jit/zend_jit_vm_helpers.c
+++ b/ext/opcache/jit/zend_jit_vm_helpers.c
@@ -696,6 +696,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                        if (rc == 1) {
 #endif
                                /* Enter into function */
+                               prev_call = NULL;
                                if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
                                        stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
                                        break;
@@ -726,6 +727,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                                level++;
                        } else {
                                /* Return from function */
+                               prev_call = EX(call);
                                if (level == 0) {
                                        if (is_toplevel) {
                                                stop = ZEND_JIT_TRACE_STOP_TOPLEVEL;
@@ -757,6 +759,10 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                                                unrolled_calls[ret_level] = &EX(func)->op_array;
                                                ret_level++;
                                                is_toplevel = EX(func)->op_array.function_name == NULL;
+
+                                               if (prev_call) {
+                                                       idx = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx);
+                                               }
 #endif
                                        } else if (start & ZEND_JIT_TRACE_START_LOOP
                                         && !zend_jit_trace_bad_loop_exit(orig_opline)) {
@@ -785,8 +791,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                        offset = jit_extension->offset;
                }
                if (EX(call) != prev_call) {
-                       if (trace_buffer[idx-1].op != ZEND_JIT_TRACE_BACK
-                        && EX(call)
+                       if (EX(call)
                         && EX(call)->prev_execute_data == prev_call) {
                                if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
                                        /* TODO: Can we continue recording ??? */
@@ -868,6 +873,13 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                }
        }
 
+       if (stop == ZEND_JIT_TRACE_STOP_LINK) {
+               /* Shrink fake INIT_CALLs */
+               while (trace_buffer[idx-1].op == ZEND_JIT_TRACE_INIT_CALL && trace_buffer[idx-1].fake) {
+                       idx--;
+               }
+       }
+
        TRACE_END(ZEND_JIT_TRACE_END, stop, opline);
 
 #ifdef HAVE_GCC_GLOBAL_REGS

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 60992a212828ed0f5a673e7d009907ad54c6d2fc..9501e08375eb58e4d1c8abcbe2df48e8189b0e10 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -9471,7 +9471,9 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const ze
                        const void *exit_addr;
                        zend_jit_trace_stack_frame *current_frame;
 
-                       trace++;
+                       do {
+                               trace++;
+                       } while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
                        ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
                        next_opline = trace->opline;
                        current_frame = JIT_G(current_frame);