Avoid unnecessary IP initiliaization on trace linking

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index bcbafad1479860d1a5146182e7099914c1d9ee35..18a3096204959affc6c59cd6118d5f0a67b70392 100644 (file)
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -348,6 +348,7 @@ typedef union _zend_jit_trace_stack {
 /* trace info flags */
 #define ZEND_JIT_TRACE_CHECK_INTERRUPT (1<<0)
 #define ZEND_JIT_TRACE_LOOP            (1<<1)
+#define ZEND_JIT_TRACE_USES_INITIAL_IP (1<<2)
 
 typedef struct _zend_jit_trace_info {
        uint32_t                  id;            /* trace id */

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index b922655e376417b089e352cd953bcf26e644d2ff..5e09b2dd7c5e50585eda57694cf602c5d117f079 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -3088,7 +3088,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
        zend_uchar res_type = IS_UNKNOWN;
        const zend_op *opline, *orig_opline;
        const zend_ssa_op *ssa_op, *orig_ssa_op;
-       const void *timeout_exit_addr = NULL;
 
        JIT_G(current_trace) = trace_buffer;
 
@@ -3142,6 +3141,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 
        if (!parent_trace) {
                zend_jit_set_last_valid_opline(opline);
+               zend_jit_track_last_valid_opline();
        } else {
                if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
                        zend_jit_trace_opline_guard(&dasm_state, opline);
@@ -3273,18 +3273,16 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
                        }
                }
 
-               if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
-                       if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
-                               uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, ZEND_JIT_EXIT_TO_VM);
-
-                               timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
-                               if (!timeout_exit_addr) {
-                                       goto jit_failure;
-                               }
-                       } else {
-                               timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
-                       }
-               }
+//             if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+//                     if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
+//                             uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+//
+//                             timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+//                             if (!timeout_exit_addr) {
+//                                     goto jit_failure;
+//                             }
+//                     }
+//             }
 
                if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
                        int last_var = op_array->last_var;
@@ -5074,11 +5072,17 @@ done:
 
        t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
 
+       if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
+               t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
+       }
+
        if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
         || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
         || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
                if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
-                       if (!zend_jit_set_ip(&dasm_state, p->opline)) {
+                       ZEND_ASSERT(!parent_trace);
+                       if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
+                        && !zend_jit_set_ip(&dasm_state, p->opline)) {
                                goto jit_failure;
                        }
                }
@@ -5087,7 +5091,25 @@ done:
                        t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
                }
                if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
+                       const void *timeout_exit_addr = NULL;
+
                        t->flags |= ZEND_JIT_TRACE_LOOP;
+
+                       if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+                               if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
+                                || (ra
+                                 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
+                                       uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+
+                                       timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+                                       if (!timeout_exit_addr) {
+                                               goto jit_failure;
+                                       }
+                               } else {
+                                       timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
+                               }
+                       }
+
                        zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
                }
        } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
@@ -5097,14 +5119,35 @@ done:
                        goto jit_failure;
                }
                if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
-                       if (!zend_jit_set_ip(&dasm_state, p->opline)) {
+                       const void *timeout_exit_addr = NULL;
+
+                       t->link = zend_jit_find_trace(p->opline->handler);
+                       if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
+                        && !zend_jit_set_ip(&dasm_state, p->opline)) {
                                goto jit_failure;
                        }
-                       t->link = zend_jit_find_trace(p->opline->handler);
-                       zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link],
-                               parent_trace &&
-                               (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT) &&
-                               zend_jit_traces[parent_trace].root == t->link);
+                       if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
+                               t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
+                       }
+                       if (parent_trace
+                        && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
+                        && zend_jit_traces[parent_trace].root == t->link) {
+                               if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
+                                       uint32_t exit_point;
+
+                                       for (i = 0; i < op_array->last_var + op_array->T; i++) {
+                                               SET_STACK_TYPE(stack, i, IS_UNKNOWN);
+                                       }
+                                       exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+                                       timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+                                       if (!timeout_exit_addr) {
+                                               goto jit_failure;
+                                       }
+                               } else {
+                                       timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
+                               }
+                       }
+                       zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
                } else {
                        zend_jit_trace_return(&dasm_state, 0);
                }

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index afb51163dd8acf4d797ae42a0196b19fcfba3bc9..566c0521813784ccac8452e7f5bcf3936e30616b 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -1340,11 +1340,12 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro SET_EX_OPLINE, op, tmp_reg
 ||     if (op == last_valid_opline) {
+||             zend_jit_use_last_valid_opline();
 |              SAVE_IP
 ||     } else {
 |              ADDR_OP2_2 mov, aword EX->opline, op, tmp_reg
 ||             if (!GCC_GLOBAL_REGS) {
-||                     last_valid_opline = NULL;
+||                     zend_jit_reset_last_valid_opline();
 ||             }
 ||     }
 |.endmacro
@@ -1578,6 +1579,7 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro UNDEFINED_OFFSET, opline
 ||     if (opline == last_valid_opline) {
+||             zend_jit_use_last_valid_opline();
 |              call ->undefined_offset_ex
 ||     } else {
 |              SET_EX_OPLINE  opline, r0
@@ -1587,6 +1589,7 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro UNDEFINED_INDEX, opline
 ||     if (opline == last_valid_opline) {
+||             zend_jit_use_last_valid_opline();
 |              call ->undefined_index_ex
 ||     } else {
 |              SET_EX_OPLINE opline, r0
@@ -1596,6 +1599,7 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro CANNOT_ADD_ELEMENT, opline
 ||     if (opline == last_valid_opline) {
+||             zend_jit_use_last_valid_opline();
 |              call ->cannot_add_element_ex
 ||     } else {
 |              SET_EX_OPLINE opline, r0
@@ -1603,31 +1607,56 @@ static void* dasm_labels[zend_lb_MAX];
 ||     }
 |.endmacro
 
-static zend_bool reuse_ip;
-static zend_bool delayed_call_chain;
-static uint32_t  delayed_call_level;
-static const zend_op *last_valid_opline;
-static int jit_return_label;
-static uint32_t current_trace_num;
+static zend_bool reuse_ip = 0;
+static zend_bool delayed_call_chain = 0;
+static uint32_t  delayed_call_level = 0;
+static const zend_op *last_valid_opline = NULL;
+static zend_bool use_last_vald_opline = 0;
+static zend_bool track_last_valid_opline = 0;
+static int jit_return_label = -1;
+static uint32_t current_trace_num = 0;
+
+static void zend_jit_track_last_valid_opline(void)
+{
+       use_last_vald_opline = 0;
+       track_last_valid_opline = 1;
+}
+
+static void zend_jit_use_last_valid_opline(void)
+{
+       if (track_last_valid_opline) {
+               use_last_vald_opline = 1;
+               track_last_valid_opline = 0;
+       }
+}
+
+static zend_bool zend_jit_trace_uses_initial_ip(void)
+{
+       return use_last_vald_opline;
+}
 
 static void zend_jit_set_last_valid_opline(const zend_op *target_opline)
 {
        if (!reuse_ip) {
+               track_last_valid_opline = 0;
                last_valid_opline = target_opline;
        }
 }
 
 static void zend_jit_reset_last_valid_opline(void)
 {
+       track_last_valid_opline = 0;
        last_valid_opline = NULL;
 }
 
-static void zend_jit_start_reuse_ip(void) {
-       last_valid_opline = NULL;
+static void zend_jit_start_reuse_ip(void)
+{
+       zend_jit_reset_last_valid_opline();
        reuse_ip = 1;
 }
 
-static void zend_jit_stop_reuse_ip(void) {
+static void zend_jit_stop_reuse_ip(void)
+{
        reuse_ip = 0;
 }
 
@@ -2804,6 +2833,8 @@ static int zend_jit_align_func(dasm_State **Dst)
        reuse_ip = 0;
        delayed_call_chain = 0;
        last_valid_opline = NULL;
+       use_last_vald_opline = 0;
+       track_last_valid_opline = 0;
        jit_return_label = -1;
        |.align 16
        return 1;
@@ -2849,16 +2880,15 @@ static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
 
 static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
 {
-       if (!last_valid_opline) {
+       if (last_valid_opline == opline) {
+               zend_jit_use_last_valid_opline();
+       } else if (GCC_GLOBAL_REGS && last_valid_opline) {
+               zend_jit_use_last_valid_opline();
+               |       ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
+       } else {
                |       LOAD_IP_ADDR opline
-       } else if (last_valid_opline != opline) {
-               if (GCC_GLOBAL_REGS) {
-                       |       ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
-               } else {
-                       |       LOAD_IP_ADDR opline
-               }
        }
-       last_valid_opline = opline;
+       zend_jit_set_last_valid_opline(opline);
 
        return 1;
 }
@@ -2890,6 +2920,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const
        if (exit_addr) {
                |       jne &exit_addr
        } else if (last_valid_opline == opline) {
+               ||              zend_jit_use_last_valid_opline();
                |       jne ->interrupt_handler
        } else {
                |       jne >1
@@ -3154,7 +3185,7 @@ static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_
        return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
 }
 
-static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, zend_bool check_interrupt)
+static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr)
 {
        const void *link_addr;
        size_t prologue_size;
@@ -3188,11 +3219,11 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t,
        }
        link_addr = (const void*)((const char*)t->code_start + prologue_size);
 
-       if (check_interrupt) {
+       if (timeout_exit_addr) {
                /* Check timeout for links to LOOP */
                |       MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
                |       je &link_addr
-               |       jmp ->interrupt_handler
+               |       jmp &timeout_exit_addr
        } else {
                |       jmp &link_addr
        }