Improved trace selection rules

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index 9b01a0fcca228f849a32d6482a891bb666802e5c..2d010abea001413497bdc934c9d915f07aa6b6d8 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -6251,6 +6251,28 @@ repeat:
        }
        stop &= ~ZEND_JIT_TRACE_HALT;
 
+       if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
+               orig_opline = trace_buffer[1].opline;
+               op_array = (zend_op_array*)trace_buffer[0].op_array;
+               jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+               offset = jit_extension->offset;
+               if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
+                       const zend_op_array *op_array = trace_buffer[0].op_array;
+                       const zend_op *opline = trace_buffer[1].opline;
+                       zend_jit_op_array_trace_extension *jit_extension =
+                               (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+                       size_t offset = jit_extension->offset;
+
+                       fprintf(stderr, "---- TRACE %d start (%s) %s() %s:%d\n",
+                               trace_num,
+                               zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
+                               op_array->function_name ?
+                                       ZSTR_VAL(op_array->function_name) : "$main",
+                               ZSTR_VAL(op_array->filename),
+                               opline->lineno);
+               }
+       }
+
        if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
                zend_jit_dump_trace(trace_buffer, NULL);
        }
@@ -6539,6 +6561,24 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
        }
        stop &= ~ZEND_JIT_TRACE_HALT;
 
+       if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
+               if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
+                       const zend_op_array *op_array = trace_buffer[0].op_array;
+                       const zend_op *opline = trace_buffer[1].opline;
+                       zend_jit_op_array_trace_extension *jit_extension =
+                               (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
+                       size_t offset = jit_extension->offset;
+
+                       fprintf(stderr, "---- TRACE %d start (%s) %s() %s:%d\n",
+                               trace_num,
+                               zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
+                               op_array->function_name ?
+                                       ZSTR_VAL(op_array->function_name) : "$main",
+                               ZSTR_VAL(op_array->filename),
+                               opline->lineno);
+               }
+       }
+
        if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
                zend_jit_dump_trace(trace_buffer, NULL);
        }
@@ -6557,7 +6597,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
                                        zend_jit_trace_stop_description[stop]);
                        }
                }
-               if (EXPECTED(stop != ZEND_JIT_TRACE_STOP_LOOP)) {
+               if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
                        stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
                } else {
                        const zend_op_array *op_array = trace_buffer[0].op_array;
@@ -6591,6 +6631,10 @@ abort:
                                        parent_num, exit_num);
                        }
                }
+               if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
+                       execute_data = EG(current_execute_data);
+                       return zend_jit_trace_hot_root(execute_data, EX(opline));
+               }
        }
 
        if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {

diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c
index 49adcfb61499df86e905b339d36f531e41375c1c..5ca1fb25aa6180fe6b5c525e0cf0e35237bc053a 100644 (file)
--- a/ext/opcache/jit/zend_jit_vm_helpers.c
+++ b/ext/opcache/jit/zend_jit_vm_helpers.c
@@ -503,6 +503,15 @@ static int zend_jit_trace_call_level(const zend_execute_data *call)
        return call_level;
 }
 
+static int zend_jit_trace_subtrace(zend_jit_trace_rec *trace_buffer, int start, int end, uint8_t event, const zend_op_array *op_array, const zend_op *opline)
+{
+       int idx;
+
+       TRACE_START(ZEND_JIT_TRACE_START, event, op_array, opline);
+       memmove(trace_buffer + idx, trace_buffer + start, (end - start) * sizeof(zend_jit_trace_rec));
+       return idx + (end - start);
+}
+
 /*
  *  Trace Linking Rules
  *  ===================
@@ -559,6 +568,9 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
        int backtrack_ret_recursion = -1;
        int backtrack_ret_recursion_level = 0;
        int loop_unroll_limit = 0;
+       int last_loop = -1;
+       int last_loop_level = -1;
+       const zend_op *last_loop_opline = NULL;
        uint32_t megamorphic = 0;
        const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
 #ifdef HAVE_GCC_GLOBAL_REGS
@@ -832,6 +844,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
 
                                                unrolled_calls[ret_level] = &EX(func)->op_array;
                                                ret_level++;
+                                               last_loop_opline = NULL;
 
                                                if (prev_call) {
                                                        int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0, &megamorphic, ret_level + level);
@@ -854,6 +867,9 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                                        }
                                } else {
                                        level--;
+                                       if (level < last_loop_level) {
+                                               last_loop_opline = NULL;
+                                       }
                                        TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
                                }
                        }
@@ -960,24 +976,44 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                                stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
                                break;
                        } else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
-                               if (start != ZEND_JIT_TRACE_START_SIDE) {
-                                       uint8_t bad_stop;
+                               uint8_t bad_stop;
 
+                               if (start != ZEND_JIT_TRACE_START_SIDE) {
                                        if (opline == orig_opline && level + ret_level == 0) {
                                                stop = ZEND_JIT_TRACE_STOP_LOOP;
                                                break;
                                        }
-                                       /* Fail to try creating a trace for inner loop first.
-                                          If this doesn't work try unroling loop. */
+                               }
+
+                               if (start != ZEND_JIT_TRACE_START_SIDE
+                                || level + ret_level != 0) {
+                                       /* First try creating a trace for inner loop.
+                                          If this doesn't work try loop unroling. */
                                        bad_stop = zend_jit_trace_bad_stop_event(opline,
                                                JIT_G(blacklist_root_trace) / 2);
                                        if (bad_stop != ZEND_JIT_TRACE_STOP_INNER_LOOP
                                         && bad_stop != ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
-                                               stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
-                                               break;
+                                               if (start == ZEND_JIT_TRACE_START_SIDE
+                                                || zend_jit_trace_bad_stop_event(orig_opline,
+                                                               JIT_G(blacklist_root_trace) / 2) != ZEND_JIT_TRACE_STOP_INNER_LOOP) {
+                                                       stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
+                                                       break;
+                                               }
                                        }
                                }
-                               if (loop_unroll_limit < JIT_G(max_loops_unroll)) {
+
+                               if (opline == last_loop_opline
+                                && level == last_loop_level) {
+                                       idx = zend_jit_trace_subtrace(trace_buffer,
+                                               last_loop, idx, ZEND_JIT_TRACE_START_LOOP, op_array, opline);
+                                       start = ZEND_JIT_TRACE_START_LOOP;
+                                       stop = ZEND_JIT_TRACE_STOP_LOOP;
+                                       ret_level = 0;
+                                       break;
+                               } else if (loop_unroll_limit < JIT_G(max_loops_unroll)) {
+                                       last_loop = idx;
+                                       last_loop_opline = opline;
+                                       last_loop_level = level;
                                        loop_unroll_limit++;
                                } else {
                                        stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;