Tracing JIT support for megamorphic calls

diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h
index a86341ba764c09f4e31879bd6865a010061e9d14..810f678d6cb0d93afbbb001916a8af79ce995136 100644 (file)
--- a/ext/opcache/jit/zend_jit.h
+++ b/ext/opcache/jit/zend_jit.h
@@ -104,6 +104,7 @@ typedef struct _zend_jit_globals {
        zend_long   max_loops_unroll;      /* max number of unrolled loops */
        zend_long   max_recursive_calls;   /* max number of recursive inlined call unrolls */
        zend_long   max_recursive_returns; /* max number of recursive inlined return unrolls */
+       zend_long   max_polymorphic_calls; /* max number of inlined polymorphic calls */
 
        zend_sym_node *symbols;            /* symbols for disassembler */
 

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index 1177bcd4002bd73bdb73e1481e6c669035963b94..7666737ab4a5118b86f453f55dbff3397e2bb0cb 100644 (file)
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -207,6 +207,7 @@ typedef enum _zend_jit_trace_stop {
 #define ZEND_JIT_EXIT_BLACKLISTED   (1<<1)
 #define ZEND_JIT_EXIT_TO_VM         (1<<2) /* exit to VM without attempt to create a side trace */
 #define ZEND_JIT_EXIT_RESTORE_CALL  (1<<3) /* deoptimizer should restore EX(call) chain */
+#define ZEND_JIT_EXIT_POLYMORPHISM  (1<<4) /* exit becaus of polymorphic call */
 
 typedef union _zend_op_trace_info {
        zend_op dummy; /* the size of this structure must be the same as zend_op */
@@ -346,6 +347,7 @@ typedef struct _zend_jit_trace_info {
        uint32_t                  exit_counters; /* offset in exit counters array */
        uint32_t                  stack_map_size;
        uint32_t                  flags;         /* See ZEND_JIT_TRACE_... defines above */
+       uint32_t                  polymorphism;  /* Counter of polymorphic calls */
        const zend_op            *opline;        /* first opline */
        const void               *code_start;    /* address of native code */
        zend_jit_trace_exit_info *exit_info;     /* info about side exits */
@@ -433,7 +435,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HAN
 
 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline);
 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
-zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start);
+zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphc);
 
 static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, zend_bool *exit_if_true)
 {
@@ -456,4 +458,25 @@ static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit
        return NULL;
 }
 
+static zend_always_inline zend_bool zend_jit_may_be_polymorphic_call(const zend_op *opline)
+{
+       if (opline->opcode == ZEND_INIT_FCALL
+        || opline->opcode == ZEND_INIT_FCALL_BY_NAME
+        || opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
+               return 0;
+       } else if (opline->opcode == ZEND_INIT_METHOD_CALL
+     || opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
+               return 1;
+       } else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
+               return (opline->op1_type != IS_CONST || opline->op2_type != IS_CONST);
+       } else if (opline->opcode == ZEND_INIT_USER_CALL) {
+               return (opline->op2_type != IS_CONST);
+       } else if (opline->opcode == ZEND_NEW) {
+               return (opline->op1_type != IS_CONST);
+       } else {
+               ZEND_ASSERT(0);
+               return 0;
+       }
+}
+
 #endif /* ZEND_JIT_INTERNAL_H */

diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index ff7120f5d72079238871e05b25478081db799fc9..a4028c926f8413dd92dc77873d6acc52ef1aa87e 100644 (file)
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -4177,6 +4177,11 @@ done:
                                                        }
                                                        call_info = call_info->next_callee;
                                                }
+                                               if (!skip_guard
+                                                && !zend_jit_may_be_polymorphic_call(init_opline)) {
+                                                       // TODO: recompilation may change target ???
+                                                       skip_guard = 1;
+                                               }
                                        }
                                        if (!skip_guard && !zend_jit_init_fcall_guard(&dasm_state, NULL, p->func, trace_buffer[1].opline)) {
                                                goto jit_failure;
@@ -4422,6 +4427,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace
                t->child_count = 0;
                t->stack_map_size = 0;
                t->flags = 0;
+               t->polymorphism = 0;
                t->opline = trace_buffer[1].opline;
                t->exit_info = exit_info;
                t->stack_map = NULL;
@@ -4770,6 +4776,9 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
                if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
                        fprintf(stderr, "/CALL");
                }
+               if (t->exit_info[i].flags & ZEND_JIT_EXIT_POLYMORPHISM) {
+                       fprintf(stderr, "/POLY");
+               }
                for (j = 0; j < stack_size; j++) {
                        zend_uchar type = STACK_TYPE(stack, j);
                        if (type != IS_UNKNOWN) {
@@ -4839,7 +4848,7 @@ repeat:
        }
 
        stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
-               ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK);
+               ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
 
        if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
                zend_jit_dump_trace(trace_buffer, NULL);
@@ -4957,7 +4966,7 @@ static zend_bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_nu
        return 0;
 }
 
-static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num)
+static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
 {
        zend_jit_trace_stop ret;
        const void *handler;
@@ -4987,6 +4996,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
                t->child_count = 0;
                t->stack_map_size = 0;
                t->flags = 0;
+               t->polymorphism = polymorphism;
                t->opline = NULL;
                t->exit_info = exit_info;
                t->stack_map = NULL;
@@ -5078,6 +5088,8 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
        int ret = 0;
        uint32_t trace_num;
        zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
+       zend_bool is_megamorphic = 0;
+       uint32_t polymorphism = 0;
 
        trace_num = ZEND_JIT_TRACE_NUM;
 
@@ -5105,7 +5117,19 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
                goto abort;
        }
 
-       stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE);
+       if (EX(call)
+        && JIT_G(max_polymorphic_calls) > 0
+        && (zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)) {
+               if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
+                       is_megamorphic = 1;
+               } else if (!zend_jit_traces[parent_num].polymorphism) {
+                       polymorphism = 1;
+               } else if (exit_num == 0) {
+                       polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
+               }
+       }
+
+       stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
 
        if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
                zend_jit_dump_trace(trace_buffer, NULL);
@@ -5129,7 +5153,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
                        }
                }
                if (EXPECTED(stop != ZEND_JIT_TRACE_STOP_LOOP)) {
-                       stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num);
+                       stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
                } else {
                        const zend_op_array *op_array = trace_buffer[0].op_array;
                        zend_jit_op_array_trace_extension *jit_extension =

diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c
index 00f2d5bb255e229e40f2e8a59b6582c6ad7cb152..734a70ac10c1d96eaca3f4c796e55c949585f852 100644 (file)
--- a/ext/opcache/jit/zend_jit_vm_helpers.c
+++ b/ext/opcache/jit/zend_jit_vm_helpers.c
@@ -457,7 +457,7 @@ static int zend_jit_trace_bad_loop_exit(const zend_op *opline)
        return 0;
 }
 
-static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx)
+static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level, uint32_t *call_level)
 {
        zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
 
@@ -466,7 +466,10 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji
                zend_jit_op_array_trace_extension *jit_extension;
 
                if (call->prev_execute_data) {
-                       idx = zend_jit_trace_record_fake_init_call(call->prev_execute_data, trace_buffer, idx);
+                       idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, megamorphic, level, call_level);
+                       if (idx < 0) {
+                               return idx;
+                       }
                }
 
                func = call->func;
@@ -483,11 +486,39 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji
                        }
                        func = (zend_function*)jit_extension->op_array;
                }
+               if (is_megamorphic
+                /* TODO: use more accurate check ??? */
+                && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
+                 || func->common.scope)) {
+                       func = NULL;
+                       *megamorphic |= (1 << (level + *call_level));
+               } else {
+                       *megamorphic &= ~(1 << (level + *call_level));
+               }
+               (*call_level)++;
                TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INIT_CALL, func);
        } while (0);
        return idx;
 }
 
+static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level)
+{
+       uint32_t call_level = 0;
+
+       return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, megamorphic, level, &call_level);
+}
+
+static int zend_jit_trace_call_level(const zend_execute_data *call)
+{
+       int call_level = 0;
+
+       while (call->prev_execute_data) {
+               call_level++;
+               call = call->prev_execute_data;
+       }
+       return call_level;
+}
+
 /*
  *  Trace Linking Rules
  *  ===================
@@ -518,7 +549,8 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji
  *
  */
 
-zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start)
+zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphic)
+
 {
 #ifdef HAVE_GCC_GLOBAL_REGS
        zend_execute_data *save_execute_data = execute_data;
@@ -528,6 +560,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
        zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
        int level = 0;
        int ret_level = 0;
+       int call_level;
        zend_vm_opcode_handler_t handler;
        const zend_op_array *op_array;
        zend_jit_op_array_trace_extension *jit_extension;
@@ -539,6 +572,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
        int backtrack_ret_recursion = -1;
        int backtrack_ret_recursion_level = 0;
        int loop_unroll_limit = 0;
+       uint32_t megamorphic = 0;
        const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
 #ifdef HAVE_GCC_GLOBAL_REGS
        zend_execute_data *prev_execute_data = ex;
@@ -571,7 +605,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
        TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
 
        if (prev_call) {
-               int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx);
+               int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic, &megamorphic, ret_level + level);
                if (ret < 0) {
                        return ZEND_JIT_TRACE_STOP_BAD_FUNC;
                }
@@ -665,17 +699,18 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                        TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
                }
 
-               switch (opline->opcode) {
-                       case ZEND_DO_FCALL:
-                       case ZEND_DO_ICALL:
-                       case ZEND_DO_UCALL:
-                       case ZEND_DO_FCALL_BY_NAME:
-                               if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
-                                       TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
-                               }
-                               break;
-                       default:
+               if (opline->opcode == ZEND_DO_FCALL
+                || opline->opcode == ZEND_DO_ICALL
+                || opline->opcode == ZEND_DO_UCALL
+                ||     opline->opcode == ZEND_DO_FCALL_BY_NAME) {
+                       call_level = zend_jit_trace_call_level(EX(call));
+                       if (megamorphic & (1 << (ret_level + level + call_level))) {
+                               stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
                                break;
+                       }
+                       if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
+                               TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
+                       }
                }
 
                handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
@@ -779,7 +814,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                                                ret_level++;
 
                                                if (prev_call) {
-                                                       int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx);
+                                                       int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0, &megamorphic, ret_level + level);
                                                        if (ret < 0) {
                                                                stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
                                                                break;
@@ -827,6 +862,22 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
                                        }
                                        func = (zend_function*)jit_extension->op_array;
                                }
+
+#ifndef HAVE_GCC_GLOBAL_REGS
+                               opline = EX(opline);
+#endif
+
+                               if (JIT_G(max_polymorphic_calls) == 0
+                                && zend_jit_may_be_polymorphic_call(opline - 1)) {
+                                       func = NULL;
+                               }
+                               call_level = zend_jit_trace_call_level(EX(call));
+                               ZEND_ASSERT(ret_level + level + call_level < 32);
+                               if (func) {
+                                       megamorphic &= ~(1 << (ret_level + level + call_level));
+                               } else {
+                                       megamorphic |= (1 << (ret_level + level + call_level));
+                               }
                                TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func);
                        }
                        prev_call = EX(call);

diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 9812cc76a749f22e1a95ffac2c40c837f169ae1d..39c99c2776020bd8153e25bc289b28cc30560e06 100644 (file)
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -8009,7 +8009,7 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, const zend_op *opline, co
                return 0;
        }
 
-       exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL, 0);
+       exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL, ZEND_JIT_EXIT_POLYMORPHISM);
        exit_addr = zend_jit_trace_get_exit_addr(exit_point);
        if (!exit_addr) {
                return 0;

diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c
index 2b0db1d18f300740ca403fa93092823286bf0b95..9b73533ce422c170a50fa68ec0ba830167aa8a79 100644 (file)
--- a/ext/opcache/zend_accelerator_module.c
+++ b/ext/opcache/zend_accelerator_module.c
@@ -303,6 +303,7 @@ ZEND_INI_BEGIN()
        STD_PHP_INI_ENTRY("opcache.jit_max_loops_unroll"          , "8",                          PHP_INI_ALL,    OnUpdateUnrollL,  max_loops_unroll,      zend_jit_globals, jit_globals)
        STD_PHP_INI_ENTRY("opcache.jit_max_recursive_calls"       , "2",                          PHP_INI_ALL,    OnUpdateUnrollC,  max_recursive_calls,   zend_jit_globals, jit_globals)
        STD_PHP_INI_ENTRY("opcache.jit_max_recursive_returns"     , "2",                          PHP_INI_ALL,    OnUpdateUnrollR,  max_recursive_returns, zend_jit_globals, jit_globals)
+       STD_PHP_INI_ENTRY("opcache.jit_max_polymorphic_calls"     , "2",                          PHP_INI_ALL,    OnUpdateLong,     max_polymorphic_calls, zend_jit_globals, jit_globals)
 #endif
 ZEND_INI_END()
 
@@ -786,6 +787,7 @@ ZEND_FUNCTION(opcache_get_configuration)
        add_assoc_long(&directives,   "opcache.jit_hot_return", JIT_G(hot_return));
        add_assoc_long(&directives,   "opcache.jit_hot_side_exit", JIT_G(hot_side_exit));
        add_assoc_long(&directives,   "opcache.jit_max_loops_unroll", JIT_G(max_loops_unroll));
+       add_assoc_long(&directives,   "opcache.jit_max_polymorphic_calls", JIT_G(max_polymorphic_calls));
        add_assoc_long(&directives,   "opcache.jit_max_recursive_calls", JIT_G(max_recursive_calls));
        add_assoc_long(&directives,   "opcache.jit_max_recursive_returns", JIT_G(max_recursive_returns));
        add_assoc_long(&directives,   "opcache.jit_max_root_traces", JIT_G(max_root_traces));