From: Dmitry Stogov Date: Tue, 9 Jun 2020 20:33:22 +0000 (+0300) Subject: Tracing JIT support for megamorphic calls X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=e9f295ac849aa72304eb6ca01c6289a928cc77dc;p=php Tracing JIT support for megamorphic calls --- diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index a86341ba76..810f678d6c 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -104,6 +104,7 @@ typedef struct _zend_jit_globals { zend_long max_loops_unroll; /* max number of unrolled loops */ zend_long max_recursive_calls; /* max number of recursive inlined call unrolls */ zend_long max_recursive_returns; /* max number of recursive inlined return unrolls */ + zend_long max_polymorphic_calls; /* max number of inlined polymorphic calls */ zend_sym_node *symbols; /* symbols for disassembler */ diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 1177bcd400..7666737ab4 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -207,6 +207,7 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_EXIT_BLACKLISTED (1<<1) #define ZEND_JIT_EXIT_TO_VM (1<<2) /* exit to VM without attempt to create a side trace */ #define ZEND_JIT_EXIT_RESTORE_CALL (1<<3) /* deoptimizer should restore EX(call) chain */ +#define ZEND_JIT_EXIT_POLYMORPHISM (1<<4) /* exit becaus of polymorphic call */ typedef union _zend_op_trace_info { zend_op dummy; /* the size of this structure must be the same as zend_op */ @@ -346,6 +347,7 @@ typedef struct _zend_jit_trace_info { uint32_t exit_counters; /* offset in exit counters array */ uint32_t stack_map_size; uint32_t flags; /* See ZEND_JIT_TRACE_... defines above */ + uint32_t polymorphism; /* Counter of polymorphic calls */ const zend_op *opline; /* first opline */ const void *code_start; /* address of native code */ zend_jit_trace_exit_info *exit_info; /* info about side exits */ @@ -433,7 +435,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HAN int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline); int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); -zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start); +zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphc); static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, zend_bool *exit_if_true) { @@ -456,4 +458,25 @@ static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit return NULL; } +static zend_always_inline zend_bool zend_jit_may_be_polymorphic_call(const zend_op *opline) +{ + if (opline->opcode == ZEND_INIT_FCALL + || opline->opcode == ZEND_INIT_FCALL_BY_NAME + || opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { + return 0; + } else if (opline->opcode == ZEND_INIT_METHOD_CALL + || opline->opcode == ZEND_INIT_DYNAMIC_CALL) { + return 1; + } else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) { + return (opline->op1_type != IS_CONST || opline->op2_type != IS_CONST); + } else if (opline->opcode == ZEND_INIT_USER_CALL) { + return (opline->op2_type != IS_CONST); + } else if (opline->opcode == ZEND_NEW) { + return (opline->op1_type != IS_CONST); + } else { + ZEND_ASSERT(0); + return 0; + } +} + #endif /* ZEND_JIT_INTERNAL_H */ diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index ff7120f5d7..a4028c926f 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -4177,6 +4177,11 @@ done: } call_info = call_info->next_callee; } + if (!skip_guard + && !zend_jit_may_be_polymorphic_call(init_opline)) { + // TODO: recompilation may change target ??? + skip_guard = 1; + } } if (!skip_guard && !zend_jit_init_fcall_guard(&dasm_state, NULL, p->func, trace_buffer[1].opline)) { goto jit_failure; @@ -4422,6 +4427,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace t->child_count = 0; t->stack_map_size = 0; t->flags = 0; + t->polymorphism = 0; t->opline = trace_buffer[1].opline; t->exit_info = exit_info; t->stack_map = NULL; @@ -4770,6 +4776,9 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) { fprintf(stderr, "/CALL"); } + if (t->exit_info[i].flags & ZEND_JIT_EXIT_POLYMORPHISM) { + fprintf(stderr, "/POLY"); + } for (j = 0; j < stack_size; j++) { zend_uchar type = STACK_TYPE(stack, j); if (type != IS_UNKNOWN) { @@ -4839,7 +4848,7 @@ repeat: } stop = zend_jit_trace_execute(execute_data, opline, trace_buffer, - ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK); + ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0); if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { zend_jit_dump_trace(trace_buffer, NULL); @@ -4957,7 +4966,7 @@ static zend_bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_nu return 0; } -static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num) +static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) { zend_jit_trace_stop ret; const void *handler; @@ -4987,6 +4996,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace t->child_count = 0; t->stack_map_size = 0; t->flags = 0; + t->polymorphism = polymorphism; t->opline = NULL; t->exit_info = exit_info; t->stack_map = NULL; @@ -5078,6 +5088,8 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 int ret = 0; uint32_t trace_num; zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; + zend_bool is_megamorphic = 0; + uint32_t polymorphism = 0; trace_num = ZEND_JIT_TRACE_NUM; @@ -5105,7 +5117,19 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 goto abort; } - stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE); + if (EX(call) + && JIT_G(max_polymorphic_calls) > 0 + && (zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)) { + if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) { + is_megamorphic = 1; + } else if (!zend_jit_traces[parent_num].polymorphism) { + polymorphism = 1; + } else if (exit_num == 0) { + polymorphism = zend_jit_traces[parent_num].polymorphism + 1; + } + } + + stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic); if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { zend_jit_dump_trace(trace_buffer, NULL); @@ -5129,7 +5153,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 } } if (EXPECTED(stop != ZEND_JIT_TRACE_STOP_LOOP)) { - stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num); + stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism); } else { const zend_op_array *op_array = trace_buffer[0].op_array; zend_jit_op_array_trace_extension *jit_extension = diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 00f2d5bb25..734a70ac10 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -457,7 +457,7 @@ static int zend_jit_trace_bad_loop_exit(const zend_op *opline) return 0; } -static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx) +static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level, uint32_t *call_level) { zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR; @@ -466,7 +466,10 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji zend_jit_op_array_trace_extension *jit_extension; if (call->prev_execute_data) { - idx = zend_jit_trace_record_fake_init_call(call->prev_execute_data, trace_buffer, idx); + idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, megamorphic, level, call_level); + if (idx < 0) { + return idx; + } } func = call->func; @@ -483,11 +486,39 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji } func = (zend_function*)jit_extension->op_array; } + if (is_megamorphic + /* TODO: use more accurate check ??? */ + && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC) + || func->common.scope)) { + func = NULL; + *megamorphic |= (1 << (level + *call_level)); + } else { + *megamorphic &= ~(1 << (level + *call_level)); + } + (*call_level)++; TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INIT_CALL, func); } while (0); return idx; } +static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level) +{ + uint32_t call_level = 0; + + return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, megamorphic, level, &call_level); +} + +static int zend_jit_trace_call_level(const zend_execute_data *call) +{ + int call_level = 0; + + while (call->prev_execute_data) { + call_level++; + call = call->prev_execute_data; + } + return call_level; +} + /* * Trace Linking Rules * =================== @@ -518,7 +549,8 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji * */ -zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start) +zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphic) + { #ifdef HAVE_GCC_GLOBAL_REGS zend_execute_data *save_execute_data = execute_data; @@ -528,6 +560,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR; int level = 0; int ret_level = 0; + int call_level; zend_vm_opcode_handler_t handler; const zend_op_array *op_array; zend_jit_op_array_trace_extension *jit_extension; @@ -539,6 +572,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, int backtrack_ret_recursion = -1; int backtrack_ret_recursion_level = 0; int loop_unroll_limit = 0; + uint32_t megamorphic = 0; const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH]; #ifdef HAVE_GCC_GLOBAL_REGS zend_execute_data *prev_execute_data = ex; @@ -571,7 +605,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline); if (prev_call) { - int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx); + int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic, &megamorphic, ret_level + level); if (ret < 0) { return ZEND_JIT_TRACE_STOP_BAD_FUNC; } @@ -665,17 +699,18 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2); } - switch (opline->opcode) { - case ZEND_DO_FCALL: - case ZEND_DO_ICALL: - case ZEND_DO_UCALL: - case ZEND_DO_FCALL_BY_NAME: - if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) { - TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func); - } - break; - default: + if (opline->opcode == ZEND_DO_FCALL + || opline->opcode == ZEND_DO_ICALL + || opline->opcode == ZEND_DO_UCALL + || opline->opcode == ZEND_DO_FCALL_BY_NAME) { + call_level = zend_jit_trace_call_level(EX(call)); + if (megamorphic & (1 << (ret_level + level + call_level))) { + stop = ZEND_JIT_TRACE_STOP_INTERPRETER; break; + } + if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) { + TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func); + } } handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; @@ -779,7 +814,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, ret_level++; if (prev_call) { - int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx); + int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0, &megamorphic, ret_level + level); if (ret < 0) { stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; break; @@ -827,6 +862,22 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } func = (zend_function*)jit_extension->op_array; } + +#ifndef HAVE_GCC_GLOBAL_REGS + opline = EX(opline); +#endif + + if (JIT_G(max_polymorphic_calls) == 0 + && zend_jit_may_be_polymorphic_call(opline - 1)) { + func = NULL; + } + call_level = zend_jit_trace_call_level(EX(call)); + ZEND_ASSERT(ret_level + level + call_level < 32); + if (func) { + megamorphic &= ~(1 << (ret_level + level + call_level)); + } else { + megamorphic |= (1 << (ret_level + level + call_level)); + } TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func); } prev_call = EX(call); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 9812cc76a7..39c99c2776 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -8009,7 +8009,7 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, const zend_op *opline, co return 0; } - exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL, 0); + exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL, ZEND_JIT_EXIT_POLYMORPHISM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index 2b0db1d18f..9b73533ce4 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -303,6 +303,7 @@ ZEND_INI_BEGIN() STD_PHP_INI_ENTRY("opcache.jit_max_loops_unroll" , "8", PHP_INI_ALL, OnUpdateUnrollL, max_loops_unroll, zend_jit_globals, jit_globals) STD_PHP_INI_ENTRY("opcache.jit_max_recursive_calls" , "2", PHP_INI_ALL, OnUpdateUnrollC, max_recursive_calls, zend_jit_globals, jit_globals) STD_PHP_INI_ENTRY("opcache.jit_max_recursive_returns" , "2", PHP_INI_ALL, OnUpdateUnrollR, max_recursive_returns, zend_jit_globals, jit_globals) + STD_PHP_INI_ENTRY("opcache.jit_max_polymorphic_calls" , "2", PHP_INI_ALL, OnUpdateLong, max_polymorphic_calls, zend_jit_globals, jit_globals) #endif ZEND_INI_END() @@ -786,6 +787,7 @@ ZEND_FUNCTION(opcache_get_configuration) add_assoc_long(&directives, "opcache.jit_hot_return", JIT_G(hot_return)); add_assoc_long(&directives, "opcache.jit_hot_side_exit", JIT_G(hot_side_exit)); add_assoc_long(&directives, "opcache.jit_max_loops_unroll", JIT_G(max_loops_unroll)); + add_assoc_long(&directives, "opcache.jit_max_polymorphic_calls", JIT_G(max_polymorphic_calls)); add_assoc_long(&directives, "opcache.jit_max_recursive_calls", JIT_G(max_recursive_calls)); add_assoc_long(&directives, "opcache.jit_max_recursive_returns", JIT_G(max_recursive_returns)); add_assoc_long(&directives, "opcache.jit_max_root_traces", JIT_G(max_root_traces));