From: Dmitry Stogov Date: Mon, 18 May 2020 16:00:38 +0000 (+0300) Subject: Keep ZEND_FUNC_INFO() for all run-time JIT triggers X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d377467d86ec74fc0311989fef5c674d1d5b1ea4;p=php Keep ZEND_FUNC_INFO() for all run-time JIT triggers --- diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 7921c1186d..6d036256d9 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -3044,15 +3044,55 @@ jit_failure: static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script) { - zend_func_info *func_info = - zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info)); + zend_func_info *func_info; - ZEND_SET_FUNC_INFO(op_array, func_info); + if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC || + JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST || + JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) { + func_info = ZEND_FUNC_INFO(op_array); + } else { + func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info)); + ZEND_SET_FUNC_INFO(op_array, func_info); + } func_info->num_args = -1; func_info->return_value_used = -1; return zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info); } +static void zend_jit_cleanup_func_info(zend_op_array *op_array) +{ + zend_func_info *func_info = ZEND_FUNC_INFO(op_array); + zend_call_info *caller_info, *callee_info; + + if (func_info) { + caller_info = func_info->caller_info; + callee_info = func_info->callee_info; + + if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC || + JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST || + JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) { + memset(func_info, 0, sizeof(zend_func_info)); + func_info->num_args = -1; + func_info->return_value_used = -1; + } else { + ZEND_SET_FUNC_INFO(op_array, NULL); + } + + while (caller_info) { + if (caller_info->caller_op_array) { + zend_jit_cleanup_func_info(caller_info->caller_op_array); + } + caller_info = caller_info->next_caller; + } + while (callee_info) { + if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) { + zend_jit_cleanup_func_info(&callee_info->callee_func->op_array); + } + callee_info = callee_info->next_callee; + } + } +} + static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline) { zend_ssa ssa; @@ -3074,7 +3114,6 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { if (zend_jit_collect_calls(op_array, script) != SUCCESS) { - ZEND_SET_FUNC_INFO(op_array, NULL); goto jit_failure; } func_info = ZEND_FUNC_INFO(op_array); @@ -3096,14 +3135,12 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons goto jit_failure; } - if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { - ZEND_SET_FUNC_INFO(op_array, NULL); - } - + zend_jit_cleanup_func_info(op_array); zend_arena_release(&CG(arena), checkpoint); return SUCCESS; jit_failure: + zend_jit_cleanup_func_info(op_array); zend_arena_release(&CG(arena), checkpoint); return FAILURE; } @@ -3114,6 +3151,7 @@ static void ZEND_FASTCALL zend_runtime_jit(void) zend_execute_data *execute_data = EG(current_execute_data); zend_op_array *op_array = &EX(func)->op_array; zend_op *opline = op_array->opcodes; + zend_jit_op_array_extension *jit_extension; zend_shared_alloc_lock(); @@ -3127,8 +3165,8 @@ static void ZEND_FASTCALL zend_runtime_jit(void) opline++; } } - opline->handler = ZEND_FUNC_INFO(op_array); - ZEND_SET_FUNC_INFO(op_array, NULL); + jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + opline->handler = jit_extension->orig_handler; /* perform real JIT for this function */ zend_real_jit_func(op_array, NULL, NULL); @@ -3147,6 +3185,7 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { zend_function *func; zend_op_array *op_array; uintptr_t counter; + zend_jit_op_array_extension *jit_extension; ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) { if (func->type == ZEND_INTERNAL_FUNCTION) { @@ -3165,8 +3204,8 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) { } counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; - opline->handler = ZEND_FUNC_INFO(op_array); - ZEND_SET_FUNC_INFO(op_array, NULL); + jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + opline->handler = jit_extension->orig_handler; if (((double)counter / (double)zend_jit_profile_counter) > ZEND_JIT_PROF_THRESHOLD) { zend_real_jit_func(op_array, NULL, NULL); } @@ -3190,7 +3229,6 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend for (i = 0; i < op_array->last; i++) { op_array->opcodes[i].handler = jit_extension->orig_handlers[i]; } - ZEND_SET_FUNC_INFO(op_array, NULL); /* perform real JIT for this function */ zend_real_jit_func(op_array, NULL, opline); @@ -3274,6 +3312,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) } if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) { + zend_jit_op_array_extension *jit_extension; zend_op *opline = op_array->opcodes; /* Set run-time JIT handler */ @@ -3283,11 +3322,17 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) opline++; } } - ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); + jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); + jit_extension->func_info.num_args = -1; + jit_extension->func_info.return_value_used = -1; + jit_extension->orig_handler = (void*)opline->handler; + ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); opline->handler = (const void*)zend_jit_runtime_jit_handler; return SUCCESS; } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) { + zend_jit_op_array_extension *jit_extension; zend_op *opline = op_array->opcodes; ZEND_ASSERT(zend_jit_profile_jit_handler != NULL); @@ -3297,7 +3342,12 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) opline++; } } - ZEND_SET_FUNC_INFO(op_array, (void*)opline->handler); + jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); + jit_extension->func_info.num_args = -1; + jit_extension->func_info.return_value_used = -1; + jit_extension->orig_handler = (void*)opline->handler; + ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); opline->handler = (const void*)zend_jit_profile_jit_handler; } @@ -3344,7 +3394,6 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { for (i = 0; i < call_graph.op_arrays_count; i++) { - ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { goto jit_failure; } @@ -3435,8 +3484,11 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) return SUCCESS; jit_failure: - for (i = 0; i < call_graph.op_arrays_count; i++) { - ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD || + JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT) { + for (i = 0; i < call_graph.op_arrays_count; i++) { + ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); + } } zend_arena_release(&CG(arena), checkpoint); return FAILURE; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 178cfb3f85..93c05be006 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -20,6 +20,11 @@ #ifndef ZEND_JIT_INTERNAL_H #define ZEND_JIT_INTERNAL_H +typedef struct _zend_jit_op_array_extension { + zend_func_info func_info; + const void *orig_handler; +} zend_jit_op_array_extension; + /* Profiler */ extern zend_ulong zend_jit_profile_counter; extern int zend_jit_profile_counter_rid; diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index d88da124d0..f5ef32bff7 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -182,7 +182,8 @@ void ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D) ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS) { zend_op_array *op_array = (zend_op_array*)EX(func); - zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_FUNC_INFO(op_array); + zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + zend_vm_opcode_handler_t handler = jit_extension->orig_handler; uintptr_t counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = (void*)(counter + 1); diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 97b926085d..3bb981ecb7 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -2166,8 +2166,10 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) # error "Unknown ZEND_MAP_PTR_KIND" #endif | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] - | // handler = (const void*)ZEND_FUNC_INFO(op_array); + | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | // handler = jit_extension->orig_handler + | mov r0, aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] | // return ((zend_vm_opcode_handler_t)handler)(); | jmp r0 return 1;