From 7f67513ca3c71c0d84a272d1572101ef45203b30 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 25 Jun 2018 19:53:58 +0300 Subject: [PATCH] Lazy function copying from op_cache SHM into process memory --- Zend/zend_API.c | 13 +++--- Zend/zend_compile.c | 4 +- Zend/zend_compile.h | 4 +- Zend/zend_execute.c | 54 ++++++++++++++++++++--- Zend/zend_execute.h | 2 + Zend/zend_execute_API.c | 7 +-- Zend/zend_object_handlers.c | 2 +- Zend/zend_vm_def.h | 20 ++++++--- Zend/zend_vm_execute.h | 20 ++++++--- ext/opcache/zend_accelerator_util_funcs.c | 4 +- ext/opcache/zend_persist.c | 7 +-- ext/opcache/zend_persist_calc.c | 2 +- ext/reflection/php_reflection.c | 4 +- 13 files changed, 107 insertions(+), 36 deletions(-) diff --git a/Zend/zend_API.c b/Zend/zend_API.c index c95ce058a4..777361d41b 100644 --- a/Zend/zend_API.c +++ b/Zend/zend_API.c @@ -3007,6 +3007,7 @@ static zend_always_inline int zend_is_callable_check_func(int check_flags, zval fcc->calling_scope = NULL; if (!ce_org) { + zend_function *func; zend_string *lmname; /* Check if function with given name exists. @@ -3015,20 +3016,20 @@ static zend_always_inline int zend_is_callable_check_func(int check_flags, zval /* Skip leading \ */ ZSTR_ALLOCA_ALLOC(lmname, Z_STRLEN_P(callable) - 1, use_heap); zend_str_tolower_copy(ZSTR_VAL(lmname), Z_STRVAL_P(callable) + 1, Z_STRLEN_P(callable)); - zv = zend_hash_find(EG(function_table), lmname); + func = zend_fetch_function(lmname); ZSTR_ALLOCA_FREE(lmname, use_heap); } else { lmname = Z_STR_P(callable); - zv = zend_hash_find(EG(function_table), lmname); - if (!zv) { + func = zend_fetch_function(lmname); + if (!func) { ZSTR_ALLOCA_ALLOC(lmname, Z_STRLEN_P(callable), use_heap); zend_str_tolower_copy(ZSTR_VAL(lmname), Z_STRVAL_P(callable), Z_STRLEN_P(callable)); - zv = zend_hash_find(EG(function_table), lmname); + func = zend_fetch_function(lmname); ZSTR_ALLOCA_FREE(lmname, use_heap); } } - if (EXPECTED(zv != NULL)) { - fcc->function_handler = Z_PTR_P(zv); + if (EXPECTED(func != NULL)) { + fcc->function_handler = func; return 1; } } diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index 9462f19f35..4b10cbf329 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -1077,7 +1077,9 @@ ZEND_API int do_bind_function(const zend_op_array *op_array, const zend_op *opli if (function->op_array.refcount) { (*function->op_array.refcount)++; } - function->op_array.static_variables = NULL; /* NULL out the unbound function */ + if (!(function->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { + function->op_array.static_variables = NULL; /* NULL out the unbound function */ + } return SUCCESS; } } diff --git a/Zend/zend_compile.h b/Zend/zend_compile.h index 5d210fb259..d194b83ce7 100644 --- a/Zend/zend_compile.h +++ b/Zend/zend_compile.h @@ -202,7 +202,6 @@ typedef struct _zend_oparray_context { * Function and method flags * * Free flags: - * 0x10 * 0x20 * 0x2000000 */ @@ -213,6 +212,9 @@ typedef struct _zend_oparray_context { #define ZEND_ACC_FINAL 0x04 #define ZEND_ACC_IMPLEMENTED_ABSTRACT 0x08 +/* Imuttable op_array (lazy loading) */ +#define ZEND_ACC_IMMUTABLE 0x10 + /* method flags (visibility) */ /* The order of those must be kept - public < protected < private */ #define ZEND_ACC_PUBLIC 0x100 diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index c64053da74..d29db27c34 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -2531,6 +2531,47 @@ static zend_never_inline void ZEND_FASTCALL init_func_run_time_cache(zend_op_arr } /* }}} */ +static zend_always_inline zend_function* ZEND_FASTCALL init_func_run_time_cache_i(zend_op_array *op_array, zval *zv) /* {{{ */ +{ + ZEND_ASSERT(op_array->run_time_cache == NULL); + if (op_array->fn_flags & ZEND_ACC_IMMUTABLE) { + zend_op_array *new_op_array = zend_arena_alloc(&CG(arena), sizeof(zend_op_array) + op_array->cache_size); + + Z_PTR_P(zv) = new_op_array; + memcpy(new_op_array, op_array, sizeof(zend_op_array)); + new_op_array->fn_flags &= ~ZEND_ACC_IMMUTABLE; + new_op_array->run_time_cache = (void**)(new_op_array + 1); + memset(new_op_array->run_time_cache, 0, new_op_array->cache_size); + return (zend_function*)new_op_array; + } else { + op_array->run_time_cache = zend_arena_alloc(&CG(arena), op_array->cache_size); + memset(op_array->run_time_cache, 0, op_array->cache_size); + return (zend_function*)op_array; + } +} +/* }}} */ + +static zend_never_inline zend_function* ZEND_FASTCALL init_func_run_time_cache_ex(zend_op_array *op_array, zval *zv) /* {{{ */ +{ + return init_func_run_time_cache_i(op_array, zv); +} +/* }}} */ + +ZEND_API zend_function * ZEND_FASTCALL zend_fetch_function(zend_string *name) /* {{{ */ +{ + zval *zv = zend_hash_find(EG(function_table), name); + + if (EXPECTED(zv != NULL)) { + zend_function *fbc = Z_FUNC_P(zv); + + if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { + fbc = (zend_function*)init_func_run_time_cache_i(&fbc->op_array, zv); + } + return fbc; + } + return NULL; +} /* }}} */ + static zend_always_inline void i_init_code_execute_data(zend_execute_data *execute_data, zend_op_array *op_array, zval *return_value) /* {{{ */ { ZEND_ASSERT(EX(func) == (zend_function*)op_array); @@ -2563,8 +2604,7 @@ ZEND_API void zend_init_func_execute_data(zend_execute_data *ex, zend_op_array * EX(prev_execute_data) = EG(current_execute_data); if (!op_array->run_time_cache) { - op_array->run_time_cache = zend_arena_alloc(&CG(arena), op_array->cache_size); - memset(op_array->run_time_cache, 0, op_array->cache_size); + init_func_run_time_cache(op_array); } i_init_func_execute_data(op_array, return_value, 1 EXECUTE_DATA_CC); @@ -2924,6 +2964,9 @@ static zend_never_inline zend_execute_data *zend_init_dynamic_call_string(zend_s return NULL; } } + if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { + init_func_run_time_cache(&fbc->op_array); + } } else { if (ZSTR_VAL(function)[0] == '\\') { lcname = zend_string_alloc(ZSTR_LEN(function) - 1, 0); @@ -2939,13 +2982,12 @@ static zend_never_inline zend_execute_data *zend_init_dynamic_call_string(zend_s zend_string_release_ex(lcname, 0); fbc = Z_FUNC_P(func); + if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { + fbc = init_func_run_time_cache_ex(&fbc->op_array, func); + } called_scope = NULL; } - if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - init_func_run_time_cache(&fbc->op_array); - } - return zend_vm_stack_push_call_frame(ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC, fbc, num_args, called_scope, NULL); } diff --git a/Zend/zend_execute.h b/Zend/zend_execute.h index 42f289568c..08a6c78a73 100644 --- a/Zend/zend_execute.h +++ b/Zend/zend_execute.h @@ -308,6 +308,8 @@ ZEND_API zend_class_entry *zend_fetch_class(zend_string *class_name, int fetch_t ZEND_API zend_class_entry *zend_fetch_class_by_name(zend_string *class_name, const zval *key, int fetch_type); void zend_verify_abstract_class(zend_class_entry *ce); +ZEND_API zend_function * ZEND_FASTCALL zend_fetch_function(zend_string *name); + ZEND_API void zend_fetch_dimension_const(zval *result, zval *container, zval *dim, int type); ZEND_API zval* zend_get_compiled_variable_value(const zend_execute_data *execute_data_ptr, uint32_t var); diff --git a/Zend/zend_execute_API.c b/Zend/zend_execute_API.c index 01f6fbf4a9..8d39de0101 100644 --- a/Zend/zend_execute_API.c +++ b/Zend/zend_execute_API.c @@ -902,9 +902,10 @@ ZEND_API zend_class_entry *zend_lookup_class_ex(zend_string *name, const zval *k } if (!EG(autoload_func)) { - zval *zv = zend_hash_find_ex(EG(function_table), ZSTR_KNOWN(ZEND_STR_MAGIC_AUTOLOAD), 1); - if (zv) { - EG(autoload_func) = (zend_function*)Z_PTR_P(zv); + zend_function *func = zend_fetch_function(ZSTR_KNOWN(ZEND_STR_MAGIC_AUTOLOAD)); + + if (func) { + EG(autoload_func) = func; } else { if (!key) { zend_string_release_ex(lc_name, 0); diff --git a/Zend/zend_object_handlers.c b/Zend/zend_object_handlers.c index 0af48993e9..86029ee4b9 100644 --- a/Zend/zend_object_handlers.c +++ b/Zend/zend_object_handlers.c @@ -1199,7 +1199,7 @@ ZEND_API zend_function *zend_get_call_trampoline_func(zend_class_entry *ce, zend func->fn_flags |= ZEND_ACC_STATIC; } func->opcodes = &EG(call_trampoline_op); - + func->run_time_cache = (void*)(intptr_t)-1; func->reserved[0] = fbc; func->scope = fbc->common.scope; /* reserve space for arguments, local and temorary variables */ diff --git a/Zend/zend_vm_def.h b/Zend/zend_vm_def.h index 0a45b09ab9..d85bf9974a 100644 --- a/Zend/zend_vm_def.h +++ b/Zend/zend_vm_def.h @@ -3258,7 +3258,7 @@ ZEND_VM_HOT_HANDLER(59, ZEND_INIT_FCALL_BY_NAME, ANY, CONST, NUM|CACHE_SLOT) } fbc = Z_FUNC_P(func); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - init_func_run_time_cache(&fbc->op_array); + fbc = init_func_run_time_cache_ex(&fbc->op_array, func); } CACHE_PTR(opline->result.num, fbc); } @@ -3421,10 +3421,10 @@ ZEND_VM_HOT_HANDLER(69, ZEND_INIT_NS_FCALL_BY_NAME, ANY, CONST, NUM|CACHE_SLOT) } } fbc = Z_FUNC_P(func); - CACHE_PTR(opline->result.num, fbc); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - init_func_run_time_cache(&fbc->op_array); + fbc = init_func_run_time_cache_ex(&fbc->op_array, func); } + CACHE_PTR(opline->result.num, fbc); } call = zend_vm_stack_push_call_frame(ZEND_CALL_NESTED_FUNCTION, @@ -3452,10 +3452,10 @@ ZEND_VM_HOT_HANDLER(61, ZEND_INIT_FCALL, NUM, CONST, NUM|CACHE_SLOT) ZEND_VM_DISPATCH_TO_HELPER(zend_undefined_function_helper, function_name, fname); } fbc = Z_FUNC_P(func); - CACHE_PTR(opline->result.num, fbc); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - init_func_run_time_cache(&fbc->op_array); + fbc = init_func_run_time_cache_ex(&fbc->op_array, func); } + CACHE_PTR(opline->result.num, fbc); } call = zend_vm_stack_push_call_frame_ex( @@ -7032,10 +7032,20 @@ ZEND_VM_HANDLER(153, ZEND_DECLARE_LAMBDA_FUNCTION, CONST, UNUSED) zval *zfunc; zval *object; zend_class_entry *called_scope; + zend_function *fbc; zfunc = zend_hash_find_ex(EG(function_table), Z_STR_P(RT_CONSTANT(opline, opline->op1)), 1); ZEND_ASSERT(zfunc != NULL && Z_FUNC_P(zfunc)->type == ZEND_USER_FUNCTION); + fbc = Z_PTR_P(zfunc); + if (fbc->common.fn_flags & ZEND_ACC_IMMUTABLE) { + zend_function *new_func = zend_arena_alloc(&CG(arena), sizeof(zend_op_array)); + + memcpy(new_func, fbc, sizeof(zend_op_array)); + new_func->common.fn_flags &= ~ZEND_ACC_IMMUTABLE; + Z_PTR_P(zfunc) = fbc = new_func; + } + if (Z_TYPE(EX(This)) == IS_OBJECT) { called_scope = Z_OBJCE(EX(This)); if (UNEXPECTED((Z_FUNC_P(zfunc)->common.fn_flags & ZEND_ACC_STATIC) || diff --git a/Zend/zend_vm_execute.h b/Zend/zend_vm_execute.h index b5bf42ec3b..aa89f097eb 100644 --- a/Zend/zend_vm_execute.h +++ b/Zend/zend_vm_execute.h @@ -2088,7 +2088,7 @@ static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_INIT_FCALL_BY_NAME } fbc = Z_FUNC_P(func); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - init_func_run_time_cache(&fbc->op_array); + fbc = init_func_run_time_cache_ex(&fbc->op_array, func); } CACHE_PTR(opline->result.num, fbc); } @@ -2176,10 +2176,10 @@ static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_INIT_NS_FCALL_BY_N } } fbc = Z_FUNC_P(func); - CACHE_PTR(opline->result.num, fbc); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - init_func_run_time_cache(&fbc->op_array); + fbc = init_func_run_time_cache_ex(&fbc->op_array, func); } + CACHE_PTR(opline->result.num, fbc); } call = zend_vm_stack_push_call_frame(ZEND_CALL_NESTED_FUNCTION, @@ -2207,10 +2207,10 @@ static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_INIT_FCALL_SPEC_CO ZEND_VM_TAIL_CALL(zend_undefined_function_helper_SPEC(fname ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } fbc = Z_FUNC_P(func); - CACHE_PTR(opline->result.num, fbc); if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) && UNEXPECTED(!fbc->op_array.run_time_cache)) { - init_func_run_time_cache(&fbc->op_array); + fbc = init_func_run_time_cache_ex(&fbc->op_array, func); } + CACHE_PTR(opline->result.num, fbc); } call = zend_vm_stack_push_call_frame_ex( @@ -9103,10 +9103,20 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_DECLARE_LAMBDA_FUNCTION_SPEC_C zval *zfunc; zval *object; zend_class_entry *called_scope; + zend_function *fbc; zfunc = zend_hash_find_ex(EG(function_table), Z_STR_P(RT_CONSTANT(opline, opline->op1)), 1); ZEND_ASSERT(zfunc != NULL && Z_FUNC_P(zfunc)->type == ZEND_USER_FUNCTION); + fbc = Z_PTR_P(zfunc); + if (fbc->common.fn_flags & ZEND_ACC_IMMUTABLE) { + zend_function *new_func = zend_arena_alloc(&CG(arena), sizeof(zend_op_array)); + + memcpy(new_func, fbc, sizeof(zend_op_array)); + new_func->common.fn_flags &= ~ZEND_ACC_IMMUTABLE; + Z_PTR_P(zfunc) = fbc = new_func; + } + if (Z_TYPE(EX(This)) == IS_OBJECT) { called_scope = Z_OBJCE(EX(This)); if (UNEXPECTED((Z_FUNC_P(zfunc)->common.fn_flags & ZEND_ACC_STATIC) || diff --git a/ext/opcache/zend_accelerator_util_funcs.c b/ext/opcache/zend_accelerator_util_funcs.c index e609421648..5e39ba1f6a 100644 --- a/ext/opcache/zend_accelerator_util_funcs.c +++ b/ext/opcache/zend_accelerator_util_funcs.c @@ -544,12 +544,12 @@ static void zend_accel_function_hash_copy_from_shm(HashTable *target, HashTable if (UNEXPECTED(t != NULL)) { if (EXPECTED(ZSTR_LEN(p->key) > 0) && EXPECTED(ZSTR_VAL(p->key)[0] == 0)) { /* Mangled key */ - zend_hash_update_ptr(target, p->key, ARENA_REALLOC(Z_PTR(p->val))); + zend_hash_update_ptr(target, p->key, Z_PTR(p->val)); } else { goto failure; } } else { - _zend_hash_append_ptr_ex(target, p->key, ARENA_REALLOC(Z_PTR(p->val)), 1); + _zend_hash_append_ptr_ex(target, p->key, Z_PTR(p->val), 1); } } target->nInternalPointer = 0; diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index d26286c33d..e19f6b0b9e 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -613,10 +613,11 @@ static void zend_persist_op_array(zval *zv) zend_op_array *op_array = Z_PTR_P(zv); ZEND_ASSERT(op_array->type == ZEND_USER_FUNCTION); - memcpy(ZCG(arena_mem), Z_PTR_P(zv), sizeof(zend_op_array)); - Z_PTR_P(zv) = ZCG(arena_mem); - ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_op_array))); + memcpy(ZCG(mem), Z_PTR_P(zv), sizeof(zend_op_array)); + Z_PTR_P(zv) = ZCG(mem); + ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(sizeof(zend_op_array))); zend_persist_op_array_ex(Z_PTR_P(zv), NULL); + ((zend_op_array*)Z_PTR_P(zv))->fn_flags |= ZEND_ACC_IMMUTABLE; } static void zend_persist_class_method(zval *zv) diff --git a/ext/opcache/zend_persist_calc.c b/ext/opcache/zend_persist_calc.c index bc8955e6e3..0291464c4e 100644 --- a/ext/opcache/zend_persist_calc.c +++ b/ext/opcache/zend_persist_calc.c @@ -256,7 +256,7 @@ static void zend_persist_op_array_calc(zval *zv) zend_op_array *op_array = Z_PTR_P(zv); ZEND_ASSERT(op_array->type == ZEND_USER_FUNCTION); - ADD_ARENA_SIZE(sizeof(zend_op_array)); + ADD_SIZE(sizeof(zend_op_array)); zend_persist_op_array_calc_ex(Z_PTR_P(zv)); } diff --git a/ext/reflection/php_reflection.c b/ext/reflection/php_reflection.c index 77b601d529..37c674718b 100644 --- a/ext/reflection/php_reflection.c +++ b/ext/reflection/php_reflection.c @@ -1559,11 +1559,11 @@ ZEND_METHOD(reflection_function, __construct) /* Ignore leading "\" */ ZSTR_ALLOCA_ALLOC(lcname, ZSTR_LEN(fname) - 1, use_heap); zend_str_tolower_copy(ZSTR_VAL(lcname), ZSTR_VAL(fname) + 1, ZSTR_LEN(fname) - 1); - fptr = zend_hash_find_ptr(EG(function_table), lcname); + fptr = zend_fetch_function(lcname); ZSTR_ALLOCA_FREE(lcname, use_heap); } else { lcname = zend_string_tolower(fname); - fptr = zend_hash_find_ptr(EG(function_table), lcname); + fptr = zend_fetch_function(lcname); zend_string_release(lcname); } -- 2.40.0