From 9b0b30272f7ba5cf38c1fb00d9feae274467f6a9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 9 Oct 2014 14:01:19 +0400 Subject: [PATCH] Copy all the arena allocated data from SHM to process memory at once --- ext/opcache/ZendAccelerator.c | 4 - ext/opcache/ZendAccelerator.h | 4 + ext/opcache/zend_accelerator_util_funcs.c | 27 ++++-- ext/opcache/zend_persist.c | 30 ++++-- ext/opcache/zend_persist_calc.c | 108 ++++++++++------------ ext/opcache/zend_shared_alloc.c | 2 +- 6 files changed, 95 insertions(+), 80 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index b5c20f91e4..d36827bde6 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -1171,10 +1171,6 @@ static zend_persistent_script *cache_script_in_shared_memory(zend_persistent_scr return new_persistent_script; } - /* cleanup after calculation */ - new_persistent_script->mem = ZCG(mem); - new_persistent_script->size = memory_used; - /* Copy into shared memory */ new_persistent_script = zend_accel_script_persist(new_persistent_script, &key, key_length TSRMLS_CC); diff --git a/ext/opcache/ZendAccelerator.h b/ext/opcache/ZendAccelerator.h index c3c7285c48..28c3d21102 100644 --- a/ext/opcache/ZendAccelerator.h +++ b/ext/opcache/ZendAccelerator.h @@ -164,6 +164,8 @@ typedef struct _zend_persistent_script { void *mem; /* shared memory area used by script structures */ size_t size; /* size of used shared memory */ + void *arena_mem; /* part that should be copied into process */ + size_t arena_size; /* All entries that shouldn't be counted in the ADLER32 * checksum must be declared in this struct @@ -234,6 +236,8 @@ typedef struct _zend_accel_globals { time_t request_time; /* preallocated shared-memory block to save current script */ void *mem; + void *arena_mem; + zend_persistent_script *current_persistent_script; /* cache to save hash lookup on the same INCLUDE opcode */ const zend_op *cache_opline; zend_persistent_script *cache_persistent_script; diff --git a/ext/opcache/zend_accelerator_util_funcs.c b/ext/opcache/zend_accelerator_util_funcs.c index 59977717f7..ee7be50619 100644 --- a/ext/opcache/zend_accelerator_util_funcs.c +++ b/ext/opcache/zend_accelerator_util_funcs.c @@ -38,6 +38,9 @@ static uint32_t zend_accel_refcount = ZEND_PROTECTED_REFCOUNT; # define accel_xlat_get(old, new) ((new) = zend_hash_str_find_ptr(&ZCG(bind_hash), (char*)&(old), sizeof(void*), (zend_ulong)(zend_uintptr_t)(old), (void**)&(new))) #endif +#define ARENA_REALLOC(ptr) \ + (void*)(((char*)(ptr)) + ((char*)ZCG(arena_mem) - (char*)ZCG(current_persistent_script)->arena_mem)) + typedef int (*id_function_t)(void *, void *); typedef void (*unique_copy_ctor_func_t)(void *pElement); @@ -469,9 +472,8 @@ static void zend_hash_clone_methods(HashTable *ht, HashTable *source, zend_class q->key = zend_clone_str(p->key TSRMLS_CC); /* Copy data */ - ZVAL_PTR(&q->val, (void *) zend_arena_alloc(&CG(arena), sizeof(zend_op_array))); + ZVAL_PTR(&q->val, ARENA_REALLOC(Z_PTR(p->val))); new_entry = (zend_op_array*)Z_PTR(q->val); - *new_entry = *(zend_op_array*)Z_PTR(p->val); /* Copy constructor */ /* we use refcount to show that op_array is referenced from several places */ @@ -546,9 +548,8 @@ static void zend_hash_clone_prop_info(HashTable *ht, HashTable *source, zend_cla q->key = zend_clone_str(p->key TSRMLS_CC); /* Copy data */ - ZVAL_PTR(&q->val, (void *) zend_arena_alloc(&CG(arena), sizeof(zend_property_info))); + ZVAL_PTR(&q->val, ARENA_REALLOC(Z_PTR(p->val))); prop_info = Z_PTR(q->val); - *prop_info = *(zend_property_info*)Z_PTR(p->val); /* Copy constructor */ prop_info->name = zend_clone_str(prop_info->name TSRMLS_CC); @@ -607,8 +608,7 @@ static void zend_class_copy_ctor(zend_class_entry **pce) zend_function *new_func; TSRMLS_FETCH(); - *pce = ce = zend_arena_alloc(&CG(arena), sizeof(zend_class_entry)); - *ce = *old_ce; + *pce = ce = ARENA_REALLOC(old_ce); ce->refcount = 1; if (old_ce->refcount != 1) { @@ -799,8 +799,7 @@ static void zend_accel_function_hash_copy(HashTable *target, HashTable *source, } } if (pCopyConstructor) { - Z_PTR_P(t) = zend_arena_alloc(&CG(arena), sizeof(zend_op_array)); - memcpy(Z_PTR_P(t), Z_PTR(p->val), sizeof(zend_op_array)); + Z_PTR_P(t) = ARENA_REALLOC(Z_PTR(p->val)); pCopyConstructor(Z_PTR_P(t)); } } @@ -876,9 +875,16 @@ zend_op_array* zend_accel_load_script(zend_persistent_script *persistent_script, op_array = (zend_op_array *) emalloc(sizeof(zend_op_array)); *op_array = persistent_script->main_op_array; - if (from_shared_memory) { + if (EXPECTED(from_shared_memory)) { zend_hash_init(&ZCG(bind_hash), 10, NULL, NULL, 0); + ZCG(current_persistent_script) = persistent_script; + ZCG(arena_mem) = NULL; + if (EXPECTED(persistent_script->arena_size)) { + ZCG(arena_mem) = zend_arena_alloc(&CG(arena), persistent_script->arena_size); + memcpy(ZCG(arena_mem), persistent_script->arena_mem, persistent_script->arena_size); + } + /* Copy all the necessary stuff from shared memory to regular memory, and protect the shared script */ if (zend_hash_num_elements(&persistent_script->class_table) > 0) { zend_accel_class_hash_copy(CG(class_table), &persistent_script->class_table, (unique_copy_ctor_func_t) zend_class_copy_ctor TSRMLS_CC); @@ -905,6 +911,7 @@ zend_op_array* zend_accel_load_script(zend_persistent_script *persistent_script, } zend_hash_destroy(&ZCG(bind_hash)); + ZCG(current_persistent_script) = NULL; } else /* if (!from_shared_memory) */ { if (zend_hash_num_elements(&persistent_script->function_table) > 0) { zend_accel_function_hash_copy(CG(function_table), &persistent_script->function_table, NULL TSRMLS_CC); @@ -921,7 +928,7 @@ zend_op_array* zend_accel_load_script(zend_persistent_script *persistent_script, CG(compiled_filename) = orig_compiled_filename; } - if (!from_shared_memory) { + if (UNEXPECTED(!from_shared_memory)) { free_persistent_script(persistent_script, 0); /* free only hashes */ } diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 6491b9d781..3e91740a71 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -486,7 +486,10 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc static void zend_persist_op_array(zval *zv TSRMLS_DC) { - Z_PTR_P(zv) = zend_accel_memdup(Z_PTR_P(zv), sizeof(zend_op_array)); + memcpy(ZCG(arena_mem), Z_PTR_P(zv), sizeof(zend_op_array)); + zend_shared_alloc_register_xlat_entry(Z_PTR_P(zv), ZCG(arena_mem)); + Z_PTR_P(zv) = ZCG(arena_mem); + ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_op_array))); zend_persist_op_array_ex(Z_PTR_P(zv), NULL TSRMLS_CC); } @@ -494,7 +497,10 @@ static void zend_persist_property_info(zval *zv TSRMLS_DC) { zend_property_info *prop; - prop = Z_PTR_P(zv) = zend_accel_memdup(Z_PTR_P(zv), sizeof(zend_property_info)); + memcpy(ZCG(arena_mem), Z_PTR_P(zv), sizeof(zend_property_info)); + zend_shared_alloc_register_xlat_entry(Z_PTR_P(zv), ZCG(arena_mem)); + prop = Z_PTR_P(zv) = ZCG(arena_mem); + ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_property_info))); zend_accel_store_interned_string(prop->name); if (prop->doc_comment) { if (ZCG(accel_directives).save_comments) { @@ -514,7 +520,10 @@ static void zend_persist_class_entry(zval *zv TSRMLS_DC) zend_class_entry *ce = Z_PTR_P(zv); if (ce->type == ZEND_USER_CLASS) { - ce = Z_PTR_P(zv) = zend_accel_memdup(ce, sizeof(zend_class_entry)); + memcpy(ZCG(arena_mem), Z_PTR_P(zv), sizeof(zend_class_entry)); + zend_shared_alloc_register_xlat_entry(Z_PTR_P(zv), ZCG(arena_mem)); + ce = Z_PTR_P(zv) = ZCG(arena_mem); + ZCG(arena_mem) = (void*)((char*)ZCG(arena_mem) + ZEND_ALIGNED_SIZE(sizeof(zend_class_entry))); zend_accel_store_interned_string(ce->name); zend_hash_persist(&ce->function_table, zend_persist_op_array TSRMLS_CC); if (ce->default_properties_table) { @@ -703,13 +712,20 @@ static void zend_accel_persist_class_table(HashTable *class_table TSRMLS_DC) zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script, char **key, unsigned int key_length TSRMLS_DC) { + script->mem = ZCG(mem); + zend_shared_alloc_clear_xlat_table(); - zend_hash_persist(&script->function_table, zend_persist_op_array TSRMLS_CC); - zend_accel_persist_class_table(&script->class_table TSRMLS_CC); - zend_persist_op_array_ex(&script->main_op_array, script TSRMLS_CC); + + zend_accel_store(script, sizeof(zend_persistent_script)); *key = zend_accel_memdup(*key, key_length + 1); zend_accel_store_string(script->full_path); - zend_accel_store(script, sizeof(zend_persistent_script)); + + script->arena_mem = ZCG(arena_mem) = ZCG(mem); + ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size); + + zend_accel_persist_class_table(&script->class_table TSRMLS_CC); + zend_hash_persist(&script->function_table, zend_persist_op_array TSRMLS_CC); + zend_persist_op_array_ex(&script->main_op_array, script TSRMLS_CC); return script; } diff --git a/ext/opcache/zend_persist_calc.c b/ext/opcache/zend_persist_calc.c index d2de94f05c..0a7b835648 100644 --- a/ext/opcache/zend_persist_calc.c +++ b/ext/opcache/zend_persist_calc.c @@ -26,10 +26,10 @@ #include "zend_shared_alloc.h" #include "zend_operators.h" -#define START_SIZE() uint memory_used = 0 -#define ADD_DUP_SIZE(m,s) memory_used += zend_shared_memdup_size((void*)m, s) -#define ADD_SIZE(m) memory_used += ZEND_ALIGNED_SIZE(m) -#define RETURN_SIZE() return memory_used +#define ADD_DUP_SIZE(m,s) ZCG(current_persistent_script)->size += zend_shared_memdup_size((void*)m, s) +#define ADD_SIZE(m) ZCG(current_persistent_script)->size += ZEND_ALIGNED_SIZE(m) + +#define ADD_ARENA_SIZE(m) ZCG(current_persistent_script)->arena_size += ZEND_ALIGNED_SIZE(m) # define ADD_STRING(str) \ ADD_DUP_SIZE((str), _STR_HEADER_SIZE + (str)->len + 1) @@ -47,16 +47,15 @@ } \ } while (0) -static uint zend_persist_zval_calc(zval *z TSRMLS_DC); +static void zend_persist_zval_calc(zval *z TSRMLS_DC); -static uint zend_hash_persist_calc(HashTable *ht, uint (*pPersistElement)(zval *pElement TSRMLS_DC) TSRMLS_DC) +static void zend_hash_persist_calc(HashTable *ht, void (*pPersistElement)(zval *pElement TSRMLS_DC) TSRMLS_DC) { uint idx; Bucket *p; - START_SIZE(); if (!ht->nTableMask) { - RETURN_SIZE(); + return; } if (ht->u.flags & HASH_FLAG_PACKED) { ADD_SIZE(sizeof(Bucket) * ht->nNumUsed); @@ -75,26 +74,23 @@ static uint zend_hash_persist_calc(HashTable *ht, uint (*pPersistElement)(zval * GC_FLAGS(p->key) |= flags; } - ADD_SIZE(pPersistElement(&p->val TSRMLS_CC)); + pPersistElement(&p->val TSRMLS_CC); } - - RETURN_SIZE(); } -static uint zend_persist_ast_calc(zend_ast *ast TSRMLS_DC) +static void zend_persist_ast_calc(zend_ast *ast TSRMLS_DC) { uint32_t i; - START_SIZE(); if (ast->kind == ZEND_AST_ZVAL) { ADD_SIZE(sizeof(zend_ast_zval)); - ADD_SIZE(zend_persist_zval_calc(zend_ast_get_zval(ast) TSRMLS_CC)); + zend_persist_zval_calc(zend_ast_get_zval(ast) TSRMLS_CC); } else if (zend_ast_is_list(ast)) { zend_ast_list *list = zend_ast_get_list(ast); ADD_SIZE(sizeof(zend_ast_list) - sizeof(zend_ast *) + sizeof(zend_ast *) * list->children); for (i = 0; i < list->children; i++) { if (list->child[i]) { - ADD_SIZE(zend_persist_ast_calc(list->child[i] TSRMLS_CC)); + zend_persist_ast_calc(list->child[i] TSRMLS_CC); } } } else { @@ -102,18 +98,16 @@ static uint zend_persist_ast_calc(zend_ast *ast TSRMLS_DC) ADD_SIZE(sizeof(zend_ast) - sizeof(zend_ast *) + sizeof(zend_ast *) * children); for (i = 0; i < children; i++) { if (ast->child[i]) { - ADD_SIZE(zend_persist_ast_calc(ast->child[i] TSRMLS_CC)); + zend_persist_ast_calc(ast->child[i] TSRMLS_CC); } } } - RETURN_SIZE(); } -static uint zend_persist_zval_calc(zval *z TSRMLS_DC) +static void zend_persist_zval_calc(zval *z TSRMLS_DC) { zend_uchar flags; uint size; - START_SIZE(); switch (Z_TYPE_P(z)) { case IS_STRING: @@ -129,38 +123,35 @@ static uint zend_persist_zval_calc(zval *z TSRMLS_DC) size = zend_shared_memdup_size(Z_ARR_P(z), sizeof(zend_array)); if (size) { ADD_SIZE(size); - ADD_SIZE(zend_hash_persist_calc(Z_ARRVAL_P(z), zend_persist_zval_calc TSRMLS_CC)); + zend_hash_persist_calc(Z_ARRVAL_P(z), zend_persist_zval_calc TSRMLS_CC); } break; case IS_REFERENCE: size = zend_shared_memdup_size(Z_REF_P(z), sizeof(zend_reference)); if (size) { ADD_SIZE(size); - ADD_SIZE(zend_persist_zval_calc(Z_REFVAL_P(z) TSRMLS_CC)); + zend_persist_zval_calc(Z_REFVAL_P(z) TSRMLS_CC); } break; case IS_CONSTANT_AST: size = zend_shared_memdup_size(Z_AST_P(z), sizeof(zend_ast_ref)); if (size) { ADD_SIZE(size); - ADD_SIZE(zend_persist_ast_calc(Z_ASTVAL_P(z) TSRMLS_CC)); + zend_persist_ast_calc(Z_ASTVAL_P(z) TSRMLS_CC); } break; } - RETURN_SIZE(); } -static uint zend_persist_op_array_calc_ex(zend_op_array *op_array TSRMLS_DC) +static void zend_persist_op_array_calc_ex(zend_op_array *op_array TSRMLS_DC) { - START_SIZE(); - if (op_array->type != ZEND_USER_FUNCTION) { - return 0; + return; } if (op_array->static_variables) { ADD_DUP_SIZE(op_array->static_variables, sizeof(HashTable)); - ADD_SIZE(zend_hash_persist_calc(op_array->static_variables, zend_persist_zval_calc TSRMLS_CC)); + zend_hash_persist_calc(op_array->static_variables, zend_persist_zval_calc TSRMLS_CC); } if (zend_shared_alloc_get_xlat_entry(op_array->opcodes)) { @@ -171,7 +162,7 @@ static uint zend_persist_op_array_calc_ex(zend_op_array *op_array TSRMLS_DC) op_array->function_name = new_name; } } - RETURN_SIZE(); + return; } if (op_array->literals) { @@ -179,7 +170,7 @@ static uint zend_persist_op_array_calc_ex(zend_op_array *op_array TSRMLS_DC) zval *end = p + op_array->last_literal; ADD_DUP_SIZE(op_array->literals, sizeof(zval) * op_array->last_literal); while (p < end) { - ADD_SIZE(zend_persist_zval_calc(p TSRMLS_CC)); + zend_persist_zval_calc(p TSRMLS_CC); p++; } } @@ -239,46 +230,39 @@ static uint zend_persist_op_array_calc_ex(zend_op_array *op_array TSRMLS_DC) ADD_INTERNED_STRING(op_array->vars[i], 0); } } - - RETURN_SIZE(); } -static uint zend_persist_op_array_calc(zval *zv TSRMLS_DC) +static void zend_persist_op_array_calc(zval *zv TSRMLS_DC) { - START_SIZE(); - ADD_SIZE(sizeof(zend_op_array)); - ADD_SIZE(zend_persist_op_array_calc_ex(Z_PTR_P(zv) TSRMLS_CC)); - RETURN_SIZE(); + ADD_ARENA_SIZE(sizeof(zend_op_array)); + zend_persist_op_array_calc_ex(Z_PTR_P(zv) TSRMLS_CC); } -static uint zend_persist_property_info_calc(zval *zv TSRMLS_DC) +static void zend_persist_property_info_calc(zval *zv TSRMLS_DC) { zend_property_info *prop = Z_PTR_P(zv); - START_SIZE(); - ADD_SIZE(sizeof(zend_property_info)); + ADD_ARENA_SIZE(sizeof(zend_property_info)); ADD_INTERNED_STRING(prop->name, 0); if (ZCG(accel_directives).save_comments && prop->doc_comment) { ADD_STRING(prop->doc_comment); } - RETURN_SIZE(); } -static uint zend_persist_class_entry_calc(zval *zv TSRMLS_DC) +static void zend_persist_class_entry_calc(zval *zv TSRMLS_DC) { zend_class_entry *ce = Z_PTR_P(zv); - START_SIZE(); if (ce->type == ZEND_USER_CLASS) { - ADD_DUP_SIZE(ce, sizeof(zend_class_entry)); + ADD_ARENA_SIZE(sizeof(zend_class_entry)); ADD_INTERNED_STRING(ce->name, 0); - ADD_SIZE(zend_hash_persist_calc(&ce->function_table, zend_persist_op_array_calc TSRMLS_CC)); + zend_hash_persist_calc(&ce->function_table, zend_persist_op_array_calc TSRMLS_CC); if (ce->default_properties_table) { int i; ADD_SIZE(sizeof(zval) * ce->default_properties_count); for (i = 0; i < ce->default_properties_count; i++) { - ADD_SIZE(zend_persist_zval_calc(&ce->default_properties_table[i] TSRMLS_CC)); + zend_persist_zval_calc(&ce->default_properties_table[i] TSRMLS_CC); } } if (ce->default_static_members_table) { @@ -286,10 +270,10 @@ static uint zend_persist_class_entry_calc(zval *zv TSRMLS_DC) ADD_SIZE(sizeof(zval) * ce->default_static_members_count); for (i = 0; i < ce->default_static_members_count; i++) { - ADD_SIZE(zend_persist_zval_calc(&ce->default_static_members_table[i] TSRMLS_CC)); + zend_persist_zval_calc(&ce->default_static_members_table[i] TSRMLS_CC); } } - ADD_SIZE(zend_hash_persist_calc(&ce->constants_table, zend_persist_zval_calc TSRMLS_CC)); + zend_hash_persist_calc(&ce->constants_table, zend_persist_zval_calc TSRMLS_CC); if (ZEND_CE_FILENAME(ce)) { ADD_STRING(ZEND_CE_FILENAME(ce)); @@ -298,7 +282,7 @@ static uint zend_persist_class_entry_calc(zval *zv TSRMLS_DC) ADD_STRING(ZEND_CE_DOC_COMMENT(ce)); } - ADD_SIZE(zend_hash_persist_calc(&ce->properties_info, zend_persist_property_info_calc TSRMLS_CC)); + zend_hash_persist_calc(&ce->properties_info, zend_persist_property_info_calc TSRMLS_CC); if (ce->trait_aliases) { int i = 0; @@ -345,24 +329,32 @@ static uint zend_persist_class_entry_calc(zval *zv TSRMLS_DC) ADD_SIZE(sizeof(zend_trait_precedence*) * (i + 1)); } } - RETURN_SIZE(); } -static uint zend_accel_persist_class_table_calc(HashTable *class_table TSRMLS_DC) +static void zend_accel_persist_class_table_calc(HashTable *class_table TSRMLS_DC) { - return zend_hash_persist_calc(class_table, zend_persist_class_entry_calc TSRMLS_CC); + zend_hash_persist_calc(class_table, zend_persist_class_entry_calc TSRMLS_CC); } uint zend_accel_script_persist_calc(zend_persistent_script *new_persistent_script, char *key, unsigned int key_length TSRMLS_DC) { - START_SIZE(); + new_persistent_script->mem = NULL; + new_persistent_script->size = 0; + new_persistent_script->arena_mem = NULL; + new_persistent_script->arena_size = 0; + ZCG(current_persistent_script) = new_persistent_script; - ADD_SIZE(zend_hash_persist_calc(&new_persistent_script->function_table, zend_persist_op_array_calc TSRMLS_CC)); - ADD_SIZE(zend_accel_persist_class_table_calc(&new_persistent_script->class_table TSRMLS_CC)); - ADD_SIZE(zend_persist_op_array_calc_ex(&new_persistent_script->main_op_array TSRMLS_CC)); + ADD_DUP_SIZE(new_persistent_script, sizeof(zend_persistent_script)); ADD_DUP_SIZE(key, key_length + 1); ADD_STRING(new_persistent_script->full_path); - ADD_DUP_SIZE(new_persistent_script, sizeof(zend_persistent_script)); - RETURN_SIZE(); + zend_accel_persist_class_table_calc(&new_persistent_script->class_table TSRMLS_CC); + zend_hash_persist_calc(&new_persistent_script->function_table, zend_persist_op_array_calc TSRMLS_CC); + zend_persist_op_array_calc_ex(&new_persistent_script->main_op_array TSRMLS_CC); + + new_persistent_script->size += new_persistent_script->arena_size; + + ZCG(current_persistent_script) = NULL; + + return new_persistent_script->size; } diff --git a/ext/opcache/zend_shared_alloc.c b/ext/opcache/zend_shared_alloc.c index 981829f5b7..43a0263ee5 100644 --- a/ext/opcache/zend_shared_alloc.c +++ b/ext/opcache/zend_shared_alloc.c @@ -343,7 +343,7 @@ void *_zend_shared_memdup(void *source, size_t size, zend_bool free_source TSRML /* we already duplicated this pointer */ return old_p; } - retval = ZCG(mem);; + retval = ZCG(mem); ZCG(mem) = (void*)(((char*)ZCG(mem)) + ZEND_ALIGNED_SIZE(size)); memcpy(retval, source, size); if (free_source) { -- 2.40.0