From 522cc6926b2b376e48e1928b806741e358f51bff Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 3 Nov 2017 18:35:03 +0300 Subject: [PATCH] erealloc() optimization --- Zend/zend_alloc.c | 223 +++++++++++++++++++++++++++++----------------- 1 file changed, 140 insertions(+), 83 deletions(-) diff --git a/Zend/zend_alloc.c b/Zend/zend_alloc.c index d13213fec9..2dc33f8cd4 100644 --- a/Zend/zend_alloc.c +++ b/Zend/zend_alloc.c @@ -1423,97 +1423,132 @@ static size_t zend_mm_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_ } } -static void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) +static zend_never_inline void *zend_mm_realloc_slow(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) +{ + void *ret; + +#if ZEND_MM_STAT + do { + size_t orig_peak = heap->peak; + size_t orig_real_peak = heap->real_peak; +#endif + ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + memcpy(ret, ptr, copy_size); + zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); +#if ZEND_MM_STAT + heap->peak = MAX(orig_peak, heap->size); + heap->real_peak = MAX(orig_real_peak, heap->real_size); + } while (0); +#endif + return ret; +} + +static zend_never_inline void *zend_mm_realloc_huge(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) { - size_t page_offset; size_t old_size; size_t new_size; - void *ret; #if ZEND_DEBUG size_t real_size; zend_mm_debug_info *dbg; #endif - page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); - if (UNEXPECTED(page_offset == 0)) { - if (UNEXPECTED(ptr == NULL)) { - return zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); - } - old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); #if ZEND_DEBUG - real_size = size; - size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)); + real_size = size; + size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)); #endif - if (size > ZEND_MM_MAX_LARGE_SIZE) { + if (size > ZEND_MM_MAX_LARGE_SIZE) { #if ZEND_DEBUG - size = real_size; + size = real_size; #endif #ifdef ZEND_WIN32 - /* On Windows we don't have ability to extend huge blocks in-place. - * We allocate them with 2MB size granularity, to avoid many - * reallocations when they are extended by small pieces - */ - new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE)); + /* On Windows we don't have ability to extend huge blocks in-place. + * We allocate them with 2MB size granularity, to avoid many + * reallocations when they are extended by small pieces + */ + new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE)); #else - new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE); + new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE); #endif - if (new_size == old_size) { + if (new_size == old_size) { #if ZEND_DEBUG - zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); #else - zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); #endif - return ptr; - } else if (new_size < old_size) { - /* unmup tail */ - if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) { + return ptr; + } else if (new_size < old_size) { + /* unmup tail */ + if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) { #if ZEND_MM_STAT || ZEND_MM_LIMIT - heap->real_size -= old_size - new_size; + heap->real_size -= old_size - new_size; #endif #if ZEND_MM_STAT - heap->size -= old_size - new_size; + heap->size -= old_size - new_size; #endif #if ZEND_DEBUG - zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); #else - zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); #endif - return ptr; - } - } else /* if (new_size > old_size) */ { + return ptr; + } + } else /* if (new_size > old_size) */ { #if ZEND_MM_LIMIT - if (UNEXPECTED(heap->real_size + (new_size - old_size) > heap->limit)) { - if (zend_mm_gc(heap) && heap->real_size + (new_size - old_size) <= heap->limit) { - /* pass */ - } else if (heap->overflow == 0) { + if (UNEXPECTED(heap->real_size + (new_size - old_size) > heap->limit)) { + if (zend_mm_gc(heap) && heap->real_size + (new_size - old_size) <= heap->limit) { + /* pass */ + } else if (heap->overflow == 0) { #if ZEND_DEBUG - zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size); + zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size); #else - zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size); + zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size); #endif - return NULL; - } + return NULL; } + } #endif - /* try to map tail right after this block */ - if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) { + /* try to map tail right after this block */ + if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) { #if ZEND_MM_STAT || ZEND_MM_LIMIT - heap->real_size += new_size - old_size; + heap->real_size += new_size - old_size; #endif #if ZEND_MM_STAT - heap->real_peak = MAX(heap->real_peak, heap->real_size); - heap->size += new_size - old_size; - heap->peak = MAX(heap->peak, heap->size); + heap->real_peak = MAX(heap->real_peak, heap->real_size); + heap->size += new_size - old_size; + heap->peak = MAX(heap->peak, heap->size); #endif #if ZEND_DEBUG - zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); #else - zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); #endif - return ptr; - } + return ptr; } } + } + + return zend_mm_realloc_slow(heap, ptr, size, MIN(old_size, copy_size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); +} + +static zend_always_inline void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, zend_bool use_copy_size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) +{ + size_t page_offset; + size_t old_size; + size_t new_size; + void *ret; +#if ZEND_DEBUG + size_t real_size; + zend_mm_debug_info *dbg; +#endif + + page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); + if (UNEXPECTED(page_offset == 0)) { + if (EXPECTED(ptr == NULL)) { + return _zend_mm_alloc(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + } else { + return zend_mm_realloc_huge(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + } } else { zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE); @@ -1527,21 +1562,56 @@ static void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, si ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted"); if (info & ZEND_MM_IS_SRUN) { int old_bin_num = ZEND_MM_SRUN_BIN_NUM(info); - old_size = bin_data_size[old_bin_num]; - if (size <= ZEND_MM_MAX_SMALL_SIZE) { - int bin_num = ZEND_MM_SMALL_SIZE_TO_BIN(size); - if (old_bin_num == bin_num) { -#if ZEND_DEBUG - dbg = zend_mm_get_debug_info(heap, ptr); - dbg->size = real_size; - dbg->filename = __zend_filename; - dbg->orig_filename = __zend_orig_filename; - dbg->lineno = __zend_lineno; - dbg->orig_lineno = __zend_orig_lineno; + + do { + old_size = bin_data_size[old_bin_num]; + + /* Check if requested size fits into current bin */ + if (size <= old_size) { + /* Check if truncation is necessary */ + if (old_bin_num > 0 && size <= bin_data_size[old_bin_num - 1]) { + /* truncation */ + ret = zend_mm_alloc_small(heap, size, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + copy_size = use_copy_size ? MIN(size, copy_size) : size; + memcpy(ret, ptr, copy_size); + zend_mm_free_small(heap, ptr, old_bin_num); + } else { + /* reallocation in-place */ + ret = ptr; + } + } else if (size <= ZEND_MM_MAX_SMALL_SIZE) { + /* small extension */ + +#if ZEND_MM_STAT + do { + size_t orig_peak = heap->peak; + size_t orig_real_peak = heap->real_peak; +#endif + ret = zend_mm_alloc_small(heap, size, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + copy_size = use_copy_size ? MIN(old_size, copy_size) : old_size; + memcpy(ret, ptr, copy_size); + zend_mm_free_small(heap, ptr, old_bin_num); +#if ZEND_MM_STAT + heap->peak = MAX(orig_peak, heap->size); + heap->real_peak = MAX(orig_real_peak, heap->real_size); + } while (0); #endif - return ptr; + } else { + /* slow reallocation */ + break; } - } + +#if ZEND_DEBUG + dbg = zend_mm_get_debug_info(heap, ret); + dbg->size = real_size; + dbg->filename = __zend_filename; + dbg->orig_filename = __zend_orig_filename; + dbg->lineno = __zend_lineno; + dbg->orig_lineno = __zend_orig_lineno; +#endif + return ret; + } while (0); + } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ { ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted"); old_size = ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE; @@ -1613,21 +1683,8 @@ static void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, si #endif } - /* Naive reallocation */ -#if ZEND_MM_STAT - do { - size_t orig_peak = heap->peak; - size_t orig_real_peak = heap->real_peak; -#endif - ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); - memcpy(ret, ptr, MIN(old_size, copy_size)); - zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); -#if ZEND_MM_STAT - heap->peak = MAX(orig_peak, heap->size); - heap->real_peak = MAX(orig_real_peak, heap->real_size); - } while (0); -#endif - return ret; + copy_size = MIN(old_size, copy_size); + return zend_mm_realloc_slow(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); } /*********************/ @@ -2277,12 +2334,12 @@ ZEND_API void ZEND_FASTCALL _zend_mm_free(zend_mm_heap *heap, void *ptr ZEND_FIL void* ZEND_FASTCALL _zend_mm_realloc(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) { - return zend_mm_realloc_heap(heap, ptr, size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + return zend_mm_realloc_heap(heap, ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); } void* ZEND_FASTCALL _zend_mm_realloc2(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) { - return zend_mm_realloc_heap(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + return zend_mm_realloc_heap(heap, ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); } ZEND_API size_t ZEND_FASTCALL _zend_mm_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) @@ -2459,7 +2516,7 @@ ZEND_API void* ZEND_FASTCALL _erealloc(void *ptr, size_t size ZEND_FILE_LINE_DC return AG(mm_heap)->custom_heap.std._realloc(ptr, size); } } - return zend_mm_realloc_heap(AG(mm_heap), ptr, size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); } ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) @@ -2472,7 +2529,7 @@ ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size return AG(mm_heap)->custom_heap.std._realloc(ptr, size); } } - return zend_mm_realloc_heap(AG(mm_heap), ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); + return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); } ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) -- 2.40.0