From f1f1f63ce1a9f984595ce0f61e69169297df7164 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 14 Nov 2017 23:08:29 +0300 Subject: [PATCH] Reimplemented MYSQLND_MEMORY_POOL to avoid allocations ouside of pool. Store all data related to result set in the pool. --- ext/mysqlnd/mysqlnd_block_alloc.c | 186 ++++++++++++++--------------- ext/mysqlnd/mysqlnd_result.c | 106 ++++++---------- ext/mysqlnd/mysqlnd_structs.h | 16 ++- ext/mysqlnd/mysqlnd_wireprotocol.c | 15 +-- 4 files changed, 143 insertions(+), 180 deletions(-) diff --git a/ext/mysqlnd/mysqlnd_block_alloc.c b/ext/mysqlnd/mysqlnd_block_alloc.c index 4dbed8b4de..2ca242d0cf 100644 --- a/ext/mysqlnd/mysqlnd_block_alloc.c +++ b/ext/mysqlnd/mysqlnd_block_alloc.c @@ -14,6 +14,7 @@ +----------------------------------------------------------------------+ | Authors: Andrey Hristov | | Ulf Wendel | + | Dmitry Stogov | +----------------------------------------------------------------------+ */ @@ -24,113 +25,110 @@ #include "mysqlnd_priv.h" +/* {{{ mysqlnd_arena_create */ +static zend_always_inline zend_arena* mysqlnd_arena_create(size_t size) +{ + zend_arena *arena = (zend_arena*)mnd_emalloc(size); + + arena->ptr = (char*) arena + ZEND_MM_ALIGNED_SIZE(sizeof(zend_arena)); + arena->end = (char*) arena + size; + arena->prev = NULL; + return arena; +} +/* }}} */ + +/* {{{ mysqlnd_arena_destroy */ +static zend_always_inline void mysqlnd_arena_destroy(zend_arena *arena) +{ + do { + zend_arena *prev = arena->prev; + mnd_efree(arena); + arena = prev; + } while (arena); +} +/* }}} */ + +/* {{{ mysqlnd_arena_alloc */ +static zend_always_inline void* mysqlnd_arena_alloc(zend_arena **arena_ptr, size_t size) +{ + zend_arena *arena = *arena_ptr; + char *ptr = arena->ptr; + + size = ZEND_MM_ALIGNED_SIZE(size); + + if (EXPECTED(size <= (size_t)(arena->end - ptr))) { + arena->ptr = ptr + size; + } else { + size_t arena_size = + UNEXPECTED((size + ZEND_MM_ALIGNED_SIZE(sizeof(zend_arena))) > (size_t)(arena->end - (char*) arena)) ? + (size + ZEND_MM_ALIGNED_SIZE(sizeof(zend_arena))) : + (size_t)(arena->end - (char*) arena); + zend_arena *new_arena = (zend_arena*)mnd_emalloc(arena_size); + + ptr = (char*) new_arena + ZEND_MM_ALIGNED_SIZE(sizeof(zend_arena)); + new_arena->ptr = (char*) new_arena + ZEND_MM_ALIGNED_SIZE(sizeof(zend_arena)) + size; + new_arena->end = (char*) new_arena + arena_size; + new_arena->prev = arena; + *arena_ptr = new_arena; + } + + return (void*) ptr; +} +/* }}} */ + /* {{{ mysqlnd_mempool_free_chunk */ static void mysqlnd_mempool_free_chunk(MYSQLND_MEMORY_POOL * pool, MYSQLND_MEMORY_POOL_CHUNK * chunk) { DBG_ENTER("mysqlnd_mempool_free_chunk"); - if (chunk->from_pool) { - /* Try to back-off and guess if this is the last block allocated */ - if (chunk->ptr == (pool->arena + (pool->arena_size - pool->free_size - chunk->size))) { - /* - This was the last allocation. Lucky us, we can free - a bit of memory from the pool. Next time we will return from the same ptr. - */ - pool->free_size += chunk->size; - } - } else { - mnd_efree(chunk->ptr); + /* Try to back-off and guess if this is the last block allocated */ + if ((char*)chunk == (char*)pool->arena->ptr - ZEND_MM_ALIGNED_SIZE(sizeof(MYSQLND_MEMORY_POOL_CHUNK) + chunk->size)) { + /* + This was the last allocation. Lucky us, we can free + a bit of memory from the pool. Next time we will return from the same ptr. + */ + pool->arena->ptr = (char*)chunk; } - mnd_efree(chunk); DBG_VOID_RETURN; } /* }}} */ /* {{{ mysqlnd_mempool_resize_chunk */ -static enum_func_status +static MYSQLND_MEMORY_POOL_CHUNK * mysqlnd_mempool_resize_chunk(MYSQLND_MEMORY_POOL * pool, MYSQLND_MEMORY_POOL_CHUNK * chunk, unsigned int size) { DBG_ENTER("mysqlnd_mempool_resize_chunk"); - if (chunk->from_pool) { - /* Try to back-off and guess if this is the last block allocated */ - if (chunk->ptr == (pool->arena + (pool->arena_size - pool->free_size - chunk->size))) { - /* - This was the last allocation. Lucky us, we can free - a bit of memory from the pool. Next time we will return from the same ptr. - */ - if ((chunk->size + pool->free_size) < size) { - zend_uchar *new_ptr; - new_ptr = mnd_emalloc(size); - if (!new_ptr) { - DBG_RETURN(FAIL); - } - memcpy(new_ptr, chunk->ptr, chunk->size); - chunk->ptr = new_ptr; - pool->free_size += chunk->size; - chunk->size = size; - chunk->from_pool = FALSE; /* now we have no pool memory */ - } else { - /* If the chunk is > than asked size then free_memory increases, otherwise decreases*/ - pool->free_size += (chunk->size - size); - } - } else { - /* Not last chunk, if the user asks for less, give it to him */ - if (chunk->size >= size) { - ; /* nop */ - } else { - zend_uchar *new_ptr; - new_ptr = mnd_emalloc(size); - if (!new_ptr) { - DBG_RETURN(FAIL); - } - memcpy(new_ptr, chunk->ptr, chunk->size); - chunk->ptr = new_ptr; - chunk->size = size; - chunk->from_pool = FALSE; /* now we have non-pool memory */ - } - } + + /* Try to back-off and guess if this is the last block allocated */ + if (((char*)chunk == (char*)pool->arena->ptr - ZEND_MM_ALIGNED_SIZE(sizeof(MYSQLND_MEMORY_POOL_CHUNK) + chunk->size)) + && (ZEND_MM_ALIGNED_SIZE(sizeof(MYSQLND_MEMORY_POOL_CHUNK) + size) <= ((char*)pool->arena->end - (char*)chunk))) { + /* + This was the last allocation. Lucky us, we can free + a bit of memory from the pool. Next time we will return from the same ptr. + */ + pool->arena->ptr = (char*)chunk + ZEND_MM_ALIGNED_SIZE(sizeof(MYSQLND_MEMORY_POOL_CHUNK) + size); } else { - zend_uchar *new_ptr = mnd_erealloc(chunk->ptr, size); - if (!new_ptr) { - DBG_RETURN(FAIL); - } - chunk->ptr = new_ptr; + MYSQLND_MEMORY_POOL_CHUNK *new_chunk = mysqlnd_arena_alloc(&pool->arena, sizeof(MYSQLND_MEMORY_POOL_CHUNK) + size); + memcpy(new_chunk, chunk, sizeof(MYSQLND_MEMORY_POOL_CHUNK) + MIN(size, chunk->size)); + chunk = new_chunk; } - DBG_RETURN(PASS); + chunk->size = size; + DBG_RETURN(chunk); } /* }}} */ /* {{{ mysqlnd_mempool_get_chunk */ -static -MYSQLND_MEMORY_POOL_CHUNK * mysqlnd_mempool_get_chunk(MYSQLND_MEMORY_POOL * pool, unsigned int size) +static MYSQLND_MEMORY_POOL_CHUNK * +mysqlnd_mempool_get_chunk(MYSQLND_MEMORY_POOL * pool, unsigned int size) { MYSQLND_MEMORY_POOL_CHUNK *chunk = NULL; DBG_ENTER("mysqlnd_mempool_get_chunk"); - chunk = mnd_emalloc(sizeof(MYSQLND_MEMORY_POOL_CHUNK)); - if (chunk) { - chunk->size = size; - /* - Should not go over MYSQLND_MAX_PACKET_SIZE, since we - expect non-arena memory in mysqlnd_wireprotocol.c . We - realloc the non-arena memory. - */ - if (size > pool->free_size) { - chunk->from_pool = FALSE; - chunk->ptr = mnd_emalloc(size); - if (!chunk->ptr) { - pool->free_chunk(pool, chunk); - chunk = NULL; - } - } else { - chunk->from_pool = TRUE; - chunk->ptr = pool->arena + (pool->arena_size - pool->free_size); - /* Last step, update free_size */ - pool->free_size -= size; - } - } + chunk = mysqlnd_arena_alloc(&pool->arena, sizeof(MYSQLND_MEMORY_POOL_CHUNK) + size); + chunk->size = size; + DBG_RETURN(chunk); } /* }}} */ @@ -140,21 +138,16 @@ MYSQLND_MEMORY_POOL_CHUNK * mysqlnd_mempool_get_chunk(MYSQLND_MEMORY_POOL * pool PHPAPI MYSQLND_MEMORY_POOL * mysqlnd_mempool_create(size_t arena_size) { - /* We calloc, because we free(). We don't mnd_calloc() for a reason. */ - MYSQLND_MEMORY_POOL * ret = mnd_ecalloc(1, sizeof(MYSQLND_MEMORY_POOL)); + zend_arena * arena; + MYSQLND_MEMORY_POOL * ret; + DBG_ENTER("mysqlnd_mempool_create"); - if (ret) { - ret->get_chunk = mysqlnd_mempool_get_chunk; - ret->free_chunk = mysqlnd_mempool_free_chunk; - ret->resize_chunk = mysqlnd_mempool_resize_chunk; - ret->free_size = ret->arena_size = arena_size ? arena_size : 0; - /* OOM ? */ - ret->arena = mnd_emalloc(ret->arena_size); - if (!ret->arena) { - mysqlnd_mempool_destroy(ret); - ret = NULL; - } - } + arena = mysqlnd_arena_create(MAX(arena_size, sizeof(zend_arena))); + ret = mysqlnd_arena_alloc(&arena, sizeof(MYSQLND_MEMORY_POOL)); + ret->arena = arena; + ret->get_chunk = mysqlnd_mempool_get_chunk; + ret->free_chunk = mysqlnd_mempool_free_chunk; + ret->resize_chunk = mysqlnd_mempool_resize_chunk; DBG_RETURN(ret); } /* }}} */ @@ -166,8 +159,7 @@ mysqlnd_mempool_destroy(MYSQLND_MEMORY_POOL * pool) { DBG_ENTER("mysqlnd_mempool_destroy"); /* mnd_free will reference LOCK_access and might crash, depending on the caller...*/ - mnd_efree(pool->arena); - mnd_efree(pool); + mysqlnd_arena_destroy(pool->arena); DBG_VOID_RETURN; } /* }}} */ diff --git a/ext/mysqlnd/mysqlnd_result.c b/ext/mysqlnd/mysqlnd_result.c index 55c68ebb8c..a59d6c228f 100644 --- a/ext/mysqlnd/mysqlnd_result.c +++ b/ext/mysqlnd/mysqlnd_result.c @@ -183,11 +183,6 @@ MYSQLND_METHOD(mysqlnd_result_unbuffered, free_result)(MYSQLND_RES_UNBUFFERED * DBG_ENTER("mysqlnd_result_unbuffered, free_result"); result->m.free_last_data(result, global_stats); - if (result->lengths) { - mnd_efree(result->lengths); - result->lengths = NULL; - } - /* must be free before because references the memory pool */ if (result->row_packet) { PACKET_FREE(result->row_packet); @@ -195,13 +190,8 @@ MYSQLND_METHOD(mysqlnd_result_unbuffered, free_result)(MYSQLND_RES_UNBUFFERED * result->row_packet = NULL; } - if (result->result_set_memory_pool) { - mysqlnd_mempool_destroy(result->result_set_memory_pool); - result->result_set_memory_pool = NULL; - } + mysqlnd_mempool_destroy(result->result_set_memory_pool); - - mnd_efree(result); DBG_VOID_RETURN; } /* }}} */ @@ -254,8 +244,6 @@ MYSQLND_METHOD(mysqlnd_result_buffered_c, free_result)(MYSQLND_RES_BUFFERED_C * static void MYSQLND_METHOD(mysqlnd_result_buffered, free_result)(MYSQLND_RES_BUFFERED * const set) { - int64_t row; - MYSQLND_MEMORY_POOL * pool; DBG_ENTER("mysqlnd_result_buffered::free_result"); DBG_INF_FMT("Freeing "MYSQLND_LLU_SPEC" row(s)", set->row_count); @@ -268,30 +256,12 @@ MYSQLND_METHOD(mysqlnd_result_buffered, free_result)(MYSQLND_RES_BUFFERED * cons MYSQLND_METHOD(mysqlnd_result_buffered_c, free_result)((MYSQLND_RES_BUFFERED_C *) set); } - pool = set->result_set_memory_pool; - for (row = set->row_count - 1; row >= 0; row--) { - MYSQLND_MEMORY_POOL_CHUNK *current_buffer = set->row_buffers[row]; - pool->free_chunk(pool, current_buffer); - } - - if (set->lengths) { - mnd_efree(set->lengths); - set->lengths = NULL; - } - if (set->row_buffers) { mnd_pefree(set->row_buffers, 0); set->row_buffers = NULL; } - if (set->result_set_memory_pool) { - mysqlnd_mempool_destroy(set->result_set_memory_pool); - set->result_set_memory_pool = NULL; - } - - set->row_count = 0; - - mnd_efree(set); + mysqlnd_mempool_destroy(set->result_set_memory_pool); DBG_VOID_RETURN; } @@ -1927,23 +1897,23 @@ PHPAPI MYSQLND_RES_UNBUFFERED * mysqlnd_result_unbuffered_init(const unsigned int field_count, const zend_bool ps) { const size_t alloc_size = sizeof(MYSQLND_RES_UNBUFFERED) + mysqlnd_plugin_count() * sizeof(void *); - MYSQLND_RES_UNBUFFERED * ret = mnd_ecalloc(1, alloc_size); + MYSQLND_MEMORY_POOL * pool; + MYSQLND_RES_UNBUFFERED * ret; DBG_ENTER("mysqlnd_result_unbuffered_init"); - if (!ret) { - DBG_RETURN(NULL); - } - if (!(ret->lengths = mnd_ecalloc(field_count, sizeof(size_t)))) { - mnd_efree(ret); - DBG_RETURN(NULL); - } - if (!(ret->result_set_memory_pool = mysqlnd_mempool_create(MYSQLND_G(mempool_default_size)))) { - mnd_efree(ret->lengths); - mnd_efree(ret); + pool = mysqlnd_mempool_create(MYSQLND_G(mempool_default_size)); + if (!pool) { DBG_RETURN(NULL); } + ret = MYSQLND_MEMORY_POOL_CHUNK_PTR(pool->get_chunk(pool, alloc_size)); + memset(ret, 0, alloc_size); + + ret->lengths = MYSQLND_MEMORY_POOL_CHUNK_PTR(pool->get_chunk(pool, field_count * sizeof(size_t))); + memset(ret->lengths, 0, field_count * sizeof(size_t)); + + ret->result_set_memory_pool = pool; ret->field_count= field_count; ret->ps = ps; @@ -1966,27 +1936,28 @@ PHPAPI MYSQLND_RES_BUFFERED_ZVAL * mysqlnd_result_buffered_zval_init(const unsigned int field_count, const zend_bool ps) { const size_t alloc_size = sizeof(MYSQLND_RES_BUFFERED_ZVAL) + mysqlnd_plugin_count() * sizeof(void *); - MYSQLND_RES_BUFFERED_ZVAL * ret = mnd_ecalloc(1, alloc_size); + MYSQLND_MEMORY_POOL * pool; + MYSQLND_RES_BUFFERED_ZVAL * ret; DBG_ENTER("mysqlnd_result_buffered_zval_init"); - if (!ret) { + pool = mysqlnd_mempool_create(MYSQLND_G(mempool_default_size)); + if (!pool) { DBG_RETURN(NULL); } + + ret = MYSQLND_MEMORY_POOL_CHUNK_PTR(pool->get_chunk(pool, alloc_size)); + memset(ret, 0, alloc_size); + if (FAIL == mysqlnd_error_info_init(&ret->error_info, 0)) { - mnd_efree(ret); - DBG_RETURN(NULL); - } - if (!(ret->lengths = mnd_ecalloc(field_count, sizeof(size_t)))) { - mnd_efree(ret); - DBG_RETURN(NULL); - } - if (!(ret->result_set_memory_pool = mysqlnd_mempool_create(MYSQLND_G(mempool_default_size)))) { - mnd_efree(ret->lengths); - mnd_efree(ret); + mysqlnd_mempool_destroy(pool); DBG_RETURN(NULL); } + ret->lengths = MYSQLND_MEMORY_POOL_CHUNK_PTR(pool->get_chunk(pool, field_count * sizeof(size_t))); + memset(ret->lengths, 0, field_count * sizeof(size_t)); + + ret->result_set_memory_pool = pool; ret->field_count= field_count; ret->ps = ps; ret->m = *mysqlnd_result_buffered_get_methods(); @@ -2012,27 +1983,28 @@ PHPAPI MYSQLND_RES_BUFFERED_C * mysqlnd_result_buffered_c_init(const unsigned int field_count, const zend_bool ps) { const size_t alloc_size = sizeof(MYSQLND_RES_BUFFERED_C) + mysqlnd_plugin_count() * sizeof(void *); - MYSQLND_RES_BUFFERED_C * ret = mnd_ecalloc(1, alloc_size); + MYSQLND_MEMORY_POOL * pool; + MYSQLND_RES_BUFFERED_C * ret; DBG_ENTER("mysqlnd_result_buffered_c_init"); - if (!ret) { + pool = mysqlnd_mempool_create(MYSQLND_G(mempool_default_size)); + if (!pool) { DBG_RETURN(NULL); } + + ret = MYSQLND_MEMORY_POOL_CHUNK_PTR(pool->get_chunk(pool, alloc_size)); + memset(ret, 0, alloc_size); + if (FAIL == mysqlnd_error_info_init(&ret->error_info, 0)) { - mnd_efree(ret); - DBG_RETURN(NULL); - } - if (!(ret->lengths = mnd_ecalloc(field_count, sizeof(size_t)))) { - mnd_efree(ret); - DBG_RETURN(NULL); - } - if (!(ret->result_set_memory_pool = mysqlnd_mempool_create(MYSQLND_G(mempool_default_size)))) { - mnd_efree(ret->lengths); - mnd_efree(ret); + mysqlnd_mempool_destroy(pool); DBG_RETURN(NULL); } + ret->lengths = MYSQLND_MEMORY_POOL_CHUNK_PTR(pool->get_chunk(pool, field_count * sizeof(size_t))); + memset(ret->lengths, 0, field_count * sizeof(size_t)); + + ret->result_set_memory_pool = pool; ret->field_count= field_count; ret->ps = ps; ret->m = *mysqlnd_result_buffered_get_methods(); diff --git a/ext/mysqlnd/mysqlnd_structs.h b/ext/mysqlnd/mysqlnd_structs.h index 094fdf9bed..60ef369999 100644 --- a/ext/mysqlnd/mysqlnd_structs.h +++ b/ext/mysqlnd/mysqlnd_structs.h @@ -57,23 +57,21 @@ typedef struct st_mysqlnd_memory_pool_chunk_llist MYSQLND_MEMORY_POOL_CHUNK_LLIS struct st_mysqlnd_memory_pool { - zend_uchar *arena; - unsigned int arena_size; - unsigned int free_size; + zend_arena *arena; MYSQLND_MEMORY_POOL_CHUNK* (*get_chunk)(MYSQLND_MEMORY_POOL * pool, unsigned int size); - enum_func_status (*resize_chunk)(MYSQLND_MEMORY_POOL * pool, MYSQLND_MEMORY_POOL_CHUNK * chunk, unsigned int size); - void (*free_chunk)(MYSQLND_MEMORY_POOL * pool, MYSQLND_MEMORY_POOL_CHUNK * chunk); + MYSQLND_MEMORY_POOL_CHUNK* (*resize_chunk)(MYSQLND_MEMORY_POOL * pool, MYSQLND_MEMORY_POOL_CHUNK * chunk, unsigned int size); + void (*free_chunk)(MYSQLND_MEMORY_POOL * pool, MYSQLND_MEMORY_POOL_CHUNK * chunk); }; struct st_mysqlnd_memory_pool_chunk { - size_t app; - zend_uchar *ptr; - unsigned int size; - zend_bool from_pool; + size_t app; + size_t size; }; +#define MYSQLND_MEMORY_POOL_CHUNK_PTR(chunk) \ + ((void*)((char*)(chunk) + sizeof(MYSQLND_MEMORY_POOL_CHUNK))) typedef struct st_mysqlnd_cmd_buffer { diff --git a/ext/mysqlnd/mysqlnd_wireprotocol.c b/ext/mysqlnd/mysqlnd_wireprotocol.c index b38fb32b42..962d40a188 100644 --- a/ext/mysqlnd/mysqlnd_wireprotocol.c +++ b/ext/mysqlnd/mysqlnd_wireprotocol.c @@ -1422,7 +1422,7 @@ php_mysqlnd_read_row_ex(MYSQLND_PFC * pfc, ret = FAIL; break; } - p = (*buffer)->ptr; + p = MYSQLND_MEMORY_POOL_CHUNK_PTR(*buffer); } else if (!first_iteration) { /* Empty packet after MYSQLND_MAX_PACKET_SIZE packet. That's ok, break */ if (!header.size) { @@ -1432,13 +1432,14 @@ php_mysqlnd_read_row_ex(MYSQLND_PFC * pfc, /* We have to realloc the buffer. */ - if (FAIL == pool->resize_chunk(pool, *buffer, *data_size + prealloc_more_bytes)) { + *buffer = pool->resize_chunk(pool, *buffer, *data_size + prealloc_more_bytes); + if (!*buffer) { SET_OOM_ERROR(error_info); ret = FAIL; break; } /* The position could have changed, recalculate */ - p = (*buffer)->ptr + (*data_size - header.size); + p = MYSQLND_MEMORY_POOL_CHUNK_PTR(*buffer) + (*data_size - header.size); } if (PASS != (ret = pfc->data->m.receive(pfc, vio, p, header.size, stats, error_info))) { @@ -1467,7 +1468,7 @@ php_mysqlnd_rowp_read_binary_protocol(MYSQLND_MEMORY_POOL_CHUNK * row_buffer, zv zend_bool as_int_or_float, MYSQLND_STATS * stats) { unsigned int i; - const zend_uchar * p = row_buffer->ptr; + const zend_uchar * p = MYSQLND_MEMORY_POOL_CHUNK_PTR(row_buffer); const zend_uchar * null_ptr; zend_uchar bit; zval *current_field, *end_field, *start_field; @@ -1559,9 +1560,9 @@ php_mysqlnd_rowp_read_text_protocol_aux(MYSQLND_MEMORY_POOL_CHUNK * row_buffer, { unsigned int i; zval *current_field, *end_field, *start_field; - zend_uchar * p = row_buffer->ptr; + zend_uchar * p = MYSQLND_MEMORY_POOL_CHUNK_PTR(row_buffer); size_t data_size = row_buffer->app; - const zend_uchar * const packet_end = (zend_uchar*) row_buffer->ptr + data_size; + const zend_uchar * const packet_end = (zend_uchar*) p + data_size; DBG_ENTER("php_mysqlnd_rowp_read_text_protocol_aux"); @@ -1773,7 +1774,7 @@ php_mysqlnd_rowp_read(MYSQLND_CONN_DATA * conn, void * _packet) packet->header.size = data_size; packet->row_buffer->app = data_size; - if (ERROR_MARKER == (*(p = packet->row_buffer->ptr))) { + if (ERROR_MARKER == (*(p = MYSQLND_MEMORY_POOL_CHUNK_PTR(packet->row_buffer)))) { /* Error message as part of the result set, not good but we should not hang. See: -- 2.40.0