Memory usage optimisation. mysqlnd is not libmysql. mysqlnd does use the

Zend allocator, which means that is easier to hit memory_limit if you
have big stored (buffered) result sets. Before with libmysql you won't
hit memory_limit because libmysql uses libc's allocator and nothing is
checked. Now, with mysqlnd the situation is stricter and it is easier to
hit memory_limit. We try to optimize for big result sets. If a result set
is larger than 10 rows we will start freeing some data to keep memory usage
after 10 rows constant. This will help in the cases where a buffered result
set is scrolled forward only and just only once, or mysqlnd will need to
decode data from the network buffers again - yes, it is a trade-off between
CPU time and memory size. The best for big result sets is of course using
unbuffered queries - for comparison : 3 Million rows with buffered take
at least 180MB, with buffered you will stay at 3MB, and unbuffered will be
just 7-8% slower.

diff --git a/ext/mysqlnd/mysqlnd_debug.c b/ext/mysqlnd/mysqlnd_debug.c
index a0527488506c36469065b56d3c920a3974cf4b7f..2b5724dbe9d849c77df57aaaf82f1221300dff71 100644 (file)
--- a/ext/mysqlnd/mysqlnd_debug.c
+++ b/ext/mysqlnd/mysqlnd_debug.c
@@ -830,6 +830,9 @@ void * _mysqlnd_perealloc(void *ptr, size_t new_size, zend_bool persistent MYSQL
 void _mysqlnd_efree(void *ptr MYSQLND_MEM_D)
 {
        DBG_ENTER(mysqlnd_efree_name);
+       if (!ptr) {
+               DBG_VOID_RETURN;
+       }
 #ifdef MYSQLND_THREADED
        if (MYSQLND_G(thread_id) != tsrm_thread_id()) {
                DBG_RETURN(_mysqlnd_pefree(ptr, 1 TSRMLS_CC ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC));

diff --git a/ext/mysqlnd/mysqlnd_result.c b/ext/mysqlnd/mysqlnd_result.c
index 8166406c6c01cc1c0f75f3272c852528aaee8ff4..5daa57333a160176f2cfa871f1351412233865b4 100644 (file)
--- a/ext/mysqlnd/mysqlnd_result.c
+++ b/ext/mysqlnd/mysqlnd_result.c
@@ -30,6 +30,9 @@
 #include "mysqlnd_debug.h"
 #include "ext/standard/basic_functions.h"
 
+#define START_FREEING_AFTER_X_ROWS 10
+static void mysqlnd_buffered_free_previous_row(MYSQLND_RES *result, int which TSRMLS_DC);
+
 #define MYSQLND_SILENT
 
 #ifdef MYSQLND_THREADED
@@ -101,11 +104,18 @@ void mysqlnd_res_initialize_result_set_rest(MYSQLND_RES * const result TSRMLS_DC
        unsigned int field_count = result->meta->field_count;
        unsigned int row_count = result->stored_data->row_count;
        DBG_ENTER("mysqlnd_res_initialize_result_set_rest");
+       DBG_INF_FMT("before heap=%lu real=%lu", zend_memory_usage(FALSE TSRMLS_CC), zend_memory_usage(TRUE TSRMLS_CC));
 
        if (!data_cursor || row_count == result->stored_data->initialized_rows) {
                DBG_VOID_RETURN;
        }
        while ((data_cursor - data_begin) < (row_count * field_count)) {
+               if (START_FREEING_AFTER_X_ROWS < ((data_cursor - data_begin) / result->field_count)) {
+                       zval **orig_data_cursor = result->stored_data->data_cursor;
+                       result->stored_data->data_cursor = data_cursor;
+                       mysqlnd_buffered_free_previous_row(result, START_FREEING_AFTER_X_ROWS TSRMLS_CC);
+                       result->stored_data->data_cursor = orig_data_cursor;
+               }
                if (NULL == data_cursor[0]) {
                        result->stored_data->initialized_rows++;
                        result->m.row_decoder(
@@ -130,6 +140,7 @@ void mysqlnd_res_initialize_result_set_rest(MYSQLND_RES * const result TSRMLS_DC
                }
                data_cursor += field_count;
        }
+       DBG_INF_FMT("after heap=%lu real=%lu", zend_memory_usage(FALSE TSRMLS_CC), zend_memory_usage(TRUE TSRMLS_CC));
        DBG_VOID_RETURN;
 }
 /* }}} */
@@ -186,6 +197,53 @@ void mysqlnd_unbuffered_free_last_data(MYSQLND_RES *result TSRMLS_DC)
 /* }}} */
 
 
+/* {{{ mysqlnd_buffered_free_previous_row */
+static
+void mysqlnd_buffered_free_previous_row(MYSQLND_RES *result, int which TSRMLS_DC)
+{
+       MYSQLND_RES_BUFFERED * set = result->stored_data;
+
+       DBG_ENTER("mysqlnd_buffered_free_previous_row");
+
+       if (!set) {
+               DBG_VOID_RETURN;
+       }
+
+       DBG_INF_FMT("which=%d result->field_count=%d data=%p data_cursor=%p", which, result->field_count, set->data, set->data_cursor);
+       if (set->data_cursor && ((set->data_cursor - (which * result->field_count) ) >= set->data)) {
+               unsigned int i, ctor_called_count = 0;
+               zend_bool copy_ctor_called;
+               MYSQLND_STATS *global_stats = result->conn? &result->conn->stats:NULL;
+               zval **current_row = set->data_cursor - (which * result->field_count);
+
+               DBG_INF_FMT("%u columns to free", result->field_count);
+               for (i = 0; i < result->field_count; i++) {
+                       if (current_row[i]) {
+                               mysqlnd_palloc_zval_ptr_dtor(&(current_row[i]),
+                                                                                        result->zval_cache, result->type,
+                                                                                        &copy_ctor_called TSRMLS_CC);
+                               if (copy_ctor_called) {
+                                       ctor_called_count++;
+                               }
+                               current_row[i] = NULL;
+                       }
+               }
+               DBG_INF_FMT("copy_ctor_called_count=%u", ctor_called_count);
+               /* By using value3 macros we hold a mutex only once, there is no value2 */
+               MYSQLND_INC_CONN_STATISTIC_W_VALUE3(global_stats,
+                                                                                       STAT_COPY_ON_WRITE_PERFORMED,
+                                                                                       ctor_called_count,
+                                                                                       STAT_COPY_ON_WRITE_SAVED,
+                                                                                       result->field_count - ctor_called_count,
+                                                                                       STAT_COPY_ON_WRITE_PERFORMED, 0);
+       }
+
+       DBG_VOID_RETURN;
+}
+/* }}} */
+
+
+
 /* {{{ mysqlnd_free_buffered_data */
 void mysqlnd_free_buffered_data(MYSQLND_RES *result TSRMLS_DC)
 {
@@ -205,16 +263,15 @@ void mysqlnd_free_buffered_data(MYSQLND_RES *result TSRMLS_DC)
 
                for (col = field_count - 1; col >= 0; --col) {
                        zend_bool copy_ctor_called;
-                       if (current_row[0] == NULL) {
-                               break;/* row that was never initialized */
-                       }
-                       mysqlnd_palloc_zval_ptr_dtor(&(current_row[col]), zval_cache,
+                       if (current_row[col] != NULL) {
+                               mysqlnd_palloc_zval_ptr_dtor(&(current_row[col]), zval_cache,
                                                                                 result->type, &copy_ctor_called TSRMLS_CC);
 #if MYSQLND_DEBUG_MEMORY
-                       DBG_INF_FMT("Copy_ctor_called=%d", copy_ctor_called);
+                               DBG_INF_FMT("Copy_ctor_called=%d", copy_ctor_called);
 #endif
-                       MYSQLND_INC_GLOBAL_STATISTIC(copy_ctor_called? STAT_COPY_ON_WRITE_PERFORMED:
-                                                                                                                  STAT_COPY_ON_WRITE_SAVED);
+                               MYSQLND_INC_GLOBAL_STATISTIC(copy_ctor_called? STAT_COPY_ON_WRITE_PERFORMED:
+                                                                                                                          STAT_COPY_ON_WRITE_SAVED);
+                       }
                }
 #if MYSQLND_DEBUG_MEMORY
                DBG_INF("Freeing current_row & current_buffer");
@@ -1099,6 +1156,10 @@ mysqlnd_fetch_row_buffered(MYSQLND_RES *result, void *param, unsigned int flags,
                zval **current_row = set->data_cursor;
                MYSQLND_FIELD *field = result->meta->fields;
                struct mysqlnd_field_hash_key *zend_hash_key = result->meta->zend_hash_keys;
+               DBG_INF_FMT("row_num=%u", (set->data_cursor - set->data) / result->meta->field_count);
+               if (START_FREEING_AFTER_X_ROWS < ((set->data_cursor - set->data) / result->field_count)) {
+                       mysqlnd_buffered_free_previous_row(result, START_FREEING_AFTER_X_ROWS TSRMLS_CC);
+               }
 
                if (NULL == current_row[0]) {
                        uint64_t row_num = (set->data_cursor - set->data) / result->meta->field_count;
@@ -1848,6 +1909,7 @@ MYSQLND_METHOD(mysqlnd_res, fetch_into)(MYSQLND_RES *result, unsigned int flags,
 
        DBG_ENTER("mysqlnd_res::fetch_into");
        DBG_INF_FMT("flags=%u mysqlnd_extension=%d", flags, extension);
+       DBG_INF_FMT("memory in use: heap_size=%lu real_size=%u", zend_memory_usage(FALSE TSRMLS_CC), zend_memory_usage(TRUE TSRMLS_CC));
 
        if (!result->m.fetch_row) {
                RETVAL_NULL();
@@ -1859,7 +1921,7 @@ MYSQLND_METHOD(mysqlnd_res, fetch_into)(MYSQLND_RES *result, unsigned int flags,
        */
        mysqlnd_array_init(return_value, mysqlnd_num_fields(result) * 2);
        if (FAIL == result->m.fetch_row(result, (void *)return_value, flags, &fetched_anything TSRMLS_CC)) {
-               php_error_docref(NULL TSRMLS_CC, E_WARNING, "Error while reading a row");
+               php_error_docref(NULL TSRMLS_CC, E_WARNING, "Error while reading/decoding a row");
                RETVAL_FALSE;
        } else if (fetched_anything == FALSE) {
                zval_dtor(return_value);
@@ -1877,6 +1939,7 @@ MYSQLND_METHOD(mysqlnd_res, fetch_into)(MYSQLND_RES *result, unsigned int flags,
          return_value is IS_NULL for no more data and an array for data. Thus it's ok
          to return here.
        */
+       DBG_INF_FMT("returning: heap_size=%lu real_size=%u", zend_memory_usage(FALSE TSRMLS_CC), zend_memory_usage(TRUE TSRMLS_CC));
        DBG_VOID_RETURN;
 }
 /* }}} */