Fix bug #80107: Handling of large compressed packets

There's two layers of packet splitting going on. First, packets
need to be split into having a payload of exactly 2^24-1 bytes or
being the last packet. If the split packet has size between 2^24-5
and 2^24-1 bytes, the compressed packets also needs to be split,
though the choice of split doesn't matter here. I'm splitting off
the first 8192 bytes, as that's what I observe libmysqlclient to be
doing.

diff --git a/NEWS b/NEWS
index 9d3d53aca10770ce5e681a7bd62fabdc57d41157..fe2bae3803352b63d171d8f225ee7cc9b9bf3790 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,8 @@ PHP                                                                        NEWS
 - MySQLnd:
   . Fixed bug #80115 (mysqlnd.debug doesn't recognize absolute paths with
     slashes). (cmb)
+  . Fixed bug #80107 (mysqli_query() fails for ~16 MB long query when
+    compression is enabled). (Nikita)
 
 - OPcache:
   . Fixed bug #80083 (Optimizer pass 6 removes variables used for ibm_db2 data

diff --git a/ext/mysqli/tests/mysqli_real_connect_compression_error.phpt b/ext/mysqli/tests/mysqli_real_connect_compression_error.phpt
index 114f819edca10d32ec5ec403daf51b5ed54e117f..df0e4dc73bd3888966dfbdf40cf9c89907ceaeb2 100644 (file)
--- a/ext/mysqli/tests/mysqli_real_connect_compression_error.phpt
+++ b/ext/mysqli/tests/mysqli_real_connect_compression_error.phpt
@@ -1,19 +1,25 @@
 --TEST--
 Bug #80107 mysqli_query() fails for ~16 MB long query when compression is enabled
---XFAIL--
-The second INSERT query fails with MySQL server has gone away
 --SKIPIF--
 <?php
 require_once('skipif.inc');
 require_once('skipifemb.inc');
-require_once('skipifconnectfailure.inc');
+require_once("connect.inc");
+$link = @my_mysqli_connect($host, $user, $passwd, $db, $port, $socket);
+if (!$link) {
+    die(sprintf("skip Can't connect to MySQL Server - [%d] %s", mysqli_connect_errno(), mysqli_connect_error()));
+}
+$result = $link->query("SHOW VARIABLES LIKE 'max_allowed_packet'");
+if ($result->fetch_assoc()['Value'] < 0xffffff) {
+    die('skip max_allowed_packet is less than 0xffffff');
+}
 ?>
---INI--
-mysqli.allow_local_infile=1
 --FILE--
 <?php
 
-include("connect.inc");
+require_once("connect.inc");
+
+$data_size = 16777174;
 
 // Insert with compression disabled:
 
@@ -22,7 +28,7 @@ $result = my_mysqli_real_connect($mysqli, $host, $user, $passwd, $db, $port, $so
 $mysqli->query("DROP TABLE IF EXISTS test");
 $mysqli->query("CREATE TABLE test (`blob` LONGBLOB NOT NULL)");
 
-$data = str_repeat("x", 16777174);
+$data = str_repeat("x", $data_size);
 $mysqli->query("INSERT INTO $db.test(`blob`) VALUE ('$data')");
 
 var_dump(mysqli_error_list($mysqli));
@@ -33,7 +39,7 @@ $mysqli->close();
 $mysqli = mysqli_init();
 $result = my_mysqli_real_connect($mysqli, $host, $user, $passwd, $db, $port, $socket, MYSQLI_CLIENT_COMPRESS);
 
-$data = str_repeat("x", 16777174);
+$data = str_repeat("x", $data_size);
 $mysqli->query("INSERT INTO $db.test(`blob`) VALUE ('$data')");
 
 var_dump(mysqli_error_list($mysqli));

diff --git a/ext/mysqlnd/mysqlnd_protocol_frame_codec.c b/ext/mysqlnd/mysqlnd_protocol_frame_codec.c
index 4f84c02c5dc8dc0f1f72c8404407a2bd2a1251dd..7b18402be1c5d9a286babae9009f0b83216188ae 100644 (file)
--- a/ext/mysqlnd/mysqlnd_protocol_frame_codec.c
+++ b/ext/mysqlnd/mysqlnd_protocol_frame_codec.c
@@ -50,6 +50,57 @@ MYSQLND_METHOD(mysqlnd_pfc, reset)(MYSQLND_PFC * const pfc, MYSQLND_STATS * cons
 #define STORE_HEADER_SIZE(safe_storage, buffer)  COPY_HEADER((safe_storage), (buffer))
 #define RESTORE_HEADER_SIZE(buffer, safe_storage) STORE_HEADER_SIZE((safe_storage), (buffer))
 
+#ifdef MYSQLND_COMPRESSION_ENABLED
+static size_t write_compressed_packet(
+               const MYSQLND_PFC *pfc, MYSQLND_VIO *vio,
+               MYSQLND_STATS *conn_stats, MYSQLND_ERROR_INFO *error_info,
+               zend_uchar *uncompressed_payload, size_t to_be_sent, zend_uchar *compress_buf) {
+       DBG_ENTER("write_compressed_packet");
+       /* here we need to compress the data and then write it, first comes the compressed header */
+       size_t tmp_complen = to_be_sent;
+       size_t payload_size;
+       if (PASS == pfc->data->m.encode((compress_buf + COMPRESSED_HEADER_SIZE + MYSQLND_HEADER_SIZE), &tmp_complen,
+                                                          uncompressed_payload, to_be_sent))
+       {
+               int3store(compress_buf + MYSQLND_HEADER_SIZE, to_be_sent);
+               payload_size = tmp_complen;
+       } else {
+               int3store(compress_buf + MYSQLND_HEADER_SIZE, 0);
+               memcpy(compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, uncompressed_payload, to_be_sent);
+               payload_size = to_be_sent;
+       }
+
+       int3store(compress_buf, payload_size);
+       int1store(compress_buf + 3, pfc->data->compressed_envelope_packet_no);
+       DBG_INF_FMT("writing "MYSQLND_SZ_T_SPEC" bytes to the network", payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE);
+
+       size_t bytes_sent = vio->data->m.network_write(vio, compress_buf, payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, conn_stats, error_info);
+       pfc->data->compressed_envelope_packet_no++;
+#ifdef WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY
+       if (res == Z_OK) {
+               size_t decompressed_size = left + MYSQLND_HEADER_SIZE;
+               zend_uchar * decompressed_data = mnd_malloc(decompressed_size);
+               int error = pfc->data->m.decode(decompressed_data, decompressed_size,
+                                                                               compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, payload_size);
+               if (error == Z_OK) {
+                       int i;
+                       DBG_INF("success decompressing");
+                       for (i = 0 ; i < decompressed_size; i++) {
+                               if (i && (i % 30 == 0)) {
+                                       printf("\n\t\t");
+                               }
+                               printf("%.2X ", (int)*((char*)&(decompressed_data[i])));
+                               DBG_INF_FMT("%.2X ", (int)*((char*)&(decompressed_data[i])));
+                       }
+               } else {
+                       DBG_INF("error decompressing");
+               }
+               mnd_free(decompressed_data);
+       }
+#endif /* WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY */
+       DBG_RETURN(bytes_sent);
+}
+#endif
 
 /* {{{ mysqlnd_pfc::send */
 /*
@@ -91,53 +142,27 @@ MYSQLND_METHOD(mysqlnd_pfc, send)(MYSQLND_PFC * const pfc, MYSQLND_VIO * const v
                DBG_INF_FMT("packet_no=%u", pfc->data->packet_no);
 #ifdef MYSQLND_COMPRESSION_ENABLED
                if (pfc->data->compressed == TRUE) {
-                       /* here we need to compress the data and then write it, first comes the compressed header */
-                       size_t tmp_complen = to_be_sent;
-                       size_t payload_size;
                        zend_uchar * uncompressed_payload = p; /* should include the header */
-
                        STORE_HEADER_SIZE(safe_storage, uncompressed_payload);
                        int3store(uncompressed_payload, to_be_sent);
                        int1store(uncompressed_payload + 3, pfc->data->packet_no);
-                       if (PASS == pfc->data->m.encode((compress_buf + COMPRESSED_HEADER_SIZE + MYSQLND_HEADER_SIZE), &tmp_complen,
-                                                                          uncompressed_payload, to_be_sent + MYSQLND_HEADER_SIZE))
-                       {
-                               int3store(compress_buf + MYSQLND_HEADER_SIZE, to_be_sent + MYSQLND_HEADER_SIZE);
-                               payload_size = tmp_complen;
+                       if (to_be_sent <= MYSQLND_MAX_PACKET_SIZE - MYSQLND_HEADER_SIZE) {
+                               bytes_sent = write_compressed_packet(
+                                       pfc, vio, conn_stats, error_info,
+                                       uncompressed_payload, to_be_sent + MYSQLND_HEADER_SIZE, compress_buf);
                        } else {
-                               int3store(compress_buf + MYSQLND_HEADER_SIZE, 0);
-                               memcpy(compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, uncompressed_payload, to_be_sent + MYSQLND_HEADER_SIZE);
-                               payload_size = to_be_sent + MYSQLND_HEADER_SIZE;
+                               /* The uncompressed size including the header would overflow. Split into two
+                                * compressed packets. The size of the first one is relatively arbitrary here. */
+                               const size_t split_off_bytes = 8192;
+                               bytes_sent = write_compressed_packet(
+                                       pfc, vio, conn_stats, error_info,
+                                       uncompressed_payload, split_off_bytes, compress_buf);
+                               bytes_sent = write_compressed_packet(
+                                       pfc, vio, conn_stats, error_info,
+                                       uncompressed_payload + split_off_bytes,
+                                       to_be_sent + MYSQLND_HEADER_SIZE - split_off_bytes, compress_buf);
                        }
                        RESTORE_HEADER_SIZE(uncompressed_payload, safe_storage);
-
-                       int3store(compress_buf, payload_size);
-                       int1store(compress_buf + 3, pfc->data->packet_no);
-                       DBG_INF_FMT("writing "MYSQLND_SZ_T_SPEC" bytes to the network", payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE);
-                       bytes_sent = vio->data->m.network_write(vio, compress_buf, payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, conn_stats, error_info);
-                       pfc->data->compressed_envelope_packet_no++;
-  #if WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY
-                       if (res == Z_OK) {
-                               size_t decompressed_size = left + MYSQLND_HEADER_SIZE;
-                               zend_uchar * decompressed_data = mnd_malloc(decompressed_size);
-                               int error = pfc->data->m.decode(decompressed_data, decompressed_size,
-                                                                                               compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, payload_size);
-                               if (error == Z_OK) {
-                                       int i;
-                                       DBG_INF("success decompressing");
-                                       for (i = 0 ; i < decompressed_size; i++) {
-                                               if (i && (i % 30 == 0)) {
-                                                       printf("\n\t\t");
-                                               }
-                                               printf("%.2X ", (int)*((char*)&(decompressed_data[i])));
-                                               DBG_INF_FMT("%.2X ", (int)*((char*)&(decompressed_data[i])));
-                                       }
-                               } else {
-                                       DBG_INF("error decompressing");
-                               }
-                               mnd_free(decompressed_data);
-                       }
-  #endif /* WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY */
                } else
 #endif /* MYSQLND_COMPRESSION_ENABLED */
                {