From: Martin Panter <vadmium+py@gmail.com>
Date: Sat, 23 Jul 2016 03:39:49 +0000 (+0000)
Subject: Issue #27130: Merge zlib 64-bit fixes from 3.5
X-Git-Tag: v3.6.0a4~136
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=525a949251686a9f8c795f0166797700ac56b77b;p=python

Issue #27130: Merge zlib 64-bit fixes from 3.5
---

525a949251686a9f8c795f0166797700ac56b77b
diff --cc Misc/NEWS
index c4cc66024b,921c89a5ff..b54592069d
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@@ -26,15 -22,20 +26,20 @@@ Core and Builtin
  Library
  -------
  
+ - Issue #27130: In the "zlib" module, fix handling of large buffers
+   (typically 4 GiB) when compressing and decompressing.  Previously, inputs
+   were limited to 4 GiB, and compression and decompression operations did not
+   properly handle results of 4 GiB.
+ 
 +- Issue #24773: Implemented PEP 495 (Local Time Disambiguation).
 +
 +- Expose the EPOLLEXCLUSIVE constant (when it is defined) in the select module.
 +
 +- Issue #27567: Expose the EPOLLRDHUP and POLLRDHUP constants in the select
 +  module.
 +
 +- Issue #1621: Avoid signed int negation overflow in the "audioop" module.
 +
  - Issue #27533: Release GIL in nt._isdir
  
  - Issue #17711: Fixed unpickling by the persistent ID with protocol 0.
diff --cc Modules/clinic/zlibmodule.c.h
index 1b2b7fed46,b1af7ce57c..506d5503c7
--- a/Modules/clinic/zlibmodule.c.h
+++ b/Modules/clinic/zlibmodule.c.h
@@@ -68,12 -65,11 +68,12 @@@ zlib_decompress(PyObject *module, PyObj
      PyObject *return_value = NULL;
      Py_buffer data = {NULL, NULL};
      int wbits = MAX_WBITS;
-     unsigned int bufsize = DEF_BUF_SIZE;
+     Py_ssize_t bufsize = DEF_BUF_SIZE;
  
      if (!PyArg_ParseTuple(args, "y*|iO&:decompress",
-         &data, &wbits, capped_uint_converter, &bufsize)) {
 -        &data, &wbits, ssize_t_converter, &bufsize))
++        &data, &wbits, ssize_t_converter, &bufsize)) {
          goto exit;
 +    }
      return_value = zlib_decompress_impl(module, &data, wbits, bufsize);
  
  exit:
@@@ -253,12 -243,11 +253,12 @@@ zlib_Decompress_decompress(compobject *
  {
      PyObject *return_value = NULL;
      Py_buffer data = {NULL, NULL};
-     unsigned int max_length = 0;
+     Py_ssize_t max_length = 0;
  
      if (!PyArg_ParseTuple(args, "y*|O&:decompress",
-         &data, capped_uint_converter, &max_length)) {
 -        &data, ssize_t_converter, &max_length))
++        &data, ssize_t_converter, &max_length)) {
          goto exit;
 +    }
      return_value = zlib_Decompress_decompress_impl(self, &data, max_length);
  
  exit:
@@@ -367,12 -354,11 +367,12 @@@ static PyObject 
  zlib_Decompress_flush(compobject *self, PyObject *args)
  {
      PyObject *return_value = NULL;
-     unsigned int length = DEF_BUF_SIZE;
+     Py_ssize_t length = DEF_BUF_SIZE;
  
      if (!PyArg_ParseTuple(args, "|O&:flush",
-         capped_uint_converter, &length)) {
 -        ssize_t_converter, &length))
++        ssize_t_converter, &length)) {
          goto exit;
 +    }
      return_value = zlib_Decompress_flush_impl(self, length);
  
  exit:
@@@ -460,4 -442,4 +460,4 @@@ exit
  #ifndef ZLIB_COMPRESS_COPY_METHODDEF
      #define ZLIB_COMPRESS_COPY_METHODDEF
  #endif /* !defined(ZLIB_COMPRESS_COPY_METHODDEF) */
- /*[clinic end generated code: output=519446af912f4e72 input=a9049054013a1b77]*/
 -/*[clinic end generated code: output=7711ef02d1d5776c input=a9049054013a1b77]*/
++/*[clinic end generated code: output=9046866b1ac5de7e input=a9049054013a1b77]*/
diff --cc Modules/zlibmodule.c
index 28f7f151ac,fccb6163e8..491bc8551c
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@@ -151,34 -209,17 +209,17 @@@ Returns a bytes object containing compr
  [clinic start generated code]*/
  
  static PyObject *
 -zlib_compress_impl(PyObject *module, Py_buffer *bytes, int level)
 -/*[clinic end generated code: output=ae64c2c3076321a0 input=be3abe9934bda4b3]*/
 +zlib_compress_impl(PyObject *module, Py_buffer *data, int level)
 +/*[clinic end generated code: output=d80906d73f6294c8 input=638d54b6315dbed3]*/
  {
-     PyObject *ReturnVal = NULL;
-     Byte *input, *output = NULL;
-     unsigned int length;
-     int err;
+     PyObject *RetVal = NULL;
+     Byte *ibuf;
+     Py_ssize_t ibuflen, obuflen = DEF_BUF_SIZE;
+     int err, flush;
      z_stream zst;
  
-     if ((size_t)data->len > UINT_MAX) {
-         PyErr_SetString(PyExc_OverflowError,
-                         "Size does not fit in an unsigned int");
-         goto error;
-     }
-     input = data->buf;
-     length = (unsigned int)data->len;
- 
-     zst.avail_out = length + length/1000 + 12 + 1;
- 
-     output = (Byte*)PyMem_Malloc(zst.avail_out);
-     if (output == NULL) {
-         PyErr_SetString(PyExc_MemoryError,
-                         "Can't allocate memory to compress data");
-         goto error;
-     }
- 
-     /* Past the point of no return.  From here on out, we need to make sure
-        we clean up mallocs & INCREFs. */
 -    ibuf = bytes->buf;
 -    ibuflen = bytes->len;
++    ibuf = data->buf;
++    ibuflen = data->len;
  
      zst.opaque = NULL;
      zst.zalloc = PyZlib_Malloc;