bpo-17560: Too small type for struct.pack/unpack in mutliprocessing.Connection (GH...

Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems.

diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index 1f3ea504fff439337426f8466f25ecff90dce2fb..c9f995e5fa7f57f1d1e058be0e2d3fba94025397 100644 (file)
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -389,23 +389,33 @@ class Connection(_ConnectionBase):
 
     def _send_bytes(self, buf):
         n = len(buf)
-        # For wire compatibility with 3.2 and lower
-        header = struct.pack("!i", n)
-        if n > 16384:
-            # The payload is large so Nagle's algorithm won't be triggered
-            # and we'd better avoid the cost of concatenation.
+        if n > 0x7fffffff:
+            pre_header = struct.pack("!i", -1)
+            header = struct.pack("!Q", n)
+            self._send(pre_header)
             self._send(header)
             self._send(buf)
         else:
-            # Issue #20540: concatenate before sending, to avoid delays due
-            # to Nagle's algorithm on a TCP socket.
-            # Also note we want to avoid sending a 0-length buffer separately,
-            # to avoid "broken pipe" errors if the other end closed the pipe.
-            self._send(header + buf)
+            # For wire compatibility with 3.7 and lower
+            header = struct.pack("!i", n)
+            if n > 16384:
+                # The payload is large so Nagle's algorithm won't be triggered
+                # and we'd better avoid the cost of concatenation.
+                self._send(header)
+                self._send(buf)
+            else:
+                # Issue #20540: concatenate before sending, to avoid delays due
+                # to Nagle's algorithm on a TCP socket.
+                # Also note we want to avoid sending a 0-length buffer separately,
+                # to avoid "broken pipe" errors if the other end closed the pipe.
+                self._send(header + buf)
 
     def _recv_bytes(self, maxsize=None):
         buf = self._recv(4)
         size, = struct.unpack("!i", buf.getvalue())
+        if size == -1:
+            buf = self._recv(8)
+            size, = struct.unpack("!Q", buf.getvalue())
         if maxsize is not None and size > maxsize:
             return None
         return self._recv(size)

diff --git a/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst b/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst
new file mode 100644 (file)
index 0000000..7cc9ed3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst
@@ -0,0 +1 @@
+Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems.
\ No newline at end of file