bpo-38319: Fix shutil._fastcopy_sendfile(): set sendfile() max block size (GH-16491)

diff --git a/Lib/shutil.py b/Lib/shutil.py
index f0d03366368edffffaf89a56701e759ff8e38405..f97de788d9d406d8cfa27dc70ac084ad2c4735f8 100644 (file)
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -135,9 +135,13 @@ def _fastcopy_sendfile(fsrc, fdst):
     # should not make any difference, also in case the file content
     # changes while being copied.
     try:
-        blocksize = max(os.fstat(infd).st_size, 2 ** 23)  # min 8MB
-    except Exception:
-        blocksize = 2 ** 27  # 128MB
+        blocksize = max(os.fstat(infd).st_size, 2 ** 23)  # min 8MiB
+    except OSError:
+        blocksize = 2 ** 27  # 128MiB
+    # On 32-bit architectures truncate to 1GiB to avoid OverflowError,
+    # see bpo-38319.
+    if sys.maxsize < 2 ** 32:
+        blocksize = min(blocksize, 2 ** 30)
 
     offset = 0
     while True:

diff --git a/Lib/socket.py b/Lib/socket.py
index e5989d9dfd40940b25dabbacd70c35951a1bd705..84a5dcb0daf29ae492ea05aab6cd403c306c530a 100755 (executable)
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -356,8 +356,8 @@ class socket(_socket.socket):
                 raise _GiveupOnSendfile(err)  # not a regular file
             if not fsize:
                 return 0  # empty file
-            blocksize = fsize if not count else count
-
+            # Truncate to 1GiB to avoid OverflowError, see bpo-38319.
+            blocksize = min(count or fsize, 2 ** 30)
             timeout = self.gettimeout()
             if timeout == 0:
                 raise ValueError("non-blocking sockets are not supported")

diff --git a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst
new file mode 100644 (file)
index 0000000..376a9e4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst
@@ -0,0 +1,2 @@
+sendfile() used in socket and shutil modules was raising OverflowError for
+files >= 2GiB on 32-bit architectures.  (patch by Giampaolo Rodola)