From 94e165096fd65e8237e60de570fb609604ab94c9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 1 Oct 2019 11:40:54 +0800 Subject: [PATCH] bpo-38319: Fix shutil._fastcopy_sendfile(): set sendfile() max block size (GH-16491) --- Lib/shutil.py | 10 +++++++--- Lib/socket.py | 4 ++-- .../Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index f0d0336636..f97de788d9 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -135,9 +135,13 @@ def _fastcopy_sendfile(fsrc, fdst): # should not make any difference, also in case the file content # changes while being copied. try: - blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MB - except Exception: - blocksize = 2 ** 27 # 128MB + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MiB + except OSError: + blocksize = 2 ** 27 # 128MiB + # On 32-bit architectures truncate to 1GiB to avoid OverflowError, + # see bpo-38319. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) offset = 0 while True: diff --git a/Lib/socket.py b/Lib/socket.py index e5989d9dfd..84a5dcb0da 100755 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -356,8 +356,8 @@ class socket(_socket.socket): raise _GiveupOnSendfile(err) # not a regular file if not fsize: return 0 # empty file - blocksize = fsize if not count else count - + # Truncate to 1GiB to avoid OverflowError, see bpo-38319. + blocksize = min(count or fsize, 2 ** 30) timeout = self.gettimeout() if timeout == 0: raise ValueError("non-blocking sockets are not supported") diff --git a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst new file mode 100644 index 0000000000..376a9e459d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst @@ -0,0 +1,2 @@ +sendfile() used in socket and shutil modules was raising OverflowError for +files >= 2GiB on 32-bit architectures. (patch by Giampaolo Rodola) -- 2.50.0