bpo-36610: shutil.copyfile(): use sendfile() on Linux only (GH-13675)

...and avoid using it on Solaris as it can raise EINVAL if offset is equal or bigger than the size of the file

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 4af5a16806088d802aece6726a0ea36a13136ecb..dcb2a16cff98cbfa7a4a6f67ea3bed8b40a82285 100644 (file)
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -420,8 +420,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``".
 
 On macOS `fcopyfile`_ is used to copy the file content (not metadata).
 
-On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports
-copies between 2 regular file descriptors :func:`os.sendfile` is used.
+On Linux :func:`os.sendfile` is used.
 
 On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB
 instead of 64 KiB) and a :func:`memoryview`-based variant of

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 5ee9cf07ebaf11b5516d8c78526a3d2cda9f73a0..98f0c3474f26ed79c12f8eb18ccd758d6ba7375a 100644 (file)
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -772,7 +772,7 @@ Optimizations
 
 * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
   :func:`shutil.copytree` and :func:`shutil.move` use platform-specific
-  "fast-copy" syscalls on Linux, macOS and Solaris in order to copy the file
+  "fast-copy" syscalls on Linux and macOS in order to copy the file
   more efficiently.
   "fast-copy" means that the copying operation occurs within the kernel,
   avoiding the use of userspace buffers in Python as in

diff --git a/Lib/shutil.py b/Lib/shutil.py
index 2dfae87c9ce9640dd5da2af400e78d703568e973..dae916b41605c6974ee44cc5ec31ee68dd1d6391 100644 (file)
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -50,7 +50,7 @@ elif _WINDOWS:
     import nt
 
 COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024
-_HAS_SENDFILE = posix and hasattr(os, "sendfile")
+_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux")
 _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile")  # macOS
 
 __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
@@ -111,7 +111,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags):
 def _fastcopy_sendfile(fsrc, fdst):
     """Copy data from one regular mmap-like fd to another by using
     high-performance sendfile(2) syscall.
-    This should work on Linux >= 2.6.33 and Solaris only.
+    This should work on Linux >= 2.6.33 only.
     """
     # Note: copyfileobj() is left alone in order to not introduce any
     # unexpected breakage. Possible risks by using zero-copy calls
@@ -122,7 +122,7 @@ def _fastcopy_sendfile(fsrc, fdst):
     #   GzipFile (which decompresses data), HTTPResponse (which decodes
     #   chunks).
     # - possibly others (e.g. encrypted fs/partition?)
-    global _HAS_SENDFILE
+    global _USE_CP_SENDFILE
     try:
         infd = fsrc.fileno()
         outfd = fdst.fileno()
@@ -152,7 +152,7 @@ def _fastcopy_sendfile(fsrc, fdst):
                 # sendfile() on this platform (probably Linux < 2.6.33)
                 # does not support copies between regular files (only
                 # sockets).
-                _HAS_SENDFILE = False
+                _USE_CP_SENDFILE = False
                 raise _GiveupOnFastCopy(err)
 
             if err.errno == errno.ENOSPC:  # filesystem is full
@@ -260,8 +260,8 @@ def copyfile(src, dst, *, follow_symlinks=True):
                     return dst
                 except _GiveupOnFastCopy:
                     pass
-            # Linux / Solaris
-            elif _HAS_SENDFILE:
+            # Linux
+            elif _USE_CP_SENDFILE:
                 try:
                     _fastcopy_sendfile(fsrc, fdst)
                     return dst

diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index eeebb97ff6921c4647a8a622d31fdf5b18c37b47..208718bb1281050c0570a9c024cc156a46291f0f 100644 (file)
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -2315,7 +2315,7 @@ class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase):
         # Emulate a case where sendfile() only support file->socket
         # fds. In such a case copyfile() is supposed to skip the
         # fast-copy attempt from then on.
-        assert shutil._HAS_SENDFILE
+        assert shutil._USE_CP_SENDFILE
         try:
             with unittest.mock.patch(
                     self.PATCHPOINT,
@@ -2324,13 +2324,13 @@ class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase):
                     with self.assertRaises(_GiveupOnFastCopy):
                         shutil._fastcopy_sendfile(src, dst)
                 assert m.called
-            assert not shutil._HAS_SENDFILE
+            assert not shutil._USE_CP_SENDFILE
 
             with unittest.mock.patch(self.PATCHPOINT) as m:
                 shutil.copyfile(TESTFN, TESTFN2)
                 assert not m.called
         finally:
-            shutil._HAS_SENDFILE = True
+            shutil._USE_CP_SENDFILE = True
 
 
 @unittest.skipIf(not MACOS, 'macOS only')

diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst
index 3d5e6336246e40bb01ddac4975529b7178598413..f4b0a048330004777372dee9b74156eb893765b9 100644 (file)
--- a/Misc/NEWS.d/3.8.0a1.rst
+++ b/Misc/NEWS.d/3.8.0a1.rst
@@ -4450,7 +4450,7 @@ data_received() being called before connection_made().
 
 :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
 :func:`shutil.copytree` and :func:`shutil.move` use platform-specific
-fast-copy syscalls on Linux, Solaris and macOS in order to copy the file
+fast-copy syscalls on Linux and macOS in order to copy the file
 more efficiently. On Windows :func:`shutil.copyfile` uses a bigger default
 buffer size (1 MiB instead of 16 KiB) and a :func:`memoryview`-based variant
 of :func:`shutil.copyfileobj` is used. The speedup for copying a 512MiB file