From: Antoine Pitrou <pitrou@free.fr>
Date: Sun, 23 Jul 2017 11:05:26 +0000 (+0200)
Subject: bpo-30919: shared memory allocation performance regression in multiprocessing (#2708)
X-Git-Tag: v3.7.0a1~386
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=3051f0b78e53d1b771b49375dc139ca13f9fd76e;p=python

bpo-30919: shared memory allocation performance regression in multiprocessing (#2708)

* Fix #30919: shared memory allocation performance regression in multiprocessing

* Change strategy for Arena directory choice

* Add blurb
---

diff --git a/Lib/multiprocessing/heap.py b/Lib/multiprocessing/heap.py
index 443321535e..ee3ed551d0 100644
--- a/Lib/multiprocessing/heap.py
+++ b/Lib/multiprocessing/heap.py
@@ -60,26 +60,32 @@ if sys.platform == 'win32':
 else:
 
     class Arena(object):
+        if sys.platform == 'linux':
+            _dir_candidates = ['/dev/shm']
+        else:
+            _dir_candidates = []
 
         def __init__(self, size, fd=-1):
             self.size = size
             self.fd = fd
             if fd == -1:
                 self.fd, name = tempfile.mkstemp(
-                     prefix='pym-%d-'%os.getpid(), dir=util.get_temp_dir())
+                     prefix='pym-%d-'%os.getpid(),
+                     dir=self._choose_dir(size))
                 os.unlink(name)
                 util.Finalize(self, os.close, (self.fd,))
-                with open(self.fd, 'wb', closefd=False) as f:
-                    bs = 1024 * 1024
-                    if size >= bs:
-                        zeros = b'\0' * bs
-                        for _ in range(size // bs):
-                            f.write(zeros)
-                        del zeros
-                    f.write(b'\0' * (size % bs))
-                    assert f.tell() == size
+                os.ftruncate(self.fd, size)
             self.buffer = mmap.mmap(self.fd, self.size)
 
+        def _choose_dir(self, size):
+            # Choose a non-storage backed directory if possible,
+            # to improve performance
+            for d in self._dir_candidates:
+                st = os.statvfs(d)
+                if st.f_bavail * st.f_frsize >= size:  # enough free space?
+                    return d
+            return util.get_temp_dir()
+
     def reduce_arena(a):
         if a.fd == -1:
             raise ValueError('Arena is unpicklable because '
diff --git a/Misc/NEWS.d/next/Library/2017-07-23-11-33-10.bpo-30919.5dYRru.rst b/Misc/NEWS.d/next/Library/2017-07-23-11-33-10.bpo-30919.5dYRru.rst
new file mode 100644
index 0000000000..44c3a22bc8
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-07-23-11-33-10.bpo-30919.5dYRru.rst
@@ -0,0 +1,4 @@
+Fix shared memory performance regression in multiprocessing in 3.x.
+
+Shared memory used anonymous memory mappings in 2.x, while 3.x mmaps actual
+files. Try to be careful to do as little disk I/O as possible.