bpo-26732: fix too many fds in processes started with the "forkserver" method (#2813)

* bpo-26732: fix too many fds in processes started with the "forkserver" method

A child process would inherit as many fds as the number of still-running children.

* Add blurb and test comment

diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py
index b9f9b9dd8b556655dd04042297ea3d56edbe3ca1..69b842aa939a3c34c3fe067977af1360730587f0 100644 (file)
--- a/Lib/multiprocessing/forkserver.py
+++ b/Lib/multiprocessing/forkserver.py
@@ -236,8 +236,11 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
                             code = 1
                             try:
                                 listener.close()
+                                selector.close()
+                                unused_fds = [alive_r, child_w, sig_r, sig_w]
+                                unused_fds.extend(pid_to_fd.values())
                                 code = _serve_one(child_r, fds,
-                                                  (alive_r, child_w, sig_r, sig_w),
+                                                  unused_fds,
                                                   old_handlers)
                             except Exception:
                                 sys.excepthook(*sys.exc_info())

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 329a6d29acadf57a10b77844ff04448727b1433b..a14fa7422e7fab679ad08dd47dc3827b73faaf9a 100644 (file)
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -494,6 +494,40 @@ class _TestProcess(BaseTestCase):
         self.assertIs(wr(), None)
         self.assertEqual(q.get(), 5)
 
+    @classmethod
+    def _test_child_fd_inflation(self, evt, q):
+        q.put(test.support.fd_count())
+        evt.wait()
+
+    def test_child_fd_inflation(self):
+        # Number of fds in child processes should not grow with the
+        # number of running children.
+        if self.TYPE == 'threads':
+            self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+        sm = multiprocessing.get_start_method()
+        if sm == 'fork':
+            # The fork method by design inherits all fds from the parent,
+            # trying to go against it is a lost battle
+            self.skipTest('test not appropriate for {}'.format(sm))
+
+        N = 5
+        evt = self.Event()
+        q = self.Queue()
+
+        procs = [self.Process(target=self._test_child_fd_inflation, args=(evt, q))
+                 for i in range(N)]
+        for p in procs:
+            p.start()
+
+        try:
+            fd_counts = [q.get() for i in range(N)]
+            self.assertEqual(len(set(fd_counts)), 1, fd_counts)
+
+        finally:
+            evt.set()
+            for p in procs:
+                p.join()
 
 #
 #

diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py
index 8e93816d965877efb289470ce9d91ca197419a05..efe52107cb4d1f4c7105a048ace6e6e3d02783ff 100644 (file)
--- a/Lib/test/libregrtest/refleak.py
+++ b/Lib/test/libregrtest/refleak.py
@@ -7,36 +7,6 @@ from inspect import isabstract
 from test import support
 
 
-try:
-    MAXFD = os.sysconf("SC_OPEN_MAX")
-except Exception:
-    MAXFD = 256
-
-
-def fd_count():
-    """Count the number of open file descriptors"""
-    if sys.platform.startswith(('linux', 'freebsd')):
-        try:
-            names = os.listdir("/proc/self/fd")
-            return len(names)
-        except FileNotFoundError:
-            pass
-
-    count = 0
-    for fd in range(MAXFD):
-        try:
-            # Prefer dup() over fstat(). fstat() can require input/output
-            # whereas dup() doesn't.
-            fd2 = os.dup(fd)
-        except OSError as e:
-            if e.errno != errno.EBADF:
-                raise
-        else:
-            os.close(fd2)
-            count += 1
-    return count
-
-
 def dash_R(the_module, test, indirect_test, huntrleaks):
     """Run a test multiple times, looking for reference leaks.
 
@@ -174,7 +144,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
     func1 = sys.getallocatedblocks
     func2 = sys.gettotalrefcount
     gc.collect()
-    return func1(), func2(), fd_count()
+    return func1(), func2(), support.fd_count()
 
 
 def clear_caches():

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 313c23004ed79780ad6d8da081d2f08efb8c0a3e..3a4d27e6b8848b4364f5d33b565d958a76d41a55 100644 (file)
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -107,7 +107,7 @@ __all__ = [
     "check_warnings", "check_no_resource_warning", "EnvironmentVarGuard",
     "run_with_locale", "swap_item",
     "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict",
-    "run_with_tz", "PGO", "missing_compiler_executable",
+    "run_with_tz", "PGO", "missing_compiler_executable", "fd_count",
     ]
 
 class Error(Exception):
@@ -2647,3 +2647,34 @@ def disable_faulthandler():
     finally:
         if is_enabled:
             faulthandler.enable(file=fd, all_threads=True)
+
+
+try:
+    MAXFD = os.sysconf("SC_OPEN_MAX")
+except Exception:
+    MAXFD = 256
+
+
+def fd_count():
+    """Count the number of open file descriptors.
+    """
+    if sys.platform.startswith(('linux', 'freebsd')):
+        try:
+            names = os.listdir("/proc/self/fd")
+            return len(names)
+        except FileNotFoundError:
+            pass
+
+    count = 0
+    for fd in range(MAXFD):
+        try:
+            # Prefer dup() over fstat(). fstat() can require input/output
+            # whereas dup() doesn't.
+            fd2 = os.dup(fd)
+        except OSError as e:
+            if e.errno != errno.EBADF:
+                raise
+        else:
+            os.close(fd2)
+            count += 1
+    return count

diff --git a/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst b/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst
new file mode 100644 (file)
index 0000000..cff0f9a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst
@@ -0,0 +1,4 @@
+Fix too many fds in processes started with the "forkserver" method.
+
+A child process would inherit as many fds as the number of still-running
+children.