bpo-37935: Added tests for os.walk(), glob.iglob() and Path.glob() (GH-15956)

Test that they do not keep too many file descriptors open for the host OS in a reasonable test scenario.

See [bpo-37935](https://bugs.python.org/issue37935).

diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py
index 767bd3764b8925fe5077879de1b0d60e93a339ff..cba8c7c60e217e7937c4d6bb4ee23e5c34764f4e 100644 (file)
--- a/Lib/test/test_glob.py
+++ b/Lib/test/test_glob.py
@@ -264,6 +264,23 @@ class GlobTests(unittest.TestCase):
                 expect += [join('sym3', 'EF')]
             eq(glob.glob(join('**', 'EF'), recursive=True), expect)
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        base = os.path.join(self.tempdir, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+        pattern = os.path.join(base, *(['*']*depth))
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        pattern = os.path.join(base, '**', 'd')
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = os.path.join(p, 'd')
+            for it in iters:
+                self.assertEqual(next(it), p)
+
 
 @skip_unless_symlink
 class SymlinkLoopGlobTests(unittest.TestCase):

diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 8ff0296fad0bb4cd519d57614b869ef55569083b..4a076e3bbf5426bbf1ef0522938e3457b8e428c0 100644 (file)
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1178,6 +1178,27 @@ class WalkTests(unittest.TestCase):
         finally:
             os.rename(path1new, path1)
 
+    def test_walk_many_open_files(self):
+        depth = 30
+        base = os.path.join(support.TESTFN, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+
+        iters = [self.walk(base, topdown=False) for j in range(100)]
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.dirname(p)
+
+        iters = [self.walk(base, topdown=True) for j in range(100)]
+        p = base
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i < depth else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.join(p, 'd')
+
 
 @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
 class FwalkTests(WalkTests):
@@ -1247,6 +1268,10 @@ class FwalkTests(WalkTests):
         self.addCleanup(os.close, newfd)
         self.assertEqual(newfd, minfd)
 
+    # fwalk() keeps file descriptors open
+    test_walk_many_open_files = None
+
+
 class BytesWalkTests(WalkTests):
     """Tests for os.walk() with bytes."""
     def walk(self, top, **kwargs):

diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index f3b385fbf36721a6686848f9f228d7ceeebb1479..f3304f01b4bee3e282c8ba4a331ab0c6ecfe1b80 100644 (file)
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1562,6 +1562,23 @@ class _BasePathTest(object):
                   }
         self.assertEqual(given, {p / x for x in expect})
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        P = self.cls
+        base = P(BASE) / 'deep'
+        p = P(base, *(['d']*depth))
+        p.mkdir(parents=True)
+        pattern = '/'.join(['*'] * depth)
+        iters = [base.glob(pattern) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        iters = [base.rglob('d') for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = p / 'd'
+            for it in iters:
+                self.assertEqual(next(it), p)
+
     def test_glob_dotdot(self):
         # ".." is not special in globs.
         P = self.cls