bpo-30776: reduce regrtest -R false positives (#2422)

* Change the regrtest --huntrleaks checker to decide if a test file
  leaks or not. Require that each run leaks at least 1 reference.
* Warmup runs are now completely ignored: ignored in the checker test
  and not used anymore to compute the sum.
* Add an unit test for a reference leak.

Example of reference differences previously considered a failure
(leak) and now considered as success (success, no leak):

    [3, 0, 0]
    [0, 1, 0]
    [8, -8, 1]

diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py
index c69fc0f147706bdb953b8bf1e88e6460cec2a958..8e18d75b217e6ea7546a4774c4fa2c8ba5d0a847 100644 (file)
--- a/Lib/test/libregrtest/refleak.py
+++ b/Lib/test/libregrtest/refleak.py
@@ -93,9 +93,21 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
         rc_before = rc_after
         fd_before = fd_after
     print(file=sys.stderr)
+
     # These checkers return False on success, True on failure
     def check_rc_deltas(deltas):
-        return any(deltas)
+        # bpo-30776: Try to ignore false positives:
+        #
+        #   [3, 0, 0]
+        #   [0, 1, 0]
+        #   [8, -8, 1]
+        #
+        # Expected leaks:
+        #
+        #   [5, 5, 6]
+        #   [10, 1, 1]
+        return all(delta >= 1 for delta in deltas)
+
     def check_alloc_deltas(deltas):
         # At least 1/3rd of 0s
         if 3 * deltas.count(0) < len(deltas):
@@ -104,14 +116,21 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
         if not set(deltas) <= {1,0,-1}:
             return True
         return False
+
+    def check_fd_deltas(deltas):
+        return any(deltas)
+
     failed = False
     for deltas, item_name, checker in [
         (rc_deltas, 'references', check_rc_deltas),
         (alloc_deltas, 'memory blocks', check_alloc_deltas),
-        (fd_deltas, 'file descriptors', check_rc_deltas)]:
+        (fd_deltas, 'file descriptors', check_fd_deltas)
+    ]:
+        # ignore warmup runs
+        deltas = deltas[nwarmup:]
         if checker(deltas):
             msg = '%s leaked %s %s, sum=%s' % (
-                test, deltas[nwarmup:], item_name, sum(deltas))
+                test, deltas, item_name, sum(deltas))
             print(msg, file=sys.stderr, flush=True)
             with open(fname, "a") as refrep:
                 print(msg, file=refrep)

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 0e676ee33a9c0c18f22c553e39ca3b4615dce121..a544f880d2be01b3f7f3bd6aaeefc5d2e4bab2bc 100644 (file)
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -783,6 +783,40 @@ class ArgsTestCase(BaseTestCase):
         output = self.run_tests('--forever', test, exitcode=2)
         self.check_executed_tests(output, [test]*3, failed=test)
 
+    def check_leak(self, code, what):
+        test = self.create_test('huntrleaks', code=code)
+
+        filename = 'reflog.txt'
+        self.addCleanup(support.unlink, filename)
+        output = self.run_tests('--huntrleaks', '3:3:', test,
+                                exitcode=2,
+                                stderr=subprocess.STDOUT)
+        self.check_executed_tests(output, [test], failed=test)
+
+        line = 'beginning 6 repetitions\n123456\n......\n'
+        self.check_line(output, re.escape(line))
+
+        line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what)
+        self.assertIn(line2, output)
+
+        with open(filename) as fp:
+            reflog = fp.read()
+            self.assertIn(line2, reflog)
+
+    @unittest.skipUnless(Py_DEBUG, 'need a debug build')
+    def test_huntrleaks(self):
+        # test --huntrleaks
+        code = textwrap.dedent("""
+            import unittest
+
+            GLOBAL_LIST = []
+
+            class RefLeakTest(unittest.TestCase):
+                def test_leak(self):
+                    GLOBAL_LIST.append(object())
+        """)
+        self.check_leak(code, 'references')
+
     @unittest.skipUnless(Py_DEBUG, 'need a debug build')
     def test_huntrleaks_fd_leak(self):
         # test --huntrleaks for file descriptor leak
@@ -807,24 +841,7 @@ class ArgsTestCase(BaseTestCase):
                     fd = os.open(__file__, os.O_RDONLY)
                     # bug: never cloes the file descriptor
         """)
-        test = self.create_test('huntrleaks', code=code)
-
-        filename = 'reflog.txt'
-        self.addCleanup(support.unlink, filename)
-        output = self.run_tests('--huntrleaks', '3:3:', test,
-                                exitcode=2,
-                                stderr=subprocess.STDOUT)
-        self.check_executed_tests(output, [test], failed=test)
-
-        line = 'beginning 6 repetitions\n123456\n......\n'
-        self.check_line(output, re.escape(line))
-
-        line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test
-        self.assertIn(line2, output)
-
-        with open(filename) as fp:
-            reflog = fp.read()
-            self.assertIn(line2, reflog)
+        self.check_leak(code, 'file descriptors')
 
     def test_list_tests(self):
         # test --list-tests