From e95ac20103437d8099fb91cd4468ab030852f3b7 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 23 Jul 2019 22:28:48 -0700 Subject: [PATCH] bpo-37667: Add regression test for regrtest. (GH-14929) Verify that it appears to find roughly the right number of tests in the stdlib's testsuite. --- Lib/test/test_regrtest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2e5df00dc5..fdab8c8ef5 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -6,6 +6,7 @@ Note: test_regrtest cannot be run twice in parallel. import contextlib import faulthandler +import glob import io import os.path import platform @@ -529,6 +530,31 @@ class BaseTestCase(unittest.TestCase): return proc.stdout +class CheckActualTests(BaseTestCase): + """ + Check that regrtest appears to find the expected set of tests. + """ + + def test_finds_expected_number_of_tests(self): + args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests'] + output = self.run_python(args) + rough_number_of_tests_found = len(output.splitlines()) + actual_testsuite_glob = os.path.join(os.path.dirname(__file__), + 'test*.py') + rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob)) + # We're not trying to duplicate test finding logic in here, + # just give a rough estimate of how many there should be and + # be near that. This is a regression test to prevent mishaps + # such as https://bugs.python.org/issue37667 in the future. + # If you need to change the values in here during some + # mythical future test suite reorganization, don't go + # overboard with logic and keep that goal in mind. + self.assertGreater(rough_number_of_tests_found, + rough_counted_test_py_files*9//10, + msg='Unexpectedly low number of tests found in:\n' + f'{", ".join(output.splitlines())}') + + class ProgramsTestCase(BaseTestCase): """ Test various ways to run the Python test suite. Use options close -- 2.40.0