From: Victor Stinner Date: Thu, 23 Nov 2017 17:34:59 +0000 (+0100) Subject: bpo-31324: Optimize support._match_test() (#4523) (#4524) X-Git-Tag: v2.7.15rc1~118 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=35d99830f1878867e3964577741d9a2d5a7fc8f7;p=python bpo-31324: Optimize support._match_test() (#4523) (#4524) * bpo-31324: Optimize support._match_test() (#4421) * Rename support._match_test() to support.match_test(): make it public * Remove support.match_tests global variable. It is replaced with a new support.set_match_tests() function, so match_test() doesn't have to check each time if patterns were modified. * Rewrite match_test(): use different code paths depending on the kind of patterns for best performances. Co-Authored-By: Serhiy Storchaka (cherry picked from commit 803ddd8ce22f0de3ab42fb98a225a704c000ef06) * bpo-31324: Fix test.support.set_match_tests(None) (#4505) (cherry picked from commit bb11c3c967afaf263e00844d4ab461b7fafd6d36) (cherry picked from commit 70b2f8797146a56a6880743424f0bedf4fc30c62) --- diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index d836468852..6df8f86d95 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1037,7 +1037,7 @@ def runtest(test, verbose, quiet, if use_resources is not None: test_support.use_resources = use_resources try: - test_support.match_tests = match_tests + test_support.set_match_tests(match_tests) if failfast: test_support.failfast = True return runtest_inner(test, verbose, quiet, huntrleaks, pgo, testdir) @@ -1580,12 +1580,12 @@ def _list_cases(suite): if isinstance(test, unittest.TestSuite): _list_cases(test) elif isinstance(test, unittest.TestCase): - if test_support._match_test(test): + if test_support.match_test(test): print(test.id()) def list_cases(testdir, selected, match_tests): test_support.verbose = False - test_support.match_tests = match_tests + test_support.set_match_tests(match_tests) save_modules = set(sys.modules) skipped = [] diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index cc594865eb..0f0da72b19 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -179,7 +179,6 @@ max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 failfast = False -match_tests = None # _original_stdout is meant to hold stdout at the time regrtest began. # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. @@ -1542,21 +1541,67 @@ def _run_suite(suite): raise TestFailed(err) -def _match_test(test): - global match_tests +# By default, don't filter tests +_match_test_func = None +_match_test_patterns = None - if match_tests is None: + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + if _match_test_func is None: return True - test_id = test.id() + else: + return _match_test_func(test.id()) - for match_test in match_tests: - if fnmatch.fnmatchcase(test_id, match_test): - return True - for name in test_id.split("."): - if fnmatch.fnmatchcase(name, match_test): +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, reject 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def set_match_tests(patterns): + global _match_test_func, _match_test_patterns + + if patterns == _match_test_patterns: + # No change: no need to recompile patterns. + return + + if not patterns: + func = None + # set_match_tests(None) behaves as set_match_tests(()) + patterns = () + elif all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect utility only uses such full test identifiers. + func = set(patterns).__contains__ + else: + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id): + if regex_match(test_id): + # The regex matchs the whole identifier like + # 'test.test_os.FileTests.test_access' return True - return False + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + func = match_test_regex + + # Create a copy since patterns can be mutable and so modified later + _match_test_patterns = tuple(patterns) + _match_test_func = func + def run_unittest(*classes): @@ -1573,7 +1618,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - _filter_suite(suite, _match_test) + _filter_suite(suite, match_test) _run_suite(suite) #======================================================================= diff --git a/Lib/test/test_test_support.py b/Lib/test/test_test_support.py index 4c4c62607c..1b62ac9902 100644 --- a/Lib/test/test_test_support.py +++ b/Lib/test/test_test_support.py @@ -330,6 +330,64 @@ class TestSupport(unittest.TestCase): del D["y"] self.assertNotIn("y", D) + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + + with support.swap_attr(support, '_match_test_func', None): + # match all + support.set_match_tests([]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match all using None + support.set_match_tests(None) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the full test identifier + support.set_match_tests([test_access.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # match the module name + support.set_match_tests(['test_os']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test '*' pattern + support.set_match_tests(['test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test case sensitivity + support.set_match_tests(['filetests']) + self.assertFalse(support.match_test(test_access)) + support.set_match_tests(['FileTests']) + self.assertTrue(support.match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + support.set_match_tests(['*test_os.*.test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Multiple patterns + support.set_match_tests([test_access.id(), test_chdir.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + support.set_match_tests(['test_access', 'DONTMATCH']) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled