From: Victor Stinner Date: Fri, 16 Jun 2017 15:30:03 +0000 (+0200) Subject: bpo-30540, bpo-30523: Add --matchfile and --list-cases options to regrtest (#2249) X-Git-Tag: v2.7.14rc1~92 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=24c2c20873dc800c99d1dabf26419b40cadfe627;p=python bpo-30540, bpo-30523: Add --matchfile and --list-cases options to regrtest (#2249) [2.7] bpo-30540, bpo-30523: Add --matchfile and --list-cases options to regrtest --- diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 521ac137b6..3bc3fa6b4f 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -38,6 +38,7 @@ Selecting tests -x/--exclude -- arguments are tests to *exclude* -s/--single -- single step through a set of tests (see below) -m/--match PAT -- match test cases and methods with glob pattern PAT +--matchfile FILENAME -- filters tests using a text file, one pattern per line -G/--failfast -- fail as soon as a test fails (only with -v or -W) -u/--use RES1,RES2,... -- specify which special resource intensive tests to run @@ -64,6 +65,8 @@ Special runs (instead of the Python stdlib test suite) --list-tests -- only write the name of tests that will be run, don't execute them +--list-cases -- only write the name of test cases that will be run, + don't execute them Additional Option Details: @@ -157,6 +160,13 @@ resources to test. Currently only the following are defined: To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the bsddb tests, give the option '-uall,-bsddb'. + +--matchfile filters tests using a text file, one pattern per line. +Pattern examples: + +- test method: test_stat_attributes +- test class: FileTests +- test identifier: test_os.FileTests.test_stat_attributes """ import StringIO @@ -316,7 +326,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', 'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo', - 'failfast', 'match=', 'testdir=', 'list-tests', 'coverage']) + 'failfast', 'match=', 'testdir=', 'list-tests', 'list-cases', + 'coverage', 'matchfile=']) except getopt.error, msg: usage(2, msg) @@ -327,6 +338,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, use_resources = [] slaveargs = None list_tests = False + list_cases_opt = False for o, a in opts: if o in ('-h', '--help'): usage(0) @@ -354,7 +366,16 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, elif o in ('-f', '--fromfile'): fromfile = a elif o in ('-m', '--match'): - match_tests = a + if match_tests is None: + match_tests = [] + match_tests.append(a) + elif o == '--matchfile': + if match_tests is None: + match_tests = [] + filename = os.path.join(test_support.SAVEDCWD, a) + with open(filename) as fp: + for line in fp: + match_tests.append(line.strip()) elif o in ('-l', '--findleaks'): findleaks = True elif o in ('-L', '--runleaks'): @@ -416,6 +437,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, testdir = a elif o == '--list-tests': list_tests = True + elif o == '--list-cases': + list_cases_opt = True else: print >>sys.stderr, ("No handler for option {}. Please " "report this as a bug at http://bugs.python.org.").format(o) @@ -534,6 +557,10 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print(name) sys.exit(0) + if list_cases_opt: + list_cases(testdir, selected) + sys.exit(0) + if trace: import trace tracer = trace.Trace(trace=False, count=True) @@ -1124,11 +1151,7 @@ def runtest_inner(test, verbose, quiet, huntrleaks=False, pgo=False, testdir=Non try: if capture_stdout: sys.stdout = capture_stdout - if test.startswith('test.') or testdir: - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test + abstest = get_abs_module(testdir, test) clear_caches() with saved_test_environment(test, verbose, quiet, pgo) as environment: start_time = time.time() @@ -1452,7 +1475,7 @@ def count(n, word): else: return "%d %ss" % (n, word) -def printlist(x, width=70, indent=4): +def printlist(x, width=70, indent=4, file=None): """Print the elements of iterable x to stdout. Optional arg width (default 70) is the maximum line length. @@ -1463,8 +1486,37 @@ def printlist(x, width=70, indent=4): from textwrap import fill blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() - print fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks) + print >>file, fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks) + +def get_abs_module(testdir, test): + if test.startswith('test.') or testdir: + return test + else: + # Always import it from the test package + return 'test.' + test + +def _list_cases(suite): + for test in suite: + if isinstance(test, unittest.TestSuite): + _list_cases(test) + elif isinstance(test, unittest.TestCase): + print(test.id()) + +def list_cases(testdir, selected): + skipped = [] + for test in selected: + abstest = get_abs_module(testdir, test) + try: + suite = unittest.defaultTestLoader.loadTestsFromName(abstest) + _list_cases(suite) + except unittest.SkipTest: + skipped.append(test) + + if skipped: + print >>sys.stderr + print >>sys.stderr, count(len(skipped), "test"), "skipped:" + printlist(skipped, file=sys.stderr) # Map sys.platform to a string containing the basenames of tests # expected to be skipped on that platform. diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 5042000788..ca55d09337 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1557,9 +1557,15 @@ def run_unittest(*classes): def case_pred(test): if match_tests is None: return True - for name in test.id().split("."): - if fnmatch.fnmatchcase(name, match_tests): + test_id = test.id() + + for match_test in match_tests: + if fnmatch.fnmatchcase(test_id, match_test): return True + + for name in test_id.split("."): + if fnmatch.fnmatchcase(name, match_test): + return True return False _filter_suite(suite, case_pred) _run_suite(suite) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2001c8ef7f..af9b1d5924 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -482,6 +482,77 @@ class ArgsTestCase(BaseTestCase): self.check_executed_tests(output, tests, failed=crash_test, randomize=True) + def parse_methods(self, output): + regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) + return [match.group(1) for match in regex.finditer(output)] + + def test_matchfile(self): + # Any code which causes a crash + code = textwrap.dedent(""" + import unittest + from test import support + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + def test_method3(self): + pass + def test_method4(self): + pass + + def test_main(): + support.run_unittest(Tests) + """) + all_methods = ['test_method1', 'test_method2', + 'test_method3', 'test_method4'] + testname = self.create_test(code=code) + + # by default, all methods should be run + output = self.run_tests("-v", testname) + methods = self.parse_methods(output) + self.assertEqual(methods, all_methods) + + # only run a subset + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + subset = [ + # only match the method name + 'test_method1', + # match the full identifier + '%s.Tests.test_method3' % testname] + with open(filename, "w") as fp: + for name in subset: + print(name, file=fp) + + output = self.run_tests("-v", "--matchfile", filename, testname) + methods = self.parse_methods(output) + subset = ['test_method1', 'test_method3'] + self.assertEqual(methods, subset) + + def test_list_cases(self): + # test --list-cases + code = textwrap.dedent(""" + import unittest + from test import support + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + + def test_main(): + support.run_unittest(Tests) + """) + testname = self.create_test(code=code) + all_methods = ['%s.Tests.test_method1' % testname, + '%s.Tests.test_method2' % testname] + output = self.run_tests('--list-cases', testname) + self.assertEqual(output.splitlines(), all_methods) + def test_main(): support.run_unittest(ProgramsTestCase, ArgsTestCase)