From 4e549e75cefc7646f02e6307b9d0cad37913187b Mon Sep 17 00:00:00 2001 From: Hans Wennborg Date: Fri, 11 Aug 2017 16:39:26 +0000 Subject: [PATCH] Merging r309044, r309071, r309120, r309122, r309140 and r309227 (PR34157) ------------------------------------------------------------------------ r309044 | mgorny | 2017-07-25 15:38:31 -0700 (Tue, 25 Jul 2017) | 14 lines [lit] Fix UnboundLocalError for invalid shtest redirects Replace the incorrect variable reference when invalid redirect is used. This fixes the following issue: File "/usr/src/llvm/utils/lit/lit/TestRunner.py", line 316, in processRedirects raise InternalShellError(cmd, "Unsupported redirect: %r" % (r,)) UnboundLocalError: local variable 'r' referenced before assignment which in turn broke shtest-shell.py and max-failures.py lit tests. The breakage was introduced during refactoring in rL307310. Differential Revision: https://reviews.llvm.org/D35857 ------------------------------------------------------------------------ ------------------------------------------------------------------------ r309071 | rnk | 2017-07-25 18:27:18 -0700 (Tue, 25 Jul 2017) | 1 line [lit] Attempt to fix Python unittest adaptor logic ------------------------------------------------------------------------ ------------------------------------------------------------------------ r309120 | modocache | 2017-07-26 07:59:36 -0700 (Wed, 26 Jul 2017) | 36 lines Revert "[lit] Remove dead code not referenced in the LLVM SVN repo." Summary: This reverts rL306623, which removed `FileBasedTest`, an abstract base class, but did not also remove the usages of that class in the lit unit tests. The revert fixes four test failures in the lit unit test suite. Test plan: As per the instructions in `utils/lit/README.txt`, run the lit unit test suite: ``` utils/lit/lit.py \ --path /path/to/your/llvm/build/bin \ utils/lit/tests ``` Verify that the following tests fail before applying this patch, and pass once the patch is applied: ``` lit :: test-data.py lit :: test-output.py lit :: xunit-output.py ``` In addition, run `check-llvm` to make sure the existing LLVM test suite executes normally. Reviewers: george.karpenkov, mgorny, dlj Reviewed By: mgorny Subscribers: llvm-commits Differential Revision: https://reviews.llvm.org/D35877 ------------------------------------------------------------------------ ------------------------------------------------------------------------ r309122 | modocache | 2017-07-26 08:02:05 -0700 (Wed, 26 Jul 2017) | 32 lines [lit] Fix type error for parallelism groups Summary: Whereas rL299560 and rL309071 call `parallelism_groups.items()`, under the assumption that `parallelism_groups` is a `dict` type, the default parameter for that attribute is a `list`. Change the default to a `dict` for type correctness. This regression in the unit tests would have been caught if the unit tests were being run continously. It also would have been caught if the lit project used a Python type checker such as `mypy`. Test Plan: As per the instructions in `utils/lit/README.txt`, run the lit unit test suite: ``` utils/lit/lit.py \ --path /path/to/your/llvm/build/bin \ utils/lit/tests ``` Verify that the test `lit :: unit/TestRunner.py` fails before applying this patch, but passes once this patch is applied. Reviewers: mgorny, rnk, rafael Reviewed By: mgorny Subscribers: llvm-commits Differential Revision: https://reviews.llvm.org/D35878 ------------------------------------------------------------------------ ------------------------------------------------------------------------ r309140 | george.karpenkov | 2017-07-26 10:19:36 -0700 (Wed, 26 Jul 2017) | 3 lines Fix LIT test breakage Differential Revision: https://reviews.llvm.org/D35867 ------------------------------------------------------------------------ ------------------------------------------------------------------------ r309227 | rnk | 2017-07-26 15:57:32 -0700 (Wed, 26 Jul 2017) | 4 lines [lit] Fix race between shtest-shell and max-failures tests Previously these tests would use the same Output directory leading to flaky non-deterministic failures. ------------------------------------------------------------------------ git-svn-id: https://llvm.org/svn/llvm-project/llvm/branches/release_50@310730 91177308-0d34-0410-b5e6-96231b3b80d8 --- utils/lit/lit/LitConfig.py | 2 +- utils/lit/lit/TestRunner.py | 2 +- utils/lit/lit/formats/__init__.py | 7 +- utils/lit/lit/formats/base.py | 157 ++++++++++++++------ utils/lit/lit/formats/shtest.py | 33 +--- utils/lit/lit/run.py | 13 +- utils/lit/tests/Inputs/max-failures/lit.cfg | 6 + utils/lit/tests/max-failures.py | 8 +- utils/lit/tests/selecting.py | 2 +- 9 files changed, 142 insertions(+), 88 deletions(-) create mode 100644 utils/lit/tests/Inputs/max-failures/lit.cfg diff --git a/utils/lit/lit/LitConfig.py b/utils/lit/lit/LitConfig.py index 2ef0a8f77ec..3351ebed54b 100644 --- a/utils/lit/lit/LitConfig.py +++ b/utils/lit/lit/LitConfig.py @@ -25,7 +25,7 @@ class LitConfig(object): params, config_prefix = None, maxIndividualTestTime = 0, maxFailures = None, - parallelism_groups = [], + parallelism_groups = {}, echo_all_commands = False): # The name of the test runner. self.progname = progname diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py index 46bcac4b306..a60a0f85487 100644 --- a/utils/lit/lit/TestRunner.py +++ b/utils/lit/lit/TestRunner.py @@ -313,7 +313,7 @@ def processRedirects(cmd, stdin_source, cmd_shenv, opened_files): elif op == ('<',): redirects[0] = [filename, 'r', None] else: - raise InternalShellError(cmd, "Unsupported redirect: %r" % (r,)) + raise InternalShellError(cmd, "Unsupported redirect: %r" % ((op, filename),)) # Open file descriptors in a second pass. std_fds = [None, None, None] diff --git a/utils/lit/lit/formats/__init__.py b/utils/lit/lit/formats/__init__.py index 7d14ca4b535..3ff46e93ead 100644 --- a/utils/lit/lit/formats/__init__.py +++ b/utils/lit/lit/formats/__init__.py @@ -1,3 +1,8 @@ -from lit.formats.base import TestFormat # noqa: F401 +from lit.formats.base import ( # noqa: F401 + TestFormat, + FileBasedTest, + OneCommandPerFileTest +) + from lit.formats.googletest import GoogleTest # noqa: F401 from lit.formats.shtest import ShTest # noqa: F401 diff --git a/utils/lit/lit/formats/base.py b/utils/lit/lit/formats/base.py index baa9ff1d3b7..6721d17e334 100644 --- a/utils/lit/lit/formats/base.py +++ b/utils/lit/lit/formats/base.py @@ -1,50 +1,117 @@ -import abc +from __future__ import absolute_import +import os + +import lit.Test +import lit.util class TestFormat(object): - """Base class for test formats. - - A TestFormat encapsulates logic for finding and executing a certain type of - test. For example, a subclass FooTestFormat would contain the logic for - finding tests written in the 'Foo' format, and the logic for running a - single one. - - TestFormat is an Abstract Base Class (ABC). It uses the Python abc.ABCMeta - type and associated @abc.abstractmethod decorator. Together, these provide - subclass behaviour which is notionally similar to C++ pure virtual classes: - only subclasses which implement all abstract methods can be instantiated - (the implementation may come from an intermediate base). - - For details on ABCs, see: https://docs.python.org/2/library/abc.html. Note - that Python ABCs have extensive abilities beyond what is used here. For - TestFormat, we only care about enforcing that abstract methods are - implemented. - """ - - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, - localConfig): - """Finds tests of this format in the given directory. - - Args: - testSuite: a Test.TestSuite object. - path_in_suite: the subpath under testSuite to look for tests. - litConfig: the LitConfig for the test suite. - localConfig: a LitConfig with local specializations. - - Returns: - An iterable of Test.Test objects. - """ - - @abc.abstractmethod + pass + +### + +class FileBasedTest(TestFormat): + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + source_path = testSuite.getSourcePath(path_in_suite) + for filename in os.listdir(source_path): + # Ignore dot files and excluded tests. + if (filename.startswith('.') or + filename in localConfig.excludes): + continue + + filepath = os.path.join(source_path, filename) + if not os.path.isdir(filepath): + base,ext = os.path.splitext(filename) + if ext in localConfig.suffixes: + yield lit.Test.Test(testSuite, path_in_suite + (filename,), + localConfig) + +### + +import re +import tempfile + +class OneCommandPerFileTest(TestFormat): + # FIXME: Refactor into generic test for running some command on a directory + # of inputs. + + def __init__(self, command, dir, recursive=False, + pattern=".*", useTempInput=False): + if isinstance(command, str): + self.command = [command] + else: + self.command = list(command) + if dir is not None: + dir = str(dir) + self.dir = dir + self.recursive = bool(recursive) + self.pattern = re.compile(pattern) + self.useTempInput = useTempInput + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + dir = self.dir + if dir is None: + dir = testSuite.getSourcePath(path_in_suite) + + for dirname,subdirs,filenames in os.walk(dir): + if not self.recursive: + subdirs[:] = [] + + subdirs[:] = [d for d in subdirs + if (d != '.svn' and + d not in localConfig.excludes)] + + for filename in filenames: + if (filename.startswith('.') or + not self.pattern.match(filename) or + filename in localConfig.excludes): + continue + + path = os.path.join(dirname,filename) + suffix = path[len(dir):] + if suffix.startswith(os.sep): + suffix = suffix[1:] + test = lit.Test.Test( + testSuite, path_in_suite + tuple(suffix.split(os.sep)), + localConfig) + # FIXME: Hack? + test.source_path = path + yield test + + def createTempInput(self, tmp, test): + raise NotImplementedError('This is an abstract method.') + def execute(self, test, litConfig): - """Runs the given 'test', which is of this format. + if test.config.unsupported: + return (lit.Test.UNSUPPORTED, 'Test is unsupported') + + cmd = list(self.command) + + # If using temp input, create a temporary file and hand it to the + # subclass. + if self.useTempInput: + tmp = tempfile.NamedTemporaryFile(suffix='.cpp') + self.createTempInput(tmp, test) + tmp.flush() + cmd.append(tmp.name) + elif hasattr(test, 'source_path'): + cmd.append(test.source_path) + else: + cmd.append(test.getSourcePath()) + + out, err, exitCode = lit.util.executeCommand(cmd) + + diags = out + err + if not exitCode and not diags.strip(): + return lit.Test.PASS,'' - Args: - test: a Test.Test object describing the test to run. - litConfig: the LitConfig for the test suite. + # Try to include some useful information. + report = """Command: %s\n""" % ' '.join(["'%s'" % a + for a in cmd]) + if self.useTempInput: + report += """Temporary File: %s\n""" % tmp.name + report += "--\n%s--\n""" % open(tmp.name).read() + report += """Output:\n--\n%s--""" % diags - Returns: - A tuple of (status:Test.ResultCode, message:str) - """ + return lit.Test.FAIL, report diff --git a/utils/lit/lit/formats/shtest.py b/utils/lit/lit/formats/shtest.py index 01ecd192092..fdc9bd0241f 100644 --- a/utils/lit/lit/formats/shtest.py +++ b/utils/lit/lit/formats/shtest.py @@ -1,13 +1,12 @@ from __future__ import absolute_import -import os - -import lit.Test import lit.TestRunner import lit.util -from .base import TestFormat -class ShTest(TestFormat): +from .base import FileBasedTest + + +class ShTest(FileBasedTest): """ShTest is a format with one file per test. This is the primary format for regression tests as described in the LLVM @@ -18,31 +17,9 @@ class ShTest(TestFormat): The ShTest files contain some number of shell-like command pipelines, along with assertions about what should be in the output. """ - - def __init__(self, execute_external = False): - """Initializer. - - The 'execute_external' argument controls whether lit uses its internal - logic for command pipelines, or passes the command to a shell - subprocess. - - Args: - execute_external: (optional) If true, use shell subprocesses instead - of lit's internal pipeline logic. - """ + def __init__(self, execute_external=False): self.execute_external = execute_external - def getTestsInDirectory(self, testSuite, path_in_suite, - litConfig, localConfig): - """Yields test files matching 'suffixes' from the localConfig.""" - file_matches = lit.util.listdir_files( - testSuite.getSourcePath(path_in_suite), - localConfig.suffixes, localConfig.excludes) - for filename in file_matches: - yield lit.Test.Test(testSuite, path_in_suite + (filename,), - localConfig) - def execute(self, test, litConfig): - """Interprets and runs the given test file, and returns the result.""" return lit.TestRunner.executeShTest(test, litConfig, self.execute_external) diff --git a/utils/lit/lit/run.py b/utils/lit/lit/run.py index 1290c142c83..3e39bdb9220 100644 --- a/utils/lit/lit/run.py +++ b/utils/lit/lit/run.py @@ -44,6 +44,12 @@ class Run(object): def __init__(self, lit_config, tests): self.lit_config = lit_config self.tests = tests + # Set up semaphores to limit parallelism of certain classes of tests. + # For example, some ASan tests require lots of virtual memory and run + # faster with less parallelism on OS X. + self.parallelism_semaphores = \ + {k: multiprocessing.Semaphore(v) for k, v in + self.lit_config.parallelism_groups.items()} def execute_test(self, test): return _execute_test_impl(test, self.lit_config, @@ -74,13 +80,6 @@ class Run(object): if not self.tests or jobs == 0: return - # Set up semaphores to limit parallelism of certain classes of tests. - # For example, some ASan tests require lots of virtual memory and run - # faster with less parallelism on OS X. - self.parallelism_semaphores = \ - {k: multiprocessing.Semaphore(v) for k, v in - self.lit_config.parallelism_groups.items()} - # Install a console-control signal handler on Windows. if win32api is not None: def console_ctrl_handler(type): diff --git a/utils/lit/tests/Inputs/max-failures/lit.cfg b/utils/lit/tests/Inputs/max-failures/lit.cfg new file mode 100644 index 00000000000..50d07566e1c --- /dev/null +++ b/utils/lit/tests/Inputs/max-failures/lit.cfg @@ -0,0 +1,6 @@ +import lit.formats +config.name = 'shtest-shell' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = os.path.dirname(__file__) + '/../shtest-shell' +config.test_exec_root = None diff --git a/utils/lit/tests/max-failures.py b/utils/lit/tests/max-failures.py index 5cc258dd08a..bc58e9a4e47 100644 --- a/utils/lit/tests/max-failures.py +++ b/utils/lit/tests/max-failures.py @@ -1,9 +1,9 @@ # Check the behavior of --max-failures option. # -# RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out -# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/shtest-shell >> %t.out -# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/shtest-shell >> %t.out -# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/shtest-shell 2>> %t.out +# RUN: not %{lit} -j 1 -v %{inputs}/max-failures > %t.out +# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/max-failures >> %t.out +# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/max-failures >> %t.out +# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/max-failures 2>> %t.out # RUN: FileCheck < %t.out %s # # END. diff --git a/utils/lit/tests/selecting.py b/utils/lit/tests/selecting.py index 19ba240f9b0..4a0d08b860b 100644 --- a/utils/lit/tests/selecting.py +++ b/utils/lit/tests/selecting.py @@ -9,7 +9,7 @@ # Check that regex-filtering based on environment variables work. # -# RUN: LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER-ENV %s +# RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER-ENV %s # CHECK-FILTER-ENV: Testing: 2 of 5 tests -- 2.49.0