]> granicus.if.org Git - llvm/commitdiff
[lit] Clean output directories before running tests.
authorZachary Turner <zturner@google.com>
Fri, 30 Jun 2017 16:01:30 +0000 (16:01 +0000)
committerZachary Turner <zturner@google.com>
Fri, 30 Jun 2017 16:01:30 +0000 (16:01 +0000)
Presently lit leaks files in the tests' output directories.
Specifically, if a test creates output files, lit makes no
effort to remove them prior to the next test run.  This is
problematic because it leads to false positives whenever a
test passes because stale  files were present.  In general
it is a source of flakiness that should be removed.

This patch addresses this by building the list of all test
directories that are part of the current run set, and then
deleting those directories and recreating them anew.  This
gives each test a clean baseline to start from.

Differential Revision: https://reviews.llvm.org/D34732

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@306832 91177308-0d34-0410-b5e6-96231b3b80d8

15 files changed:
test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.cpp [new file with mode: 0644]
test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.dwo
test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.o
test/DebugInfo/Inputs/split-dwarf-multiple-cu.dwo
test/DebugInfo/Inputs/split-dwarf-multiple-cu.o
test/DebugInfo/Inputs/split-dwarf-multiple-cu1.cpp [new file with mode: 0644]
test/DebugInfo/Inputs/split-dwarf-multiple-cu2.cpp [new file with mode: 0644]
test/DebugInfo/Inputs/split-dwarf-test
test/DebugInfo/Inputs/split-dwarf-test.dwo
test/DebugInfo/llvm-symbolizer.test
test/Other/lit-globbing.ll
test/Unit/lit.cfg
utils/lit/lit/Test.py
utils/lit/lit/TestRunner.py
utils/lit/lit/run.py

diff --git a/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.cpp b/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.cpp
new file mode 100644 (file)
index 0000000..d7361f4
--- /dev/null
@@ -0,0 +1,7 @@
+void f1();
+__attribute__((always_inline)) void f2() {
+  f1();
+}
+void f3() {
+  f2();
+}
index 2a3bc57caa6de0845e445c1b9d072d8671989023..44bdaf4bffe0de44c6926eea479b35e4d988b60d 100644 (file)
Binary files a/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.dwo and b/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.dwo differ
index b6993c6cae20930edf7a25e27ea4ed786d0b1e69..540302785c2e6bd83866649df92e2634974777f6 100644 (file)
Binary files a/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.o and b/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.o differ
index 4df9894b089a0d4537f6ee7d62d5763dec16b092..94cc3e9d0f7a7bf049ccf7e26fac242e74aaed4c 100644 (file)
Binary files a/test/DebugInfo/Inputs/split-dwarf-multiple-cu.dwo and b/test/DebugInfo/Inputs/split-dwarf-multiple-cu.dwo differ
index aa4ab4bc76f7c1d7cf9253dcf5fe9389256b3a73..dd0cda4a8ff22ebd211ccac886a6acf53f75ccf9 100644 (file)
Binary files a/test/DebugInfo/Inputs/split-dwarf-multiple-cu.o and b/test/DebugInfo/Inputs/split-dwarf-multiple-cu.o differ
diff --git a/test/DebugInfo/Inputs/split-dwarf-multiple-cu1.cpp b/test/DebugInfo/Inputs/split-dwarf-multiple-cu1.cpp
new file mode 100644 (file)
index 0000000..9bed4bf
--- /dev/null
@@ -0,0 +1,2 @@
+extern int i;
+int i;
diff --git a/test/DebugInfo/Inputs/split-dwarf-multiple-cu2.cpp b/test/DebugInfo/Inputs/split-dwarf-multiple-cu2.cpp
new file mode 100644 (file)
index 0000000..1d17cf3
--- /dev/null
@@ -0,0 +1,7 @@
+void f1();
+inline __attribute__((always_inline)) void f2() {
+  f1();
+}
+void f3() {
+  f2();
+}
index 4d6aa0cd98a34dbda578f74b1174b56542859fad..0e9d4c880bb08676aed49e826ebfcbfe5fc830b1 100755 (executable)
Binary files a/test/DebugInfo/Inputs/split-dwarf-test and b/test/DebugInfo/Inputs/split-dwarf-test differ
index 588374d7d21e1830d12b1bee2dfa67ce640322c4..fb5eb9c0ffa9901c1721206b8b9e93d7ecef9224 100644 (file)
Binary files a/test/DebugInfo/Inputs/split-dwarf-test.dwo and b/test/DebugInfo/Inputs/split-dwarf-test.dwo differ
index 2c64804659fceee6d6da36e696c364c2687d7e4a..49da075a451e29cf22f93fb334c1562d3d78af7f 100644 (file)
@@ -20,8 +20,8 @@ RUN: echo "%p/Inputs/llvm-symbolizer-dwo-test 0x400514" >> %t.input
 RUN: echo "%p/Inputs/fission-ranges.elf-x86_64 0x720" >> %t.input
 RUN: echo "%p/Inputs/arange-overlap.elf-x86_64 0x714" >> %t.input
 RUN: cp %p/Inputs/split-dwarf-test.dwo %T
-RUN: echo "%p/Inputs/split-dwarf-test 0x4005d4" >> %t.input
-RUN: echo "%p/Inputs/split-dwarf-test 0x4005c4" >> %t.input
+RUN: echo "%p/Inputs/split-dwarf-test 0x400504" >> %t.input
+RUN: echo "%p/Inputs/split-dwarf-test 0x4004f0" >> %t.input
 RUN: echo "%p/Inputs/cross-cu-inlining.x86_64-macho.o 0x17" >> %t.input
 RUN: cp %p/Inputs/split-dwarf-multiple-cu.dwo %T
 RUN: echo "%p/Inputs/split-dwarf-multiple-cu.o 0x4" >> %t.input
@@ -141,9 +141,9 @@ CHECK-NEXT: main
 CHECK-NEXT: /tmp{{[/\\]}}cross-cu-inlining.c:11:0
 
 CHECK:      f2
-CHECK-NEXT: b.cpp:3:3
+CHECK-NEXT: split-dwarf-multiple-cu2.cpp:3:3
 CHECK-NEXT: f3
-CHECK-NEXT: b.cpp:6:0
+CHECK-NEXT: split-dwarf-multiple-cu2.cpp:6:0
 
 CHECK:      f2
 CHECK-NEXT: split-dwarf-addr-object-relocation.cpp:3:3
index 5a668a90a40b304f6ed12d45cc77b61d546b3285..b60510ab8b22f599273044eab961cc9910f6fa26 100644 (file)
@@ -1,28 +1,28 @@
-RUN: echo TA > %T/TA.txt\r
-RUN: echo TB > %T/TB.txt\r
-RUN: echo TAB > %T/TAB.txt\r
-\r
-RUN: echo %T/TA* | FileCheck -check-prefix=STAR %s\r
-RUN: echo %T/'TA'* | FileCheck -check-prefix=STAR %s\r
-RUN: echo %T/T'A'* | FileCheck -check-prefix=STAR %s\r
-\r
-RUN: echo %T/T?.txt | FileCheck -check-prefix=QUESTION %s\r
-RUN: echo %T/'T'?.txt | FileCheck -check-prefix=QUESTION %s\r
-\r
-RUN: echo %T/T??.txt | FileCheck -check-prefix=QUESTION2 %s\r
-RUN: echo %T/'T'??.txt | FileCheck -check-prefix=QUESTION2 %s\r
-\r
-RUN: echo 'T*' 'T?.txt' 'T??.txt' | FileCheck -check-prefix=QUOTEDARGS %s\r
-\r
-STAR-NOT: TB.txt\r
-STAR: {{(TA.txt.*TAB.txt|TAB.txt.*TA.txt)}}\r
-\r
-QUESTION-NOT: TAB.txt\r
-QUESTION: {{(TA.txt.*TB.txt|TB.txt.*TA.txt)}}\r
-\r
-QUESTION2-NOT: TA.txt\r
-QUESTION2-NOT: TB.txt\r
-QUESTION2: TAB.txt\r
-\r
-QUOTEDARGS-NOT: .txt\r
-QUOTEDARGS: T* T?.txt T??.txt\r
+RUN: echo XXA > %T/XXA.txt
+RUN: echo XXB > %T/XXB.txt
+RUN: echo XXAB > %T/XXAB.txt
+
+RUN: echo %T/XXA* | FileCheck -check-prefix=STAR %s
+RUN: echo %T/'XXA'* | FileCheck -check-prefix=STAR %s
+RUN: echo %T/XX'A'* | FileCheck -check-prefix=STAR %s
+
+RUN: echo %T/XX?.txt | FileCheck -check-prefix=QUESTION %s
+RUN: echo %T/'XX'?.txt | FileCheck -check-prefix=QUESTION %s
+
+RUN: echo %T/XX??.txt | FileCheck -check-prefix=QUESTION2 %s
+RUN: echo %T/'XX'??.txt | FileCheck -check-prefix=QUESTION2 %s
+
+RUN: echo 'XX*' 'XX?.txt' 'XX??.txt' | FileCheck -check-prefix=QUOTEDARGS %s
+
+STAR-NOT: XXB.txt
+STAR: {{(XXA.txt.*XXAB.txt|XXAB.txt.*XXA.txt)}}
+
+QUESTION-NOT: XXAB.txt
+QUESTION: {{(XXA.txt.*XXB.txt|XXB.txt.*XXA.txt)}}
+
+QUESTION2-NOT: XXA.txt
+QUESTION2-NOT: XXB.txt
+QUESTION2: XXAB.txt
+
+QUOTEDARGS-NOT: .txt
+QUOTEDARGS: XX* XX?.txt XX??.txt
index dac0bf829ba6ff3c632106cdc741a366afd7f851..a3c00d6d18cd9d04ea0d950a981b8529c825282d 100644 (file)
@@ -39,9 +39,11 @@ for symbolizer in ['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']:
         config.environment[symbolizer] = os.environ[symbolizer]
 
 # Win32 seeks DLLs along %PATH%.
-if sys.platform in ['win32', 'cygwin'] and os.path.isdir(config.shlibdir):
-    config.environment['PATH'] = os.path.pathsep.join((
-            config.shlibdir, config.environment['PATH']))
+if sys.platform in ['win32', 'cygwin']:
+       shlibdir = getattr(config, 'shlibdir', None)
+       if shlibdir is not None and os.path.isdir(shlibdir):
+           config.environment['PATH'] = os.path.pathsep.join((
+                   config.shlibdir, config.environment['PATH']))
 
 # Win32 may use %SYSTEMDRIVE% during file system shell operations, so propogate.
 if sys.platform == 'win32' and 'SYSTEMDRIVE' in os.environ:
index 1a9e3fe80fb39da913773a5b1659e0ab4b65b573..dc82670358a1331c412144731cc5e6da27275df4 100644 (file)
@@ -172,7 +172,7 @@ class TestSuite:
         return os.path.join(self.source_root, *components)
 
     def getExecPath(self, components):
-        return os.path.join(self.exec_root, *components)
+        return os.path.join(self.exec_root, "Output", *components)
 
 class Test:
     """Test - Information on a single test instance."""
@@ -222,10 +222,13 @@ class Test:
             # Syntax error in an XFAIL line.
             self.result.code = UNRESOLVED
             self.result.output = str(e)
-        
+
     def getFullName(self):
         return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
 
+    def getTestBaseName(self):
+        return self.path_in_suite[-1]
+
     def getFilePath(self):
         if self.file_path:
             return self.file_path
@@ -234,8 +237,11 @@ class Test:
     def getSourcePath(self):
         return self.suite.getSourcePath(self.path_in_suite)
 
-    def getExecPath(self):
-        return self.suite.getExecPath(self.path_in_suite)
+    def getTempFilePrefix(self):
+        return self.suite.getExecPath(self.path_in_suite) + ".tmp"
+
+    def getTempFileDir(self):
+        return os.path.dirname(self.getTempFilePrefix())
 
     def isExpectedToFail(self):
         """
@@ -347,7 +353,7 @@ class Test:
         safe_name = self.suite.name.replace(".","-")
 
         if safe_test_path:
-            class_name = safe_name + "." + "/".join(safe_test_path) 
+            class_name = safe_name + "." + "/".join(safe_test_path)
         else:
             class_name = safe_name + "." + safe_name
 
index 37b03cc19f850580736f3ae445a3a8e9a48a3959..e72be0253085073bfa75bad30a6c0961e0569a08 100644 (file)
@@ -123,7 +123,7 @@ class ShellCommandResult(object):
         self.exitCode = exitCode
         self.timeoutReached = timeoutReached
         self.outputFiles = list(outputFiles)
-               
+
 def executeShCmd(cmd, shenv, results, timeout=0):
     """
         Wrapper around _executeShCmd that handles
@@ -501,7 +501,7 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
                         data = None
                     if data is not None:
                         output_files.append((name, path, data))
-            
+
         results.append(ShellCommandResult(
             cmd.commands[i], out, err, res, timeoutHelper.timeoutReached(),
             output_files))
@@ -573,7 +573,7 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
                 else:
                     out += data
                 out += "\n"
-                    
+
         if result.stdout.strip():
             out += '# command output:\n%s\n' % (result.stdout,)
         if result.stderr.strip():
@@ -690,37 +690,28 @@ def parseIntegratedTestScriptCommands(source_path, keywords):
     finally:
         f.close()
 
-def getTempPaths(test):
-    """Get the temporary location, this is always relative to the test suite
-    root, not test source root."""
-    execpath = test.getExecPath()
-    execdir,execbase = os.path.split(execpath)
-    tmpDir = os.path.join(execdir, 'Output')
-    tmpBase = os.path.join(tmpDir, execbase)
-    return tmpDir, tmpBase
-
-def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
+def getDefaultSubstitutions(test, normalize_slashes=False):
     sourcepath = test.getSourcePath()
     sourcedir = os.path.dirname(sourcepath)
+    tmpDir = test.getTempFileDir()
+    tmpPrefix = test.getTempFilePrefix()
+    baseName = test.getTestBaseName()
 
     # Normalize slashes, if requested.
     if normalize_slashes:
         sourcepath = sourcepath.replace('\\', '/')
         sourcedir = sourcedir.replace('\\', '/')
         tmpDir = tmpDir.replace('\\', '/')
-        tmpBase = tmpBase.replace('\\', '/')
 
     # We use #_MARKER_# to hide %% while we do the other substitutions.
     substitutions = []
     substitutions.extend([('%%', '#_MARKER_#')])
     substitutions.extend(test.config.substitutions)
-    tmpName = tmpBase + '.tmp'
-    baseName = os.path.basename(tmpBase)
     substitutions.extend([('%s', sourcepath),
                           ('%S', sourcedir),
                           ('%p', sourcedir),
                           ('%{pathsep}', os.pathsep),
-                          ('%t', tmpName),
+                          ('%t', tmpPrefix),
                           ('%basename_t', baseName),
                           ('%T', tmpDir),
                           ('#_MARKER_#', '%')])
@@ -730,7 +721,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
             ('%/s', sourcepath.replace('\\', '/')),
             ('%/S', sourcedir.replace('\\', '/')),
             ('%/p', sourcedir.replace('\\', '/')),
-            ('%/t', tmpBase.replace('\\', '/') + '.tmp'),
+            ('%/t', tmpPrefix.replace('\\', '/')),
             ('%/T', tmpDir.replace('\\', '/')),
             ])
 
@@ -740,7 +731,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
                 ('%:s', re.sub(r'^(.):', r'\1', sourcepath)),
                 ('%:S', re.sub(r'^(.):', r'\1', sourcedir)),
                 ('%:p', re.sub(r'^(.):', r'\1', sourcedir)),
-                ('%:t', re.sub(r'^(.):', r'\1', tmpBase) + '.tmp'),
+                ('%:t', re.sub(r'^(.):', r'\1', tmpPrefix)),
                 ('%:T', re.sub(r'^(.):', r'\1', tmpDir)),
                 ])
     else:
@@ -748,7 +739,7 @@ def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
                 ('%:s', sourcepath),
                 ('%:S', sourcedir),
                 ('%:p', sourcedir),
-                ('%:t', tmpBase + '.tmp'),
+                ('%:t', tmpPrefix),
                 ('%:T', tmpDir),
                 ])
     return substitutions
@@ -779,7 +770,7 @@ class ParserKind(object):
     TAG: A keyword taking no value. Ex 'END.'
     COMMAND: A keyword taking a list of shell commands. Ex 'RUN:'
     LIST: A keyword taking a comma-separated list of values.
-    BOOLEAN_EXPR: A keyword taking a comma-separated list of 
+    BOOLEAN_EXPR: A keyword taking a comma-separated list of
         boolean expressions. Ex 'XFAIL:'
     CUSTOM: A keyword with custom parsing semantics.
     """
@@ -951,14 +942,14 @@ def parseIntegratedTestScript(test, additional_parsers=[],
         IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR,
                                     initial_value=test.requires),
         IntegratedTestKeywordParser('REQUIRES-ANY:', ParserKind.CUSTOM,
-                                    IntegratedTestKeywordParser._handleRequiresAny, 
-                                    initial_value=test.requires), 
+                                    IntegratedTestKeywordParser._handleRequiresAny,
+                                    initial_value=test.requires),
         IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
                                     initial_value=test.unsupported),
         IntegratedTestKeywordParser('END.', ParserKind.TAG)
     ]
     keyword_parsers = {p.keyword: p for p in builtin_parsers}
-    
+
     # Install user-defined additional parsers.
     for parser in additional_parsers:
         if not isinstance(parser, IntegratedTestKeywordParser):
@@ -968,7 +959,7 @@ def parseIntegratedTestScript(test, additional_parsers=[],
             raise ValueError("Parser for keyword '%s' already exists"
                              % parser.keyword)
         keyword_parsers[parser.keyword] = parser
-        
+
     # Collect the test lines from the script.
     sourcepath = test.getSourcePath()
     for line_number, command_type, ln in \
@@ -1014,12 +1005,8 @@ def parseIntegratedTestScript(test, additional_parsers=[],
 
     return script
 
-
 def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
-    # Create the output directory if it does not already exist.
-    lit.util.mkdir_p(os.path.dirname(tmpBase))
-
-    execdir = os.path.dirname(test.getExecPath())
+    execdir = os.path.dirname(test.getTempFileDir())
     if useExternalSh:
         res = executeScript(test, litConfig, tmpBase, script, execdir)
     else:
@@ -1063,10 +1050,8 @@ def executeShTest(test, litConfig, useExternalSh,
         return script
     if litConfig.noExecute:
         return lit.Test.Result(Test.PASS)
-
-    tmpDir, tmpBase = getTempPaths(test)
     substitutions = list(extra_substitutions)
-    substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
+    substitutions += getDefaultSubstitutions(test,
                                              normalize_slashes=useExternalSh)
     script = applySubstitutions(script, substitutions)
 
@@ -1075,7 +1060,8 @@ def executeShTest(test, litConfig, useExternalSh,
     if hasattr(test.config, 'test_retry_attempts'):
         attempts += test.config.test_retry_attempts
     for i in range(attempts):
-        res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
+        res = _runShTest(test, litConfig, useExternalSh, script,
+                         test.getTempFilePrefix())
         if res.code != Test.FAIL:
             break
     # If we had to run the test more than once, count it as a flaky pass. These
index 1290c142c834e709c438d628b13e209f027a98ec..bdaf1415a1cc326cdaeef1ff0c1dfc34b2317db1 100644 (file)
@@ -1,4 +1,5 @@
 import os
+import shutil
 import sys
 import threading
 import time
@@ -74,6 +75,25 @@ class Run(object):
         if not self.tests or jobs == 0:
             return
 
+        # Create fresh output directories for each test we're going to run.
+        # This guarantees that test runs will not remnants of previous test
+        # runs' output.
+        clean_paths = set()
+        for test in self.tests:
+            clean_paths.add(os.path.normpath(test.getTempFileDir()))
+        clean_paths = list(clean_paths)
+        # Sort by number of path components, to ensure that parent directories
+        # get deleted and re-created before child directories.
+        clean_paths.sort(key=lambda x: len(x.split(os.sep)))
+        for base in clean_paths:
+            if os.path.exists(base):
+                if not os.path.islink(base) and os.path.isdir(base):
+                    shutil.rmtree(base, True)
+                else:
+                    os.unlink(os.path)
+            if not os.path.exists(base):
+                lit.util.mkdir_p(base)
+
         # Set up semaphores to limit parallelism of certain classes of tests.
         # For example, some ASan tests require lots of virtual memory and run
         # faster with less parallelism on OS X.