[analyzer] Update SATestBuild.py to enable a 'download and patch' model for projects.

Currently the SATestBuild.py and SATestAdd.py buildbot scripts expect project
sources to be checked into the project repository. This commit changes these
scripts to additionally support a model where project sources are downloaded
rather than checked into the repository. Sometimes projects may need to be
modified (for example, to support a newer versions of clang), so the updated scripts
also allow for an optional patch file that will be applied to the downloaded
project source before analysis.

To support this workflow, this commit changes the expected layout of
a project in the repository. The project-specific helper scripts will stay
in the root of each project directory, but the benchmark source itself (if
checked into the repo) should now be stored in a subdirectory named
'CachedSource':

  project_name/
    cleanup_run_static_analyzer.sh [optional]
    run_static_analyzer.cmd [required]
    download_project.sh [optional]
    CachedSource/ [optional]
    changes_for_analyzer.patch [optional]

If the 'CachedSource' source directory is not present, the download script will
be executed. This script should download the project source into 'CachedSource'.
Then, if 'changes_for_analyzer.patch' is present its changes will
be applied to a copy of 'CachedSource' before analysis.

Differential Revision: http://reviews.llvm.org/D14345

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@252410 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/utils/analyzer/SATestAdd.py b/utils/analyzer/SATestAdd.py
index 28050fddb819dd12fca9c417f4e0ca5cef1adc3a..4b94a109ce6451f5697a1c383dcc9607cc36622a 100755 (executable)
--- a/utils/analyzer/SATestAdd.py
+++ b/utils/analyzer/SATestAdd.py
@@ -10,11 +10,35 @@ the Repository Directory.
    have the same name as the project ID
 
  The project should use the following files for set up:
-      - pre_run_static_analyzer.sh - prepare the build environment.
+      - cleanup_run_static_analyzer.sh - prepare the build environment.
                                      Ex: make clean can be a part of it.
       - run_static_analyzer.cmd - a list of commands to run through scan-build.
                                      Each command should be on a separate line.
                                      Choose from: configure, make, xcodebuild
+      - download_project.sh - download the project into the CachedSource/
+                                     directory. For example, download a zip of
+                                     the project source from GitHub, unzip it,
+                                     and rename the unzipped directory to
+                                     'CachedSource'. This script is not called
+                                     when 'CachedSource' is already present,
+                                     so an alternative is to check the
+                                     'CachedSource' directory into the
+                                     repository directly.
+      - CachedSource/ - An optional directory containing the source of the
+                                     project being analyzed. If present,
+                                     download_project.sh will not be called.
+      - changes_for_analyzer.patch - An optional patch file for any local changes
+                                     (e.g., to adapt to newer version of clang)
+                                     that should be applied to CachedSource
+                                     before analysis. To construct this patch,
+                                     run the the download script to download
+                                     the project to CachedSource, copy the
+                                     CachedSource to another directory (for
+                                     example, PatchedSource) and make any needed
+                                     modifications to the the copied source.
+                                     Then run:
+                                          diff -ur CachedSource PatchedSource \
+                                              > changes_for_analyzer.patch
 """
 import SATestBuild
 

diff --git a/utils/analyzer/SATestBuild.py b/utils/analyzer/SATestBuild.py
index 21cf4524bc751d07db2d3a8d3bceee5f713ef8f9..d0503c6389c9ee63d2aae07c377f7fb2ff86283f 100755 (executable)
--- a/utils/analyzer/SATestBuild.py
+++ b/utils/analyzer/SATestBuild.py
@@ -154,6 +154,8 @@ Jobs = int(math.ceil(detectCPUs() * 0.75))
 ProjectMapFile = "projectMap.csv"
 
 # Names of the project specific scripts.
+# The script that downloads the project.
+DownloadScript = "download_project.sh"
 # The script that needs to be executed before the build can start.
 CleanupScript = "cleanup_run_static_analyzer.sh"
 # This is a file containing commands for scan-build.
@@ -173,6 +175,21 @@ DiffsSummaryFileName = "diffs.txt"
 SBOutputDirName = "ScanBuildResults"
 SBOutputDirReferencePrefix = "Ref"
 
+# The name of the directory storing the cached project source. If this directory
+# does not exist, the download script will be executed. That script should
+# create the "CachedSource" directory and download the project source into it.
+CachedSourceDirName = "CachedSource"
+
+# The name of the directory containing the source code that will be analyzed.
+# Each time a project is analyzed, a fresh copy of its CachedSource directory
+# will be copied to the PatchedSource directory and then the local patches
+# in PatchfileName will be applied (if PatchfileName exists).
+PatchedSourceDirName = "PatchedSource"
+
+# The name of the patchfile specifying any changes that should be applied
+# to the CachedSource before analyzing.
+PatchfileName = "changes_for_analyzer.patch"
+
 # The list of checkers used during analyzes.
 # Currently, consists of all the non-experimental checkers, plus a few alpha
 # checkers we don't want to regress on.
@@ -186,23 +203,73 @@ Verbose = 1
 
 # Run pre-processing script if any.
 def runCleanupScript(Dir, PBuildLogFile):
+    Cwd = os.path.join(Dir, PatchedSourceDirName)
     ScriptPath = os.path.join(Dir, CleanupScript)
+    runScript(ScriptPath, PBuildLogFile, Cwd)
+
+# Run the script to download the project, if it exists.
+def runDownloadScript(Dir, PBuildLogFile):
+    ScriptPath = os.path.join(Dir, DownloadScript)
+    runScript(ScriptPath, PBuildLogFile, Dir)
+
+# Run the provided script if it exists.
+def runScript(ScriptPath, PBuildLogFile, Cwd):
     if os.path.exists(ScriptPath):
         try:
             if Verbose == 1:
                 print "  Executing: %s" % (ScriptPath,)
-            check_call("chmod +x %s" % ScriptPath, cwd = Dir,
+            check_call("chmod +x %s" % ScriptPath, cwd = Cwd,
                                               stderr=PBuildLogFile,
                                               stdout=PBuildLogFile,
                                               shell=True)
-            check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile,
+            check_call(ScriptPath, cwd = Cwd, stderr=PBuildLogFile,
                                               stdout=PBuildLogFile,
                                               shell=True)
         except:
-            print "Error: The pre-processing step failed. See ", \
-                  PBuildLogFile.name, " for details."
+            print "Error: Running %s failed. See %s for details." % (ScriptPath,
+                PBuildLogFile.name)
             sys.exit(-1)
 
+# Download the project and apply the local patchfile if it exists.
+def downloadAndPatch(Dir, PBuildLogFile):
+    CachedSourceDirPath = os.path.join(Dir, CachedSourceDirName)
+
+    # If the we don't already have the cached source, run the project's
+    # download script to download it.
+    if not os.path.exists(CachedSourceDirPath):
+      runDownloadScript(Dir, PBuildLogFile)
+      if not os.path.exists(CachedSourceDirPath):
+        print "Error: '%s' not found after download." % (CachedSourceDirPath)
+        exit(-1)
+
+    PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
+
+    # Remove potentially stale patched source.
+    if os.path.exists(PatchedSourceDirPath):
+        shutil.rmtree(PatchedSourceDirPath)
+
+    # Copy the cached source and apply any patches to the copy.
+    shutil.copytree(CachedSourceDirPath, PatchedSourceDirPath, symlinks=True)
+    applyPatch(Dir, PBuildLogFile)
+
+def applyPatch(Dir, PBuildLogFile):
+    PatchfilePath = os.path.join(Dir, PatchfileName)
+    PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
+    if not os.path.exists(PatchfilePath):
+        print "  No local patches."
+        return
+
+    print "  Applying patch."
+    try:
+        check_call("patch -p1 < %s" % (PatchfilePath),
+                    cwd = PatchedSourceDirPath,
+                    stderr=PBuildLogFile,
+                    stdout=PBuildLogFile,
+                    shell=True)
+    except:
+        print "Error: Patch failed. See %s for details." % (PBuildLogFile.name)
+        sys.exit(-1)
+
 # Build the project with scan-build by reading in the commands and
 # prefixing them with the scan-build options.
 def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
@@ -215,6 +282,9 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
     if os.environ.has_key('SA_ADDITIONAL_CHECKERS'):
         AllCheckers = AllCheckers + ',' + os.environ['SA_ADDITIONAL_CHECKERS']
 
+    # Run scan-build from within the patched source directory.
+    SBCwd = os.path.join(Dir, PatchedSourceDirName)
+
     SBOptions = "--use-analyzer " + Clang + " "
     SBOptions += "-plist-html -o " + SBOutputDir + " "
     SBOptions += "-enable-checker " + AllCheckers + " "
@@ -238,9 +308,9 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
             SBCommand = SBPrefix + Command
             if Verbose == 1:
                 print "  Executing: %s" % (SBCommand,)
-            check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile,
-                                             stdout=PBuildLogFile,
-                                             shell=True)
+            check_call(SBCommand, cwd = SBCwd, stderr=PBuildLogFile,
+                                               stdout=PBuildLogFile,
+                                               shell=True)
     except:
         print "Error: scan-build failed. See ",PBuildLogFile.name,\
               " for details."
@@ -355,9 +425,9 @@ def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
 
     # Build and analyze the project.
     try:
-        runCleanupScript(Dir, PBuildLogFile)
-
         if (ProjectBuildMode == 1):
+            downloadAndPatch(Dir, PBuildLogFile)
+            runCleanupScript(Dir, PBuildLogFile)
             runScanBuild(Dir, SBOutputDir, PBuildLogFile)
         else:
             runAnalyzePreprocessed(Dir, SBOutputDir, ProjectBuildMode)
@@ -372,8 +442,12 @@ def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
                         continue
                     Plist = os.path.join(DirPath, F)
                     Data = plistlib.readPlist(Plist)
-                    Paths = [SourceFile[len(Dir)+1:] if SourceFile.startswith(Dir)\
-                            else SourceFile for SourceFile in Data['files']]
+                    PathPrefix = Dir
+                    if (ProjectBuildMode == 1):
+                        PathPrefix = os.path.join(Dir, PatchedSourceDirName)
+                    Paths = [SourceFile[len(PathPrefix)+1:]\
+                              if SourceFile.startswith(PathPrefix)\
+                              else SourceFile for SourceFile in Data['files']]
                     Data['files'] = Paths
                     plistlib.writePlist(Data, Plist)
 
@@ -489,7 +563,8 @@ def runCmpResults(Dir, Strictness = 0):
             print "  Comparing Results: %s %s" % (RefDir, NewDir)
 
         DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
-        Opts = CmpRuns.CmpOptions(DiffsPath, "", Dir)
+        PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
+        Opts = CmpRuns.CmpOptions(DiffsPath, "", PatchedSourceDirPath)
         # Discard everything coming out of stdout (CmpRun produces a lot of them).
         OLD_STDOUT = sys.stdout
         sys.stdout = Discarder()