From 68170291648f0112957a8b3d6912a1a1fed81965 Mon Sep 17 00:00:00 2001 From: Devin Coughlin Date: Sat, 7 Nov 2015 18:27:35 +0000 Subject: [PATCH] [analyzer] Update SATestBuild.py to enable a 'download and patch' model for projects. Currently the SATestBuild.py and SATestAdd.py buildbot scripts expect project sources to be checked into the project repository. This commit changes these scripts to additionally support a model where project sources are downloaded rather than checked into the repository. Sometimes projects may need to be modified (for example, to support a newer versions of clang), so the updated scripts also allow for an optional patch file that will be applied to the downloaded project source before analysis. To support this workflow, this commit changes the expected layout of a project in the repository. The project-specific helper scripts will stay in the root of each project directory, but the benchmark source itself (if checked into the repo) should now be stored in a subdirectory named 'CachedSource': project_name/ cleanup_run_static_analyzer.sh [optional] run_static_analyzer.cmd [required] download_project.sh [optional] CachedSource/ [optional] changes_for_analyzer.patch [optional] If the 'CachedSource' source directory is not present, the download script will be executed. This script should download the project source into 'CachedSource'. Then, if 'changes_for_analyzer.patch' is present its changes will be applied to a copy of 'CachedSource' before analysis. Differential Revision: http://reviews.llvm.org/D14345 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@252410 91177308-0d34-0410-b5e6-96231b3b80d8 --- utils/analyzer/SATestAdd.py | 26 ++++++++- utils/analyzer/SATestBuild.py | 99 ++++++++++++++++++++++++++++++----- 2 files changed, 112 insertions(+), 13 deletions(-) diff --git a/utils/analyzer/SATestAdd.py b/utils/analyzer/SATestAdd.py index 28050fddb8..4b94a109ce 100755 --- a/utils/analyzer/SATestAdd.py +++ b/utils/analyzer/SATestAdd.py @@ -10,11 +10,35 @@ the Repository Directory. have the same name as the project ID The project should use the following files for set up: - - pre_run_static_analyzer.sh - prepare the build environment. + - cleanup_run_static_analyzer.sh - prepare the build environment. Ex: make clean can be a part of it. - run_static_analyzer.cmd - a list of commands to run through scan-build. Each command should be on a separate line. Choose from: configure, make, xcodebuild + - download_project.sh - download the project into the CachedSource/ + directory. For example, download a zip of + the project source from GitHub, unzip it, + and rename the unzipped directory to + 'CachedSource'. This script is not called + when 'CachedSource' is already present, + so an alternative is to check the + 'CachedSource' directory into the + repository directly. + - CachedSource/ - An optional directory containing the source of the + project being analyzed. If present, + download_project.sh will not be called. + - changes_for_analyzer.patch - An optional patch file for any local changes + (e.g., to adapt to newer version of clang) + that should be applied to CachedSource + before analysis. To construct this patch, + run the the download script to download + the project to CachedSource, copy the + CachedSource to another directory (for + example, PatchedSource) and make any needed + modifications to the the copied source. + Then run: + diff -ur CachedSource PatchedSource \ + > changes_for_analyzer.patch """ import SATestBuild diff --git a/utils/analyzer/SATestBuild.py b/utils/analyzer/SATestBuild.py index 21cf4524bc..d0503c6389 100755 --- a/utils/analyzer/SATestBuild.py +++ b/utils/analyzer/SATestBuild.py @@ -154,6 +154,8 @@ Jobs = int(math.ceil(detectCPUs() * 0.75)) ProjectMapFile = "projectMap.csv" # Names of the project specific scripts. +# The script that downloads the project. +DownloadScript = "download_project.sh" # The script that needs to be executed before the build can start. CleanupScript = "cleanup_run_static_analyzer.sh" # This is a file containing commands for scan-build. @@ -173,6 +175,21 @@ DiffsSummaryFileName = "diffs.txt" SBOutputDirName = "ScanBuildResults" SBOutputDirReferencePrefix = "Ref" +# The name of the directory storing the cached project source. If this directory +# does not exist, the download script will be executed. That script should +# create the "CachedSource" directory and download the project source into it. +CachedSourceDirName = "CachedSource" + +# The name of the directory containing the source code that will be analyzed. +# Each time a project is analyzed, a fresh copy of its CachedSource directory +# will be copied to the PatchedSource directory and then the local patches +# in PatchfileName will be applied (if PatchfileName exists). +PatchedSourceDirName = "PatchedSource" + +# The name of the patchfile specifying any changes that should be applied +# to the CachedSource before analyzing. +PatchfileName = "changes_for_analyzer.patch" + # The list of checkers used during analyzes. # Currently, consists of all the non-experimental checkers, plus a few alpha # checkers we don't want to regress on. @@ -186,23 +203,73 @@ Verbose = 1 # Run pre-processing script if any. def runCleanupScript(Dir, PBuildLogFile): + Cwd = os.path.join(Dir, PatchedSourceDirName) ScriptPath = os.path.join(Dir, CleanupScript) + runScript(ScriptPath, PBuildLogFile, Cwd) + +# Run the script to download the project, if it exists. +def runDownloadScript(Dir, PBuildLogFile): + ScriptPath = os.path.join(Dir, DownloadScript) + runScript(ScriptPath, PBuildLogFile, Dir) + +# Run the provided script if it exists. +def runScript(ScriptPath, PBuildLogFile, Cwd): if os.path.exists(ScriptPath): try: if Verbose == 1: print " Executing: %s" % (ScriptPath,) - check_call("chmod +x %s" % ScriptPath, cwd = Dir, + check_call("chmod +x %s" % ScriptPath, cwd = Cwd, stderr=PBuildLogFile, stdout=PBuildLogFile, shell=True) - check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile, + check_call(ScriptPath, cwd = Cwd, stderr=PBuildLogFile, stdout=PBuildLogFile, shell=True) except: - print "Error: The pre-processing step failed. See ", \ - PBuildLogFile.name, " for details." + print "Error: Running %s failed. See %s for details." % (ScriptPath, + PBuildLogFile.name) sys.exit(-1) +# Download the project and apply the local patchfile if it exists. +def downloadAndPatch(Dir, PBuildLogFile): + CachedSourceDirPath = os.path.join(Dir, CachedSourceDirName) + + # If the we don't already have the cached source, run the project's + # download script to download it. + if not os.path.exists(CachedSourceDirPath): + runDownloadScript(Dir, PBuildLogFile) + if not os.path.exists(CachedSourceDirPath): + print "Error: '%s' not found after download." % (CachedSourceDirPath) + exit(-1) + + PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) + + # Remove potentially stale patched source. + if os.path.exists(PatchedSourceDirPath): + shutil.rmtree(PatchedSourceDirPath) + + # Copy the cached source and apply any patches to the copy. + shutil.copytree(CachedSourceDirPath, PatchedSourceDirPath, symlinks=True) + applyPatch(Dir, PBuildLogFile) + +def applyPatch(Dir, PBuildLogFile): + PatchfilePath = os.path.join(Dir, PatchfileName) + PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) + if not os.path.exists(PatchfilePath): + print " No local patches." + return + + print " Applying patch." + try: + check_call("patch -p1 < %s" % (PatchfilePath), + cwd = PatchedSourceDirPath, + stderr=PBuildLogFile, + stdout=PBuildLogFile, + shell=True) + except: + print "Error: Patch failed. See %s for details." % (PBuildLogFile.name) + sys.exit(-1) + # Build the project with scan-build by reading in the commands and # prefixing them with the scan-build options. def runScanBuild(Dir, SBOutputDir, PBuildLogFile): @@ -215,6 +282,9 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile): if os.environ.has_key('SA_ADDITIONAL_CHECKERS'): AllCheckers = AllCheckers + ',' + os.environ['SA_ADDITIONAL_CHECKERS'] + # Run scan-build from within the patched source directory. + SBCwd = os.path.join(Dir, PatchedSourceDirName) + SBOptions = "--use-analyzer " + Clang + " " SBOptions += "-plist-html -o " + SBOutputDir + " " SBOptions += "-enable-checker " + AllCheckers + " " @@ -238,9 +308,9 @@ def runScanBuild(Dir, SBOutputDir, PBuildLogFile): SBCommand = SBPrefix + Command if Verbose == 1: print " Executing: %s" % (SBCommand,) - check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile, - stdout=PBuildLogFile, - shell=True) + check_call(SBCommand, cwd = SBCwd, stderr=PBuildLogFile, + stdout=PBuildLogFile, + shell=True) except: print "Error: scan-build failed. See ",PBuildLogFile.name,\ " for details." @@ -355,9 +425,9 @@ def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild): # Build and analyze the project. try: - runCleanupScript(Dir, PBuildLogFile) - if (ProjectBuildMode == 1): + downloadAndPatch(Dir, PBuildLogFile) + runCleanupScript(Dir, PBuildLogFile) runScanBuild(Dir, SBOutputDir, PBuildLogFile) else: runAnalyzePreprocessed(Dir, SBOutputDir, ProjectBuildMode) @@ -372,8 +442,12 @@ def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild): continue Plist = os.path.join(DirPath, F) Data = plistlib.readPlist(Plist) - Paths = [SourceFile[len(Dir)+1:] if SourceFile.startswith(Dir)\ - else SourceFile for SourceFile in Data['files']] + PathPrefix = Dir + if (ProjectBuildMode == 1): + PathPrefix = os.path.join(Dir, PatchedSourceDirName) + Paths = [SourceFile[len(PathPrefix)+1:]\ + if SourceFile.startswith(PathPrefix)\ + else SourceFile for SourceFile in Data['files']] Data['files'] = Paths plistlib.writePlist(Data, Plist) @@ -489,7 +563,8 @@ def runCmpResults(Dir, Strictness = 0): print " Comparing Results: %s %s" % (RefDir, NewDir) DiffsPath = os.path.join(NewDir, DiffsSummaryFileName) - Opts = CmpRuns.CmpOptions(DiffsPath, "", Dir) + PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) + Opts = CmpRuns.CmpOptions(DiffsPath, "", PatchedSourceDirPath) # Discard everything coming out of stdout (CmpRun produces a lot of them). OLD_STDOUT = sys.stdout sys.stdout = Discarder() -- 2.40.0