From 337c8c682f2e5c55f6ed66fa770602c95e7b223b Mon Sep 17 00:00:00 2001
From: Dmitri Gribenko <gribozavr@gmail.com>
Date: Wed, 12 Feb 2014 10:33:14 +0000
Subject: [PATCH] Add an option to allow Clang verify source files for a module
 only once during the build

When Clang loads the module, it verifies the user source files that the module
was built from.  If any file was changed, the module is rebuilt.  There are two
problems with this:
1. correctness: we don't verify system files (there are too many of them, and
   stat'ing all of them would take a lot of time);
2. performance: the same module file is verified again and again during a
   single build.

This change allows the build system to optimize source file verification.  The
idea is based on the fact that while the project is being built, the source
files don't change.  This allows us to verify the module only once during a
single build session.  The build system passes a flag,
-fbuild-session-timestamp=, to inform Clang of the time when the build started.
The build system also requests to enable this feature by passing
-fmodules-validate-once-per-build-session.  If these flags are not passed, the
behavior is not changed.  When Clang verifies the module the first time, it
writes out a timestamp file.  Then, when Clang loads the module the second
time, it finds a timestamp file, so it can compare the verification timestamp
of the module with the time when the build started.  If the verification
timestamp is too old, the module is verified again, and the timestamp file is
updated.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@201224 91177308-0d34-0410-b5e6-96231b3b80d8
---
 include/clang-c/Index.h                      |  2 +-
 include/clang/Basic/DiagnosticDriverKinds.td |  4 ++
 include/clang/Driver/Options.td              |  7 +++
 include/clang/Frontend/Utils.h               | 11 +++++
 include/clang/Lex/HeaderSearchOptions.h      | 15 +++++-
 include/clang/Serialization/Module.h         | 10 ++++
 lib/Driver/Tools.cpp                         | 10 ++++
 lib/Frontend/CompilerInstance.cpp            | 25 +++++-----
 lib/Frontend/CompilerInvocation.cpp          | 26 +++++++++--
 lib/Serialization/ASTReader.cpp              | 49 ++++++++++++++++++--
 lib/Serialization/ModuleManager.cpp          | 10 ++++
 test/Driver/modules.m                        | 10 ++++
 tools/c-index-test/c-index-test.c            | 10 ++++
 tools/libclang/CMakeLists.txt                |  1 +
 tools/libclang/libclang.exports              |  1 +
 15 files changed, 166 insertions(+), 25 deletions(-)

diff --git a/include/clang-c/Index.h b/include/clang-c/Index.h
index d98c1e66fd..0f15216900 100644
--- a/include/clang-c/Index.h
+++ b/include/clang-c/Index.h
@@ -30,7 +30,7 @@
  * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable.
  */
 #define CINDEX_VERSION_MAJOR 0
-#define CINDEX_VERSION_MINOR 21
+#define CINDEX_VERSION_MINOR 22
 
 #define CINDEX_VERSION_ENCODE(major, minor) ( \
       ((major) * 10000)                       \
diff --git a/include/clang/Basic/DiagnosticDriverKinds.td b/include/clang/Basic/DiagnosticDriverKinds.td
index dbc5ce994c..cc7d8969f3 100644
--- a/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/include/clang/Basic/DiagnosticDriverKinds.td
@@ -166,4 +166,8 @@ def err_analyzer_config_no_value : Error<
   "analyzer-config option '%0' has a key but no value">;
 def err_analyzer_config_multiple_values : Error<
   "analyzer-config option '%0' should contain only one '='">;
+
+def err_drv_modules_validate_once_requires_timestamp : Error<
+  "option '-fmodules-validate-once-per-build-session' requires "
+  "'-fbuild-session-timestamp=<seconds since Epoch>'">;
 }
diff --git a/include/clang/Driver/Options.td b/include/clang/Driver/Options.td
index 52cd9efdd4..ed0fc0315e 100644
--- a/include/clang/Driver/Options.td
+++ b/include/clang/Driver/Options.td
@@ -581,6 +581,13 @@ def fmodules_prune_interval : Joined<["-"], "fmodules-prune-interval=">, Group<i
 def fmodules_prune_after : Joined<["-"], "fmodules-prune-after=">, Group<i_Group>,
   Flags<[CC1Option]>, MetaVarName<"<seconds>">,
   HelpText<"Specify the interval (in seconds) after which a module file will be considered unused">;
+def fbuild_session_timestamp : Joined<["-"], "fbuild-session-timestamp=">,
+  Group<i_Group>, Flags<[CC1Option]>, MetaVarName<"<time since Epoch in seconds>">,
+  HelpText<"Time when the current build session started">;
+def fmodules_validate_once_per_build_session : Flag<["-"], "fmodules-validate-once-per-build-session">,
+  Group<i_Group>, Flags<[CC1Option]>,
+  HelpText<"Don't verify input files for the modules if the module has been "
+           "successfully validate or loaded during this build session">;
 def fmodules : Flag <["-"], "fmodules">, Group<f_Group>,
   Flags<[DriverOption, CC1Option]>,
   HelpText<"Enable the 'modules' language feature">;
diff --git a/include/clang/Frontend/Utils.h b/include/clang/Frontend/Utils.h
index db52d5cf0d..471d9ac82b 100644
--- a/include/clang/Frontend/Utils.h
+++ b/include/clang/Frontend/Utils.h
@@ -123,6 +123,17 @@ inline int getLastArgIntValue(const llvm::opt::ArgList &Args,
   return getLastArgIntValue(Args, Id, Default, &Diags);
 }
 
+uint64_t getLastArgUInt64Value(const llvm::opt::ArgList &Args,
+                               llvm::opt::OptSpecifier Id, uint64_t Default,
+                               DiagnosticsEngine *Diags = 0);
+
+inline uint64_t getLastArgUInt64Value(const llvm::opt::ArgList &Args,
+                                      llvm::opt::OptSpecifier Id,
+                                      uint64_t Default,
+                                      DiagnosticsEngine &Diags) {
+  return getLastArgUInt64Value(Args, Id, Default, &Diags);
+}
+
 // When Clang->getFrontendOpts().DisableFree is set we don't delete some of the
 // global objects, but we don't want LeakDetectors to complain, so we bury them
 // in a globally visible array.
diff --git a/include/clang/Lex/HeaderSearchOptions.h b/include/clang/Lex/HeaderSearchOptions.h
index 0b21c0dd34..b471b8cafe 100644
--- a/include/clang/Lex/HeaderSearchOptions.h
+++ b/include/clang/Lex/HeaderSearchOptions.h
@@ -116,6 +116,12 @@ public:
   /// regenerated often.
   unsigned ModuleCachePruneAfter;
 
+  /// \brief The time in seconds when the build session started.
+  ///
+  /// This time is used by other optimizations in header search and module
+  /// loading.
+  uint64_t BuildSessionTimestamp;
+
   /// \brief The set of macro names that should be ignored for the purposes
   /// of computing the module hash.
   llvm::SetVector<std::string> ModulesIgnoreMacros;
@@ -138,14 +144,21 @@ public:
   /// Whether header search information should be output as for -v.
   unsigned Verbose : 1;
 
+  /// \brief If true, skip verifying input files used by modules if the
+  /// module was already verified during this build session (see
+  /// \c BuildSessionTimestamp).
+  unsigned ModulesValidateOncePerBuildSession : 1;
+
 public:
   HeaderSearchOptions(StringRef _Sysroot = "/")
     : Sysroot(_Sysroot), DisableModuleHash(0), ModuleMaps(0),
       ModuleCachePruneInterval(7*24*60*60),
       ModuleCachePruneAfter(31*24*60*60),
+      BuildSessionTimestamp(0),
       UseBuiltinIncludes(true),
       UseStandardSystemIncludes(true), UseStandardCXXIncludes(true),
-      UseLibcxx(false), Verbose(false) {}
+      UseLibcxx(false), Verbose(false),
+      ModulesValidateOncePerBuildSession(false) {}
 
   /// AddPath - Add the \p Path path to the specified \p Group list.
   void AddPath(StringRef Path, frontend::IncludeDirGroup Group,
diff --git a/include/clang/Serialization/Module.h b/include/clang/Serialization/Module.h
index f4c07553d6..821c9cdfb5 100644
--- a/include/clang/Serialization/Module.h
+++ b/include/clang/Serialization/Module.h
@@ -115,6 +115,10 @@ public:
   /// \brief The file name of the module file.
   std::string FileName;
 
+  std::string getTimestampFilename() const {
+    return FileName + ".timestamp";
+  }
+
   /// \brief The original source file name that was used to build the
   /// primary AST file, which may have been modified for
   /// relocatable-pch support.
@@ -185,6 +189,12 @@ public:
   /// \brief The input files that have been loaded from this AST file.
   std::vector<InputFile> InputFilesLoaded;
 
+  /// \brief If non-zero, specifies the time when we last validated input
+  /// files.  Zero means we never validated them.
+  ///
+  /// The time is specified in seconds since the start of the Epoch.
+  uint64_t InputFilesValidationTimestamp;
+
   // === Source Locations ===
 
   /// \brief Cursor used to read source location entries.
diff --git a/lib/Driver/Tools.cpp b/lib/Driver/Tools.cpp
index 9e71c31c62..73437f68f3 100644
--- a/lib/Driver/Tools.cpp
+++ b/lib/Driver/Tools.cpp
@@ -3289,6 +3289,16 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_interval);
   Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_after);
 
+  Args.AddLastArg(CmdArgs, options::OPT_fbuild_session_timestamp);
+
+  if (Args.getLastArg(options::OPT_fmodules_validate_once_per_build_session)) {
+    if (!Args.getLastArg(options::OPT_fbuild_session_timestamp))
+      D.Diag(diag::err_drv_modules_validate_once_requires_timestamp);
+
+    Args.AddLastArg(CmdArgs,
+                    options::OPT_fmodules_validate_once_per_build_session);
+  }
+
   // -faccess-control is default.
   if (Args.hasFlag(options::OPT_fno_access_control,
                    options::OPT_faccess_control,
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index d121180125..61fe26c922 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -1057,41 +1057,38 @@ static void pruneModuleCache(const HeaderSearchOptions &HSOpts) {
       continue;
 
     // Walk all of the files within this directory.
-    bool RemovedAllFiles = true;
     for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
          File != FileEnd && !EC; File.increment(EC)) {
       // We only care about module and global module index files.
-      if (llvm::sys::path::extension(File->path()) != ".pcm" &&
-          llvm::sys::path::filename(File->path()) != "modules.idx") {
-        RemovedAllFiles = false;
+      StringRef Extension = llvm::sys::path::extension(File->path());
+      if (Extension != ".pcm" && Extension != ".timestamp" &&
+          llvm::sys::path::filename(File->path()) != "modules.idx")
         continue;
-      }
 
       // Look at this file. If we can't stat it, there's nothing interesting
       // there.
-      if (::stat(File->path().c_str(), &StatBuf)) {
-        RemovedAllFiles = false;
+      if (::stat(File->path().c_str(), &StatBuf))
         continue;
-      }
 
       // If the file has been used recently enough, leave it there.
       time_t FileAccessTime = StatBuf.st_atime;
       if (CurrentTime - FileAccessTime <=
               time_t(HSOpts.ModuleCachePruneAfter)) {
-        RemovedAllFiles = false;
         continue;
       }
 
       // Remove the file.
-      bool Existed;
-      if (llvm::sys::fs::remove(File->path(), Existed) || !Existed) {
-        RemovedAllFiles = false;
-      }
+      llvm::sys::fs::remove(File->path());
+
+      // Remove the timestamp file.
+      std::string TimpestampFilename = File->path() + ".timestamp";
+      llvm::sys::fs::remove(TimpestampFilename);
     }
 
     // If we removed all of the files in the directory, remove the directory
     // itself.
-    if (RemovedAllFiles)
+    if (llvm::sys::fs::directory_iterator(Dir->path(), EC) ==
+            llvm::sys::fs::directory_iterator() && !EC)
       llvm::sys::fs::remove(Dir->path());
   }
 }
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index 5320415ecb..5b26df037f 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -924,6 +924,10 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
       getLastArgIntValue(Args, OPT_fmodules_prune_interval, 7 * 24 * 60 * 60);
   Opts.ModuleCachePruneAfter =
       getLastArgIntValue(Args, OPT_fmodules_prune_after, 31 * 24 * 60 * 60);
+  Opts.ModulesValidateOncePerBuildSession =
+      Args.hasArg(OPT_fmodules_validate_once_per_build_session);
+  Opts.BuildSessionTimestamp =
+      getLastArgUInt64Value(Args, OPT_fbuild_session_timestamp, 0);
   for (arg_iterator it = Args.filtered_begin(OPT_fmodules_ignore_macro),
                     ie = Args.filtered_end();
        it != ie; ++it) {
@@ -1837,10 +1841,11 @@ std::string CompilerInvocation::getModuleHash() const {
 
 namespace clang {
 
-// Declared in clang/Frontend/Utils.h.
-int getLastArgIntValue(const ArgList &Args, OptSpecifier Id, int Default,
-                       DiagnosticsEngine *Diags) {
-  int Res = Default;
+template<typename IntTy>
+static IntTy getLastArgIntValueImpl(const ArgList &Args, OptSpecifier Id,
+                                    IntTy Default,
+                                    DiagnosticsEngine *Diags) {
+  IntTy Res = Default;
   if (Arg *A = Args.getLastArg(Id)) {
     if (StringRef(A->getValue()).getAsInteger(10, Res)) {
       if (Diags)
@@ -1851,6 +1856,19 @@ int getLastArgIntValue(const ArgList &Args, OptSpecifier Id, int Default,
   return Res;
 }
 
+
+// Declared in clang/Frontend/Utils.h.
+int getLastArgIntValue(const ArgList &Args, OptSpecifier Id, int Default,
+                       DiagnosticsEngine *Diags) {
+  return getLastArgIntValueImpl<int>(Args, Id, Default, Diags);
+}
+
+uint64_t getLastArgUInt64Value(const ArgList &Args, OptSpecifier Id,
+                               uint64_t Default,
+                               DiagnosticsEngine *Diags) {
+  return getLastArgIntValueImpl<uint64_t>(Args, Id, Default, Diags);
+}
+
 void BuryPointer(const void *Ptr) {
   // This function may be called only a small fixed amount of times per each
   // invocation, otherwise we do actually have a leak which we want to report.
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index 674aefeb1c..fb1d7edcba 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -49,6 +49,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SaveAndRestore.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
 #include <algorithm>
 #include <cstdio>
@@ -1827,16 +1828,28 @@ ASTReader::ReadControlBlock(ModuleFile &F,
     case llvm::BitstreamEntry::Error:
       Error("malformed block record in AST file");
       return Failure;
-    case llvm::BitstreamEntry::EndBlock:
-      // Validate all of the non-system input files.
-      if (!DisableValidation) {
+    case llvm::BitstreamEntry::EndBlock: {
+      // Validate input files.
+      const HeaderSearchOptions &HSOpts =
+          PP.getHeaderSearchInfo().getHeaderSearchOpts();
+      if (!DisableValidation &&
+          (!HSOpts.ModulesValidateOncePerBuildSession ||
+           F.InputFilesValidationTimestamp <= HSOpts.BuildSessionTimestamp)) {
         bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
         // All user input files reside at the index range [0, Record[1]), and
         // system input files reside at [Record[1], Record[0]).
         // Record is the one from INPUT_FILE_OFFSETS.
+        //
+        // If we are reading a module, we will create a verification timestamp,
+        // so we verify all input files.  Otherwise, verify only user input
+        // files.
         unsigned NumInputs = Record[0];
         unsigned NumUserInputs = Record[1];
-        unsigned N = ValidateSystemInputs ? NumInputs : NumUserInputs;
+        unsigned N = ValidateSystemInputs ||
+                             (HSOpts.ModulesValidateOncePerBuildSession &&
+                              F.Kind == MK_Module)
+                         ? NumInputs
+                         : NumUserInputs;
         for (unsigned I = 0; I < N; ++I) {
           InputFile IF = getInputFile(F, I+1, Complain);
           if (!IF.getFile() || IF.isOutOfDate())
@@ -1844,7 +1857,8 @@ ASTReader::ReadControlBlock(ModuleFile &F,
         }
       }
       return Success;
-      
+    }
+
     case llvm::BitstreamEntry::SubBlock:
       switch (Entry.ID) {
       case INPUT_FILES_BLOCK_ID:
@@ -2922,6 +2936,16 @@ bool ASTReader::isGlobalIndexUnavailable() const {
          !hasGlobalIndex() && TriedLoadingGlobalIndex;
 }
 
+static void updateModuleTimestamp(ModuleFile &MF) {
+  // Overwrite the timestamp file contents so that file's mtime changes.
+  std::string TimestampFilename = MF.getTimestampFilename();
+  std::string ErrorInfo;
+  llvm::raw_fd_ostream OS(TimestampFilename.c_str(), ErrorInfo);
+  if (!ErrorInfo.empty())
+    return;
+  OS << "Timestamp file\n";
+}
+
 ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
                                             ModuleKind Type,
                                             SourceLocation ImportLoc,
@@ -3080,6 +3104,21 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
                        PreviousGeneration);
   }
 
+  if (PP.getHeaderSearchInfo()
+          .getHeaderSearchOpts()
+          .ModulesValidateOncePerBuildSession) {
+    // Now we are certain that the module and all modules it depends on are
+    // up to date.  Create or update timestamp files for modules that are
+    // located in the module cache (not for PCH files that could be anywhere
+    // in the filesystem).
+    for (unsigned I = 0, N = Loaded.size(); I != N; ++I) {
+      ImportedModule &M = Loaded[I];
+      if (M.Mod->Kind == MK_Module) {
+        updateModuleTimestamp(*M.Mod);
+      }
+    }
+  }
+
   return Success;
 }
 
diff --git a/lib/Serialization/ModuleManager.cpp b/lib/Serialization/ModuleManager.cpp
index 9c4b3d92be..711afcd2dc 100644
--- a/lib/Serialization/ModuleManager.cpp
+++ b/lib/Serialization/ModuleManager.cpp
@@ -86,6 +86,16 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
     NewModule = true;
     ModuleEntry = New;
 
+    New->InputFilesValidationTimestamp = 0;
+    if (New->Kind == MK_Module) {
+      std::string TimestampFilename = New->getTimestampFilename();
+      llvm::sys::fs::file_status Status;
+      // A cached stat value would be fine as well.
+      if (!FileMgr.getNoncachedStatValue(TimestampFilename, Status))
+        New->InputFilesValidationTimestamp =
+            Status.getLastModificationTime().toEpochTime();
+    }
+
     // Load the contents of the module
     if (llvm::MemoryBuffer *Buffer = lookupBuffer(FileName)) {
       // The buffer was already provided for us.
diff --git a/test/Driver/modules.m b/test/Driver/modules.m
index b93054dbf8..7b14f21443 100644
--- a/test/Driver/modules.m
+++ b/test/Driver/modules.m
@@ -4,3 +4,13 @@
 // RUN: %clang -fmodules -fno-modules -fmodules -### %s 2>&1 | FileCheck -check-prefix=CHECK-HAS-MODULES %s
 // CHECK-HAS-MODULES: -fmodules
 
+// RUN: %clang -fbuild-session-timestamp=123 -### %s 2>&1 | FileCheck -check-prefix=TIMESTAMP_ONLY %s
+// TIMESTAMP_ONLY: -fbuild-session-timestamp=123
+
+// RUN: %clang -fbuild-session-timestamp=123 -fmodules-validate-once-per-build-session -### %s 2>&1 | FileCheck -check-prefix=MODULES_VALIDATE_ONCE %s
+// MODULES_VALIDATE_ONCE: -fbuild-session-timestamp=123
+// MODULES_VALIDATE_ONCE: -fmodules-validate-once-per-build-session
+
+// RUN: %clang -fmodules-validate-once-per-build-session -### %s 2>&1 | FileCheck -check-prefix=MODULES_VALIDATE_ONCE_ERR %s
+// MODULES_VALIDATE_ONCE_ERR: option '-fmodules-validate-once-per-build-session' requires '-fbuild-session-timestamp=<seconds since Epoch>'
+
diff --git a/tools/c-index-test/c-index-test.c b/tools/c-index-test/c-index-test.c
index 9288f92d17..7f5bf4c2ed 100644
--- a/tools/c-index-test/c-index-test.c
+++ b/tools/c-index-test/c-index-test.c
@@ -2,6 +2,7 @@
 
 #include "clang-c/Index.h"
 #include "clang-c/CXCompilationDatabase.h"
+#include "clang-c/BuildSystem.h"
 #include "llvm/Config/config.h"
 #include <ctype.h>
 #include <stdlib.h>
@@ -3800,6 +3801,11 @@ static int read_diagnostics(const char *filename) {
   return 0;
 }
 
+static int perform_print_build_session_timestamp(void) {
+  printf("%lld\n", clang_getBuildSessionTimestamp());
+  return 0;
+}
+
 /******************************************************************************/
 /* Command line processing.                                                   */
 /******************************************************************************/
@@ -3855,6 +3861,8 @@ static void print_usage(void) {
     "       c-index-test -write-pch <file> <compiler arguments>\n");
   fprintf(stderr,
     "       c-index-test -compilation-db [lookup <filename>] database\n");
+  fprintf(stderr,
+    "       c-index-test -print-build-session-timestamp\n");
   fprintf(stderr,
     "       c-index-test -read-diagnostics <file>\n\n");
   fprintf(stderr,
@@ -3955,6 +3963,8 @@ int cindextest_main(int argc, const char **argv) {
     return write_pch_file(argv[2], argc - 3, argv + 3);
   else if (argc > 2 && strcmp(argv[1], "-compilation-db") == 0)
     return perform_test_compilation_db(argv[argc-1], argc - 3, argv + 2);
+  else if (argc == 2 && strcmp(argv[1], "-print-build-session-timestamp") == 0)
+    return perform_print_build_session_timestamp();
 
   print_usage();
   return 1;
diff --git a/tools/libclang/CMakeLists.txt b/tools/libclang/CMakeLists.txt
index 97203b0df4..95a8a6ccba 100644
--- a/tools/libclang/CMakeLists.txt
+++ b/tools/libclang/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
 
 set(SOURCES
   ARCMigrate.cpp
+  BuildSystem.cpp
   CIndex.cpp
   CIndexCXX.cpp
   CIndexCodeCompletion.cpp
diff --git a/tools/libclang/libclang.exports b/tools/libclang/libclang.exports
index 809cf03a10..975993adb3 100644
--- a/tools/libclang/libclang.exports
+++ b/tools/libclang/libclang.exports
@@ -119,6 +119,7 @@ clang_formatDiagnostic
 clang_getArgType
 clang_getArrayElementType
 clang_getArraySize
+clang_getBuildSessionTimestamp
 clang_getCString
 clang_getCXTUResourceUsage
 clang_getCXXAccessSpecifier
-- 
2.50.1