[ThinLTO] Add option to emit imports files for distributed backends

Summary:
Add support for emission of plaintext lists of the imported files for
each distributed backend compilation. Used for distributed build file
staging.

Invoked with new gold-plugin thinlto-emit-imports-files option, which is
only valid with thinlto-index-only (i.e. for distributed builds), or
from llvm-lto with new -thinlto-action=emitimports value.

Depends on D19556.

Reviewers: joker.eph

Subscribers: llvm-commits, joker.eph

Differential Revision: http://reviews.llvm.org/D19636

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@269067 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/LTO/ThinLTOCodeGenerator.h b/include/llvm/LTO/ThinLTOCodeGenerator.h
index 7a8face2542183da09d23206ef8a189aee66113a..4a47c1fbc968048b971622edc7d762bbba2b70f7 100644 (file)
--- a/include/llvm/LTO/ThinLTOCodeGenerator.h
+++ b/include/llvm/LTO/ThinLTOCodeGenerator.h
@@ -201,6 +201,12 @@ public:
    */
   void promote(Module &Module, ModuleSummaryIndex &Index);
 
+  /**
+   * Compute and emit the imported files for module at \p ModulePath.
+   */
+  static void emitImports(StringRef ModulePath, StringRef OutputName,
+                          ModuleSummaryIndex &Index);
+
   /**
    * Perform cross-module importing for the module identified by
    * ModuleIdentifier.

diff --git a/include/llvm/Transforms/IPO/FunctionImport.h b/include/llvm/Transforms/IPO/FunctionImport.h
index 6e13141d3755bffadb867dac35af341c84556905..276185c9082a6ae518011fa501d0adc0f5fe5f96 100644 (file)
--- a/include/llvm/Transforms/IPO/FunctionImport.h
+++ b/include/llvm/Transforms/IPO/FunctionImport.h
@@ -103,6 +103,10 @@ void gatherImportedSummariesForModule(
     const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
     const StringMap<FunctionImporter::ImportMapTy> &ImportLists,
     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex);
+
+std::error_code
+EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename,
+                 const StringMap<FunctionImporter::ImportMapTy> &ImportLists);
 }
 
 #endif // LLVM_FUNCTIONIMPORT_H

diff --git a/lib/LTO/ThinLTOCodeGenerator.cpp b/lib/LTO/ThinLTOCodeGenerator.cpp
index 68d7a714b6b1cc49a8716b81a94d0b3bd47d519f..93a3f3e84ad09e4e4af74c609fe42b43f486c46a 100644 (file)
--- a/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -742,6 +742,30 @@ void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
                                          ModuleToSummariesForIndex);
 }
 
+/**
+ * Emit the list of files needed for importing into module.
+ */
+void ThinLTOCodeGenerator::emitImports(StringRef ModulePath,
+                                       StringRef OutputName,
+                                       ModuleSummaryIndex &Index) {
+  auto ModuleCount = Index.modulePaths().size();
+
+  // Collect for each module the list of function it defines (GUID -> Summary).
+  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
+  Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
+
+  // Generate import/export list
+  StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
+  StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
+  ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
+                           ExportLists);
+
+  std::error_code EC;
+  if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists)))
+    report_fatal_error(Twine("Failed to open ") + OutputName +
+                       " to save imports lists\n");
+}
+
 /**
  * Perform internalization.
  */

diff --git a/lib/Transforms/IPO/FunctionImport.cpp b/lib/Transforms/IPO/FunctionImport.cpp
index 6c571f440b9ab85c1bc572db21cc534c37c974a9..3ce3b7b841ef719bed5b4f882c87f689fe8bce07 100644 (file)
--- a/lib/Transforms/IPO/FunctionImport.cpp
+++ b/lib/Transforms/IPO/FunctionImport.cpp
@@ -445,6 +445,21 @@ void llvm::gatherImportedSummariesForModule(
   }
 }
 
+/// Emit the files \p ModulePath will import from into \p OutputFilename.
+std::error_code llvm::EmitImportsFiles(
+    StringRef ModulePath, StringRef OutputFilename,
+    const StringMap<FunctionImporter::ImportMapTy> &ImportLists) {
+  auto ModuleImports = ImportLists.find(ModulePath);
+  std::error_code EC;
+  raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None);
+  if (EC)
+    return EC;
+  if (ModuleImports != ImportLists.end())
+    for (auto &ILI : ModuleImports->second)
+      ImportsOS << ILI.first() << "\n";
+  return std::error_code();
+}
+
 // Automatically import functions in Module \p DestModule based on the summaries
 // index.
 //

diff --git a/test/ThinLTO/X86/Inputs/emit_imports.ll b/test/ThinLTO/X86/Inputs/emit_imports.ll
new file mode 100644 (file)
index 0000000..4e0840f
--- /dev/null
+++ b/test/ThinLTO/X86/Inputs/emit_imports.ll
@@ -0,0 +1,4 @@
+define void @g() {
+entry:
+  ret void
+}

diff --git a/test/ThinLTO/X86/emit_imports.ll b/test/ThinLTO/X86/emit_imports.ll
new file mode 100644 (file)
index 0000000..3fcdf87
--- /dev/null
+++ b/test/ThinLTO/X86/emit_imports.ll
@@ -0,0 +1,21 @@
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %p/Inputs/emit_imports.ll -o %t2.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc
+; RUN: llvm-lto -thinlto-action=emitimports -thinlto-index %t.index.bc %t1.bc %t2.bc
+
+; The imports file for this module contains the bitcode file for
+; Inputs/emit_imports.ll
+; RUN: cat %t1.bc.imports | count 1
+; RUN: cat %t1.bc.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: emit_imports.ll.tmp2.bc
+
+; The imports file for Input/emit_imports.ll is empty as it does not import anything.
+; RUN: cat %t2.bc.imports | count 0
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}

diff --git a/test/tools/gold/X86/thinlto_emit_imports.ll b/test/tools/gold/X86/thinlto_emit_imports.ll
new file mode 100644 (file)
index 0000000..845b691
--- /dev/null
+++ b/test/tools/gold/X86/thinlto_emit_imports.ll
@@ -0,0 +1,27 @@
+; Generate summary sections and test gold handling.
+; RUN: opt -module-summary %s -o %t.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
+
+; Ensure gold generates imports files if requested for distributed backends.
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=thinlto-index-only \
+; RUN:    --plugin-opt=thinlto-emit-imports-files \
+; RUN:    -shared %t.o %t2.o -o %t3
+
+; The imports file for this module contains the bitcode file for
+; Inputs/thinlto.ll
+; RUN: cat %t.o.imports | count 1
+; RUN: cat %t.o.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: test/tools/gold/X86/Output/thinlto_emit_imports.ll.tmp2.o
+
+; The imports file for Input/thinlto.ll is empty as it does not import anything.
+; RUN: cat %t2.o.imports | count 0
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}

diff --git a/tools/gold/gold-plugin.cpp b/tools/gold/gold-plugin.cpp
index fb433beb6586c0152c80a79cba803f4d92e9fb56..44df9d535ec822a20d51bcc0e5cedc436bfad692 100644 (file)
--- a/tools/gold/gold-plugin.cpp
+++ b/tools/gold/gold-plugin.cpp
@@ -184,6 +184,11 @@ namespace options {
   // the import decisions, and exit afterwards. The assumption is
   // that the build system will launch the backend processes.
   static bool thinlto_index_only = false;
+  // If true, when generating individual index files for distributed backends,
+  // also generate a "${bitcodefile}.imports" file at the same location for each
+  // bitcode file, listing the files it imports from in plain text. This is to
+  // support distributed build file staging.
+  static bool thinlto_emit_imports_files = false;
   // Additional options to pass into the code generator.
   // Note: This array will contain all plugin options which are not claimed
   // as plugin exclusive to pass to the code generator.
@@ -217,6 +222,8 @@ namespace options {
       thinlto = true;
     } else if (opt == "thinlto-index-only") {
       thinlto_index_only = true;
+    } else if (opt == "thinlto-emit-imports-files") {
+      thinlto_emit_imports_files = true;
     } else if (opt.size() == 2 && opt[0] == 'O') {
       if (opt[1] < '0' || opt[1] > '3')
         message(LDPL_FATAL, "Optimization level must be between 0 and 3");
@@ -1209,6 +1216,10 @@ static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) {
       CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
   }
 
+  if (options::thinlto_emit_imports_files && !options::thinlto_index_only)
+    message(LDPL_WARNING,
+            "thinlto-emit-imports-files ignored unless thinlto-index-only");
+
   if (options::thinlto_index_only) {
     // Collect for each module the list of function it defines (GUID ->
     // Summary).
@@ -1244,6 +1255,15 @@ static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) {
                                        ModuleToDefinedGVSummaries, ImportLists,
                                        ModuleToSummariesForIndex);
       WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
+
+      if (options::thinlto_emit_imports_files) {
+        if ((EC = EmitImportsFiles(
+                 InputFile.file().name,
+                 (Twine(InputFile.file().name) + ".imports").str(),
+                 ImportLists)))
+          message(LDPL_FATAL, "Unable to open %s.imports",
+                  InputFile.file().name, EC.message().c_str());
+      }
     }
 
     cleanup_hook();

diff --git a/tools/llvm-lto/llvm-lto.cpp b/tools/llvm-lto/llvm-lto.cpp
index 395988c0ebfdbfd1e48396f4b948ee1598ab6c82..655e8c8379e6e0ce2c86641bc1689d1ff7f38229 100644 (file)
--- a/tools/llvm-lto/llvm-lto.cpp
+++ b/tools/llvm-lto/llvm-lto.cpp
@@ -67,6 +67,7 @@ static cl::opt<bool>
 enum ThinLTOModes {
   THINLINK,
   THINDISTRIBUTE,
+  THINEMITIMPORTS,
   THINPROMOTE,
   THINIMPORT,
   THININTERNALIZE,
@@ -83,6 +84,8 @@ cl::opt<ThinLTOModes> ThinLTOMode(
             "ThinLink: produces the index by linking only the summaries."),
         clEnumValN(THINDISTRIBUTE, "distributedindexes",
                    "Produces individual indexes for distributed backends."),
+        clEnumValN(THINEMITIMPORTS, "emitimports",
+                   "Emit imports files for distributed backends."),
         clEnumValN(THINPROMOTE, "promote",
                    "Perform pre-import promotion (requires -thinlto-index)."),
         clEnumValN(THINIMPORT, "import", "Perform both promotion and "
@@ -359,6 +362,8 @@ public:
       return thinLink();
     case THINDISTRIBUTE:
       return distributedIndexes();
+    case THINEMITIMPORTS:
+      return emitImports();
     case THINPROMOTE:
       return promote();
     case THINIMPORT:
@@ -431,6 +436,25 @@ private:
     }
   }
 
+  /// Load the combined index from disk, compute the imports, and emit
+  /// the import file lists for each module to disk.
+  void emitImports() {
+    if (InputFilenames.size() != 1 && !OutputFilename.empty())
+      report_fatal_error("Can't handle a single output filename and multiple "
+                         "input files, do not provide an output filename and "
+                         "the output files will be suffixed from the input "
+                         "ones.");
+
+    auto Index = loadCombinedIndex();
+    for (auto &Filename : InputFilenames) {
+      std::string OutputName = OutputFilename;
+      if (OutputName.empty()) {
+        OutputName = Filename + ".imports";
+      }
+      ThinLTOCodeGenerator::emitImports(Filename, OutputName, *Index);
+    }
+  }
+
   /// Load the combined index from disk, then load every file referenced by
   /// the index and add them to the generator, finally perform the promotion
   /// on the files mentioned on the command line (these must match the index