From c9756de38dea38310fab3b6b9264d7d7a4142a32 Mon Sep 17 00:00:00 2001 From: Sergey Dmitriev Date: Mon, 26 Aug 2019 19:48:43 +0000 Subject: [PATCH] [Clang][Bundler] Use llvm-objcopy for creating fat object files clang-offload-bundler currently uses partial linking for creating fat object files, but such technique cannot be used on Windows due to the absence of partial linking support in the linker. This patch changes implementation to use llvm-objcopy for merging device and host objects instead of doing partial linking. This is one step forward towards enabling OpenMP offload on Windows. Differential Revision: https://reviews.llvm.org/D66485 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@369955 91177308-0d34-0410-b5e6-96231b3b80d8 --- test/CMakeLists.txt | 1 + test/Driver/clang-offload-bundler.c | 9 +- tools/clang-offload-bundler/CMakeLists.txt | 2 +- .../ClangOffloadBundler.cpp | 142 +++--------------- 4 files changed, 29 insertions(+), 125 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6998649897..175a50ef8c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -111,6 +111,7 @@ if( NOT CLANG_BUILT_STANDALONE ) llvm-lto2 llvm-modextract llvm-nm + llvm-objcopy llvm-objdump llvm-profdata llvm-readelf diff --git a/test/Driver/clang-offload-bundler.c b/test/Driver/clang-offload-bundler.c index 4eea550eea..fe8d51e7c6 100644 --- a/test/Driver/clang-offload-bundler.c +++ b/test/Driver/clang-offload-bundler.c @@ -229,12 +229,9 @@ // tests. // -// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o -### -dump-temporary-files 2>&1 \ -// RUN: | FileCheck %s --check-prefix CK-OBJ-CMD -// CK-OBJ-CMD: private constant [{{[0-9]+}} x i8] c"{{.+}}", section "__CLANG_OFFLOAD_BUNDLE__host-[[HOST:.+]]" -// CK-OBJ-CMD: private constant [{{[0-9]+}} x i8] c"Content of device file 1{{.+}}", section "__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu" -// CK-OBJ-CMD: private constant [{{[0-9]+}} x i8] c"Content of device file 2{{.+}}", section "__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu" -// CK-OBJ-CMD: clang{{(.exe)?}}" "-r" "-target" "[[HOST]]" "-o" "{{.+}}.o" "{{.+}}.o" "{{.+}}.bc" "-nostdlib" +// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o -### 2>&1 \ +// RUN: | FileCheck %s -DHOST=%itanium_abi_triple -DINOBJ1=%t.o -DINOBJ2=%t.tgt1 -DINOBJ3=%t.tgt2 -DOUTOBJ=%t.bundle3.o --check-prefix CK-OBJ-CMD +// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=[[INOBJ1]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=[[INOBJ2]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=[[INOBJ3]]" "[[INOBJ1]]" "[[OUTOBJ]]" // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle diff --git a/tools/clang-offload-bundler/CMakeLists.txt b/tools/clang-offload-bundler/CMakeLists.txt index 465bef040a..fd8f0d219a 100644 --- a/tools/clang-offload-bundler/CMakeLists.txt +++ b/tools/clang-offload-bundler/CMakeLists.txt @@ -1,4 +1,4 @@ -set(LLVM_LINK_COMPONENTS BitWriter Core Object Support) +set(LLVM_LINK_COMPONENTS Object Support) if(NOT CLANG_BUILT_STANDALONE) set(tablegen_deps intrinsics_gen) diff --git a/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/tools/clang-offload-bundler/ClangOffloadBundler.cpp index 8f26687aa7..9f52790d55 100644 --- a/tools/clang-offload-bundler/ClangOffloadBundler.cpp +++ b/tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -21,12 +21,6 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" -#include "llvm/Bitcode/BitcodeWriter.h" -#include "llvm/IR/Constant.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Casting.h" @@ -39,6 +33,7 @@ #include "llvm/Support/Program.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/Signals.h" +#include "llvm/Support/StringSaver.h" #include #include #include @@ -94,11 +89,6 @@ static cl::opt PrintExternalCommands( "instead of actually executing them - for testing purposes.\n"), cl::init(false), cl::cat(ClangOffloadBundlerCategory)); -static cl::opt DumpTemporaryFiles( - "dump-temporary-files", - cl::desc("Dumps any temporary files created - for testing purposes.\n"), - cl::init(false), cl::cat(ClangOffloadBundlerCategory)); - /// Magic string that marks the existence of offloading data. #define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" @@ -116,12 +106,6 @@ static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, OffloadKind = KindTriplePair.first; Triple = KindTriplePair.second; } -static StringRef getTriple(StringRef Target) { - StringRef OffloadKind; - StringRef Triple; - getOffloadKindAndTriple(Target, OffloadKind, Triple); - return Triple; -} static bool hasHostKind(StringRef Target) { StringRef OffloadKind; StringRef Triple; @@ -410,19 +394,6 @@ class ObjectFileHandler final : public FileHandler { /// read from the buffers. unsigned NumberOfProcessedInputs = 0; - /// LLVM context used to create the auxiliary modules. - LLVMContext VMContext; - - /// LLVM module used to create an object with all the bundle - /// components. - std::unique_ptr AuxModule; - - /// The current triple we are working with. - StringRef CurrentTriple; - - /// The name of the main input file. - StringRef MainInputFileName; - /// Iterator of the current and next section. section_iterator CurrentSection; section_iterator NextSection; @@ -476,19 +447,10 @@ public: // Record number of inputs. NumberOfInputs = Inputs.size(); - - // Create an LLVM module to have the content we need to bundle. - auto *M = new Module("clang-offload-bundle", VMContext); - M->setTargetTriple(getTriple(TargetNames[HostInputIndex])); - AuxModule.reset(M); } void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final { ++NumberOfProcessedInputs; - - // Record the triple we are using, that will be used to name the section we - // will create. - CurrentTriple = TargetTriple; } bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final { @@ -500,76 +462,39 @@ public: if (NumberOfProcessedInputs != NumberOfInputs) return false; - // Create the bitcode file name to write the resulting code to. Keep it if - // save-temps is active. - SmallString<128> BitcodeFileName; - if (sys::fs::createTemporaryFile("clang-offload-bundler", "bc", - BitcodeFileName)) { - errs() << "error: unable to create temporary file.\n"; + // Find llvm-objcopy in order to create the bundle binary. + ErrorOr Objcopy = sys::findProgramByName( + "llvm-objcopy", sys::path::parent_path(BundlerExecutable)); + if (!Objcopy) { + errs() << "error: unable to find 'llvm-objcopy' in path.\n"; return true; } - // Dump the contents of the temporary file if that was requested. - if (DumpTemporaryFiles) { - errs() << ";\n; Object file bundler IR file.\n;\n"; - AuxModule.get()->print(errs(), nullptr, - /*ShouldPreserveUseListOrder=*/false, - /*IsForDebug=*/true); - errs() << '\n'; - } - - // Find clang in order to create the bundle binary. - StringRef Dir = sys::path::parent_path(BundlerExecutable); - - auto ClangBinary = sys::findProgramByName("clang", Dir); - if (ClangBinary.getError()) { - // Remove bitcode file. - sys::fs::remove(BitcodeFileName); - - errs() << "error: unable to find 'clang' in path.\n"; - return true; - } - - // Do the incremental linking. We write to the output file directly. So, we - // close it and use the name to pass down to clang. + // We write to the output file directly. So, we close it and use the name + // to pass down to llvm-objcopy. OS.close(); - SmallString<128> TargetName = getTriple(TargetNames[HostInputIndex]); - std::vector ClangArgs = {"clang", - "-r", - "-target", - TargetName.c_str(), - "-o", - OutputFileNames.front().c_str(), - InputFileNames[HostInputIndex].c_str(), - BitcodeFileName.c_str(), - "-nostdlib"}; + + // Compose command line for the objcopy tool. + BumpPtrAllocator Alloc; + StringSaver SS{Alloc}; + SmallVector ObjcopyArgs{"llvm-objcopy"}; + for (unsigned I = 0; I < NumberOfInputs; ++I) + ObjcopyArgs.push_back(SS.save(Twine("--add-section=") + + OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] + + "=" + InputFileNames[I])); + ObjcopyArgs.push_back(InputFileNames[HostInputIndex]); + ObjcopyArgs.push_back(OutputFileNames.front()); // If the user asked for the commands to be printed out, we do that instead // of executing it. if (PrintExternalCommands) { - errs() << "\"" << ClangBinary.get() << "\""; - for (StringRef Arg : ClangArgs) + errs() << "\"" << Objcopy.get() << "\""; + for (StringRef Arg : drop_begin(ObjcopyArgs, 1)) errs() << " \"" << Arg << "\""; errs() << "\n"; } else { - // Write the bitcode contents to the temporary file. - { - std::error_code EC; - raw_fd_ostream BitcodeFile(BitcodeFileName, EC, sys::fs::OF_None); - if (EC) { - errs() << "error: unable to open temporary file.\n"; - return true; - } - WriteBitcodeToFile(*AuxModule, BitcodeFile); - } - - bool Failed = sys::ExecuteAndWait(ClangBinary.get(), ClangArgs); - - // Remove bitcode file. - sys::fs::remove(BitcodeFileName); - - if (Failed) { - errs() << "error: incremental linking by external tool failed.\n"; + if (sys::ExecuteAndWait(Objcopy.get(), ObjcopyArgs)) { + errs() << "error: llvm-objcopy tool failed.\n"; return true; } } @@ -577,26 +502,7 @@ public: return false; } - void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { - Module *M = AuxModule.get(); - - // Create the new section name, it will consist of the reserved prefix - // concatenated with the triple. - std::string SectionName = OFFLOAD_BUNDLER_MAGIC_STR; - SectionName += CurrentTriple; - - // Create the constant with the content of the section. - auto *Content = ConstantDataArray::get( - VMContext, ArrayRef(reinterpret_cast( - Input.getBufferStart()), - Input.getBufferSize())); - - // Create the global in the desired section. We don't want these globals - // in the symbol table, so we mark them private. - auto *GV = new GlobalVariable(*M, Content->getType(), /*IsConstant=*/true, - GlobalVariable::PrivateLinkage, Content); - GV->setSection(SectionName); - } + void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {} }; /// Handler for text files. The bundled file will have the following format. -- 2.40.0