From 4d355e22e150ad96f50beed1c1ae5a5a8370c3f2 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Fri, 20 Feb 2015 23:34:26 +0000 Subject: [PATCH] Wrap clang module files in a Mach-O, ELF, or COFF container. This is a necessary prerequisite for debugging with modules. The .pcm files become containers that hold the serialized AST which allows us to store debug information in the module file that can be shared by all object files that were built importing the module. rdar://problem/19104245 This reapplies r230044 with a fixed configure+make build and updated dependencies. Take 2. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@230089 91177308-0d34-0410-b5e6-96231b3b80d8 --- docs/PCHInternals.rst | 3 + .../clang/CodeGen/CodeGenModuleContainer.h | 34 ++++ include/clang/Frontend/FrontendActions.h | 8 + include/clang/Serialization/ASTReader.h | 4 + include/clang/Serialization/ASTWriter.h | 18 ++- lib/CodeGen/CMakeLists.txt | 2 + lib/CodeGen/CodeGenModuleContainer.cpp | 150 ++++++++++++++++++ lib/Frontend/ASTUnit.cpp | 18 ++- lib/Frontend/CMakeLists.txt | 1 + lib/Frontend/ChainedIncludesSource.cpp | 16 +- lib/Frontend/FrontendActions.cpp | 40 ++++- lib/Frontend/MultiplexConsumer.cpp | 27 ++++ lib/Serialization/ASTReader.cpp | 34 +++- lib/Serialization/CMakeLists.txt | 1 + lib/Serialization/GeneratePCH.cpp | 18 +-- lib/Serialization/GlobalModuleIndex.cpp | 5 +- lib/Serialization/ModuleManager.cpp | 9 +- test/CMakeLists.txt | 2 +- test/Modules/module_container.m | 14 ++ test/PCH/floating-literal.c | 1 + tools/arcmt-test/Makefile | 5 +- tools/c-index-test/Makefile | 4 +- tools/clang-check/Makefile | 5 +- tools/diagtool/Makefile | 2 +- tools/libclang/CMakeLists.txt | 1 + tools/libclang/Makefile | 4 +- unittests/AST/Makefile | 5 +- unittests/ASTMatchers/Dynamic/Makefile | 11 +- unittests/ASTMatchers/Makefile | 5 +- unittests/CodeGen/Makefile | 4 +- unittests/Frontend/Makefile | 3 +- unittests/Sema/Makefile | 5 +- unittests/Tooling/Makefile | 5 +- 33 files changed, 399 insertions(+), 65 deletions(-) create mode 100644 include/clang/CodeGen/CodeGenModuleContainer.h create mode 100644 lib/CodeGen/CodeGenModuleContainer.cpp create mode 100644 test/Modules/module_container.m diff --git a/docs/PCHInternals.rst b/docs/PCHInternals.rst index a36e65c199..d247d3ed0a 100644 --- a/docs/PCHInternals.rst +++ b/docs/PCHInternals.rst @@ -65,6 +65,9 @@ be included at the beginning of the translation unit. The extensions to the AST file format required for modules are discussed in the section on :ref:`modules `. +Clang's AST files are Mach-O, ELF, or COFF containers that contain a +``__clangast`` section which holds the AST bitstream. + Clang's AST files are designed with a compact on-disk representation, which minimizes both creation time and the time required to initially load the AST file. The AST file itself contains a serialized representation of Clang's diff --git a/include/clang/CodeGen/CodeGenModuleContainer.h b/include/clang/CodeGen/CodeGenModuleContainer.h new file mode 100644 index 0000000000..3c5456fc18 --- /dev/null +++ b/include/clang/CodeGen/CodeGenModuleContainer.h @@ -0,0 +1,34 @@ +//===--- CodeGen/ModuleContainerGenerator.h - Emit .pcm files ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_CODEGEN_MODULE_CONTAINER_H +#define LLVM_CLANG_CODEGEN_MODULE_CONTAINER_H + +#include "ModuleBuilder.h" +#include + +namespace llvm { +class raw_ostream; +} + +namespace clang { + +class PCHGenerator; +class TargetOptions; + +/// \brief Create a CodeGenerator instance. +/// It is the responsibility of the caller to call delete on +/// the allocated CodeGenerator instance. +CodeGenerator *CreateModuleContainerGenerator( + DiagnosticsEngine &Diags, const std::string &ModuleName, + const CodeGenOptions &CGO, const TargetOptions &TO, const LangOptions &LO, + llvm::raw_ostream *OS, PCHGenerator *PCHGen); +} + +#endif diff --git a/include/clang/Frontend/FrontendActions.h b/include/clang/Frontend/FrontendActions.h index 850f87c073..b96b9f559e 100644 --- a/include/clang/Frontend/FrontendActions.h +++ b/include/clang/Frontend/FrontendActions.h @@ -69,6 +69,14 @@ protected: StringRef InFile) override; }; +/// \brief Emits the output of a GeneratePCHAction or GenerateModuleAction into +/// a Mach-O/ELF/COFF container. +class GeneratePCMContainerAction : public FrontendAction { +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override; +}; + class GeneratePCHAction : public ASTFrontendAction { protected: std::unique_ptr CreateASTConsumer(CompilerInstance &CI, diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h index 27af9995aa..754273688b 100644 --- a/include/clang/Serialization/ASTReader.h +++ b/include/clang/Serialization/ASTReader.h @@ -1127,6 +1127,10 @@ private: public: void ResolveImportedPath(ModuleFile &M, std::string &Filename); static void ResolveImportedPath(std::string &Filename, StringRef Prefix); + /// \brief Initialize a BitstreamReader with the `__clangast` section from an + /// object file container found in Buffer. + static void InitStreamFileWithModule(llvm::MemoryBufferRef Buffer, + llvm::BitstreamReader &StreamFile); private: struct ImportedModule { diff --git a/include/clang/Serialization/ASTWriter.h b/include/clang/Serialization/ASTWriter.h index 1d2fa556e2..4b8668b3c6 100644 --- a/include/clang/Serialization/ASTWriter.h +++ b/include/clang/Serialization/ASTWriter.h @@ -823,10 +823,13 @@ class PCHGenerator : public SemaConsumer { std::string OutputFile; clang::Module *Module; std::string isysroot; - raw_ostream *Out; Sema *SemaPtr; - SmallVector Buffer; + // This buffer is always large, but BitstreamWriter really wants a + // SmallVectorImpl. + SmallVector Buffer; llvm::BitstreamWriter Stream; + std::function*)> + SerializationFinishedCallback; ASTWriter Writer; bool AllowASTWithErrors; bool HasEmittedPCH; @@ -836,16 +839,21 @@ protected: const ASTWriter &getWriter() const { return Writer; } public: - PCHGenerator(const Preprocessor &PP, StringRef OutputFile, + PCHGenerator(const Preprocessor &PP, + StringRef OutputFile, clang::Module *Module, - StringRef isysroot, raw_ostream *Out, + StringRef isysroot, bool AllowASTWithErrors = false); ~PCHGenerator(); void InitializeSema(Sema &S) override { SemaPtr = &S; } void HandleTranslationUnit(ASTContext &Ctx) override; ASTMutationListener *GetASTMutationListener() override; ASTDeserializationListener *GetASTDeserializationListener() override; - + /// \brief Register a callback to be invoked when the serialization is done. + void RegisterSerializationFinishedCallback( + const std::function*)> Fn) { + SerializationFinishedCallback = Fn; + } bool hasEmittedPCH() const { return HasEmittedPCH; } }; diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index 18f505d8c5..dae14bd3b1 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -1,4 +1,5 @@ set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} Analysis BitReader BitWriter @@ -63,6 +64,7 @@ add_clang_library(clangCodeGen CodeGenAction.cpp CodeGenFunction.cpp CodeGenModule.cpp + CodeGenModuleContainer.cpp CodeGenPGO.cpp CodeGenTBAA.cpp CodeGenTypes.cpp diff --git a/lib/CodeGen/CodeGenModuleContainer.cpp b/lib/CodeGen/CodeGenModuleContainer.cpp new file mode 100644 index 0000000000..b1162ccf19 --- /dev/null +++ b/lib/CodeGen/CodeGenModuleContainer.cpp @@ -0,0 +1,150 @@ +//===--- CodeGenModuleContainer.cpp - Emit .pcm files ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/CodeGen/CodeGenModuleContainer.h" +#include "CodeGenModule.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/DeclObjC.h" +#include "clang/AST/Expr.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/CodeGen/BackendUtil.h" +#include "clang/Frontend/CodeGenOptions.h" +#include "clang/Serialization/ASTWriter.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/TargetRegistry.h" +#include +using namespace clang; + +namespace { +class ModuleContainerGenerator : public CodeGenerator { + DiagnosticsEngine &Diags; + std::unique_ptr TD; + ASTContext *Ctx; + const CodeGenOptions CodeGenOpts; + const TargetOptions TargetOpts; + const LangOptions LangOpts; + llvm::LLVMContext VMContext; + std::unique_ptr M; + std::unique_ptr Builder; + raw_ostream *OS; + SmallVectorImpl *SerializedASTBuffer; + +public: + ModuleContainerGenerator(DiagnosticsEngine &diags, + const std::string &ModuleName, + const CodeGenOptions &CGO, const TargetOptions &TO, + const LangOptions &LO, raw_ostream *OS, + PCHGenerator *PCHGen) + : Diags(diags), CodeGenOpts(CGO), TargetOpts(TO), LangOpts(LO), + M(new llvm::Module(ModuleName, VMContext)), OS(OS) { + PCHGen->RegisterSerializationFinishedCallback( + [&](SmallVectorImpl *Buf){ + SerializedASTBuffer = Buf; + }); + } + + virtual ~ModuleContainerGenerator() {} + llvm::Module *GetModule() override { return M.get(); } + llvm::Module *ReleaseModule() override { return M.release(); } + + /// Lifted from ModuleBuilder. + const Decl *GetDeclForMangledName(StringRef MangledName) override { + GlobalDecl Result; + if (!Builder->lookupRepresentativeDecl(MangledName, Result)) + return nullptr; + const Decl *D = Result.getCanonicalDecl().getDecl(); + if (auto FD = dyn_cast(D)) { + if (FD->hasBody(FD)) + return FD; + } else if (auto TD = dyn_cast(D)) { + if (auto Def = TD->getDefinition()) + return Def; + } + return D; + } + + void Initialize(ASTContext &Context) override { + Ctx = &Context; + M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple()); + M->setDataLayout(Ctx->getTargetInfo().getTargetDescription()); + TD.reset(new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription())); + Builder.reset( + new CodeGen::CodeGenModule(Context, CodeGenOpts, *M, *TD, Diags)); + } + + /// Emit a container holding the serialized AST. + void HandleTranslationUnit(ASTContext &Ctx) override { + if (Diags.hasErrorOccurred()) { + if (Builder) + Builder->clear(); + M.reset(); + return; + } + + // Finalize the Builder. + if (Builder) + Builder->Release(); + + // Initialize the backend if we haven't done so already. + LLVMInitializeAllTargetInfos(); + LLVMInitializeAllTargets(); + LLVMInitializeAllAsmPrinters(); + LLVMInitializeAllTargetMCs(); + + // Ensure the target exists. + std::string Error; + auto Triple = Ctx.getTargetInfo().getTriple(); + if (!llvm::TargetRegistry::lookupTarget(Triple.getTriple(), Error)) + llvm::report_fatal_error(Error); + + // Emit the serialized Clang AST into its own section. + auto Size = SerializedASTBuffer->size(); + auto Int8Ty = llvm::Type::getInt8Ty(VMContext); + auto *Ty = llvm::ArrayType::get(Int8Ty, Size); + auto *Data = llvm::ConstantDataArray::getString(VMContext, + StringRef(SerializedASTBuffer->data(), Size), /*AddNull=*/false); + auto *ASTSym = new llvm::GlobalVariable(*M, Ty, /*constant*/ true, + llvm::GlobalVariable::InternalLinkage, Data, "__clang_ast"); + ASTSym->setAlignment(8); + if (Triple.isOSBinFormatMachO()) + // Include Mach-O segment name. + ASTSym->setSection("__CLANG,__clangast"); + else if (Triple.isOSBinFormatCOFF()) + // Adhere to COFF eight-character limit. + ASTSym->setSection("clangast"); + else + ASTSym->setSection("__clangast"); + + // Use the LLVM backend to emit the pcm. + EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts, + Ctx.getTargetInfo().getTargetDescription(), M.get(), + BackendAction::Backend_EmitObj, OS); + + // Make sure the module container hits disk now. + OS->flush(); + + // Free up some memory, in case the process is kept alive. + SerializedASTBuffer->clear(); + } +}; +} + +CodeGenerator *clang::CreateModuleContainerGenerator( + DiagnosticsEngine &Diags, const std::string &ModuleName, + const CodeGenOptions &CGO, const TargetOptions &TO, const LangOptions &LO, + llvm::raw_ostream *OS, PCHGenerator *PCHGen) { + return + new ModuleContainerGenerator(Diags, ModuleName, CGO, TO, LO, OS, PCHGen); +} diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp index bfb1efe352..58a5f98008 100644 --- a/lib/Frontend/ASTUnit.cpp +++ b/lib/Frontend/ASTUnit.cpp @@ -914,13 +914,20 @@ class PrecompilePreambleConsumer : public PCHGenerator { unsigned &Hash; std::vector TopLevelDecls; PrecompilePreambleAction *Action; + raw_ostream *Out; + SmallVectorImpl *SerializedASTBuffer; public: PrecompilePreambleConsumer(ASTUnit &Unit, PrecompilePreambleAction *Action, const Preprocessor &PP, StringRef isysroot, raw_ostream *Out) - : PCHGenerator(PP, "", nullptr, isysroot, Out, /*AllowASTWithErrors=*/true), - Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action) { + : PCHGenerator(PP, "", nullptr, isysroot, /*AllowASTWithErrors=*/true), + Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action), + Out(Out) { + RegisterSerializationFinishedCallback( + [&](SmallVectorImpl *Buf){ + SerializedASTBuffer = Buf; + }); Hash = 0; } @@ -941,6 +948,13 @@ public: void HandleTranslationUnit(ASTContext &Ctx) override { PCHGenerator::HandleTranslationUnit(Ctx); if (hasEmittedPCH()) { + // Write the generated bitstream to "Out". + Out->write((char *)&SerializedASTBuffer->front(), + SerializedASTBuffer->size()); + // Make sure it hits disk now. + Out->flush(); + SerializedASTBuffer->clear(); + // Translate the top-level declarations we captured during // parsing into declaration IDs in the precompiled // preamble. This will allow us to deserialize those top-level diff --git a/lib/Frontend/CMakeLists.txt b/lib/Frontend/CMakeLists.txt index 7c5fca54d1..6c9085d65d 100644 --- a/lib/Frontend/CMakeLists.txt +++ b/lib/Frontend/CMakeLists.txt @@ -45,6 +45,7 @@ add_clang_library(clangFrontend LINK_LIBS clangAST clangBasic + clangCodeGen clangDriver clangEdit clangLex diff --git a/lib/Frontend/ChainedIncludesSource.cpp b/lib/Frontend/ChainedIncludesSource.cpp index cb260b4f4c..1ecc0bfc6b 100644 --- a/lib/Frontend/ChainedIncludesSource.cpp +++ b/lib/Frontend/ChainedIncludesSource.cpp @@ -156,11 +156,13 @@ IntrusiveRefCntPtr clang::createChainedIncludesSource( &Clang->getPreprocessor()); Clang->createASTContext(); - SmallVector serialAST; - llvm::raw_svector_ostream OS(serialAST); - auto consumer = - llvm::make_unique(Clang->getPreprocessor(), "-", nullptr, - /*isysroot=*/"", &OS); + auto consumer = llvm::make_unique(Clang->getPreprocessor(), + "-", nullptr, /*isysroot=*/""); + SmallVectorImpl *serialAST; + consumer->RegisterSerializationFinishedCallback( + [&](SmallVectorImpl *Buf){ + serialAST = Buf; + }); Clang->getASTContext().setASTMutationListener( consumer->GetASTMutationListener()); Clang->setASTConsumer(std::move(consumer)); @@ -197,7 +199,9 @@ IntrusiveRefCntPtr clang::createChainedIncludesSource( ParseAST(Clang->getSema()); Clang->getDiagnosticClient().EndSourceFile(); - SerialBufs.push_back(llvm::MemoryBuffer::getMemBufferCopy(OS.str())); + SerialBufs.push_back(llvm::MemoryBuffer:: + getMemBufferCopy(StringRef(serialAST->data(), serialAST->size()))); + serialAST->clear(); source->CIs.push_back(Clang.release()); } diff --git a/lib/Frontend/FrontendActions.cpp b/lib/Frontend/FrontendActions.cpp index a55a325785..3e0f525e65 100644 --- a/lib/Frontend/FrontendActions.cpp +++ b/lib/Frontend/FrontendActions.cpp @@ -10,10 +10,13 @@ #include "clang/Frontend/FrontendActions.h" #include "clang/AST/ASTConsumer.h" #include "clang/Basic/FileManager.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/CodeGen/CodeGenModuleContainer.h" #include "clang/Frontend/ASTConsumers.h" #include "clang/Frontend/ASTUnit.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/MultiplexConsumer.h" #include "clang/Frontend/Utils.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Pragma.h" @@ -85,8 +88,23 @@ GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { if (!CI.getFrontendOpts().RelocatablePCH) Sysroot.clear(); - return llvm::make_unique(CI.getPreprocessor(), OutputFile, - nullptr, Sysroot, OS); + + std::vector> Consumers; + Consumers.push_back(llvm::make_unique(CI.getPreprocessor(), + OutputFile, nullptr, + Sysroot)); + + auto CGOpts = CI.getCodeGenOpts(); + // The debug info emitted by ModuleContainerGenerator is not affected by the + // optimization level. + CGOpts.OptimizationLevel = 0; + CGOpts.setDebugInfo(CodeGenOptions::LimitedDebugInfo); + Consumers.push_back(std::unique_ptr( + CreateModuleContainerGenerator(CI.getDiagnostics(), "PCH", CGOpts, + CI.getTargetOpts(), CI.getLangOpts(), OS, + cast(Consumers[0].get())))); + + return llvm::make_unique(std::move(Consumers)); } bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI, @@ -122,8 +140,22 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, if (ComputeASTConsumerArguments(CI, InFile, Sysroot, OutputFile, OS)) return nullptr; - return llvm::make_unique(CI.getPreprocessor(), OutputFile, - Module, Sysroot, OS); + std::vector> Consumers; + Consumers.push_back(llvm::make_unique(CI.getPreprocessor(), + OutputFile, Module, + Sysroot)); + + auto CGOpts = CI.getCodeGenOpts(); + // The debug info emitted by ModuleContainerGenerator is not affected by the + // optimization level. + CGOpts.OptimizationLevel = 0; + CGOpts.setDebugInfo(CodeGenOptions::LimitedDebugInfo); + Consumers.push_back( + std::unique_ptr(CreateModuleContainerGenerator( + CI.getDiagnostics(), Module->getFullModuleName(), CGOpts, + CI.getTargetOpts(), CI.getLangOpts(), OS, + cast(Consumers[0].get())))); + return llvm::make_unique(std::move(Consumers)); } static SmallVectorImpl & diff --git a/lib/Frontend/MultiplexConsumer.cpp b/lib/Frontend/MultiplexConsumer.cpp index 3c4fed1d18..bb9c907b2e 100644 --- a/lib/Frontend/MultiplexConsumer.cpp +++ b/lib/Frontend/MultiplexConsumer.cpp @@ -33,11 +33,14 @@ public: void ReaderInitialized(ASTReader *Reader) override; void IdentifierRead(serialization::IdentID ID, IdentifierInfo *II) override; + void MacroRead(serialization::MacroID ID, MacroInfo *MI) override; void TypeRead(serialization::TypeIdx Idx, QualType T) override; void DeclRead(serialization::DeclID ID, const Decl *D) override; void SelectorRead(serialization::SelectorID iD, Selector Sel) override; void MacroDefinitionRead(serialization::PreprocessedEntityID, MacroDefinition *MD) override; + void ModuleRead(serialization::SubmoduleID ID, Module *Mod) override; + private: std::vector Listeners; }; @@ -59,6 +62,12 @@ void MultiplexASTDeserializationListener::IdentifierRead( Listeners[i]->IdentifierRead(ID, II); } +void MultiplexASTDeserializationListener::MacroRead( + serialization::MacroID ID, MacroInfo *MI) { + for (auto &Listener : Listeners) + Listener->MacroRead(ID, MI); +} + void MultiplexASTDeserializationListener::TypeRead( serialization::TypeIdx Idx, QualType T) { for (size_t i = 0, e = Listeners.size(); i != e; ++i) @@ -83,6 +92,12 @@ void MultiplexASTDeserializationListener::MacroDefinitionRead( Listeners[i]->MacroDefinitionRead(ID, MD); } +void MultiplexASTDeserializationListener::ModuleRead( + serialization::SubmoduleID ID, Module *Mod) { + for (auto &Listener : Listeners) + Listener->ModuleRead(ID, Mod); +} + // This ASTMutationListener forwards its notifications to a set of // child listeners. class MultiplexASTMutationListener : public ASTMutationListener { @@ -98,11 +113,13 @@ public: const VarTemplateSpecializationDecl *D) override; void AddedCXXTemplateSpecialization(const FunctionTemplateDecl *TD, const FunctionDecl *D) override; + void ResolvedExceptionSpec(const FunctionDecl *FD) override; void DeducedReturnType(const FunctionDecl *FD, QualType ReturnType) override; void CompletedImplicitDefinition(const FunctionDecl *D) override; void StaticDataMemberInstantiated(const VarDecl *D) override; void AddedObjCCategoryToInterface(const ObjCCategoryDecl *CatD, const ObjCInterfaceDecl *IFD) override; + void FunctionDefinitionInstantiated(const FunctionDecl *D) override; void AddedObjCPropertyInClassExtension(const ObjCPropertyDecl *Prop, const ObjCPropertyDecl *OrigProp, const ObjCCategoryDecl *ClassExt) override; @@ -149,6 +166,11 @@ void MultiplexASTMutationListener::AddedCXXTemplateSpecialization( for (size_t i = 0, e = Listeners.size(); i != e; ++i) Listeners[i]->AddedCXXTemplateSpecialization(TD, D); } +void MultiplexASTMutationListener::ResolvedExceptionSpec( + const FunctionDecl *FD) { + for (auto &Listener : Listeners) + Listener->ResolvedExceptionSpec(FD); +} void MultiplexASTMutationListener::DeducedReturnType(const FunctionDecl *FD, QualType ReturnType) { for (size_t i = 0, e = Listeners.size(); i != e; ++i) @@ -170,6 +192,11 @@ void MultiplexASTMutationListener::AddedObjCCategoryToInterface( for (size_t i = 0, e = Listeners.size(); i != e; ++i) Listeners[i]->AddedObjCCategoryToInterface(CatD, IFD); } +void MultiplexASTMutationListener::FunctionDefinitionInstantiated( + const FunctionDecl *D) { + for (auto &Listener : Listeners) + Listener->FunctionDefinitionInstantiated(D); +} void MultiplexASTMutationListener::AddedObjCPropertyInClassExtension( const ObjCPropertyDecl *Prop, const ObjCPropertyDecl *OrigProp, diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp index 0ee2b2b221..dd73bbae37 100644 --- a/lib/Serialization/ASTReader.cpp +++ b/lib/Serialization/ASTReader.cpp @@ -46,6 +46,8 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ObjectFile.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" @@ -632,6 +634,27 @@ void PCHValidator::ReadCounter(const ModuleFile &M, unsigned Value) { // AST reader implementation //===----------------------------------------------------------------------===// +void ASTReader::InitStreamFileWithModule(llvm::MemoryBufferRef Buffer, + llvm::BitstreamReader &StreamFile) { + if (auto OF = llvm::object::ObjectFile::createObjectFile(Buffer)) { + bool IsCOFF = isa(OF.get().get()); + // Find the clang AST section in the container. + for (auto &Section : OF->get()->sections()) { + StringRef Name; + Section.getName(Name); + if ((!IsCOFF && Name == "__clangast") || + ( IsCOFF && Name == "clangast")) { + StringRef Buf; + Section.getContents(Buf); + return StreamFile.init((const unsigned char*)Buf.begin(), + (const unsigned char*)Buf.end()); + } + } + } + StreamFile.init((const unsigned char *)Buffer.getBufferStart(), + (const unsigned char *)Buffer.getBufferEnd()); +} + void ASTReader::setDeserializationListener(ASTDeserializationListener *Listener, bool TakeOwnership) { DeserializationListener = Listener; @@ -3883,9 +3906,10 @@ ASTReader::ReadASTCore(StringRef FileName, ModuleFile &F = *M; BitstreamCursor &Stream = F.Stream; + InitStreamFileWithModule(F.Buffer->getMemBufferRef(), F.StreamFile); Stream.init(&F.StreamFile); - F.SizeInBits = F.Buffer->getBufferSize() * 8; - + F.SizeInBits = F.StreamFile.getBitcodeBytes().getExtent() * 8; + // Sniff for the signature. if (Stream.Read(8) != 'C' || Stream.Read(8) != 'P' || @@ -4174,8 +4198,7 @@ std::string ASTReader::getOriginalSourceFile(const std::string &ASTFileName, // Initialize the stream llvm::BitstreamReader StreamFile; - StreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(), - (const unsigned char *)(*Buffer)->getBufferEnd()); + InitStreamFileWithModule((*Buffer)->getMemBufferRef(), StreamFile); BitstreamCursor Stream(StreamFile); // Sniff for the signature. @@ -4270,8 +4293,7 @@ bool ASTReader::readASTFileControlBlock(StringRef Filename, // Initialize the stream llvm::BitstreamReader StreamFile; - StreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(), - (const unsigned char *)(*Buffer)->getBufferEnd()); + InitStreamFileWithModule((*Buffer)->getMemBufferRef(), StreamFile); BitstreamCursor Stream(StreamFile); // Sniff for the signature. diff --git a/lib/Serialization/CMakeLists.txt b/lib/Serialization/CMakeLists.txt index d885db2297..a1a5ad4abe 100644 --- a/lib/Serialization/CMakeLists.txt +++ b/lib/Serialization/CMakeLists.txt @@ -1,5 +1,6 @@ set(LLVM_LINK_COMPONENTS BitReader + Object Support ) diff --git a/lib/Serialization/GeneratePCH.cpp b/lib/Serialization/GeneratePCH.cpp index b5031fdf92..0cd01dc6f3 100644 --- a/lib/Serialization/GeneratePCH.cpp +++ b/lib/Serialization/GeneratePCH.cpp @@ -19,7 +19,6 @@ #include "clang/Lex/Preprocessor.h" #include "clang/Sema/SemaConsumer.h" #include "llvm/Bitcode/BitstreamWriter.h" -#include "llvm/Support/raw_ostream.h" #include using namespace clang; @@ -28,10 +27,11 @@ PCHGenerator::PCHGenerator(const Preprocessor &PP, StringRef OutputFile, clang::Module *Module, StringRef isysroot, - raw_ostream *OS, bool AllowASTWithErrors) + bool AllowASTWithErrors) : PP(PP), OutputFile(OutputFile), Module(Module), - isysroot(isysroot.str()), Out(OS), - SemaPtr(nullptr), Stream(Buffer), Writer(Stream), + isysroot(isysroot.str()), + SemaPtr(nullptr), Stream(Buffer), + Writer(Stream), AllowASTWithErrors(AllowASTWithErrors), HasEmittedPCH(false) { } @@ -52,14 +52,8 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) { assert(SemaPtr && "No Sema?"); Writer.WriteAST(*SemaPtr, OutputFile, Module, isysroot, hasErrors); - // Write the generated bitstream to "Out". - Out->write((char *)&Buffer.front(), Buffer.size()); - - // Make sure it hits disk now. - Out->flush(); - - // Free up some memory, in case the process is kept alive. - Buffer.clear(); + if (SerializationFinishedCallback) + SerializationFinishedCallback(&Buffer); HasEmittedPCH = true; } diff --git a/lib/Serialization/GlobalModuleIndex.cpp b/lib/Serialization/GlobalModuleIndex.cpp index 4791388045..68a23ea870 100644 --- a/lib/Serialization/GlobalModuleIndex.cpp +++ b/lib/Serialization/GlobalModuleIndex.cpp @@ -15,6 +15,7 @@ #include "clang/Basic/FileManager.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Serialization/ASTBitCodes.h" +#include "clang/Serialization/ASTReader.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/Module.h" #include "llvm/ADT/DenseMap.h" @@ -501,8 +502,8 @@ bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) { // Initialize the input stream llvm::BitstreamReader InStreamFile; - InStreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(), - (const unsigned char *)(*Buffer)->getBufferEnd()); + ASTReader::InitStreamFileWithModule((*Buffer)->getMemBufferRef(), + InStreamFile); llvm::BitstreamCursor InStream(InStreamFile); // Sniff for the signature. diff --git a/lib/Serialization/ModuleManager.cpp b/lib/Serialization/ModuleManager.cpp index ac98ca0b87..8f1473f3a3 100644 --- a/lib/Serialization/ModuleManager.cpp +++ b/lib/Serialization/ModuleManager.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/ModuleMap.h" +#include "clang/Serialization/ASTReader.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/ModuleManager.h" #include "llvm/Support/MemoryBuffer.h" @@ -135,10 +136,10 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, New->Buffer = std::move(*Buf); } - - // Initialize the stream - New->StreamFile.init((const unsigned char *)New->Buffer->getBufferStart(), - (const unsigned char *)New->Buffer->getBufferEnd()); + + // Initialize the stream. + ASTReader::InitStreamFileWithModule(New->Buffer->getMemBufferRef(), + New->StreamFile); } if (ExpectedSignature) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 591a9272ae..526f129bab 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -58,7 +58,7 @@ set(CLANG_TEST_PARAMS if( NOT CLANG_BUILT_STANDALONE ) list(APPEND CLANG_TEST_DEPS llvm-config - llc opt FileCheck count not llvm-symbolizer llvm-profdata + llc opt FileCheck count not llvm-symbolizer llvm-profdata llvm-objdump ) endif() diff --git a/test/Modules/module_container.m b/test/Modules/module_container.m new file mode 100644 index 0000000000..348a869ad2 --- /dev/null +++ b/test/Modules/module_container.m @@ -0,0 +1,14 @@ +@import DependsOnModule; +// REQUIRES: x86-registered-target +// RUN: rm -rf %t-MachO %t-ELF %t-COFF +// RUN: %clang_cc1 -triple=x86_64-apple-darwin -fmodules -fdisable-module-hash -fmodules-cache-path=%t-MachO -F %S/Inputs %s +// RUN: %clang_cc1 -triple=x86_64-linux-elf -fmodules -fdisable-module-hash -fmodules-cache-path=%t-ELF -F %S/Inputs %s +// RUN: %clang_cc1 -triple=x86_64-windows-coff -fmodules -fdisable-module-hash -fmodules-cache-path=%t-COFF -F %S/Inputs %s + +// RUN: llvm-objdump -section-headers %t-MachO/DependsOnModule.pcm %t-ELF/DependsOnModule.pcm %t-COFF/DependsOnModule.pcm | FileCheck %s +// CHECK: file format Mach-O 64-bit x86-64 +// CHECK: __clangast {{[0-9a-f]+}} {{[0-9a-f]+}} DATA +// CHECK: file format ELF64-x86-64 +// CHECK: __clangast {{[0-9a-f]+}} {{[0-9a-f]+}} DATA +// CHECK: file format COFF-x86-64 +// CHECK: clangast {{[0-9a-f]+}} {{[0-9a-f]+}} diff --git a/test/PCH/floating-literal.c b/test/PCH/floating-literal.c index 738e45a415..b5ff6fe84b 100644 --- a/test/PCH/floating-literal.c +++ b/test/PCH/floating-literal.c @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -triple mips64-none-linux-gnu -emit-pch -o %t %s +// REQUIRES: mips-registered-target // RUN: %clang_cc1 -x ast -ast-print %t | FileCheck %s // Make sure the semantics of FloatingLiterals are stored correctly in diff --git a/tools/arcmt-test/Makefile b/tools/arcmt-test/Makefile index d9d44bb05b..642f2e0567 100644 --- a/tools/arcmt-test/Makefile +++ b/tools/arcmt-test/Makefile @@ -17,10 +17,11 @@ TOOL_NO_EXPORTS = 1 NO_INSTALL = 1 include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation ipo mc objcarcopts option support USEDLIBS = clangARCMigrate.a clangRewrite.a \ clangFrontend.a clangDriver.a clangSerialization.a clangParse.a \ clangSema.a clangEdit.a clangAnalysis.a clangAST.a clangLex.a \ - clangBasic.a + clangBasic.a clangCodeGen.a include $(CLANG_LEVEL)/Makefile diff --git a/tools/c-index-test/Makefile b/tools/c-index-test/Makefile index 62bc9348db..068fce19be 100644 --- a/tools/c-index-test/Makefile +++ b/tools/c-index-test/Makefile @@ -22,12 +22,14 @@ TOOL_NO_EXPORTS = 1 # LINK_COMPONENTS before including Makefile.rules include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation ipo support mc objcarcopts option # Note that 'USEDLIBS' must include all of the core clang libraries # when -static is given to linker on cygming. USEDLIBS = clang.a \ clangIndex.a clangFormat.a clangRewrite.a \ + clangCodeGen.a \ clangFrontend.a clangDriver.a \ clangTooling.a \ clangToolingCore.a \ diff --git a/tools/clang-check/Makefile b/tools/clang-check/Makefile index e98a131de5..3f54a5b5b1 100644 --- a/tools/clang-check/Makefile +++ b/tools/clang-check/Makefile @@ -15,9 +15,10 @@ TOOLNAME = clang-check TOOL_NO_EXPORTS = 1 include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation ipo mc option objcarcopts support USEDLIBS = clangFrontend.a clangSerialization.a clangDriver.a \ - clangTooling.a clangParse.a clangSema.a \ + clangCodeGen.a clangTooling.a clangParse.a clangSema.a \ clangStaticAnalyzerFrontend.a clangStaticAnalyzerCheckers.a \ clangStaticAnalyzerCore.a clangAnalysis.a clangRewriteFrontend.a \ clangRewrite.a clangEdit.a clangAST.a clangLex.a clangBasic.a diff --git a/tools/diagtool/Makefile b/tools/diagtool/Makefile index d49e976e64..1a74506006 100644 --- a/tools/diagtool/Makefile +++ b/tools/diagtool/Makefile @@ -20,7 +20,7 @@ include $(CLANG_LEVEL)/../../Makefile.config LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option USEDLIBS = clangFrontend.a clangDriver.a clangSerialization.a clangParse.a \ clangSema.a clangAnalysis.a clangEdit.a clangAST.a clangLex.a \ - clangBasic.a + clangBasic.a clangCodeGen.a include $(CLANG_LEVEL)/Makefile diff --git a/tools/libclang/CMakeLists.txt b/tools/libclang/CMakeLists.txt index 26f88a97c5..1cc4cb748d 100644 --- a/tools/libclang/CMakeLists.txt +++ b/tools/libclang/CMakeLists.txt @@ -40,6 +40,7 @@ set(SOURCES set(LIBS clangAST clangBasic + clangCodeGen clangFrontend clangIndex clangLex diff --git a/tools/libclang/Makefile b/tools/libclang/Makefile index 97f663cde4..141a347e9c 100644 --- a/tools/libclang/Makefile +++ b/tools/libclang/Makefile @@ -16,11 +16,13 @@ LINK_LIBS_IN_SHARED = 1 SHARED_LIBRARY = 1 include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := AsmParser BitReader Core MC MCParser Option Support +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter core \ + instrumentation mc mcparser objcarcopts option support USEDLIBS = clangIndex.a clangARCMigrate.a \ clangRewriteFrontend.a \ clangFormat.a \ clangTooling.a clangToolingCore.a \ + clangCodeGen.a \ clangFrontend.a clangDriver.a \ clangSerialization.a \ clangParse.a clangSema.a \ diff --git a/unittests/AST/Makefile b/unittests/AST/Makefile index e3b3d7dc33..db6e5010f5 100644 --- a/unittests/AST/Makefile +++ b/unittests/AST/Makefile @@ -10,9 +10,10 @@ CLANG_LEVEL = ../.. TESTNAME = AST include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation mc option objcarcopts support USEDLIBS = clangTooling.a clangFrontend.a clangSerialization.a clangDriver.a \ - clangRewrite.a clangRewriteFrontend.a \ + clangCodeGen.a clangRewrite.a clangRewriteFrontend.a \ clangParse.a clangSema.a clangAnalysis.a \ clangEdit.a clangAST.a clangASTMatchers.a clangLex.a clangBasic.a diff --git a/unittests/ASTMatchers/Dynamic/Makefile b/unittests/ASTMatchers/Dynamic/Makefile index dfd0086c88..dab58be39b 100644 --- a/unittests/ASTMatchers/Dynamic/Makefile +++ b/unittests/ASTMatchers/Dynamic/Makefile @@ -11,10 +11,11 @@ CLANG_LEVEL = ../../.. TESTNAME = DynamicASTMatchers include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option -USEDLIBS = clangTooling.a clangFrontend.a clangSerialization.a clangDriver.a \ - clangRewrite.a clangRewriteFrontend.a clangParse.a clangSema.a \ - clangAnalysis.a clangEdit.a clangAST.a clangASTMatchers.a \ - clangLex.a clangBasic.a clangDynamicASTMatchers.a +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation mc option objcarcopts support +USEDLIBS = clangTooling.a clangFrontend.a clangSerialization.a clangDriver.a \ + clangCodeGen.a clangRewrite.a clangRewriteFrontend.a clangParse.a \ + clangSema.a clangAnalysis.a clangEdit.a clangAST.a \ + clangASTMatchers.a clangLex.a clangBasic.a clangDynamicASTMatchers.a include $(CLANG_LEVEL)/unittests/Makefile diff --git a/unittests/ASTMatchers/Makefile b/unittests/ASTMatchers/Makefile index 92f2fa0e5d..6eacbc261b 100644 --- a/unittests/ASTMatchers/Makefile +++ b/unittests/ASTMatchers/Makefile @@ -13,9 +13,10 @@ PARALLEL_DIRS = Dynamic TESTNAME = ASTMatchers include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation mc option objcarcopts support USEDLIBS = clangTooling.a clangFrontend.a clangSerialization.a clangDriver.a \ - clangRewrite.a clangRewriteFrontend.a \ + clangCodeGen.a clangRewrite.a clangRewriteFrontend.a \ clangParse.a clangSema.a clangAnalysis.a \ clangEdit.a clangAST.a clangASTMatchers.a clangLex.a clangBasic.a diff --git a/unittests/CodeGen/Makefile b/unittests/CodeGen/Makefile index de347e1afd..2c825850c3 100644 --- a/unittests/CodeGen/Makefile +++ b/unittests/CodeGen/Makefile @@ -10,8 +10,8 @@ CLANG_LEVEL = ../.. TESTNAME = CodeGen include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader mc option \ - profiledata support +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation mc option objcarcopts profiledata support USEDLIBS = clangCodeGen.a clangFrontend.a clangSerialization.a \ clangDriver.a \ clangParse.a clangSema.a clangAnalysis.a \ diff --git a/unittests/Frontend/Makefile b/unittests/Frontend/Makefile index a63ae18245..2604f6e5cc 100644 --- a/unittests/Frontend/Makefile +++ b/unittests/Frontend/Makefile @@ -10,7 +10,8 @@ CLANG_LEVEL = ../.. TESTNAME = Frontend include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation mc option objcarcopts support USEDLIBS = clangFrontendTool.a clangFrontend.a clangDriver.a \ clangSerialization.a clangCodeGen.a clangParse.a clangSema.a \ clangStaticAnalyzerCheckers.a clangStaticAnalyzerCore.a \ diff --git a/unittests/Sema/Makefile b/unittests/Sema/Makefile index 7fd5c27ad6..65ab9bc8dc 100644 --- a/unittests/Sema/Makefile +++ b/unittests/Sema/Makefile @@ -10,9 +10,10 @@ CLANG_LEVEL = ../.. TESTNAME = Sema include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation mc option objcarcopts support USEDLIBS = clangTooling.a clangFrontend.a clangSerialization.a clangDriver.a \ - clangRewrite.a clangRewriteFrontend.a \ + clangCodeGen.a clangRewrite.a clangRewriteFrontend.a \ clangParse.a clangSema.a clangAnalysis.a \ clangEdit.a clangAST.a clangASTMatchers.a clangLex.a clangBasic.a diff --git a/unittests/Tooling/Makefile b/unittests/Tooling/Makefile index 514e80bd03..86c1c34483 100644 --- a/unittests/Tooling/Makefile +++ b/unittests/Tooling/Makefile @@ -10,9 +10,10 @@ CLANG_LEVEL = ../.. TESTNAME = Tooling include $(CLANG_LEVEL)/../../Makefile.config -LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader support mc option +LINK_COMPONENTS := $(TARGETS_TO_BUILD) asmparser bitreader bitwriter \ + instrumentation mc option objcarcopts support USEDLIBS = clangTooling.a clangToolingCore.a clangFrontend.a \ - clangSerialization.a clangDriver.a \ + clangCodeGen.a clangSerialization.a clangDriver.a \ clangParse.a clangRewrite.a clangRewriteFrontend.a \ clangSema.a clangAnalysis.a clangEdit.a \ clangAST.a clangASTMatchers.a clangLex.a clangBasic.a -- 2.40.0