From b0f4b9a558933b307073f7cd7753602f94354ae9 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Wed, 9 Mar 2011 17:21:42 +0000 Subject: [PATCH] Introduce '-chain-include' option to specify headers that will be converted to chained PCHs in memory without having to use multiple runs and intermediate files. Intended for testing & debugging of chained PCH. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@127339 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Driver/CC1Options.td | 2 + include/clang/Frontend/ASTUnit.h | 11 +- include/clang/Frontend/PreprocessorOptions.h | 3 + include/clang/Serialization/ASTReader.h | 11 + .../Serialization/ChainedIncludesSource.h | 63 ++++++ lib/Frontend/ASTUnit.cpp | 33 ++- lib/Frontend/CompilerInvocation.cpp | 10 + lib/Frontend/FrontendAction.cpp | 13 +- lib/Serialization/ASTReader.cpp | 34 +-- lib/Serialization/ChainedIncludesSource.cpp | 207 ++++++++++++++++++ test/PCH/chain-cxx.cpp | 4 +- 11 files changed, 366 insertions(+), 25 deletions(-) create mode 100644 include/clang/Serialization/ChainedIncludesSource.h create mode 100644 lib/Serialization/ChainedIncludesSource.cpp diff --git a/include/clang/Driver/CC1Options.td b/include/clang/Driver/CC1Options.td index 12f2556b9e..c0fb23deb0 100644 --- a/include/clang/Driver/CC1Options.td +++ b/include/clang/Driver/CC1Options.td @@ -560,6 +560,8 @@ def include_pch : Separate<"-include-pch">, MetaVarName<"">, HelpText<"Include precompiled header file">; def include_pth : Separate<"-include-pth">, MetaVarName<"">, HelpText<"Include file before parsing">; +def chain_include : Separate<"-chain-include">, MetaVarName<"">, + HelpText<"Include and chain a header file after turning it into PCH">; def preamble_bytes_EQ : Joined<"-preamble-bytes=">, HelpText<"Assume that the precompiled header is a precompiled preamble " "covering the first N bytes of the main file">; diff --git a/include/clang/Frontend/ASTUnit.h b/include/clang/Frontend/ASTUnit.h index 3138d7db46..a0c99f12e9 100644 --- a/include/clang/Frontend/ASTUnit.h +++ b/include/clang/Frontend/ASTUnit.h @@ -540,7 +540,11 @@ public: /// \brief A mapping from a file name to the memory buffer that stores the /// remapped contents of that file. typedef std::pair RemappedFile; - + + /// \brief Create a ASTUnit. Gets ownership of the passed CompilerInvocation. + static ASTUnit *create(CompilerInvocation *CI, + llvm::IntrusiveRefCntPtr Diags); + /// \brief Create a ASTUnit from an AST file. /// /// \param Filename - The AST file to load. @@ -656,6 +660,11 @@ public: /// /// \returns True if an error occurred, false otherwise. bool Save(llvm::StringRef File); + + /// \brief Serialize this translation unit with the given output stream. + /// + /// \returns True if an error occurred, false otherwise. + bool serialize(llvm::raw_ostream &OS); }; } // namespace clang diff --git a/include/clang/Frontend/PreprocessorOptions.h b/include/clang/Frontend/PreprocessorOptions.h index 078b22e719..e875ec1fef 100644 --- a/include/clang/Frontend/PreprocessorOptions.h +++ b/include/clang/Frontend/PreprocessorOptions.h @@ -44,6 +44,9 @@ public: /// The implicit PCH included at the start of the translation unit, or empty. std::string ImplicitPCHInclude; + /// \brief Headers that will be converted to chained PCHs in memory. + std::vector ChainedIncludes; + /// \brief When true, disables most of the normal validation performed on /// precompiled headers. bool DisablePCHValidation; diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h index 424e78c391..835a0dbb10 100644 --- a/include/clang/Serialization/ASTReader.h +++ b/include/clang/Serialization/ASTReader.h @@ -213,6 +213,10 @@ private: /// \brief The AST consumer. ASTConsumer *Consumer; + /// \brief AST buffers for chained PCHs created and stored in memory. + /// First (not depending on another) PCH in chain is in front. + std::deque ASTBuffers; + /// \brief Information that is needed for every module. struct PerFileData { PerFileData(ASTFileType Ty); @@ -886,6 +890,13 @@ public: /// \brief Sets and initializes the given Context. void InitializeContext(ASTContext &Context); + /// \brief Set AST buffers for chained PCHs created and stored in memory. + /// First (not depending on another) PCH in chain is first in array. + void setASTMemoryBuffers(llvm::MemoryBuffer **bufs, unsigned numBufs) { + ASTBuffers.clear(); + ASTBuffers.insert(ASTBuffers.begin(), bufs, bufs + numBufs); + } + /// \brief Retrieve the name of the named (primary) AST file const std::string &getFileName() const { return Chain[0]->FileName; } diff --git a/include/clang/Serialization/ChainedIncludesSource.h b/include/clang/Serialization/ChainedIncludesSource.h new file mode 100644 index 0000000000..90e74d913c --- /dev/null +++ b/include/clang/Serialization/ChainedIncludesSource.h @@ -0,0 +1,63 @@ +//===- ChainedIncludesSource.h - Chained PCHs in Memory ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the ChainedIncludesSource class, which converts headers +// to chained PCHs in memory, mainly used for testing. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_SERIALIZATION_CHAINEDINCLUDESSOURCE_H +#define LLVM_CLANG_SERIALIZATION_CHAINEDINCLUDESSOURCE_H + +#include "clang/AST/ExternalASTSource.h" +#include + +namespace clang { + class CompilerInstance; + +class ChainedIncludesSource : public ExternalASTSource { +public: + virtual ~ChainedIncludesSource(); + + static ChainedIncludesSource *create(CompilerInstance &CI); + +private: + ExternalASTSource &getFinalReader() const { return *FinalReader; } + + std::vector CIs; + llvm::OwningPtr FinalReader; + + +protected: + +//===----------------------------------------------------------------------===// +// ExternalASTSource interface. +//===----------------------------------------------------------------------===// + + virtual Decl *GetExternalDecl(uint32_t ID); + virtual Selector GetExternalSelector(uint32_t ID); + virtual uint32_t GetNumExternalSelectors(); + virtual Stmt *GetExternalDeclStmt(uint64_t Offset); + virtual CXXBaseSpecifier *GetExternalCXXBaseSpecifiers(uint64_t Offset); + virtual DeclContextLookupResult + FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name); + virtual void MaterializeVisibleDecls(const DeclContext *DC); + virtual bool FindExternalLexicalDecls(const DeclContext *DC, + bool (*isKindWeWant)(Decl::Kind), + llvm::SmallVectorImpl &Result); + virtual void CompleteType(TagDecl *Tag); + virtual void CompleteType(ObjCInterfaceDecl *Class); + virtual void StartedDeserializing(); + virtual void FinishedDeserializing(); + virtual void StartTranslationUnit(ASTConsumer *Consumer); + virtual void PrintStats(); +}; + +} + +#endif diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp index 5e534b3d57..41f2442a11 100644 --- a/lib/Frontend/ASTUnit.cpp +++ b/lib/Frontend/ASTUnit.cpp @@ -92,7 +92,8 @@ const unsigned DefaultPreambleRebuildInterval = 5; static llvm::sys::cas_flag ActiveASTUnitObjects; ASTUnit::ASTUnit(bool _MainFileIsAST) - : CaptureDiagnostics(false), MainFileIsAST(_MainFileIsAST), + : OnlyLocalDecls(false), CaptureDiagnostics(false), + MainFileIsAST(_MainFileIsAST), CompleteTranslationUnit(true), WantTiming(getenv("LIBCLANG_TIMING")), OwnsRemappedFileBuffers(true), NumStoredDiagnosticsFromDriver(0), @@ -1522,6 +1523,19 @@ llvm::StringRef ASTUnit::getMainFileName() const { return Invocation->getFrontendOpts().Inputs[0].second; } +ASTUnit *ASTUnit::create(CompilerInvocation *CI, + llvm::IntrusiveRefCntPtr Diags) { + llvm::OwningPtr AST; + AST.reset(new ASTUnit(false)); + ConfigureDiags(Diags, 0, 0, *AST, /*CaptureDiagnostics=*/false); + AST->Diagnostics = Diags; + AST->Invocation.reset(CI); + AST->FileMgr.reset(new FileManager(CI->getFileSystemOpts())); + AST->SourceMgr.reset(new SourceManager(*Diags, *AST->FileMgr)); + + return AST.take(); +} + bool ASTUnit::LoadFromCompilerInvocation(bool PrecompilePreamble) { if (!Invocation) return true; @@ -2148,7 +2162,16 @@ bool ASTUnit::Save(llvm::StringRef File) { llvm::raw_fd_ostream::F_Binary); if (!ErrorInfo.empty() || Out.has_error()) return true; - + + serialize(Out); + Out.close(); + return Out.has_error(); +} + +bool ASTUnit::serialize(llvm::raw_ostream &OS) { + if (getDiagnostics().hasErrorOccurred()) + return true; + std::vector Buffer; llvm::BitstreamWriter Stream(Buffer); ASTWriter Writer(Stream); @@ -2156,7 +2179,7 @@ bool ASTUnit::Save(llvm::StringRef File) { // Write the generated bitstream to "Out". if (!Buffer.empty()) - Out.write((char *)&Buffer.front(), Buffer.size()); - Out.close(); - return Out.has_error(); + OS.write((char *)&Buffer.front(), Buffer.size()); + + return false; } diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 8f99403923..d8de7c08b4 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -704,6 +704,10 @@ static void PreprocessorOptsToArgs(const PreprocessorOptions &Opts, assert(Opts.ImplicitPTHInclude == Opts.TokenCache && "Unsupported option combination!"); } + for (unsigned i = 0, e = Opts.ChainedIncludes.size(); i != e; ++i) { + Res.push_back("-chain-include"); + Res.push_back(Opts.ChainedIncludes[i]); + } for (unsigned i = 0, e = Opts.RemappedFiles.size(); i != e; ++i) { Res.push_back("-remap-file"); Res.push_back(Opts.RemappedFiles[i].first + ";" + @@ -1562,6 +1566,12 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args, Opts.Includes.push_back(A->getValue(Args)); } + for (arg_iterator it = Args.filtered_begin(OPT_chain_include), + ie = Args.filtered_end(); it != ie; ++it) { + const Arg *A = *it; + Opts.ChainedIncludes.push_back(A->getValue(Args)); + } + // Include 'altivec.h' if -faltivec option present if (Args.hasArg(OPT_faltivec)) Opts.Includes.push_back("altivec.h"); diff --git a/lib/Frontend/FrontendAction.cpp b/lib/Frontend/FrontendAction.cpp index e3d8b85941..5c34e9491b 100644 --- a/lib/Frontend/FrontendAction.cpp +++ b/lib/Frontend/FrontendAction.cpp @@ -20,6 +20,7 @@ #include "clang/Frontend/MultiplexConsumer.h" #include "clang/Parse/ParseAST.h" #include "clang/Serialization/ASTDeserializationListener.h" +#include "clang/Serialization/ChainedIncludesSource.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Timer.h" #include "llvm/Support/ErrorHandling.h" @@ -209,8 +210,16 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, CI.getASTContext().setASTMutationListener(Consumer->GetASTMutationListener()); - /// Use PCH? - if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) { + if (!CI.getPreprocessorOpts().ChainedIncludes.empty()) { + // Convert headers to PCH and chain them. + llvm::OwningPtr source; + source.reset(ChainedIncludesSource::create(CI)); + if (!source) + goto failure; + CI.getASTContext().setExternalSource(source); + + } else if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) { + // Use PCH. assert(hasPCHSupport() && "This action does not have PCH support!"); ASTDeserializationListener *DeserialListener = CI.getInvocation().getFrontendOpts().ChainedPCH ? diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp index 4717cd6268..030e652169 100644 --- a/lib/Serialization/ASTReader.cpp +++ b/lib/Serialization/ASTReader.cpp @@ -2461,20 +2461,26 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName, if (CurrentDir.empty()) CurrentDir = "."; } - // Open the AST file. - // - // FIXME: This shouldn't be here, we should just take a raw_ostream. - std::string ErrStr; - llvm::error_code ec; - if (FileName == "-") { - ec = llvm::MemoryBuffer::getSTDIN(F.Buffer); - if (ec) - ErrStr = ec.message(); - } else - F.Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrStr)); - if (!F.Buffer) { - Error(ErrStr.c_str()); - return IgnorePCH; + if (!ASTBuffers.empty()) { + F.Buffer.reset(ASTBuffers.front()); + ASTBuffers.pop_front(); + assert(F.Buffer && "Passed null buffer"); + } else { + // Open the AST file. + // + // FIXME: This shouldn't be here, we should just take a raw_ostream. + std::string ErrStr; + llvm::error_code ec; + if (FileName == "-") { + ec = llvm::MemoryBuffer::getSTDIN(F.Buffer); + if (ec) + ErrStr = ec.message(); + } else + F.Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrStr)); + if (!F.Buffer) { + Error(ErrStr.c_str()); + return IgnorePCH; + } } // Initialize the stream diff --git a/lib/Serialization/ChainedIncludesSource.cpp b/lib/Serialization/ChainedIncludesSource.cpp new file mode 100644 index 0000000000..da34743cee --- /dev/null +++ b/lib/Serialization/ChainedIncludesSource.cpp @@ -0,0 +1,207 @@ +//===- ChainedIncludesSource.cpp - Chained PCHs in Memory -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the ChainedIncludesSource class, which converts headers +// to chained PCHs in memory, mainly used for testing. +// +//===----------------------------------------------------------------------===// + +#include "clang/Serialization/ChainedIncludesSource.h" +#include "clang/Serialization/ASTReader.h" +#include "clang/Serialization/ASTWriter.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/ASTUnit.h" +#include "clang/Parse/ParseAST.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Basic/TargetInfo.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace clang; + +static ASTReader *createASTReader(CompilerInstance &CI, + llvm::StringRef pchFile, + llvm::MemoryBuffer **memBufs, + unsigned numBufs, + ASTDeserializationListener *deserialListener = 0) { + Preprocessor &PP = CI.getPreprocessor(); + llvm::OwningPtr Reader; + Reader.reset(new ASTReader(PP, &CI.getASTContext(), /*isysroot=*/0, + /*DisableValidation=*/true)); + Reader->setASTMemoryBuffers(memBufs, numBufs); + Reader->setDeserializationListener(deserialListener); + switch (Reader->ReadAST(pchFile, ASTReader::PCH)) { + case ASTReader::Success: + // Set the predefines buffer as suggested by the PCH reader. + PP.setPredefines(Reader->getSuggestedPredefines()); + return Reader.take(); + + case ASTReader::Failure: + case ASTReader::IgnorePCH: + break; + } + return 0; +} + +ChainedIncludesSource::~ChainedIncludesSource() { + for (unsigned i = 0, e = CIs.size(); i != e; ++i) + delete CIs[i]; +} + +ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) { + + std::vector &includes = CI.getPreprocessorOpts().ChainedIncludes; + assert(!includes.empty() && "No '-chain-include' in options!"); + + llvm::OwningPtr source(new ChainedIncludesSource()); + InputKind IK = CI.getFrontendOpts().Inputs[0].first; + + llvm::SmallVector serialBufs; + + for (unsigned i = 0, e = includes.size(); i != e; ++i) { + bool firstInclude = (i == 0); + llvm::OwningPtr CInvok; + CInvok.reset(new CompilerInvocation(CI.getInvocation())); + + CInvok->getPreprocessorOpts().ChainedIncludes.clear(); + CInvok->getPreprocessorOpts().ImplicitPCHInclude.clear(); + CInvok->getPreprocessorOpts().ImplicitPTHInclude.clear(); + CInvok->getPreprocessorOpts().DisablePCHValidation = true; + CInvok->getPreprocessorOpts().Includes.clear(); + CInvok->getPreprocessorOpts().MacroIncludes.clear(); + CInvok->getPreprocessorOpts().Macros.clear(); + + CInvok->getFrontendOpts().Inputs.clear(); + CInvok->getFrontendOpts().Inputs.push_back(std::make_pair(IK, includes[i])); + + TextDiagnosticPrinter *DiagClient = + new TextDiagnosticPrinter(llvm::errs(), DiagnosticOptions()); + llvm::IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + llvm::IntrusiveRefCntPtr Diags(new Diagnostic(DiagID, + DiagClient)); + + llvm::OwningPtr Clang(new CompilerInstance()); + Clang->setInvocation(CInvok.take()); + Clang->setDiagnostics(Diags.getPtr()); + Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(), + Clang->getTargetOpts())); + Clang->createFileManager(); + Clang->createSourceManager(Clang->getFileManager()); + Clang->createPreprocessor(); + Clang->createASTContext(); + + llvm::SmallVector serialAST; + llvm::raw_svector_ostream OS(serialAST); + llvm::OwningPtr consumer; + consumer.reset(new PCHGenerator(Clang->getPreprocessor(), "-", + /*Chaining=*/!firstInclude, + /*isysroot=*/0, &OS)); + Clang->getASTContext().setASTMutationListener( + consumer->GetASTMutationListener()); + Clang->setASTConsumer(consumer.take()); + Clang->createSema(/*CompleteTranslationUnit=*/false, 0); + + if (firstInclude) { + Preprocessor &PP = Clang->getPreprocessor(); + PP.getBuiltinInfo().InitializeBuiltins(PP.getIdentifierTable(), + PP.getLangOptions()); + } else { + assert(!serialBufs.empty()); + llvm::SmallVector bufs; + for (unsigned i = 0, e = serialBufs.size(); i != e; ++i) { + bufs.push_back(llvm::MemoryBuffer::getMemBufferCopy( + llvm::StringRef(serialBufs[i]->getBufferStart(), + serialBufs[i]->getBufferSize()))); + } + std::string pchName = includes[i-1]; + llvm::raw_string_ostream os(pchName); + os << ".pch" << i-1; + os.flush(); + llvm::OwningPtr Reader; + Reader.reset(createASTReader(*Clang, pchName, bufs.data(), bufs.size(), + Clang->getASTConsumer().GetASTDeserializationListener())); + if (!Reader) + return 0; + Clang->getASTContext().setExternalSource(Reader); + } + + if (!Clang->InitializeSourceManager(includes[i])) + return 0; + + ParseAST(Clang->getSema()); + OS.flush(); + serialBufs.push_back( + llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(serialAST.data(), + serialAST.size()))); + source->CIs.push_back(Clang.take()); + } + + assert(!serialBufs.empty()); + std::string pchName = includes.back() + ".pch-final"; + llvm::OwningPtr Reader; + Reader.reset(createASTReader(CI, pchName, + serialBufs.data(), serialBufs.size())); + if (!Reader) + return 0; + + source->FinalReader.reset(Reader.take()); + return source.take(); +} + +//===----------------------------------------------------------------------===// +// ExternalASTSource interface. +//===----------------------------------------------------------------------===// + +Decl *ChainedIncludesSource::GetExternalDecl(uint32_t ID) { + return getFinalReader().GetExternalDecl(ID); +} +Selector ChainedIncludesSource::GetExternalSelector(uint32_t ID) { + return getFinalReader().GetExternalSelector(ID); +} +uint32_t ChainedIncludesSource::GetNumExternalSelectors() { + return getFinalReader().GetNumExternalSelectors(); +} +Stmt *ChainedIncludesSource::GetExternalDeclStmt(uint64_t Offset) { + return getFinalReader().GetExternalDeclStmt(Offset); +} +CXXBaseSpecifier * +ChainedIncludesSource::GetExternalCXXBaseSpecifiers(uint64_t Offset) { + return getFinalReader().GetExternalCXXBaseSpecifiers(Offset); +} +DeclContextLookupResult +ChainedIncludesSource::FindExternalVisibleDeclsByName(const DeclContext *DC, + DeclarationName Name) { + return getFinalReader().FindExternalVisibleDeclsByName(DC, Name); +} +void ChainedIncludesSource::MaterializeVisibleDecls(const DeclContext *DC) { + return getFinalReader().MaterializeVisibleDecls(DC); +} +bool ChainedIncludesSource::FindExternalLexicalDecls(const DeclContext *DC, + bool (*isKindWeWant)(Decl::Kind), + llvm::SmallVectorImpl &Result) { + return getFinalReader().FindExternalLexicalDecls(DC, isKindWeWant, Result); +} +void ChainedIncludesSource::CompleteType(TagDecl *Tag) { + return getFinalReader().CompleteType(Tag); +} +void ChainedIncludesSource::CompleteType(ObjCInterfaceDecl *Class) { + return getFinalReader().CompleteType(Class); +} +void ChainedIncludesSource::StartedDeserializing() { + return getFinalReader().StartedDeserializing(); +} +void ChainedIncludesSource::FinishedDeserializing() { + return getFinalReader().FinishedDeserializing(); +} +void ChainedIncludesSource::StartTranslationUnit(ASTConsumer *Consumer) { + return getFinalReader().StartTranslationUnit(Consumer); +} +void ChainedIncludesSource::PrintStats() { + return getFinalReader().PrintStats(); +} diff --git a/test/PCH/chain-cxx.cpp b/test/PCH/chain-cxx.cpp index bf37acb298..107ddfee1c 100644 --- a/test/PCH/chain-cxx.cpp +++ b/test/PCH/chain-cxx.cpp @@ -4,9 +4,7 @@ // RUN: %clang_cc1 -fsyntax-only -verify -include %s -include %s %s // With PCH -// RUN: %clang_cc1 -x c++-header -emit-pch -o %t1 %s -// RUN: %clang_cc1 -x c++-header -emit-pch -o %t2 %s -include-pch %t1 -chained-pch -// RUN: %clang_cc1 -fsyntax-only -verify -include-pch %t2 %s +// RUN: %clang_cc1 -fsyntax-only -verify %s -chain-include %s -chain-include %s #ifndef HEADER1 #define HEADER1 -- 2.40.0