From 1a6f43b655762d6b70594e47013ad8f14a12480d Mon Sep 17 00:00:00 2001 From: Richard Smith Date: Mon, 5 Jun 2017 18:10:11 +0000 Subject: [PATCH] Rather than rejecting attempts to run preprocessor-only actions on AST files, replay the steps taken to create the AST file with the preprocessor-only action installed to produce preprocessed output. This can be used to produce the preprocessed text for an existing .pch or .pcm file. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@304726 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Basic/LangOptions.h | 1 + include/clang/Basic/SourceManager.h | 4 ++ include/clang/Frontend/ASTUnit.h | 7 ++- include/clang/Frontend/FrontendAction.h | 4 +- lib/Basic/SourceManager.cpp | 35 +++++++++++ lib/Frontend/ASTUnit.cpp | 36 ++++++++++-- lib/Frontend/FrontendAction.cpp | 78 ++++++++++++++++++++----- lib/Serialization/ASTReader.cpp | 1 + test/Modules/preprocess-module.cpp | 7 ++- 9 files changed, 147 insertions(+), 26 deletions(-) diff --git a/include/clang/Basic/LangOptions.h b/include/clang/Basic/LangOptions.h index 2513de70e7..8488515d2b 100644 --- a/include/clang/Basic/LangOptions.h +++ b/include/clang/Basic/LangOptions.h @@ -58,6 +58,7 @@ public: SOB_Trapping // -ftrapv }; + // FIXME: Unify with TUKind. enum CompilingModuleKind { CMK_None, ///< Not compiling a module interface at all. CMK_ModuleMap, ///< Compiling a module from a module map. diff --git a/include/clang/Basic/SourceManager.h b/include/clang/Basic/SourceManager.h index eda8029340..5e01f64167 100644 --- a/include/clang/Basic/SourceManager.h +++ b/include/clang/Basic/SourceManager.h @@ -722,6 +722,10 @@ public: void clearIDTables(); + /// Initialize this source manager suitably to replay the compilation + /// described by \p Old. Requires that \p Old outlive \p *this. + void initializeForReplay(const SourceManager &Old); + DiagnosticsEngine &getDiagnostics() const { return Diag; } FileManager &getFileManager() const { return FileMgr; } diff --git a/include/clang/Frontend/ASTUnit.h b/include/clang/Frontend/ASTUnit.h index fb0a5e8acd..a2a1796cd3 100644 --- a/include/clang/Frontend/ASTUnit.h +++ b/include/clang/Frontend/ASTUnit.h @@ -51,6 +51,7 @@ class DiagnosticsEngine; class FileEntry; class FileManager; class HeaderSearch; +class InputKind; class MemoryBufferCache; class Preprocessor; class PCHContainerOperations; @@ -305,9 +306,6 @@ private: /// (likely to change while trying to use them). bool UserFilesAreVolatile : 1; - /// \brief The language options used when we load an AST file. - LangOptions ASTFileLangOpts; - static void ConfigureDiags(IntrusiveRefCntPtr Diags, ASTUnit &AST, bool CaptureDiagnostics); @@ -702,6 +700,9 @@ public: /// \brief Determine what kind of translation unit this AST represents. TranslationUnitKind getTranslationUnitKind() const { return TUKind; } + /// \brief Determine the input kind this AST unit represents. + InputKind getInputKind() const; + /// \brief A mapping from a file name to the memory buffer that stores the /// remapped contents of that file. typedef std::pair RemappedFile; diff --git a/include/clang/Frontend/FrontendAction.h b/include/clang/Frontend/FrontendAction.h index 8d690a448f..07dfa5e6b7 100644 --- a/include/clang/Frontend/FrontendAction.h +++ b/include/clang/Frontend/FrontendAction.h @@ -176,10 +176,10 @@ public: virtual TranslationUnitKind getTranslationUnitKind() { return TU_Complete; } /// \brief Does this action support use with PCH? - virtual bool hasPCHSupport() const { return !usesPreprocessorOnly(); } + virtual bool hasPCHSupport() const { return true; } /// \brief Does this action support use with AST files? - virtual bool hasASTFileSupport() const { return !usesPreprocessorOnly(); } + virtual bool hasASTFileSupport() const { return true; } /// \brief Does this action support use with IR files? virtual bool hasIRSupport() const { return false; } diff --git a/lib/Basic/SourceManager.cpp b/lib/Basic/SourceManager.cpp index c5cff74ac9..c0cd2a8d2a 100644 --- a/lib/Basic/SourceManager.cpp +++ b/lib/Basic/SourceManager.cpp @@ -345,6 +345,41 @@ void SourceManager::clearIDTables() { createExpansionLoc(SourceLocation(),SourceLocation(),SourceLocation(), 1); } +void SourceManager::initializeForReplay(const SourceManager &Old) { + assert(MainFileID.isInvalid() && "expected uninitialized SourceManager"); + + auto CloneContentCache = [&](const ContentCache *Cache) -> ContentCache * { + auto *Clone = new (ContentCacheAlloc.Allocate()) ContentCache; + Clone->OrigEntry = Cache->OrigEntry; + Clone->ContentsEntry = Cache->ContentsEntry; + Clone->BufferOverridden = Cache->BufferOverridden; + Clone->IsSystemFile = Cache->IsSystemFile; + Clone->IsTransient = Cache->IsTransient; + Clone->replaceBuffer(Cache->getRawBuffer(), /*DoNotFree*/true); + return Clone; + }; + + // Set up our main file ID as a copy of the old source manager's main file. + const SLocEntry &OldMainFile = Old.getSLocEntry(Old.getMainFileID()); + assert(OldMainFile.isFile() && "main file is macro expansion?"); + setMainFileID(createFileID( + CloneContentCache(OldMainFile.getFile().getContentCache()), + SourceLocation(), OldMainFile.getFile().getFileCharacteristic(), 0, 0)); + + // Ensure all SLocEntries are loaded from the external source. + for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I) + if (!Old.SLocEntryLoaded[I]) + Old.loadSLocEntry(I, nullptr); + + // Inherit any content cache data from the old source manager. + for (auto &FileInfo : Old.FileInfos) { + SrcMgr::ContentCache *&Slot = FileInfos[FileInfo.first]; + if (Slot) + continue; + Slot = CloneContentCache(FileInfo.second); + } +} + /// getOrCreateContentCache - Create or return a cached ContentCache for the /// specified file. const ContentCache * diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp index 01f7ca8aba..1e5fd8b40a 100644 --- a/lib/Frontend/ASTUnit.cpp +++ b/lib/Frontend/ASTUnit.cpp @@ -667,6 +667,7 @@ std::unique_ptr ASTUnit::LoadFromASTFile( ConfigureDiags(Diags, *AST, CaptureDiagnostics); + AST->LangOpts = std::make_shared(); AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->Diagnostics = Diags; @@ -682,7 +683,7 @@ std::unique_ptr ASTUnit::LoadFromASTFile( AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts, AST->getSourceManager(), AST->getDiagnostics(), - AST->ASTFileLangOpts, + AST->getLangOpts(), /*Target=*/nullptr)); auto PPOpts = std::make_shared(); @@ -696,13 +697,13 @@ std::unique_ptr ASTUnit::LoadFromASTFile( unsigned Counter; AST->PP = std::make_shared( - std::move(PPOpts), AST->getDiagnostics(), AST->ASTFileLangOpts, + std::move(PPOpts), AST->getDiagnostics(), *AST->LangOpts, AST->getSourceManager(), *AST->PCMCache, HeaderInfo, *AST, /*IILookup=*/nullptr, /*OwnsHeaderSearch=*/false); Preprocessor &PP = *AST->PP; - AST->Ctx = new ASTContext(AST->ASTFileLangOpts, AST->getSourceManager(), + AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(), PP.getIdentifierTable(), PP.getSelectorTable(), PP.getBuiltinInfo()); ASTContext &Context = *AST->Ctx; @@ -716,7 +717,7 @@ std::unique_ptr ASTUnit::LoadFromASTFile( AllowPCHWithCompilerErrors); AST->Reader->setListener(llvm::make_unique( - *AST->PP, Context, AST->ASTFileLangOpts, AST->TargetOpts, AST->Target, + *AST->PP, Context, *AST->LangOpts, AST->TargetOpts, AST->Target, Counter)); // Attach the AST reader to the AST context as an external AST @@ -2879,7 +2880,32 @@ const FileEntry *ASTUnit::getPCHFile() { } bool ASTUnit::isModuleFile() { - return isMainFileAST() && ASTFileLangOpts.isCompilingModule(); + return isMainFileAST() && getLangOpts().isCompilingModule(); +} + +InputKind ASTUnit::getInputKind() const { + auto &LangOpts = getLangOpts(); + + InputKind::Language Lang; + if (LangOpts.OpenCL) + Lang = InputKind::OpenCL; + else if (LangOpts.CUDA) + Lang = InputKind::CUDA; + else if (LangOpts.RenderScript) + Lang = InputKind::RenderScript; + else if (LangOpts.CPlusPlus) + Lang = LangOpts.ObjC1 ? InputKind::ObjCXX : InputKind::CXX; + else + Lang = LangOpts.ObjC1 ? InputKind::ObjC : InputKind::C; + + InputKind::Format Fmt = InputKind::Source; + if (LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap) + Fmt = InputKind::ModuleMap; + + // We don't know if input was preprocessed. Assume not. + bool PP = false; + + return InputKind(Lang, Fmt, PP); } void ASTUnit::PreambleData::countLines() const { diff --git a/lib/Frontend/FrontendAction.cpp b/lib/Frontend/FrontendAction.cpp index e2fbe96534..ef98172b55 100644 --- a/lib/Frontend/FrontendAction.cpp +++ b/lib/Frontend/FrontendAction.cpp @@ -387,8 +387,7 @@ static std::error_code collectModuleHeaderIncludes( return std::error_code(); } -static bool loadModuleMapForModuleBuild(CompilerInstance &CI, - StringRef Filename, bool IsSystem, +static bool loadModuleMapForModuleBuild(CompilerInstance &CI, bool IsSystem, bool IsPreprocessed, std::string &PresumedModuleMapFile, unsigned &Offset) { @@ -523,7 +522,8 @@ getInputBufferForModule(CompilerInstance &CI, Module *M) { } bool FrontendAction::BeginSourceFile(CompilerInstance &CI, - const FrontendInputFile &Input) { + const FrontendInputFile &RealInput) { + FrontendInputFile Input(RealInput); assert(!Instance && "Already processing a source file!"); assert(!Input.isEmpty() && "Unexpected empty filename!"); setCurrentInput(Input); @@ -531,15 +531,69 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, StringRef InputFile = Input.getFile(); bool HasBegunSourceFile = false; + bool ReplayASTFile = Input.getKind().getFormat() == InputKind::Precompiled && + usesPreprocessorOnly(); if (!BeginInvocation(CI)) goto failure; + // If we're replaying the build of an AST file, import it and set up + // the initial state from its build. + if (ReplayASTFile) { + IntrusiveRefCntPtr Diags(&CI.getDiagnostics()); + + // The AST unit populates its own diagnostics engine rather than ours. + IntrusiveRefCntPtr ASTDiags( + new DiagnosticsEngine(Diags->getDiagnosticIDs(), + &Diags->getDiagnosticOptions())); + ASTDiags->setClient(Diags->getClient(), /*OwnsClient*/false); + + std::unique_ptr AST = ASTUnit::LoadFromASTFile( + InputFile, CI.getPCHContainerReader(), ASTDiags, CI.getFileSystemOpts(), + CI.getCodeGenOpts().DebugTypeExtRefs); + if (!AST) + goto failure; + + // Options relating to how we treat the input (but not what we do with it) + // are inherited from the AST unit. + CI.getLangOpts() = AST->getLangOpts(); + + // Preload all the module files loaded transitively by the AST unit. + if (auto ASTReader = AST->getASTReader()) { + auto &MM = ASTReader->getModuleManager(); + for (ModuleFile &MF : MM) + if (&MF != &MM.getPrimaryModule()) + CI.getFrontendOpts().ModuleFiles.push_back(MF.FileName); + } + + // Set the shared objects, these are reset when we finish processing the + // file, otherwise the CompilerInstance will happily destroy them. + CI.setFileManager(&AST->getFileManager()); + CI.createSourceManager(CI.getFileManager()); + CI.getSourceManager().initializeForReplay(AST->getSourceManager()); + CI.createPreprocessor(getTranslationUnitKind()); + + // Set up the input file for replay purposes. + auto Kind = AST->getInputKind(); + if (Kind.getFormat() == InputKind::ModuleMap) { + Module *ASTModule = + AST->getPreprocessor().getHeaderSearchInfo().lookupModule( + AST->getLangOpts().CurrentModule, /*AllowSearch*/ false); + Input = FrontendInputFile(ASTModule->PresumedModuleMapFile, Kind); + } else { + auto &SM = CI.getSourceManager(); + FileID ID = SM.getMainFileID(); + if (auto *File = SM.getFileEntryForID(ID)) + Input = FrontendInputFile(File->getName(), Kind); + else + Input = FrontendInputFile(SM.getBuffer(ID), Kind); + } + setCurrentInput(Input, std::move(AST)); + } + // AST files follow a very different path, since they share objects via the // AST unit. if (Input.getKind().getFormat() == InputKind::Precompiled) { - // FIXME: We should not be asserting on bad command-line arguments. - assert(!usesPreprocessorOnly() && - "Attempt to pass AST file to preprocessor only action!"); + assert(!usesPreprocessorOnly() && "this case was handled above"); assert(hasASTFileSupport() && "This action does not have AST file support!"); @@ -680,7 +734,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, std::string PresumedModuleMapFile; unsigned OffsetToContents; - if (loadModuleMapForModuleBuild(CI, Input.getFile(), Input.isSystem(), + if (loadModuleMapForModuleBuild(CI, Input.isSystem(), Input.isPreprocessed(), PresumedModuleMapFile, OffsetToContents)) goto failure; @@ -829,14 +883,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // If we failed, reset state since the client will not end up calling the // matching EndSourceFile(). - failure: - if (isCurrentFileAST()) { - CI.setASTContext(nullptr); - CI.setPreprocessor(nullptr); - CI.setSourceManager(nullptr); - CI.setFileManager(nullptr); - } - +failure: if (HasBegunSourceFile) CI.getDiagnosticClient().EndSourceFile(); CI.clearOutputFiles(/*EraseFiles=*/true); @@ -914,6 +961,7 @@ void FrontendAction::EndSourceFile() { CI.resetAndLeakPreprocessor(); CI.resetAndLeakSourceManager(); CI.resetAndLeakFileManager(); + BuryPointer(CurrentASTUnit.release()); } else { CI.setPreprocessor(nullptr); CI.setSourceManager(nullptr); diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp index e16a9b3ee3..c1c625a743 100644 --- a/lib/Serialization/ASTReader.cpp +++ b/lib/Serialization/ASTReader.cpp @@ -4918,6 +4918,7 @@ ASTReader::ReadSubmoduleBlock(ModuleFile &F, unsigned ClientLoadCapabilities) { } CurrentModule->setASTFile(F.File); + CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; } CurrentModule->Kind = ModuleKind; diff --git a/test/Modules/preprocess-module.cpp b/test/Modules/preprocess-module.cpp index 1e9458e667..9d0af5a768 100644 --- a/test/Modules/preprocess-module.cpp +++ b/test/Modules/preprocess-module.cpp @@ -34,10 +34,15 @@ // RUN: rm %t/fwd.h %t/file.h %t/file2.h %t/module.modulemap // RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -x c++-module-map-cpp-output %t/copy.ii -emit-module -o %t/copy.pcm -// Finally, check that our module contains correct mapping information for the headers. +// Check that our module contains correct mapping information for the headers. // RUN: cp %S/Inputs/preprocess/fwd.h %S/Inputs/preprocess/file.h %S/Inputs/preprocess/file2.h %S/Inputs/preprocess/module.modulemap %t // RUN: %clang_cc1 -fmodules -fmodule-file=%t/copy.pcm %s -I%t -verify -fno-modules-error-recovery -DCOPY -DINCLUDE +// Check that we can preprocess from a .pcm file and that we get the same result as preprocessing from the original sources. +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess -x c++-module-map %S/Inputs/preprocess/module.modulemap -emit-module -o %t/file.pcm +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess %t/file.pcm -E -frewrite-includes -o %t/file.rewrite.ii +// RUN: cmp %t/rewrite.ii %t/file.rewrite.ii + // == module map // CHECK: # 1 "{{.*}}module.modulemap" // CHECK: module file { -- 2.40.0