From 9f36327e3df8d2a4e89109b86eb610d84a13b4b9 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Fri, 7 Mar 2014 06:40:32 +0000 Subject: [PATCH] Add dependencies from imported modules with -MD Add module dependencies to the dependency files created by -MD/-MMD/etc. by attaching an ASTReaderListener that will call into the dependency file generator when a module input file is seen in the serialized AST. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@203208 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Frontend/CompilerInstance.h | 3 + include/clang/Frontend/Utils.h | 17 +++- include/clang/Serialization/ASTReader.h | 51 ++++++++++- lib/Frontend/CompilerInstance.cpp | 6 +- lib/Frontend/DependencyFile.cpp | 84 +++++++++++------ lib/Lex/ModuleMap.cpp | 3 +- lib/Serialization/ASTReader.cpp | 105 +++++++++++++++++++--- lib/Serialization/ASTWriter.cpp | 29 +----- test/Modules/dependency-gen.m | 20 +++++ 9 files changed, 245 insertions(+), 73 deletions(-) create mode 100644 test/Modules/dependency-gen.m diff --git a/include/clang/Frontend/CompilerInstance.h b/include/clang/Frontend/CompilerInstance.h index c8ffd32624..3079a06413 100644 --- a/include/clang/Frontend/CompilerInstance.h +++ b/include/clang/Frontend/CompilerInstance.h @@ -105,6 +105,9 @@ class CompilerInstance : public ModuleLoader { /// \brief The ASTReader, if one exists. IntrusiveRefCntPtr ModuleManager; + /// \brief The dependency file generator. + OwningPtr TheDependencyFileGenerator; + /// \brief The set of top-level modules that has already been loaded, /// along with the module map llvm::DenseMap KnownModules; diff --git a/include/clang/Frontend/Utils.h b/include/clang/Frontend/Utils.h index 471d9ac82b..63eecb108f 100644 --- a/include/clang/Frontend/Utils.h +++ b/include/clang/Frontend/Utils.h @@ -30,6 +30,7 @@ class ArgList; namespace clang { class ASTConsumer; +class ASTReader; class CompilerInstance; class CompilerInvocation; class Decl; @@ -72,10 +73,18 @@ void ProcessWarningOptions(DiagnosticsEngine &Diags, void DoPrintPreprocessedInput(Preprocessor &PP, raw_ostream* OS, const PreprocessorOutputOptions &Opts); -/// AttachDependencyFileGen - Create a dependency file generator, and attach -/// it to the given preprocessor. This takes ownership of the output stream. -void AttachDependencyFileGen(Preprocessor &PP, - const DependencyOutputOptions &Opts); +/// Builds a depdenency file when attached to a Preprocessor (for includes) and +/// ASTReader (for module imports), and writes it out at the end of processing +/// a source file. Users should attach to the ast reader whenever a module is +/// loaded. +class DependencyFileGenerator { + void *Impl; // Opaque implementation + DependencyFileGenerator(void *Impl); +public: + static DependencyFileGenerator *CreateAndAttachToPreprocessor( + Preprocessor &PP, const DependencyOutputOptions &Opts); + void AttachToASTReader(ASTReader &R); +}; /// AttachDependencyGraphGen - Create a dependency graph generator, and attach /// it to the given preprocessor. diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h index 040243cd2c..f77505d1b0 100644 --- a/include/clang/Serialization/ASTReader.h +++ b/include/clang/Serialization/ASTReader.h @@ -174,14 +174,50 @@ public: /// \brief Returns true if this \c ASTReaderListener wants to receive the /// input files of the AST file via \c visitInputFile, false otherwise. virtual bool needsInputFileVisitation() { return false; } - - /// \brief if \c needsInputFileVisitation returns true, this is called for each - /// input file of the AST file. + /// \brief Returns true if this \c ASTReaderListener wants to receive the + /// system input files of the AST file via \c visitInputFile, false otherwise. + virtual bool needsSystemInputFileVisitation() { return false; } + /// \brief if \c needsInputFileVisitation returns true, this is called for + /// each non-system input file of the AST File. If + /// \c needsSystemInputFileVisitation is true, then it is called for all + /// system input files as well. /// /// \returns true to continue receiving the next input file, false to stop. virtual bool visitInputFile(StringRef Filename, bool isSystem) { return true;} }; +/// \brief Simple wrapper class for chaining listeners. +class ChainedASTReaderListener : public ASTReaderListener { + OwningPtr First; + OwningPtr Second; +public: + /// Takes ownership of \p First and \p Second. + ChainedASTReaderListener(ASTReaderListener *First, ASTReaderListener *Second) + : First(First), Second(Second) { } + + virtual bool ReadFullVersionInformation(StringRef FullVersion); + virtual bool ReadLanguageOptions(const LangOptions &LangOpts, + bool Complain); + virtual bool ReadTargetOptions(const TargetOptions &TargetOpts, + bool Complain); + virtual bool ReadDiagnosticOptions(const DiagnosticOptions &DiagOpts, + bool Complain); + virtual bool ReadFileSystemOptions(const FileSystemOptions &FSOpts, + bool Complain); + + virtual bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, + bool Complain); + virtual bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts, + bool Complain, + std::string &SuggestedPredefines); + + virtual void ReadCounter(const serialization::ModuleFile &M, + unsigned Value); + virtual bool needsInputFileVisitation(); + virtual bool needsSystemInputFileVisitation(); + virtual bool visitInputFile(StringRef Filename, bool isSystem); +}; + /// \brief ASTReaderListener implementation to validate the information of /// the PCH file against an initialized Preprocessor. class PCHValidator : public ASTReaderListener { @@ -1277,6 +1313,15 @@ public: Listener.reset(listener); } + /// \brief Add an AST callbak listener. + /// + /// Takes ownership of \p L. + void addListener(ASTReaderListener *L) { + if (Listener) + L = new ChainedASTReaderListener(L, Listener.take()); + Listener.reset(L); + } + /// \brief Set the AST deserialization listener. void setDeserializationListener(ASTDeserializationListener *Listener); diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp index e07eb5295d..f10311a4ea 100644 --- a/lib/Frontend/CompilerInstance.cpp +++ b/lib/Frontend/CompilerInstance.cpp @@ -269,7 +269,8 @@ void CompilerInstance::createPreprocessor() { // Handle generating dependencies, if requested. const DependencyOutputOptions &DepOpts = getDependencyOutputOpts(); if (!DepOpts.OutputFile.empty()) - AttachDependencyFileGen(*PP, DepOpts); + TheDependencyFileGenerator.reset( + DependencyFileGenerator::CreateAndAttachToPreprocessor(*PP, DepOpts)); if (!DepOpts.DOTOutputFile.empty()) AttachDependencyGraphGen(*PP, DepOpts.DOTOutputFile, getHeaderSearchOpts().Sysroot); @@ -1160,6 +1161,9 @@ CompilerInstance::loadModule(SourceLocation ImportLoc, ModuleManager->StartTranslationUnit(&getASTConsumer()); } + if (TheDependencyFileGenerator) + TheDependencyFileGenerator->AttachToASTReader(*ModuleManager); + // Try to load the module file. unsigned ARRFlags = ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing; switch (ModuleManager->ReadAST(ModuleFileName, serialization::MK_Module, diff --git a/lib/Frontend/DependencyFile.cpp b/lib/Frontend/DependencyFile.cpp index 39ccdd736c..175a2c9a26 100644 --- a/lib/Frontend/DependencyFile.cpp +++ b/lib/Frontend/DependencyFile.cpp @@ -20,6 +20,7 @@ #include "clang/Lex/LexDiagnostic.h" #include "clang/Lex/PPCallbacks.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Serialization/ASTReader.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -28,7 +29,8 @@ using namespace clang; namespace { -class DependencyFileCallback : public PPCallbacks { +/// Private implementation for DependencyFileGenerator +class DFGImpl : public PPCallbacks { std::vector Files; llvm::StringSet<> FilesSet; const Preprocessor *PP; @@ -41,12 +43,10 @@ class DependencyFileCallback : public PPCallbacks { private: bool FileMatchesDepCriteria(const char *Filename, SrcMgr::CharacteristicKind FileType); - void AddFilename(StringRef Filename); void OutputDependencyFile(); public: - DependencyFileCallback(const Preprocessor *_PP, - const DependencyOutputOptions &Opts) + DFGImpl(const Preprocessor *_PP, const DependencyOutputOptions &Opts) : PP(_PP), OutputFile(Opts.OutputFile), Targets(Opts.Targets), IncludeSystemHeaders(Opts.IncludeSystemHeaders), PhonyTarget(Opts.UsePhonyTargets), @@ -69,27 +69,54 @@ public: virtual void EndOfMainFile() { OutputDependencyFile(); } + + void AddFilename(StringRef Filename); + bool includeSystemHeaders() const { return IncludeSystemHeaders; } +}; + +class DFGASTReaderListener : public ASTReaderListener { + DFGImpl &Parent; +public: + DFGASTReaderListener(DFGImpl &Parent) + : Parent(Parent) { } + virtual bool needsInputFileVisitation() { return true; } + virtual bool needsSystemInputFileVisitation() { + return Parent.includeSystemHeaders(); + } + virtual bool visitInputFile(StringRef Filename, bool isSystem); }; } -void clang::AttachDependencyFileGen(Preprocessor &PP, - const DependencyOutputOptions &Opts) { +DependencyFileGenerator::DependencyFileGenerator(void *Impl) +: Impl(Impl) { } + +DependencyFileGenerator *DependencyFileGenerator::CreateAndAttachToPreprocessor( + clang::Preprocessor &PP, const clang::DependencyOutputOptions &Opts) { + if (Opts.Targets.empty()) { PP.getDiagnostics().Report(diag::err_fe_dependency_file_requires_MT); - return; + return NULL; } // Disable the "file not found" diagnostic if the -MG option was given. if (Opts.AddMissingHeaderDeps) PP.SetSuppressIncludeNotFoundError(true); - PP.addPPCallbacks(new DependencyFileCallback(&PP, Opts)); + DFGImpl *Callback = new DFGImpl(&PP, Opts); + PP.addPPCallbacks(Callback); // PP owns the Callback + return new DependencyFileGenerator(Callback); +} + +void DependencyFileGenerator::AttachToASTReader(ASTReader &R) { + DFGImpl *I = reinterpret_cast(Impl); + assert(I && "missing implementation"); + R.addListener(new DFGASTReaderListener(*I)); } /// FileMatchesDepCriteria - Determine whether the given Filename should be /// considered as a dependency. -bool DependencyFileCallback::FileMatchesDepCriteria(const char *Filename, - SrcMgr::CharacteristicKind FileType) { +bool DFGImpl::FileMatchesDepCriteria(const char *Filename, + SrcMgr::CharacteristicKind FileType) { if (strcmp("", Filename) == 0) return false; @@ -99,10 +126,10 @@ bool DependencyFileCallback::FileMatchesDepCriteria(const char *Filename, return FileType == SrcMgr::C_User; } -void DependencyFileCallback::FileChanged(SourceLocation Loc, - FileChangeReason Reason, - SrcMgr::CharacteristicKind FileType, - FileID PrevFID) { +void DFGImpl::FileChanged(SourceLocation Loc, + FileChangeReason Reason, + SrcMgr::CharacteristicKind FileType, + FileID PrevFID) { if (Reason != PPCallbacks::EnterFile) return; @@ -130,15 +157,15 @@ void DependencyFileCallback::FileChanged(SourceLocation Loc, AddFilename(Filename); } -void DependencyFileCallback::InclusionDirective(SourceLocation HashLoc, - const Token &IncludeTok, - StringRef FileName, - bool IsAngled, - CharSourceRange FilenameRange, - const FileEntry *File, - StringRef SearchPath, - StringRef RelativePath, - const Module *Imported) { +void DFGImpl::InclusionDirective(SourceLocation HashLoc, + const Token &IncludeTok, + StringRef FileName, + bool IsAngled, + CharSourceRange FilenameRange, + const FileEntry *File, + StringRef SearchPath, + StringRef RelativePath, + const Module *Imported) { if (!File) { if (AddMissingHeaderDeps) AddFilename(FileName); @@ -147,7 +174,7 @@ void DependencyFileCallback::InclusionDirective(SourceLocation HashLoc, } } -void DependencyFileCallback::AddFilename(StringRef Filename) { +void DFGImpl::AddFilename(StringRef Filename) { if (FilesSet.insert(Filename)) Files.push_back(Filename); } @@ -164,7 +191,7 @@ static void PrintFilename(raw_ostream &OS, StringRef Filename) { } } -void DependencyFileCallback::OutputDependencyFile() { +void DFGImpl::OutputDependencyFile() { if (SeenMissingHeader) { llvm::sys::fs::remove(OutputFile); return; @@ -234,3 +261,10 @@ void DependencyFileCallback::OutputDependencyFile() { } } +bool DFGASTReaderListener::visitInputFile(llvm::StringRef Filename, + bool IsSystem) { + assert(!IsSystem || needsSystemInputFileVisitation()); + Parent.AddFilename(Filename); + return true; +} + diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp index 14bc3baba6..beb0bcbbd3 100644 --- a/lib/Lex/ModuleMap.cpp +++ b/lib/Lex/ModuleMap.cpp @@ -2213,7 +2213,8 @@ bool ModuleMap::parseModuleMapFile(const FileEntry *File, bool IsSystem) { return Known->second; assert(Target != 0 && "Missing target information"); - FileID ID = SourceMgr.createFileID(File, SourceLocation(), SrcMgr::C_User); + auto FileCharacter = IsSystem ? SrcMgr::C_System : SrcMgr::C_User; + FileID ID = SourceMgr.createFileID(File, SourceLocation(), FileCharacter); const llvm::MemoryBuffer *Buffer = SourceMgr.getBuffer(ID); if (!Buffer) return ParsedModuleMap[File] = true; diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp index 0afe262396..60734db8ab 100644 --- a/lib/Serialization/ASTReader.cpp +++ b/lib/Serialization/ASTReader.cpp @@ -60,6 +60,70 @@ using namespace clang::serialization; using namespace clang::serialization::reader; using llvm::BitstreamCursor; + +//===----------------------------------------------------------------------===// +// ChainedASTReaderListener implementation +//===----------------------------------------------------------------------===// + +bool +ChainedASTReaderListener::ReadFullVersionInformation(StringRef FullVersion) { + return First->ReadFullVersionInformation(FullVersion) || + Second->ReadFullVersionInformation(FullVersion); +} +bool ChainedASTReaderListener::ReadLanguageOptions(const LangOptions &LangOpts, + bool Complain) { + return First->ReadLanguageOptions(LangOpts, Complain) || + Second->ReadLanguageOptions(LangOpts, Complain); +} +bool +ChainedASTReaderListener::ReadTargetOptions(const TargetOptions &TargetOpts, + bool Complain) { + return First->ReadTargetOptions(TargetOpts, Complain) || + Second->ReadTargetOptions(TargetOpts, Complain); +} +bool ChainedASTReaderListener::ReadDiagnosticOptions( + const DiagnosticOptions &DiagOpts, bool Complain) { + return First->ReadDiagnosticOptions(DiagOpts, Complain) || + Second->ReadDiagnosticOptions(DiagOpts, Complain); +} +bool +ChainedASTReaderListener::ReadFileSystemOptions(const FileSystemOptions &FSOpts, + bool Complain) { + return First->ReadFileSystemOptions(FSOpts, Complain) || + Second->ReadFileSystemOptions(FSOpts, Complain); +} + +bool ChainedASTReaderListener::ReadHeaderSearchOptions( + const HeaderSearchOptions &HSOpts, bool Complain) { + return First->ReadHeaderSearchOptions(HSOpts, Complain) || + Second->ReadHeaderSearchOptions(HSOpts, Complain); +} +bool ChainedASTReaderListener::ReadPreprocessorOptions( + const PreprocessorOptions &PPOpts, bool Complain, + std::string &SuggestedPredefines) { + return First->ReadPreprocessorOptions(PPOpts, Complain, + SuggestedPredefines) || + Second->ReadPreprocessorOptions(PPOpts, Complain, SuggestedPredefines); +} +void ChainedASTReaderListener::ReadCounter(const serialization::ModuleFile &M, + unsigned Value) { + First->ReadCounter(M, Value); + Second->ReadCounter(M, Value); +} +bool ChainedASTReaderListener::needsInputFileVisitation() { + return First->needsInputFileVisitation() || + Second->needsInputFileVisitation(); +} +bool ChainedASTReaderListener::needsSystemInputFileVisitation() { + return First->needsSystemInputFileVisitation() || + Second->needsSystemInputFileVisitation(); +} +bool ChainedASTReaderListener::visitInputFile(StringRef Filename, + bool isSystem) { + return First->visitInputFile(Filename, isSystem) || + Second->visitInputFile(Filename, isSystem); +} + //===----------------------------------------------------------------------===// // PCH validator implementation //===----------------------------------------------------------------------===// @@ -2004,30 +2068,44 @@ ASTReader::ReadControlBlock(ModuleFile &F, // Validate input files. const HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts(); + + // All user input files reside at the index range [0, Record[1]), and + // system input files reside at [Record[1], Record[0]). + // Record is the one from INPUT_FILE_OFFSETS. + unsigned NumInputs = Record[0]; + unsigned NumUserInputs = Record[1]; + if (!DisableValidation && (!HSOpts.ModulesValidateOncePerBuildSession || F.InputFilesValidationTimestamp <= HSOpts.BuildSessionTimestamp)) { bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0; - // All user input files reside at the index range [0, Record[1]), and - // system input files reside at [Record[1], Record[0]). - // Record is the one from INPUT_FILE_OFFSETS. - // + // If we are reading a module, we will create a verification timestamp, // so we verify all input files. Otherwise, verify only user input // files. - unsigned NumInputs = Record[0]; - unsigned NumUserInputs = Record[1]; - unsigned N = ValidateSystemInputs || - (HSOpts.ModulesValidateOncePerBuildSession && - F.Kind == MK_Module) - ? NumInputs - : NumUserInputs; + + unsigned N = NumUserInputs; + if (ValidateSystemInputs || + (Listener && Listener->needsInputFileVisitation()) || + (HSOpts.ModulesValidateOncePerBuildSession && F.Kind == MK_Module)) + N = NumInputs; + for (unsigned I = 0; I < N; ++I) { InputFile IF = getInputFile(F, I+1, Complain); + if (const FileEntry *F = IF.getFile()) + Listener->visitInputFile(F->getName(), I >= NumUserInputs); if (!IF.getFile() || IF.isOutOfDate()) return OutOfDate; } } + + if (Listener && Listener->needsInputFileVisitation()) { + unsigned N = Listener->needsSystemInputFileVisitation() ? NumInputs + : NumUserInputs; + for (unsigned I = 0; I < N; ++I) + Listener->visitInputFile(getInputFileName(F, I+1), I >= NumUserInputs); + } + return Success; } @@ -3729,6 +3807,7 @@ bool ASTReader::readASTFileControlBlock(StringRef Filename, return true; bool NeedsInputFiles = Listener.needsInputFileVisitation(); + bool NeedsSystemInputFiles = Listener.needsSystemInputFileVisitation(); BitstreamCursor InputFilesCursor; if (NeedsInputFiles) { InputFilesCursor = Stream; @@ -3815,6 +3894,10 @@ bool ASTReader::readASTFileControlBlock(StringRef Filename, for (unsigned I = 0; I != NumInputFiles; ++I) { // Go find this input file. bool isSystemFile = I >= NumUserFiles; + + if (isSystemFile && !NeedsSystemInputFiles) + break; // the rest are system input files + BitstreamCursor &Cursor = InputFilesCursor; SavedStreamPosition SavedPosition(Cursor); Cursor.JumpToBit(InputFileOffs[I]); diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp index 2769b9fda1..3e6b719bc6 100644 --- a/lib/Serialization/ASTWriter.cpp +++ b/lib/Serialization/ASTWriter.cpp @@ -1300,33 +1300,6 @@ void ASTWriter::WriteInputFiles(SourceManager &SourceMgr, SortedFiles.push_front(Entry); } - // If we have an isysroot for a Darwin SDK, include its SDKSettings.plist in - // the set of (non-system) input files. This is simple heuristic for - // detecting whether the system headers may have changed, because it is too - // expensive to stat() all of the system headers. - FileManager &FileMgr = SourceMgr.getFileManager(); - if (!HSOpts.Sysroot.empty() && !Chain) { - llvm::SmallString<128> SDKSettingsFileName(HSOpts.Sysroot); - llvm::sys::path::append(SDKSettingsFileName, "SDKSettings.plist"); - if (const FileEntry *SDKSettingsFile = FileMgr.getFile(SDKSettingsFileName)) { - InputFileEntry Entry = { SDKSettingsFile, false, false }; - SortedFiles.push_front(Entry); - } - } - - // Add the compiler's own module.map in the set of (non-system) input files. - // This is a simple heuristic for detecting whether the compiler's headers - // have changed, because we don't want to stat() all of them. - if (Modules && !Chain) { - SmallString<128> P = StringRef(HSOpts.ResourceDir); - llvm::sys::path::append(P, "include"); - llvm::sys::path::append(P, "module.map"); - if (const FileEntry *ModuleMapFile = FileMgr.getFile(P)) { - InputFileEntry Entry = { ModuleMapFile, false, false }; - SortedFiles.push_front(Entry); - } - } - unsigned UserFilesNum = 0; // Write out all of the input files. std::vector InputFileOffsets; @@ -1363,7 +1336,7 @@ void ASTWriter::WriteInputFiles(SourceManager &SourceMgr, // Ask the file manager to fixup the relative path for us. This will // honor the working directory. - FileMgr.FixupRelativePath(FilePath); + SourceMgr.getFileManager().FixupRelativePath(FilePath); // FIXME: This call to make_absolute shouldn't be necessary, the // call to FixupRelativePath should always return an absolute path. diff --git a/test/Modules/dependency-gen.m b/test/Modules/dependency-gen.m new file mode 100644 index 0000000000..471ce14413 --- /dev/null +++ b/test/Modules/dependency-gen.m @@ -0,0 +1,20 @@ +// RUN: rm -rf %t-mcp +// RUN: mkdir -p %t-mcp + +// RUN: %clang_cc1 -x objective-c -dependency-file %t.d.1 -MT %s.o -I %S/Inputs -fsyntax-only -fmodules -fmodules-cache-path=%t-mcp %s +// RUN: FileCheck %s < %t.d.1 +// CHECK: dependency-gen.m +// CHECK: Inputs/diamond_top.h +// CHECK: Inputs/module.map +// CHECK-NOT: string.h + + +// RUN: %clang_cc1 -x objective-c -dependency-file %t.d.2 -MT %s.o -I %S/Inputs -sys-header-deps -fsyntax-only -fmodules -fmodules-cache-path=%t-mcp %s +// RUN: FileCheck %s -check-prefix=CHECK-SYS < %t.d.2 +// CHECK-SYS: dependency-gen.m +// CHECK-SYS: Inputs/diamond_top.h +// CHECK-SYS: Inputs/module.map +// CHECK-SYS: string.h + +#import "diamond_top.h" +#import "string.h" -- 2.40.0