From: Douglas Gregor Date: Mon, 4 Jan 2010 19:18:44 +0000 (+0000) Subject: Teach Preprocessor::macro_begin/macro_end to lazily load all macro X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=88a35862fbe473f2a4f0c19f24dbe536937e1dc6;p=clang Teach Preprocessor::macro_begin/macro_end to lazily load all macro definitions from a precompiled header. This ensures that code-completion with macro names behaves the same with or without precompiled headers. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@92497 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/clang/Frontend/PCHReader.h b/include/clang/Frontend/PCHReader.h index 7e2c65690f..9665ce189f 100644 --- a/include/clang/Frontend/PCHReader.h +++ b/include/clang/Frontend/PCHReader.h @@ -20,6 +20,7 @@ #include "clang/AST/DeclObjC.h" #include "clang/AST/Type.h" #include "clang/AST/TemplateBase.h" +#include "clang/Lex/ExternalPreprocessorSource.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/SourceManager.h" @@ -146,7 +147,8 @@ public: /// required when traversing the AST. Only those AST nodes that are /// actually required will be de-serialized. class PCHReader - : public ExternalSemaSource, + : public ExternalPreprocessorSource, + public ExternalSemaSource, public IdentifierInfoLookup, public ExternalIdentifierLookup, public ExternalSLocEntrySource { @@ -183,6 +185,10 @@ private: llvm::BitstreamReader StreamFile; llvm::BitstreamCursor Stream; + /// \brief The cursor to the start of the preprocessor block, which stores + /// all of the macro definitions. + llvm::BitstreamCursor MacroCursor; + /// DeclsCursor - This is a cursor to the start of the DECLS_BLOCK block. It /// has read all the abbreviations at the start of the block and is ready to /// jump around with these in context. @@ -634,8 +640,7 @@ public: /// This routine builds a new IdentifierInfo for the given identifier. If any /// declarations with this name are visible from translation unit scope, their /// declarations will be deserialized and introduced into the declaration - /// chain of the identifier. FIXME: if this identifier names a macro, - /// deserialize the macro. + /// chain of the identifier. virtual IdentifierInfo* get(const char *NameStart, const char *NameEnd); IdentifierInfo* get(llvm::StringRef Name) { return get(Name.begin(), Name.end()); @@ -712,6 +717,9 @@ public: /// \brief Reads the macro record located at the given offset. void ReadMacroRecord(uint64_t Offset); + /// \brief Read the set of macros defined by this external macro source. + virtual void ReadDefinedMacros(); + /// \brief Retrieve the AST context that this PCH reader /// supplements. ASTContext *getContext() { return Context; } diff --git a/include/clang/Lex/ExternalPreprocessorSource.h b/include/clang/Lex/ExternalPreprocessorSource.h new file mode 100644 index 0000000000..e545b90e09 --- /dev/null +++ b/include/clang/Lex/ExternalPreprocessorSource.h @@ -0,0 +1,34 @@ +//===- ExternalPreprocessorSource.h - Abstract Macro Interface --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the ExternalPreprocessorSource interface, which enables +// construction of macro definitions from some external source. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_LEX_EXTERNAL_PREPROCESSOR_SOURCE_H +#define LLVM_CLANG_LEX_EXTERNAL_PREPROCESSOR_SOURCE_H + +namespace clang { + +/// \brief Abstract interface for external sources of preprocessor +/// information. +/// +/// This abstract class allows an external sources (such as the \c PCHReader) +/// to provide additional macro definitions. +class ExternalPreprocessorSource { +public: + virtual ~ExternalPreprocessorSource(); + + /// \brief Read the set of macros defined by this external macro source. + virtual void ReadDefinedMacros() = 0; +}; + +} + +#endif // LLVM_CLANG_LEX_EXTERNAL_PREPROCESSOR_SOURCE_H \ No newline at end of file diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h index 7c838ff862..a73d829f9e 100644 --- a/include/clang/Lex/Preprocessor.h +++ b/include/clang/Lex/Preprocessor.h @@ -32,6 +32,7 @@ namespace clang { class SourceManager; +class ExternalPreprocessorSource; class FileManager; class FileEntry; class HeaderSearch; @@ -42,7 +43,7 @@ class ScratchBuffer; class TargetInfo; class PPCallbacks; class DirectoryLookup; - + /// Preprocessor - This object engages in a tight little dance with the lexer to /// efficiently preprocess tokens. Lexers know only about tokens within a /// single source file, and don't know anything about preprocessor-level issues @@ -57,6 +58,9 @@ class Preprocessor { ScratchBuffer *ScratchBuf; HeaderSearch &HeaderInfo; + /// \brief External source of macros. + ExternalPreprocessorSource *ExternalSource; + /// PTH - An optional PTHManager object used for getting tokens from /// a token cache rather than lexing the original source file. llvm::OwningPtr PTH; @@ -99,6 +103,9 @@ class Preprocessor { /// DisableMacroExpansion - True if macro expansion is disabled. bool DisableMacroExpansion : 1; + /// \brief Whether we have already loaded macros from the external source. + mutable bool ReadMacrosFromExternalSource : 1; + /// Identifiers - This is mapping/lookup information for all identifiers in /// the program, including program keywords. mutable IdentifierTable Identifiers; @@ -247,6 +254,14 @@ public: PTHManager *getPTHManager() { return PTH.get(); } + void setExternalSource(ExternalPreprocessorSource *Source) { + ExternalSource = Source; + } + + ExternalPreprocessorSource *getExternalSource() const { + return ExternalSource; + } + /// SetCommentRetentionState - Control whether or not the preprocessor retains /// comments in output. void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) { @@ -296,11 +311,9 @@ public: /// state of the macro table. This visits every currently-defined macro. typedef llvm::DenseMap::const_iterator macro_iterator; - macro_iterator macro_begin() const { return Macros.begin(); } - macro_iterator macro_end() const { return Macros.end(); } - - - + macro_iterator macro_begin(bool IncludeExternalMacros = true) const; + macro_iterator macro_end(bool IncludeExternalMacros = true) const; + const std::string &getPredefines() const { return Predefines; } /// setPredefines - Set the predefines for this Preprocessor. These /// predefines are automatically injected when parsing the main file. diff --git a/lib/Frontend/PCHReader.cpp b/lib/Frontend/PCHReader.cpp index d8fd791b19..4c6e5f44a0 100644 --- a/lib/Frontend/PCHReader.cpp +++ b/lib/Frontend/PCHReader.cpp @@ -1079,6 +1079,61 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) { } } +void PCHReader::ReadDefinedMacros() { + // If there was no preprocessor block, do nothing. + if (!MacroCursor.getBitStreamReader()) + return; + + llvm::BitstreamCursor Cursor = MacroCursor; + if (Cursor.EnterSubBlock(pch::PREPROCESSOR_BLOCK_ID)) { + Error("malformed preprocessor block record in PCH file"); + return; + } + + RecordData Record; + while (true) { + unsigned Code = Cursor.ReadCode(); + if (Code == llvm::bitc::END_BLOCK) { + if (Cursor.ReadBlockEnd()) + Error("error at end of preprocessor block in PCH file"); + return; + } + + if (Code == llvm::bitc::ENTER_SUBBLOCK) { + // No known subblocks, always skip them. + Cursor.ReadSubBlockID(); + if (Cursor.SkipBlock()) { + Error("malformed block record in PCH file"); + return; + } + continue; + } + + if (Code == llvm::bitc::DEFINE_ABBREV) { + Cursor.ReadAbbrevRecord(); + continue; + } + + // Read a record. + const char *BlobStart; + unsigned BlobLen; + Record.clear(); + switch (Cursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) { + default: // Default behavior: ignore. + break; + + case pch::PP_MACRO_OBJECT_LIKE: + case pch::PP_MACRO_FUNCTION_LIKE: + DecodeIdentifierInfo(Record[0]); + break; + + case pch::PP_TOKEN: + // Ignore tokens. + break; + } + } +} + /// \brief If we are loading a relocatable PCH file, and the filename is /// not an absolute path, add the system root to the beginning of the file /// name. @@ -1140,6 +1195,10 @@ PCHReader::ReadPCHBlock() { break; case pch::PREPROCESSOR_BLOCK_ID: + MacroCursor = Stream; + if (PP) + PP->setExternalSource(this); + if (Stream.SkipBlock()) { Error("malformed block record in PCH file"); return Failure; @@ -1494,7 +1553,8 @@ void PCHReader::InitializeContext(ASTContext &Ctx) { assert(PP && "Forgot to set Preprocessor ?"); PP->getIdentifierTable().setExternalIdentifierLookup(this); PP->getHeaderSearchInfo().SetExternalLookup(this); - + PP->setExternalSource(this); + // Load the translation unit declaration ReadDeclRecord(DeclOffsets[0], 0); diff --git a/lib/Frontend/PCHWriter.cpp b/lib/Frontend/PCHWriter.cpp index 2875f0930c..39e5d6f46b 100644 --- a/lib/Frontend/PCHWriter.cpp +++ b/lib/Frontend/PCHWriter.cpp @@ -1148,7 +1148,6 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) { // Loop over all the macro definitions that are live at the end of the file, // emitting each to the PP section. - // FIXME: Make sure that this sees macros defined in included PCH files. for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end(); I != E; ++I) { // FIXME: This emits macros in hash table order, we should do it in a stable @@ -1160,7 +1159,6 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) { if (MI->isBuiltinMacro()) continue; - // FIXME: Remove this identifier reference? AddIdentifierRef(I->first, Record); MacroOffsets[I->first] = Stream.GetCurrentBitNo(); Record.push_back(MI->getDefinitionLoc().getRawEncoding()); diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp index ce1b19ca7c..0b26ccbecb 100644 --- a/lib/Lex/PPLexerChange.cpp +++ b/lib/Lex/PPLexerChange.cpp @@ -249,7 +249,8 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { // diagnostic is enabled, look for macros that have not been used. if (getDiagnostics().getDiagnosticLevel(diag::pp_macro_not_used) != Diagnostic::Ignored) { - for (macro_iterator I = macro_begin(), E = macro_end(); I != E; ++I) + for (macro_iterator I = macro_begin(false), E = macro_end(false); + I != E; ++I) if (!I->second->isUsed()) Diag(I->second->getDefinitionLoc(), diag::pp_macro_not_used); } diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 81966cb2b9..26bb3a90da 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -27,6 +27,7 @@ #include "clang/Lex/Preprocessor.h" #include "MacroArgs.h" +#include "clang/Lex/ExternalPreprocessorSource.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/MacroInfo.h" #include "clang/Lex/Pragma.h" @@ -43,6 +44,7 @@ using namespace clang; //===----------------------------------------------------------------------===// +ExternalPreprocessorSource::~ExternalPreprocessorSource() { } Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts, const TargetInfo &target, SourceManager &SM, @@ -50,9 +52,9 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts, IdentifierInfoLookup* IILookup, bool OwnsHeaders) : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()), - SourceMgr(SM), HeaderInfo(Headers), Identifiers(opts, IILookup), - BuiltinInfo(Target), CodeCompletionFile(0), CurPPLexer(0), CurDirLookup(0), - Callbacks(0), MacroArgCache(0) { + SourceMgr(SM), HeaderInfo(Headers), ExternalSource(0), + Identifiers(opts, IILookup), BuiltinInfo(Target), CodeCompletionFile(0), + CurPPLexer(0), CurDirLookup(0), Callbacks(0), MacroArgCache(0) { ScratchBuf = new ScratchBuffer(SourceMgr); CounterValue = 0; // __COUNTER__ starts at 0. OwnsHeaderSearch = OwnsHeaders; @@ -77,6 +79,9 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts, CachedLexPos = 0; + // We haven't read anything from the external source. + ReadMacrosFromExternalSource = false; + // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. // This gets unpoisoned where it is allowed. (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); @@ -194,6 +199,28 @@ void Preprocessor::PrintStats() { << NumFastTokenPaste << " on the fast path.\n"; } +Preprocessor::macro_iterator +Preprocessor::macro_begin(bool IncludeExternalMacros) const { + if (IncludeExternalMacros && ExternalSource && + !ReadMacrosFromExternalSource) { + ReadMacrosFromExternalSource = true; + ExternalSource->ReadDefinedMacros(); + } + + return Macros.begin(); +} + +Preprocessor::macro_iterator +Preprocessor::macro_end(bool IncludeExternalMacros) const { + if (IncludeExternalMacros && ExternalSource && + !ReadMacrosFromExternalSource) { + ReadMacrosFromExternalSource = true; + ExternalSource->ReadDefinedMacros(); + } + + return Macros.end(); +} + bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, unsigned TruncateAtLine, unsigned TruncateAtColumn) { diff --git a/test/CodeCompletion/Inputs/macros.h b/test/CodeCompletion/Inputs/macros.h new file mode 100644 index 0000000000..98b5ac6510 --- /dev/null +++ b/test/CodeCompletion/Inputs/macros.h @@ -0,0 +1,4 @@ +#define FOO +#define BAR(X, Y) X, Y +#define IDENTITY(X) X +#define WIBBLE(...) diff --git a/test/CodeCompletion/macros.c b/test/CodeCompletion/macros.c index 0ba2f065c4..f8861e912b 100644 --- a/test/CodeCompletion/macros.c +++ b/test/CodeCompletion/macros.c @@ -1,8 +1,3 @@ -#define FOO -#define BAR(X, Y) X, Y -#define IDENTITY(X) X -#define WIBBLE(...) - enum Color { Red, Green, Blue }; @@ -13,11 +8,17 @@ struct Point { }; void test(struct Point *p) { - // RUN: %clang_cc1 -fsyntax-only -code-completion-macros -code-completion-at=%s:17:14 %s -o - | FileCheck -check-prefix=CC1 %s + // RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:12:14 %s -o - | FileCheck -check-prefix=CC1 %s switch (p->IDENTITY(color)) { - // RUN: %clang_cc1 -fsyntax-only -code-completion-macros -code-completion-at=%s:19:9 %s -o - | FileCheck -check-prefix=CC2 %s + // RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s case } + + // Run the same tests, this time with macros loaded from the PCH file. + // RUN: %clang_cc1 -emit-pch -o %t %S/Inputs/macros.h + // RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:12:14 %s -o - | FileCheck -check-prefix=CC1 %s + // RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s + // CC1: color // CC1: x // CC1: y