From 4d7e0ced7f16a04aabe2d8d91cbbb52fb1162810 Mon Sep 17 00:00:00 2001 From: Alexander Kornienko Date: Tue, 25 Sep 2012 17:18:14 +0000 Subject: [PATCH] Macro history (de-)serialization. Deserialization currently reads only the latest macro definition. Needs more work. Summary: Passes all tests (+ the new one with code completion), but needs a thorough review in part related to modules. Reviewers: doug.gregor Reviewed By: alexfh CC: cfe-commits, rsmith Differential Revision: http://llvm-reviews.chandlerc.com/D41 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@164610 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Basic/IdentifierTable.h | 17 ++- include/clang/Lex/Preprocessor.h | 12 +- lib/Basic/IdentifierTable.cpp | 1 + lib/Lex/PPMacroExpansion.cpp | 9 +- lib/Serialization/ASTReader.cpp | 33 +++-- lib/Serialization/ASTWriter.cpp | 192 ++++++++++++++------------ test/CodeCompletion/Inputs/macros.h | 7 + test/CodeCompletion/macros.c | 12 ++ 8 files changed, 169 insertions(+), 114 deletions(-) diff --git a/include/clang/Basic/IdentifierTable.h b/include/clang/Basic/IdentifierTable.h index e0f4510f26..b3b1842aad 100644 --- a/include/clang/Basic/IdentifierTable.h +++ b/include/clang/Basic/IdentifierTable.h @@ -54,6 +54,7 @@ class IdentifierInfo { // are for builtins. unsigned ObjCOrBuiltinID :11; bool HasMacro : 1; // True if there is a #define for this. + bool HadMacro : 1; // True if there was a #define for this. bool IsExtension : 1; // True if identifier is a lang extension. bool IsCXX11CompatKeyword : 1; // True if identifier is a keyword in C++11. bool IsPoisoned : 1; // True if identifier is poisoned. @@ -70,8 +71,8 @@ class IdentifierInfo { // stored externally. bool IsModulesImport : 1; // True if this is the 'import' contextual // keyword. - // 1 bit left in 32-bit word. - + // 32-bit word is filled. + void *FETokenInfo; // Managed by the language front-end. llvm::StringMapEntry *Entry; @@ -133,10 +134,18 @@ public: if (HasMacro == Val) return; HasMacro = Val; - if (Val) + if (Val) { NeedsHandleIdentifier = 1; - else + HadMacro = true; + } else { RecomputeNeedsHandleIdentifier(); + } + } + /// \brief Returns true if this identifier was \#defined to some value at any + /// moment. In this case there should be an entry for the identifier in the + /// macro history table in Preprocessor. + bool hadMacroDefinition() const { + return HadMacro; } /// getTokenID - If this is a source-language token (e.g. 'for'), this API diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h index 76d8fdb259..c957906b03 100644 --- a/include/clang/Lex/Preprocessor.h +++ b/include/clang/Lex/Preprocessor.h @@ -366,8 +366,6 @@ private: // Cached tokens state. /// allocation. MacroInfoChain *MICache; - MacroInfo *getInfoForMacro(IdentifierInfo *II) const; - public: Preprocessor(DiagnosticsEngine &diags, LangOptions &opts, const TargetInfo *target, @@ -467,9 +465,17 @@ public: if (!II->hasMacroDefinition()) return 0; - return getInfoForMacro(II); + MacroInfo *MI = getMacroInfoHistory(II); + assert(MI->getUndefLoc().isInvalid() && "Macro is undefined!"); + return MI; } + /// \brief Given an identifier, return the (probably #undef'd) MacroInfo + /// representing the most recent macro definition. One can iterate over all + /// previous macro definitions from it. This method should only be called for + /// identifiers that hadMacroDefinition(). + MacroInfo *getMacroInfoHistory(IdentifierInfo *II) const; + /// \brief Specify a macro for this identifier. void setMacroInfo(IdentifierInfo *II, MacroInfo *MI, bool LoadedFromAST = false); diff --git a/lib/Basic/IdentifierTable.cpp b/lib/Basic/IdentifierTable.cpp index 082c768dfc..55c45cfb23 100644 --- a/lib/Basic/IdentifierTable.cpp +++ b/lib/Basic/IdentifierTable.cpp @@ -33,6 +33,7 @@ IdentifierInfo::IdentifierInfo() { TokenID = tok::identifier; ObjCOrBuiltinID = 0; HasMacro = false; + HadMacro = false; IsExtension = false; IsCXX11CompatKeyword = false; IsPoisoned = false; diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index b2d597713b..dcaa5a6636 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -32,8 +32,8 @@ #include using namespace clang; -MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const { - assert(II->hasMacroDefinition() && "Identifier is not a macro!"); +MacroInfo *Preprocessor::getMacroInfoHistory(IdentifierInfo *II) const { + assert(II->hadMacroDefinition() && "Identifier has not been not a macro!"); macro_iterator Pos = Macros.find(II); if (Pos == Macros.end()) { @@ -42,7 +42,6 @@ MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const { Pos = Macros.find(II); } assert(Pos != Macros.end() && "Identifier macro info is missing!"); - assert(Pos->second->getUndefLoc().isInvalid() && "Macro is undefined!"); return Pos->second; } @@ -51,9 +50,11 @@ MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const { void Preprocessor::setMacroInfo(IdentifierInfo *II, MacroInfo *MI, bool LoadedFromAST) { assert(MI && "MacroInfo should be non-zero!"); + assert((LoadedFromAST || MI->getUndefLoc().isInvalid()) && + "Undefined macros can only be registered when just LoadedFromAST"); MI->setPreviousDefinition(Macros[II]); Macros[II] = MI; - II->setHasMacroDefinition(true); + II->setHasMacroDefinition(MI->getUndefLoc().isInvalid()); if (II->isFromAST() && !LoadedFromAST) II->setChangedSinceDeserialization(); } diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp index 6de0c65ba3..c57e933668 100644 --- a/lib/Serialization/ASTReader.cpp +++ b/lib/Serialization/ASTReader.cpp @@ -527,6 +527,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k, return II; } + unsigned ObjCOrBuiltinID = ReadUnalignedLE16(d); unsigned Bits = ReadUnalignedLE16(d); bool CPlusPlusOperatorKeyword = Bits & 0x01; Bits >>= 1; @@ -536,13 +537,13 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k, Bits >>= 1; bool ExtensionToken = Bits & 0x01; Bits >>= 1; + bool hadMacroDefinition = Bits & 0x01; + Bits >>= 1; bool hasMacroDefinition = Bits & 0x01; Bits >>= 1; - unsigned ObjCOrBuiltinID = Bits & 0x7FF; - Bits >>= 11; assert(Bits == 0 && "Extra bits in the identifier?"); - DataLen -= 6; + DataLen -= 8; // Build the IdentifierInfo itself and link the identifier ID with // the new IdentifierInfo. @@ -570,7 +571,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k, // If this identifier is a macro, deserialize the macro // definition. - if (hasMacroDefinition) { + if (hadMacroDefinition) { // FIXME: Check for conflicts? uint32_t Offset = ReadUnalignedLE32(d); unsigned LocalSubmoduleID = ReadUnalignedLE32(d); @@ -590,10 +591,10 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k, // module is not yet visible. Reader.HiddenNamesMap[Owner].push_back(II); } - } + } } - - Reader.setIdentifierIsMacro(II, F, Offset, Visible); + + Reader.setIdentifierIsMacro(II, F, Offset, Visible && hasMacroDefinition); DataLen -= 8; } @@ -1312,18 +1313,21 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) { Error("macro must have a name in AST file"); return; } - - SourceLocation Loc = ReadSourceLocation(F, Record[1]); - bool isUsed = Record[2]; + unsigned NextIndex = 1; + SourceLocation Loc = ReadSourceLocation(F, Record, NextIndex); MacroInfo *MI = PP.AllocateMacroInfo(Loc); - MI->setIsUsed(isUsed); + + SourceLocation UndefLoc = ReadSourceLocation(F, Record, NextIndex); + if (UndefLoc.isValid()) + MI->setUndefLoc(UndefLoc); + + MI->setIsUsed(Record[NextIndex++]); MI->setIsFromAST(); - bool IsPublic = Record[3]; - unsigned NextIndex = 4; + bool IsPublic = Record[NextIndex++]; MI->setVisibility(IsPublic, ReadSourceLocation(F, Record, NextIndex)); - + if (RecType == PP_MACRO_FUNCTION_LIKE) { // Decode function-like macro info. bool isC99VarArgs = Record[NextIndex++]; @@ -2551,6 +2555,7 @@ void ASTReader::makeNamesVisible(const HiddenNames &Names) { D->Hidden = false; else { IdentifierInfo *II = Names[I].get(); + // FIXME: Check if this works correctly with macro history. if (!II->hasMacroDefinition()) { // Make sure that this macro hasn't been #undef'd in the mean-time. llvm::DenseMap::iterator Known diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp index 59d2dec853..5ec5936f64 100644 --- a/lib/Serialization/ASTWriter.cpp +++ b/lib/Serialization/ASTWriter.cpp @@ -1674,102 +1674,112 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) { SmallVector, 2> MacrosToEmit; llvm::SmallPtrSet MacroDefinitionsSeen; - for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0), + for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0), E = PP.macro_end(Chain == 0); I != E; ++I) { - // FIXME: We'll need to store macro history in PCH. - if (I->first->hasMacroDefinition()) { - if (!IsModule || I->second->isPublic()) { - MacroDefinitionsSeen.insert(I->first); - MacrosToEmit.push_back(std::make_pair(I->first, I->second)); - } + if (!IsModule || I->second->isPublic()) { + MacroDefinitionsSeen.insert(I->first); + MacrosToEmit.push_back(std::make_pair(I->first, I->second)); } } - + // Sort the set of macro definitions that need to be serialized by the // name of the macro, to provide a stable ordering. - llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(), + llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(), &compareMacroDefinitions); - + // Resolve any identifiers that defined macros at the time they were // deserialized, adding them to the list of macros to emit (if appropriate). for (unsigned I = 0, N = DeserializedMacroNames.size(); I != N; ++I) { IdentifierInfo *Name = const_cast(DeserializedMacroNames[I]); - if (Name->hasMacroDefinition() && MacroDefinitionsSeen.insert(Name)) + if (Name->hadMacroDefinition() && MacroDefinitionsSeen.insert(Name)) MacrosToEmit.push_back(std::make_pair(Name, PP.getMacroInfo(Name))); } - + for (unsigned I = 0, N = MacrosToEmit.size(); I != N; ++I) { const IdentifierInfo *Name = MacrosToEmit[I].first; MacroInfo *MI = MacrosToEmit[I].second; if (!MI) continue; - - // Don't emit builtin macros like __LINE__ to the AST file unless they have - // been redefined by the header (in which case they are not isBuiltinMacro). - // Also skip macros from a AST file if we're chaining. - - // FIXME: There is a (probably minor) optimization we could do here, if - // the macro comes from the original PCH but the identifier comes from a - // chained PCH, by storing the offset into the original PCH rather than - // writing the macro definition a second time. - if (MI->isBuiltinMacro() || - (Chain && - Name->isFromAST() && !Name->hasChangedSinceDeserialization() && - MI->isFromAST() && !MI->hasChangedAfterLoad())) - continue; - AddIdentifierRef(Name, Record); - MacroOffsets[Name] = Stream.GetCurrentBitNo(); - Record.push_back(MI->getDefinitionLoc().getRawEncoding()); - Record.push_back(MI->isUsed()); - Record.push_back(MI->isPublic()); - AddSourceLocation(MI->getVisibilityLocation(), Record); - unsigned Code; - if (MI->isObjectLike()) { - Code = PP_MACRO_OBJECT_LIKE; - } else { - Code = PP_MACRO_FUNCTION_LIKE; - - Record.push_back(MI->isC99Varargs()); - Record.push_back(MI->isGNUVarargs()); - Record.push_back(MI->getNumArgs()); - for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end(); - I != E; ++I) - AddIdentifierRef(*I, Record); + // History of macro definitions for this identifier in chronological order. + SmallVector MacroHistory; + while (MI) { + MacroHistory.push_back(MI); + MI = MI->getPreviousDefinition(); } - // If we have a detailed preprocessing record, record the macro definition - // ID that corresponds to this macro. - if (PPRec) - Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]); + while (!MacroHistory.empty()) { + MI = MacroHistory.pop_back_val(); + + // Don't emit builtin macros like __LINE__ to the AST file unless they + // have been redefined by the header (in which case they are not + // isBuiltinMacro). + // Also skip macros from a AST file if we're chaining. + + // FIXME: There is a (probably minor) optimization we could do here, if + // the macro comes from the original PCH but the identifier comes from a + // chained PCH, by storing the offset into the original PCH rather than + // writing the macro definition a second time. + if (MI->isBuiltinMacro() || + (Chain && + Name->isFromAST() && !Name->hasChangedSinceDeserialization() && + MI->isFromAST() && !MI->hasChangedAfterLoad())) + continue; - Stream.EmitRecord(Code, Record); - Record.clear(); + AddIdentifierRef(Name, Record); + MacroOffsets[Name] = Stream.GetCurrentBitNo(); + AddSourceLocation(MI->getDefinitionLoc(), Record); + AddSourceLocation(MI->getUndefLoc(), Record); + Record.push_back(MI->isUsed()); + Record.push_back(MI->isPublic()); + AddSourceLocation(MI->getVisibilityLocation(), Record); + unsigned Code; + if (MI->isObjectLike()) { + Code = PP_MACRO_OBJECT_LIKE; + } else { + Code = PP_MACRO_FUNCTION_LIKE; + + Record.push_back(MI->isC99Varargs()); + Record.push_back(MI->isGNUVarargs()); + Record.push_back(MI->getNumArgs()); + for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end(); + I != E; ++I) + AddIdentifierRef(*I, Record); + } - // Emit the tokens array. - for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) { - // Note that we know that the preprocessor does not have any annotation - // tokens in it because they are created by the parser, and thus can't be - // in a macro definition. - const Token &Tok = MI->getReplacementToken(TokNo); - - Record.push_back(Tok.getLocation().getRawEncoding()); - Record.push_back(Tok.getLength()); - - // FIXME: When reading literal tokens, reconstruct the literal pointer if - // it is needed. - AddIdentifierRef(Tok.getIdentifierInfo(), Record); - // FIXME: Should translate token kind to a stable encoding. - Record.push_back(Tok.getKind()); - // FIXME: Should translate token flags to a stable encoding. - Record.push_back(Tok.getFlags()); - - Stream.EmitRecord(PP_TOKEN, Record); + // If we have a detailed preprocessing record, record the macro definition + // ID that corresponds to this macro. + if (PPRec) + Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]); + + Stream.EmitRecord(Code, Record); Record.clear(); + + // Emit the tokens array. + for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) { + // Note that we know that the preprocessor does not have any annotation + // tokens in it because they are created by the parser, and thus can't + // be in a macro definition. + const Token &Tok = MI->getReplacementToken(TokNo); + + Record.push_back(Tok.getLocation().getRawEncoding()); + Record.push_back(Tok.getLength()); + + // FIXME: When reading literal tokens, reconstruct the literal pointer + // if it is needed. + AddIdentifierRef(Tok.getIdentifierInfo(), Record); + // FIXME: Should translate token kind to a stable encoding. + Record.push_back(Tok.getKind()); + // FIXME: Should translate token flags to a stable encoding. + Record.push_back(Tok.getFlags()); + + Stream.EmitRecord(PP_TOKEN, Record); + Record.clear(); + } + ++NumMacros; } - ++NumMacros; } Stream.ExitBlock(); } @@ -2496,17 +2506,17 @@ class ASTIdentifierTableTrait { II->getFETokenInfo()) return true; - return hasMacroDefinition(II, Macro); + return hadMacroDefinition(II, Macro); } - - bool hasMacroDefinition(IdentifierInfo *II, MacroInfo *&Macro) { - if (!II->hasMacroDefinition()) + + bool hadMacroDefinition(IdentifierInfo *II, MacroInfo *&Macro) { + if (!II->hadMacroDefinition()) return false; - - if (Macro || (Macro = PP.getMacroInfo(II))) + + if (Macro || (Macro = PP.getMacroInfoHistory(II))) return !Macro->isBuiltinMacro() && (!IsModule || Macro->isPublic()); - - return false; + + return false; } public: @@ -2530,10 +2540,11 @@ public: unsigned DataLen = 4; // 4 bytes for the persistent ID << 1 MacroInfo *Macro = 0; if (isInterestingIdentifier(II, Macro)) { - DataLen += 2; // 2 bytes for builtin ID, flags - if (hasMacroDefinition(II, Macro)) + DataLen += 2; // 2 bytes for builtin ID + DataLen += 2; // 2 bytes for flags + if (hadMacroDefinition(II, Macro)) DataLen += 8; - + for (IdentifierResolver::iterator D = IdResolver.begin(II), DEnd = IdResolver.end(); D != DEnd; ++D) @@ -2564,23 +2575,26 @@ public: } clang::io::Emit32(Out, (ID << 1) | 0x01); - uint32_t Bits = 0; - bool HasMacroDefinition = hasMacroDefinition(II, Macro); - Bits = (uint32_t)II->getObjCOrBuiltinID(); - assert((Bits & 0x7ff) == Bits && "ObjCOrBuiltinID too big for ASTReader."); + uint32_t Bits = (uint32_t)II->getObjCOrBuiltinID(); + assert((Bits & 0xffff) == Bits && "ObjCOrBuiltinID too big for ASTReader."); + clang::io::Emit16(Out, Bits); + Bits = 0; + bool HadMacroDefinition = hadMacroDefinition(II, Macro); + bool HasMacroDefinition = HadMacroDefinition && II->hasMacroDefinition(); Bits = (Bits << 1) | unsigned(HasMacroDefinition); + Bits = (Bits << 1) | unsigned(HadMacroDefinition); Bits = (Bits << 1) | unsigned(II->isExtensionToken()); Bits = (Bits << 1) | unsigned(II->isPoisoned()); Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier()); Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword()); clang::io::Emit16(Out, Bits); - if (HasMacroDefinition) { + if (HadMacroDefinition) { clang::io::Emit32(Out, Writer.getMacroOffset(II)); - clang::io::Emit32(Out, + clang::io::Emit32(Out, Writer.inferSubmoduleIDFromLocation(Macro->getDefinitionLoc())); } - + // Emit the declaration IDs in reverse order, because the // IdentifierResolver provides the declarations as they would be // visible (e.g., the function "stat" would come before the struct @@ -4447,7 +4461,7 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) { IdentifierIDs[II] = ID; - if (II->hasMacroDefinition()) + if (II->hadMacroDefinition()) DeserializedMacroNames.push_back(II); } diff --git a/test/CodeCompletion/Inputs/macros.h b/test/CodeCompletion/Inputs/macros.h index 98b5ac6510..5f15dfc2be 100644 --- a/test/CodeCompletion/Inputs/macros.h +++ b/test/CodeCompletion/Inputs/macros.h @@ -2,3 +2,10 @@ #define BAR(X, Y) X, Y #define IDENTITY(X) X #define WIBBLE(...) +#define DEAD_MACRO +#undef DEAD_MACRO +#define MACRO_WITH_HISTORY a +#undef MACRO_WITH_HISTORY +#define MACRO_WITH_HISTORY b, c +#undef MACRO_WITH_HISTORY +#define MACRO_WITH_HISTORY(X, Y) X->Y diff --git a/test/CodeCompletion/macros.c b/test/CodeCompletion/macros.c index 0758bbf768..28f69b2e1b 100644 --- a/test/CodeCompletion/macros.c +++ b/test/CodeCompletion/macros.c @@ -13,11 +13,15 @@ void test(struct Point *p) { // RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s case } + // RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:17:7 %s -o - | FileCheck -check-prefix=CC3 %s +#ifdef Q +#endif // Run the same tests, this time with macros loaded from the PCH file. // RUN: %clang_cc1 -emit-pch -o %t %S/Inputs/macros.h // RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:12:14 %s -o - | FileCheck -check-prefix=CC1 %s // RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s + // RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:17:7 %s -o - | FileCheck -check-prefix=CC3 %s // CC1: color // CC1: x @@ -29,6 +33,14 @@ void test(struct Point *p) { // CC2: FOO // CC2: Green // CC2: IDENTITY(<#X#>) + // CC2: MACRO_WITH_HISTORY(<#X#>, <#Y#>) // CC2: Red // CC2: WIBBLE + + // CC3: BAR + // CC3: DEAD_MACRO + // CC3: FOO + // CC3: IDENTITY + // CC3: MACRO_WITH_HISTORY + // CC3: WIBBLE } -- 2.40.0