From: Argyrios Kyrtzidis Date: Thu, 15 Sep 2011 18:02:56 +0000 (+0000) Subject: [PCH] Overhaul how preprocessed entities are [de]serialized. X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=e24692b30adbe8144597678a0e3354912e99c747;p=clang [PCH] Overhaul how preprocessed entities are [de]serialized. -Use an array of offsets for all preprocessed entities -Get rid of the separate array of offsets for just macro definitions; for references to macro definitions use an index inside the preprocessed entities array. -Deserialize each preprocessed entity lazily, at first request; not in bulk. Paves the way for binary searching of preprocessed entities that will offer efficiency and will simplify things on the libclang side a lot. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@139809 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/clang/Lex/PreprocessingRecord.h b/include/clang/Lex/PreprocessingRecord.h index e98a1bad22..0cc6cc5377 100644 --- a/include/clang/Lex/PreprocessingRecord.h +++ b/include/clang/Lex/PreprocessingRecord.h @@ -44,6 +44,9 @@ namespace clang { public: /// \brief The kind of preprocessed entity an object describes. enum EntityKind { + /// \brief Indicates a problem trying to load the preprocessed entity. + InvalidKind, + /// \brief A macro expansion. MacroExpansionKind, @@ -74,7 +77,9 @@ namespace clang { protected: PreprocessedEntity(EntityKind Kind, SourceRange Range) : Kind(Kind), Range(Range) { } - + + friend class PreprocessingRecord; + public: /// \brief Retrieve the kind of preprocessed entity stored in this object. EntityKind getKind() const { return Kind; } @@ -82,7 +87,11 @@ namespace clang { /// \brief Retrieve the source range that covers this entire preprocessed /// entity. SourceRange getSourceRange() const { return Range; } - + + /// \brief Returns true if there was a problem loading the preprocessed + /// entity. + bool isInvalid() const { return Kind == InvalidKind; } + // Implement isa/cast/dyncast/etc. static bool classof(const PreprocessedEntity *) { return true; } @@ -256,10 +265,12 @@ namespace clang { public: virtual ~ExternalPreprocessingRecordSource(); - /// \brief Read any preallocated preprocessed entities from the external - /// source. - virtual void ReadPreprocessedEntities() = 0; - + /// \brief Read a preallocated preprocessed entity from the external source. + /// + /// \returns null if an error occurred that prevented the preprocessed + /// entity from being loaded. + virtual PreprocessedEntity *ReadPreprocessedEntity(unsigned Index) = 0; + /// \brief Read the preprocessed entity at the given offset. virtual PreprocessedEntity * ReadPreprocessedEntityAtOffset(uint64_t Offset) = 0; @@ -286,17 +297,47 @@ namespace clang { /// The entries in this vector are loaded lazily from the external source, /// and are referenced by the iterator using negative indices. std::vector LoadedPreprocessedEntities; - + + /// \brief Global (loaded or local) ID for a preprocessed entity. + /// Negative values are used to indicate preprocessed entities + /// loaded from the external source while non-negative values are used to + /// indicate preprocessed entities introduced by the current preprocessor. + /// If M is the number of loaded preprocessed entities, value -M + /// corresponds to element 0 in the loaded entities vector, position -M+1 + /// corresponds to element 1 in the loaded entities vector, etc. + typedef int PPEntityID; + + PPEntityID getPPEntityID(unsigned Index, bool isLoaded) const { + return isLoaded ? PPEntityID(Index) - LoadedPreprocessedEntities.size() + : Index; + } + /// \brief Mapping from MacroInfo structures to their definitions. - llvm::DenseMap MacroDefinitions; + llvm::DenseMap MacroDefinitions; /// \brief External source of preprocessed entities. ExternalPreprocessingRecordSource *ExternalSource; + + /// \brief Retrieve the preprocessed entity at the given ID. + PreprocessedEntity *getPreprocessedEntity(PPEntityID PPID); + + /// \brief Retrieve the loaded preprocessed entity at the given index. + PreprocessedEntity *getLoadedPreprocessedEntity(unsigned Index); - /// \brief Whether we have already loaded all of the preallocated entities. - mutable bool LoadedPreallocatedEntities; + /// \brief Determine the number of preprocessed entities that were + /// loaded (or can be loaded) from an external source. + unsigned getNumLoadedPreprocessedEntities() const { + return LoadedPreprocessedEntities.size(); + } - void MaybeLoadPreallocatedEntities() const ; + /// \brief Allocate space for a new set of loaded preprocessed entities. + /// + /// \returns The index into the set of loaded preprocessed entities, which + /// corresponds to the first newly-allocated entity. + unsigned allocateLoadedEntities(unsigned NumEntities); + + /// \brief Register a new macro definition. + void RegisterMacroDefinition(MacroInfo *Macro, PPEntityID PPID); public: /// \brief Construct a new preprocessing record. @@ -328,7 +369,7 @@ namespace clang { /// corresponds to element 0 in the loaded entities vector, position -M+1 /// corresponds to element 1 in the loaded entities vector, etc. This /// gives us a reasonably efficient, source-order walk. - int Position; + PPEntityID Position; public: typedef PreprocessedEntity *value_type; @@ -342,20 +383,11 @@ namespace clang { iterator(PreprocessingRecord *Self, int Position) : Self(Self), Position(Position) { } - reference operator*() const { - if (Position < 0) - return Self->LoadedPreprocessedEntities.end()[Position]; - return Self->PreprocessedEntities[Position]; - } - - pointer operator->() const { - if (Position < 0) - return &Self->LoadedPreprocessedEntities.end()[Position]; - - return &Self->PreprocessedEntities[Position]; + value_type operator*() const { + return Self->getPreprocessedEntity(Position); } - reference operator[](difference_type D) { + value_type operator[](difference_type D) { return *(*this + D); } @@ -450,33 +482,6 @@ namespace clang { return ExternalSource; } - /// \brief Allocate space for a new set of loaded preprocessed entities. - /// - /// \returns The index into the set of loaded preprocessed entities, which - /// corresponds to the first newly-allocated entity. - unsigned allocateLoadedEntities(unsigned NumEntities); - - /// \brief Set the preallocated entry at the given index to the given - /// preprocessed entity, which was loaded from the external source. - void setLoadedPreallocatedEntity(unsigned Index, - PreprocessedEntity *Entity); - - /// \brief Register a new macro definition. - void RegisterMacroDefinition(MacroInfo *Macro, MacroDefinition *MD); - - /// \brief Retrieve the loaded preprocessed entity at the given index. - PreprocessedEntity *getLoadedPreprocessedEntity(unsigned Index) { - assert(Index < LoadedPreprocessedEntities.size() && - "Out-of bounds loaded preprocessed entity"); - return LoadedPreprocessedEntities[Index]; - } - - /// \brief Determine the number of preprocessed entities that were - /// loaded (or can be loaded) from an external source. - unsigned getNumLoadedPreprocessedEntities() const { - return LoadedPreprocessedEntities.size(); - } - /// \brief Retrieve the macro definition that corresponds to the given /// \c MacroInfo. MacroDefinition *findMacroDefinition(const MacroInfo *MI); @@ -493,6 +498,9 @@ namespace clang { SourceLocation EndLoc, StringRef SearchPath, StringRef RelativePath); + + friend class ASTReader; + friend class ASTWriter; }; } // end namespace clang diff --git a/include/clang/Serialization/ASTBitCodes.h b/include/clang/Serialization/ASTBitCodes.h index 0c3e72c081..f3dd8dc860 100644 --- a/include/clang/Serialization/ASTBitCodes.h +++ b/include/clang/Serialization/ASTBitCodes.h @@ -127,12 +127,6 @@ namespace clang { /// \brief The number of predefined identifier IDs. const unsigned int NUM_PREDEF_IDENT_IDS = 1; - /// \brief An ID number that refers to a macro in an AST file. - typedef uint32_t MacroID; - - /// \brief The number of predefined macro IDs. - const unsigned int NUM_PREDEF_MACRO_IDS = 1; - /// \brief An ID number that refers to an ObjC selctor in an AST file. typedef uint32_t SelectorID; @@ -312,9 +306,9 @@ namespace clang { /// \brief Record code for the array of unused file scoped decls. UNUSED_FILESCOPED_DECLS = 22, - /// \brief Record code for the table of offsets to macro definition - /// entries in the preprocessing record. - MACRO_DEFINITION_OFFSETS = 23, + /// \brief Record code for the table of offsets to entries in the + /// preprocessing record. + PPD_ENTITIES_OFFSETS = 23, /// \brief Record code for the array of VTable uses. VTABLE_USES = 24, diff --git a/include/clang/Serialization/ASTDeserializationListener.h b/include/clang/Serialization/ASTDeserializationListener.h index f8cdebe5a9..588fe0e63c 100644 --- a/include/clang/Serialization/ASTDeserializationListener.h +++ b/include/clang/Serialization/ASTDeserializationListener.h @@ -45,7 +45,7 @@ public: /// \brief A selector was read from the AST file. virtual void SelectorRead(serialization::SelectorID iD, Selector Sel) { } /// \brief A macro definition was read from the AST file. - virtual void MacroDefinitionRead(serialization::MacroID, + virtual void MacroDefinitionRead(serialization::PreprocessedEntityID, MacroDefinition *MD) { } }; diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h index 86bb62ae19..9f3973bc24 100644 --- a/include/clang/Serialization/ASTReader.h +++ b/include/clang/Serialization/ASTReader.h @@ -357,17 +357,6 @@ private: /// global selector ID to produce a local ID. GlobalSelectorMapType GlobalSelectorMap; - /// \brief The macro definitions we have already loaded. - SmallVector MacroDefinitionsLoaded; - - typedef ContinuousRangeMap - GlobalMacroDefinitionMapType; - - /// \brief Mapping from global macro definition IDs to the module in which the - /// selector resides along with the offset that should be added to the - /// global selector ID to produce a local ID. - GlobalMacroDefinitionMapType GlobalMacroDefinitionMap; - /// \brief Mapping from identifiers that represent macros whose definitions /// have not yet been deserialized to the global offset where the macro /// record resides. @@ -776,9 +765,12 @@ public: /// which contains a (typically-empty) subset of the predefines /// build prior to including the precompiled header. const std::string &getSuggestedPredefines() { return SuggestedPredefines; } - - /// \brief Read preprocessed entities into the preprocessing record. - virtual void ReadPreprocessedEntities(); + + /// \brief Read a preallocated preprocessed entity from the external source. + /// + /// \returns null if an error occurred that prevented the preprocessed + /// entity from being loaded. + virtual PreprocessedEntity *ReadPreprocessedEntity(unsigned Index); /// \brief Read the preprocessed entity at the given offset. virtual PreprocessedEntity *ReadPreprocessedEntityAtOffset(uint64_t Offset); @@ -819,16 +811,11 @@ public: unsigned Result = 0; for (ModuleConstIterator I = ModuleMgr.begin(), E = ModuleMgr.end(); I != E; ++I) { - Result += (*I)->NumPreallocatedPreprocessingEntities; + Result += (*I)->NumPreprocessedEntities; } return Result; } - - /// \brief Returns the number of macro definitions found in the chain. - unsigned getTotalNumMacroDefinitions() const { - return static_cast(MacroDefinitionsLoaded.size()); - } /// \brief Returns the number of C++ base specifiers found in the chain. unsigned getTotalNumCXXBaseSpecifiers() const { @@ -1245,20 +1232,6 @@ public: /// into the unread macro record offsets table. void LoadMacroDefinition( llvm::DenseMap::iterator Pos); - - /// \brief Retrieve the macro definition with the given ID. - MacroDefinition *getMacroDefinition(serialization::MacroID ID); - - /// \brief Retrieve the global macro definition ID that corresponds to the - /// local macro definition ID within a given module. - serialization::MacroID getGlobalMacroDefinitionID(Module &M, - unsigned LocalID); - - /// \brief Deserialize a macro definition that is local to the given - /// module. - MacroDefinition *getLocalMacroDefinition(Module &M, unsigned LocalID) { - return getMacroDefinition(getGlobalMacroDefinitionID(M, LocalID)); - } /// \brief Retrieve the AST context that this AST reader supplements. ASTContext &getContext() { return Context; } diff --git a/include/clang/Serialization/ASTWriter.h b/include/clang/Serialization/ASTWriter.h index d872ce260b..1ecab72afa 100644 --- a/include/clang/Serialization/ASTWriter.h +++ b/include/clang/Serialization/ASTWriter.h @@ -205,21 +205,11 @@ private: /// \brief The set of identifiers that had macro definitions at some point. std::vector DeserializedMacroNames; - - /// \brief The first ID number we can use for our own macro definitions. - serialization::MacroID FirstMacroID; - - /// \brief The decl ID that will be assigned to the next new macro definition. - serialization::MacroID NextMacroID; /// \brief Mapping from macro definitions (as they occur in the preprocessing /// record) to the macro IDs. - llvm::DenseMap + llvm::DenseMap MacroDefinitions; - - /// \brief Mapping from the macro definition indices in \c MacroDefinitions - /// to the corresponding offsets within the preprocessor block. - std::vector MacroDefinitionOffsets; typedef SmallVector UpdateRecord; typedef llvm::DenseMap DeclUpdateMap; @@ -461,10 +451,6 @@ public: "Identifier does not name a macro"); return MacroOffsets[II]; } - - /// \brief Retrieve the ID number corresponding to the given macro - /// definition. - serialization::MacroID getMacroDefinitionID(MacroDefinition *MD); /// \brief Emit a reference to a type. void AddTypeRef(QualType T, RecordDataImpl &Record); @@ -626,7 +612,8 @@ public: void TypeRead(serialization::TypeIdx Idx, QualType T); void DeclRead(serialization::DeclID ID, const Decl *D); void SelectorRead(serialization::SelectorID ID, Selector Sel); - void MacroDefinitionRead(serialization::MacroID ID, MacroDefinition *MD); + void MacroDefinitionRead(serialization::PreprocessedEntityID ID, + MacroDefinition *MD); // ASTMutationListener implementation. virtual void CompletedTagDefinition(const TagDecl *D); diff --git a/include/clang/Serialization/Module.h b/include/clang/Serialization/Module.h index c35a4f0096..9f063316c1 100644 --- a/include/clang/Serialization/Module.h +++ b/include/clang/Serialization/Module.h @@ -177,19 +177,8 @@ public: /// \brief Remapping table for preprocessed entity IDs in this module. ContinuousRangeMap PreprocessedEntityRemap; - /// \brief The number of macro definitions in this file. - unsigned LocalNumMacroDefinitions; - - /// \brief Offsets of all of the macro definitions in the preprocessing - /// record in the AST file. - const uint32_t *MacroDefinitionOffsets; - - /// \brief Base macro definition ID for macro definitions local to this - /// module. - serialization::MacroID BaseMacroDefinitionID; - - /// \brief Remapping table for macro definition IDs in this module. - ContinuousRangeMap MacroDefinitionRemap; + const uint32_t *PreprocessedEntityOffsets; + unsigned NumPreprocessedEntities; // === Header search information === @@ -309,10 +298,6 @@ public: /// The dynamic type of this stat cache is always ASTStatCache void *StatCache; - /// \brief The number of preallocated preprocessing entities in the - /// preprocessing record. - unsigned NumPreallocatedPreprocessingEntities; - /// \brief List of modules which depend on this module llvm::SetVector ImportedBy; diff --git a/lib/Frontend/MultiplexConsumer.cpp b/lib/Frontend/MultiplexConsumer.cpp index 5aa65d7a60..8e746f65a9 100644 --- a/lib/Frontend/MultiplexConsumer.cpp +++ b/lib/Frontend/MultiplexConsumer.cpp @@ -37,7 +37,7 @@ public: virtual void TypeRead(serialization::TypeIdx Idx, QualType T); virtual void DeclRead(serialization::DeclID ID, const Decl *D); virtual void SelectorRead(serialization::SelectorID iD, Selector Sel); - virtual void MacroDefinitionRead(serialization::MacroID, + virtual void MacroDefinitionRead(serialization::PreprocessedEntityID, MacroDefinition *MD); private: std::vector Listeners; @@ -79,7 +79,7 @@ void MultiplexASTDeserializationListener::SelectorRead( } void MultiplexASTDeserializationListener::MacroDefinitionRead( - serialization::MacroID ID, MacroDefinition *MD) { + serialization::PreprocessedEntityID ID, MacroDefinition *MD) { for (size_t i = 0, e = Listeners.size(); i != e; ++i) Listeners[i]->MacroDefinitionRead(ID, MD); } diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp index d02db25ae3..8d96fb00ca 100644 --- a/lib/Lex/PreprocessingRecord.cpp +++ b/lib/Lex/PreprocessingRecord.cpp @@ -37,17 +37,9 @@ InclusionDirective::InclusionDirective(PreprocessingRecord &PPRec, this->FileName = StringRef(Memory, FileName.size()); } -void PreprocessingRecord::MaybeLoadPreallocatedEntities() const { - if (!ExternalSource || LoadedPreallocatedEntities) - return; - - LoadedPreallocatedEntities = true; - ExternalSource->ReadPreprocessedEntities(); -} - PreprocessingRecord::PreprocessingRecord(bool IncludeNestedMacroExpansions) : IncludeNestedMacroExpansions(IncludeNestedMacroExpansions), - ExternalSource(0), LoadedPreallocatedEntities(false) + ExternalSource(0) { } @@ -56,14 +48,10 @@ PreprocessingRecord::begin(bool OnlyLocalEntities) { if (OnlyLocalEntities) return iterator(this, 0); - MaybeLoadPreallocatedEntities(); return iterator(this, -(int)LoadedPreprocessedEntities.size()); } PreprocessingRecord::iterator PreprocessingRecord::end(bool OnlyLocalEntities) { - if (!OnlyLocalEntities) - MaybeLoadPreallocatedEntities(); - return iterator(this, PreprocessedEntities.size()); } @@ -85,26 +73,49 @@ unsigned PreprocessingRecord::allocateLoadedEntities(unsigned NumEntities) { return Result; } -void -PreprocessingRecord::setLoadedPreallocatedEntity(unsigned Index, - PreprocessedEntity *Entity) { - assert(Index < LoadedPreprocessedEntities.size() && - "Out-of-bounds preallocated entity"); - LoadedPreprocessedEntities[Index] = Entity; +void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro, + PPEntityID PPID) { + MacroDefinitions[Macro] = PPID; } -void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro, - MacroDefinition *MD) { - MacroDefinitions[Macro] = MD; +/// \brief Retrieve the preprocessed entity at the given ID. +PreprocessedEntity *PreprocessingRecord::getPreprocessedEntity(PPEntityID PPID){ + if (PPID < 0) { + assert(unsigned(-PPID-1) < LoadedPreprocessedEntities.size() && + "Out-of bounds loaded preprocessed entity"); + return getLoadedPreprocessedEntity(LoadedPreprocessedEntities.size()+PPID); + } + assert(unsigned(PPID) < PreprocessedEntities.size() && + "Out-of bounds local preprocessed entity"); + return PreprocessedEntities[PPID]; +} + +/// \brief Retrieve the loaded preprocessed entity at the given index. +PreprocessedEntity * +PreprocessingRecord::getLoadedPreprocessedEntity(unsigned Index) { + assert(Index < LoadedPreprocessedEntities.size() && + "Out-of bounds loaded preprocessed entity"); + assert(ExternalSource && "No external source to load from"); + PreprocessedEntity *&Entity = LoadedPreprocessedEntities[Index]; + if (!Entity) { + Entity = ExternalSource->ReadPreprocessedEntity(Index); + if (!Entity) // Failed to load. + Entity = new (*this) + PreprocessedEntity(PreprocessedEntity::InvalidKind, SourceRange()); + } + return Entity; } MacroDefinition *PreprocessingRecord::findMacroDefinition(const MacroInfo *MI) { - llvm::DenseMap::iterator Pos + llvm::DenseMap::iterator Pos = MacroDefinitions.find(MI); if (Pos == MacroDefinitions.end()) return 0; - return Pos->second; + PreprocessedEntity *Entity = getPreprocessedEntity(Pos->second); + if (Entity->isInvalid()) + return 0; + return cast(Entity); } void PreprocessingRecord::MacroExpands(const Token &Id, const MacroInfo* MI, @@ -127,13 +138,14 @@ void PreprocessingRecord::MacroDefined(const Token &Id, = new (*this) MacroDefinition(Id.getIdentifierInfo(), MI->getDefinitionLoc(), R); - MacroDefinitions[MI] = Def; PreprocessedEntities.push_back(Def); + MacroDefinitions[MI] = getPPEntityID(PreprocessedEntities.size()-1, + /*isLoaded=*/false); } void PreprocessingRecord::MacroUndefined(const Token &Id, const MacroInfo *MI) { - llvm::DenseMap::iterator Pos + llvm::DenseMap::iterator Pos = MacroDefinitions.find(MI); if (Pos != MacroDefinitions.end()) MacroDefinitions.erase(Pos); diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp index 77f4fb84ed..30c44291ee 100644 --- a/lib/Serialization/ASTReader.cpp +++ b/lib/Serialization/ASTReader.cpp @@ -1310,10 +1310,14 @@ void ASTReader::ReadMacroRecord(Module &F, uint64_t Offset) { // form its body to it. Macro = MI; - if (NextIndex + 1 == Record.size() && PP.getPreprocessingRecord()) { - // We have a macro definition. Load it now. - PP.getPreprocessingRecord()->RegisterMacroDefinition(Macro, - getLocalMacroDefinition(F, Record[NextIndex])); + if (NextIndex + 1 == Record.size() && PP.getPreprocessingRecord() && + Record[NextIndex]) { + // We have a macro definition. Register the association + PreprocessedEntityID + GlobalID = getGlobalPreprocessedEntityID(F, Record[NextIndex]); + PreprocessingRecord &PPRec = *PP.getPreprocessingRecord(); + PPRec.RegisterMacroDefinition(Macro, + PPRec.getPPEntityID(GlobalID-1, /*isLoaded=*/true)); } ++NumMacrosRead; @@ -1375,61 +1379,43 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(Module &F) { Code, Record, BlobStart, BlobLen); switch (RecType) { case PPD_MACRO_EXPANSION: { - PreprocessedEntityID GlobalID = getGlobalPreprocessedEntityID(F, Record[0]); - if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(GlobalID-1)) - return PE; - bool isBuiltin = Record[3]; MacroExpansion *ME; - if (isBuiltin) + if (isBuiltin) { ME = new (PPRec) MacroExpansion(getLocalIdentifier(F, Record[4]), SourceRange(ReadSourceLocation(F, Record[1]), ReadSourceLocation(F, Record[2]))); - else - ME = new (PPRec) MacroExpansion(getLocalMacroDefinition(F, Record[4]), + } else { + PreprocessedEntityID + GlobalID = getGlobalPreprocessedEntityID(F, Record[4]); + ME = new (PPRec) MacroExpansion( + cast(PPRec.getLoadedPreprocessedEntity(GlobalID-1)), SourceRange(ReadSourceLocation(F, Record[1]), ReadSourceLocation(F, Record[2]))); - PPRec.setLoadedPreallocatedEntity(GlobalID - 1, ME); + } return ME; } case PPD_MACRO_DEFINITION: { PreprocessedEntityID GlobalID = getGlobalPreprocessedEntityID(F, Record[0]); - if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(GlobalID-1)) - return PE; - - unsigned MacroDefID = getGlobalMacroDefinitionID(F, Record[1]); - if (MacroDefID > MacroDefinitionsLoaded.size()) { - Error("out-of-bounds macro definition record"); - return 0; - } // Decode the identifier info and then check again; if the macro is // still defined and associated with the identifier, - IdentifierInfo *II = getLocalIdentifier(F, Record[4]); - if (!MacroDefinitionsLoaded[MacroDefID - 1]) { - MacroDefinition *MD - = new (PPRec) MacroDefinition(II, - ReadSourceLocation(F, Record[5]), - SourceRange( - ReadSourceLocation(F, Record[2]), - ReadSourceLocation(F, Record[3]))); - - PPRec.setLoadedPreallocatedEntity(GlobalID - 1, MD); - MacroDefinitionsLoaded[MacroDefID - 1] = MD; - - if (DeserializationListener) - DeserializationListener->MacroDefinitionRead(MacroDefID, MD); - } - - return MacroDefinitionsLoaded[MacroDefID - 1]; + IdentifierInfo *II = getLocalIdentifier(F, Record[3]); + MacroDefinition *MD + = new (PPRec) MacroDefinition(II, + ReadSourceLocation(F, Record[4]), + SourceRange( + ReadSourceLocation(F, Record[1]), + ReadSourceLocation(F, Record[2]))); + + if (DeserializationListener) + DeserializationListener->MacroDefinitionRead(GlobalID, MD); + + return MD; } case PPD_INCLUSION_DIRECTIVE: { - PreprocessedEntityID GlobalID = getGlobalPreprocessedEntityID(F, Record[0]); - if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(GlobalID-1)) - return PE; - const char *FullFileNameStart = BlobStart + Record[3]; const FileEntry *File = PP.getFileManager().getFile(StringRef(FullFileNameStart, @@ -1445,7 +1431,6 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(Module &F) { File, SourceRange(ReadSourceLocation(F, Record[1]), ReadSourceLocation(F, Record[2]))); - PPRec.setLoadedPreallocatedEntity(GlobalID - 1, ID); return ID; } } @@ -1606,24 +1591,6 @@ void ASTReader::LoadMacroDefinition(IdentifierInfo *II) { LoadMacroDefinition(Pos); } -MacroDefinition *ASTReader::getMacroDefinition(MacroID ID) { - if (ID == 0 || ID > MacroDefinitionsLoaded.size()) - return 0; - - if (!MacroDefinitionsLoaded[ID - 1]) { - GlobalMacroDefinitionMapType::iterator I =GlobalMacroDefinitionMap.find(ID); - assert(I != GlobalMacroDefinitionMap.end() && - "Corrupted global macro definition map"); - Module &M = *I->second; - unsigned Index = ID - 1 - M.BaseMacroDefinitionID; - SavedStreamPosition SavedPosition(M.PreprocessorDetailCursor); - M.PreprocessorDetailCursor.JumpToBit(M.MacroDefinitionOffsets[Index]); - LoadPreprocessedEntity(M); - } - - return MacroDefinitionsLoaded[ID - 1]; -} - const FileEntry *ASTReader::getFileEntry(StringRef filenameStrRef) { std::string Filename = filenameStrRef; MaybeAddSystemRootToFilename(Filename); @@ -1640,18 +1607,6 @@ const FileEntry *ASTReader::getFileEntry(StringRef filenameStrRef) { return File; } -MacroID ASTReader::getGlobalMacroDefinitionID(Module &M, unsigned LocalID) { - if (LocalID < NUM_PREDEF_MACRO_IDS) - return LocalID; - - ContinuousRangeMap::iterator I - = M.MacroDefinitionRemap.find(LocalID - NUM_PREDEF_MACRO_IDS); - assert(I != M.MacroDefinitionRemap.end() && - "Invalid index into macro definition ID remap"); - - return LocalID + I->second; -} - /// \brief If we are loading a relocatable PCH file, and the filename is /// not an absolute path, add the system root to the beginning of the file /// name. @@ -2088,8 +2043,6 @@ ASTReader::ReadASTBlock(Module &F) { IdentifierRemap(F.IdentifierRemap); ContinuousRangeMap::Builder PreprocessedEntityRemap(F.PreprocessedEntityRemap); - ContinuousRangeMap::Builder - MacroDefinitionRemap(F.MacroDefinitionRemap); ContinuousRangeMap::Builder SelectorRemap(F.SelectorRemap); ContinuousRangeMap::Builder DeclRemap(F.DeclRemap); @@ -2108,7 +2061,6 @@ ASTReader::ReadASTBlock(Module &F) { uint32_t SLocOffset = io::ReadUnalignedLE32(Data); uint32_t IdentifierIDOffset = io::ReadUnalignedLE32(Data); uint32_t PreprocessedEntityIDOffset = io::ReadUnalignedLE32(Data); - uint32_t MacroDefinitionIDOffset = io::ReadUnalignedLE32(Data); uint32_t SelectorIDOffset = io::ReadUnalignedLE32(Data); uint32_t DeclIDOffset = io::ReadUnalignedLE32(Data); uint32_t TypeIndexOffset = io::ReadUnalignedLE32(Data); @@ -2122,9 +2074,6 @@ ASTReader::ReadASTBlock(Module &F) { PreprocessedEntityRemap.insert( std::make_pair(PreprocessedEntityIDOffset, OM->BasePreprocessedEntityID - PreprocessedEntityIDOffset)); - MacroDefinitionRemap.insert( - std::make_pair(MacroDefinitionIDOffset, - OM->BaseMacroDefinitionID - MacroDefinitionIDOffset)); SelectorRemap.insert(std::make_pair(SelectorIDOffset, OM->BaseSelectorID - SelectorIDOffset)); DeclRemap.insert(std::make_pair(DeclIDOffset, @@ -2251,12 +2200,12 @@ ASTReader::ReadASTBlock(Module &F) { break; } - case MACRO_DEFINITION_OFFSETS: { - F.MacroDefinitionOffsets = (const uint32_t *)BlobStart; - F.NumPreallocatedPreprocessingEntities = Record[0]; - unsigned LocalBasePreprocessedEntityID = Record[1]; - F.LocalNumMacroDefinitions = Record[2]; - unsigned LocalBaseMacroID = Record[3]; + case PPD_ENTITIES_OFFSETS: { + F.PreprocessedEntityOffsets = (const uint32_t *)BlobStart; + assert(BlobLen % sizeof(uint32_t) == 0); + F.NumPreprocessedEntities = BlobLen / sizeof(uint32_t); + + unsigned LocalBasePreprocessedEntityID = Record[0]; unsigned StartingID; if (!PP.getPreprocessingRecord()) @@ -2265,11 +2214,10 @@ ASTReader::ReadASTBlock(Module &F) { PP.getPreprocessingRecord()->SetExternalSource(*this); StartingID = PP.getPreprocessingRecord() - ->allocateLoadedEntities(F.NumPreallocatedPreprocessingEntities); - F.BaseMacroDefinitionID = getTotalNumMacroDefinitions(); + ->allocateLoadedEntities(F.NumPreprocessedEntities); F.BasePreprocessedEntityID = StartingID; - if (F.NumPreallocatedPreprocessingEntities > 0) { + if (F.NumPreprocessedEntities > 0) { // Introduce the global -> local mapping for preprocessed entities in // this module. GlobalPreprocessedEntityMap.insert(std::make_pair(StartingID, &F)); @@ -2280,24 +2228,7 @@ ASTReader::ReadASTBlock(Module &F) { std::make_pair(LocalBasePreprocessedEntityID, F.BasePreprocessedEntityID - LocalBasePreprocessedEntityID)); } - - if (F.LocalNumMacroDefinitions > 0) { - // Introduce the global -> local mapping for macro definitions within - // this module. - GlobalMacroDefinitionMap.insert( - std::make_pair(getTotalNumMacroDefinitions() + 1, &F)); - - // Introduce the local -> global mapping for macro definitions within - // this module. - F.MacroDefinitionRemap.insert( - std::make_pair(LocalBaseMacroID, - F.BaseMacroDefinitionID - LocalBaseMacroID)); - - MacroDefinitionsLoaded.resize( - MacroDefinitionsLoaded.size() + F.LocalNumMacroDefinitions); - } - break; } @@ -2936,37 +2867,17 @@ bool ASTReader::ParseLanguageOptions( return false; } -namespace { - /// \brief Visitor used by ASTReader::ReadPreprocessedEntities() to load - /// all of the preprocessed entities within a module. - class ReadPreprocessedEntitiesVisitor { - ASTReader &Reader; - - public: - explicit ReadPreprocessedEntitiesVisitor(ASTReader &Reader) - : Reader(Reader) { } - - static bool visit(Module &M, bool Preorder, void *UserData) { - if (Preorder) - return false; - - ReadPreprocessedEntitiesVisitor *This - = static_cast(UserData); - - if (!M.PreprocessorDetailCursor.getBitStreamReader()) - return false; - - SavedStreamPosition SavedPosition(M.PreprocessorDetailCursor); - M.PreprocessorDetailCursor.JumpToBit(M.PreprocessorDetailStartOffset); - while (This->Reader.LoadPreprocessedEntity(M)) { } - return false; - } - }; -} +PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) { + GlobalPreprocessedEntityMapType::iterator + I = GlobalPreprocessedEntityMap.find(Index); + assert(I != GlobalPreprocessedEntityMap.end() && + "Corrupted global preprocessed entity map"); + Module &M = *I->second; + unsigned LocalIndex = Index - M.BasePreprocessedEntityID; -void ASTReader::ReadPreprocessedEntities() { - ReadPreprocessedEntitiesVisitor Visitor(*this); - ModuleMgr.visitDepthFirst(&ReadPreprocessedEntitiesVisitor::visit, &Visitor); + SavedStreamPosition SavedPosition(M.PreprocessorDetailCursor); + M.PreprocessorDetailCursor.JumpToBit(M.PreprocessedEntityOffsets[LocalIndex]); + return LoadPreprocessedEntity(M); } PreprocessedEntity *ASTReader::ReadPreprocessedEntityAtOffset(uint64_t Offset) { @@ -4280,7 +4191,6 @@ void ASTReader::dump() { dumpModuleIDMap("Global declaration map", GlobalDeclMap); dumpModuleIDMap("Global identifier map", GlobalIdentifierMap); dumpModuleIDMap("Global selector map", GlobalSelectorMap); - dumpModuleIDMap("Global macro definition map", GlobalMacroDefinitionMap); dumpModuleIDMap("Global preprocessed entity map", GlobalPreprocessedEntityMap); diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp index cb1a7d0389..32fb89a8d6 100644 --- a/lib/Serialization/ASTWriter.cpp +++ b/lib/Serialization/ASTWriter.cpp @@ -764,7 +764,7 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(STAT_CACHE); RECORD(EXT_VECTOR_DECLS); RECORD(VERSION_CONTROL_BRANCH_REVISION); - RECORD(MACRO_DEFINITION_OFFSETS); + RECORD(PPD_ENTITIES_OFFSETS); RECORD(IMPORTS); RECORD(REFERENCED_SELECTOR_POOL); RECORD(TU_UPDATE_LEXICAL); @@ -1586,6 +1586,10 @@ static int compareMacroDefinitions(const void *XPtr, const void *YPtr) { /// preprocessor. /// void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) { + PreprocessingRecord *PPRec = PP.getPreprocessingRecord(); + if (PPRec) + WritePreprocessorDetail(*PPRec); + RecordData Record; // If the preprocessor __COUNTER__ value has been bumped, remember it. @@ -1606,7 +1610,6 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) { // Loop over all the macro definitions that are live at the end of the file, // emitting each to the PP section. - PreprocessingRecord *PPRec = PP.getPreprocessingRecord(); // Construct the list of macro definitions that need to be serialized. SmallVector, 2> @@ -1676,7 +1679,7 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) { // If we have a detailed preprocessing record, record the macro definition // ID that corresponds to this macro. if (PPRec) - Record.push_back(getMacroDefinitionID(PPRec->findMacroDefinition(MI))); + Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]); Stream.EmitRecord(Code, Record); Record.clear(); @@ -1705,15 +1708,14 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) { ++NumMacros; } Stream.ExitBlock(); - - if (PPRec) - WritePreprocessorDetail(*PPRec); } void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) { if (PPRec.begin(Chain) == PPRec.end(Chain)) return; - + + SmallVector PreprocessedEntityOffsets; + // Enter the preprocessor block. Stream.EnterSubblock(PREPROCESSOR_DETAIL_BLOCK_ID, 3); @@ -1748,30 +1750,18 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) { (void)++E, ++NumPreprocessingRecords, ++NextPreprocessorEntityID) { Record.clear(); + PreprocessedEntityOffsets.push_back(Stream.GetCurrentBitNo()); + if (MacroDefinition *MD = dyn_cast(*E)) { - // Record this macro definition's location. - MacroID ID = getMacroDefinitionID(MD); - - // Don't write the macro definition if it is from another AST file. - if (ID < FirstMacroID) - continue; + // Record this macro definition's ID. + MacroDefinitions[MD] = NextPreprocessorEntityID; // Notify the serialization listener that we're serializing this entity. if (SerializationListener) SerializationListener->SerializedPreprocessedEntity(*E, BitsInChain + Stream.GetCurrentBitNo()); - - unsigned Position = ID - FirstMacroID; - if (Position != MacroDefinitionOffsets.size()) { - if (Position > MacroDefinitionOffsets.size()) - MacroDefinitionOffsets.resize(Position + 1); - - MacroDefinitionOffsets[Position] = Stream.GetCurrentBitNo(); - } else - MacroDefinitionOffsets.push_back(Stream.GetCurrentBitNo()); Record.push_back(NextPreprocessorEntityID); - Record.push_back(ID); AddSourceLocation(MD->getSourceRange().getBegin(), Record); AddSourceLocation(MD->getSourceRange().getEnd(), Record); AddIdentifierRef(MD->getName(), Record); @@ -1793,7 +1783,7 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) { if (ME->isBuiltinMacro()) AddIdentifierRef(ME->getName(), Record); else - Record.push_back(getMacroDefinitionID(ME->getDefinition())); + Record.push_back(MacroDefinitions[ME->getDefinition()]); Stream.EmitRecord(PPD_MACRO_EXPANSION, Record); continue; } @@ -1819,25 +1809,21 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) { // Write the offsets table for the preprocessing record. if (NumPreprocessingRecords > 0) { + assert(PreprocessedEntityOffsets.size() == NumPreprocessingRecords); + // Write the offsets table for identifier IDs. using namespace llvm; BitCodeAbbrev *Abbrev = new BitCodeAbbrev(); - Abbrev->Add(BitCodeAbbrevOp(MACRO_DEFINITION_OFFSETS)); - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of records + Abbrev->Add(BitCodeAbbrevOp(PPD_ENTITIES_OFFSETS)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first pp entity - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macro defs - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first macro def Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - unsigned MacroDefOffsetAbbrev = Stream.EmitAbbrev(Abbrev); + unsigned PPEOffsetAbbrev = Stream.EmitAbbrev(Abbrev); Record.clear(); - Record.push_back(MACRO_DEFINITION_OFFSETS); - Record.push_back(NumPreprocessingRecords); + Record.push_back(PPD_ENTITIES_OFFSETS); Record.push_back(FirstPreprocessorEntityID - NUM_PREDEF_PP_ENTITY_IDS); - Record.push_back(MacroDefinitionOffsets.size()); - Record.push_back(FirstMacroID - NUM_PREDEF_MACRO_IDS); - Stream.EmitRecordWithBlob(MacroDefOffsetAbbrev, Record, - data(MacroDefinitionOffsets)); + Stream.EmitRecordWithBlob(PPEOffsetAbbrev, Record, + data(PreprocessedEntityOffsets)); } } @@ -2737,7 +2723,6 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream) FirstTypeID(NUM_PREDEF_TYPE_IDS), NextTypeID(FirstTypeID), FirstIdentID(NUM_PREDEF_IDENT_IDS), NextIdentID(FirstIdentID), FirstSelectorID(NUM_PREDEF_SELECTOR_IDS), NextSelectorID(FirstSelectorID), - FirstMacroID(NUM_PREDEF_MACRO_IDS), NextMacroID(FirstMacroID), CollectedStmts(&StmtsToEmit), NumStatements(0), NumMacros(0), NumLexicalDeclContexts(0), NumVisibleDeclContexts(0), @@ -2950,7 +2935,6 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls, io::Emit32(Out, (*M)->SLocEntryBaseOffset); io::Emit32(Out, (*M)->BaseIdentifierID); io::Emit32(Out, (*M)->BasePreprocessedEntityID); - io::Emit32(Out, (*M)->BaseMacroDefinitionID); io::Emit32(Out, (*M)->BaseSelectorID); io::Emit32(Out, (*M)->BaseDeclID); io::Emit32(Out, (*M)->BaseTypeIndex); @@ -3231,16 +3215,6 @@ IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) { return ID; } -MacroID ASTWriter::getMacroDefinitionID(MacroDefinition *MD) { - if (MD == 0) - return 0; - - MacroID &ID = MacroDefinitions[MD]; - if (ID == 0) - ID = NextMacroID++; - return ID; -} - void ASTWriter::AddSelectorRef(const Selector SelRef, RecordDataImpl &Record) { Record.push_back(getSelectorRef(SelRef)); } @@ -3853,7 +3827,6 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { FirstTypeID == NextTypeID && FirstIdentID == NextIdentID && FirstSelectorID == NextSelectorID && - FirstMacroID == NextMacroID && "Setting chain after writing has started."); Chain = Reader; @@ -3862,12 +3835,10 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes(); FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers(); FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors(); - FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacroDefinitions(); NextDeclID = FirstDeclID; NextTypeID = FirstTypeID; NextIdentID = FirstIdentID; NextSelectorID = FirstSelectorID; - NextMacroID = FirstMacroID; } void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) { @@ -3895,8 +3866,9 @@ void ASTWriter::SelectorRead(SelectorID ID, Selector S) { SelectorIDs[S] = ID; } -void ASTWriter::MacroDefinitionRead(serialization::MacroID ID, +void ASTWriter::MacroDefinitionRead(serialization::PreprocessedEntityID ID, MacroDefinition *MD) { + assert(MacroDefinitions.find(MD) == MacroDefinitions.end()); MacroDefinitions[MD] = ID; } diff --git a/lib/Serialization/Module.cpp b/lib/Serialization/Module.cpp index b5a96bb98b..0a721c4365 100644 --- a/lib/Serialization/Module.cpp +++ b/lib/Serialization/Module.cpp @@ -27,16 +27,15 @@ Module::Module(ModuleKind Kind) SLocFileOffsets(0), LocalNumIdentifiers(0), IdentifierOffsets(0), BaseIdentifierID(0), IdentifierTableData(0), IdentifierLookupTable(0), BasePreprocessedEntityID(0), - LocalNumMacroDefinitions(0), MacroDefinitionOffsets(0), - BaseMacroDefinitionID(0), LocalNumHeaderFileInfos(0), + PreprocessedEntityOffsets(0), NumPreprocessedEntities(0), + LocalNumHeaderFileInfos(0), HeaderFileInfoTableData(0), HeaderFileInfoTable(0), HeaderFileFrameworkStrings(0), LocalNumSelectors(0), SelectorOffsets(0), BaseSelectorID(0), SelectorLookupTableData(0), SelectorLookupTable(0), LocalNumDecls(0), DeclOffsets(0), BaseDeclID(0), LocalNumCXXBaseSpecifiers(0), CXXBaseSpecifiersOffsets(0), - LocalNumTypes(0), TypeOffsets(0), BaseTypeIndex(0), StatCache(0), - NumPreallocatedPreprocessingEntities(0) + LocalNumTypes(0), TypeOffsets(0), BaseTypeIndex(0), StatCache(0) {} Module::~Module() { @@ -96,17 +95,10 @@ void Module::dump() { llvm::errs() << " Base preprocessed entity ID: " << BasePreprocessedEntityID << '\n' << " Number of preprocessed entities: " - << NumPreallocatedPreprocessingEntities << '\n'; + << NumPreprocessedEntities << '\n'; dumpLocalRemap("Preprocessed entity ID local -> global map", PreprocessedEntityRemap); - llvm::errs() << " Base macro definition ID: " << BaseMacroDefinitionID - << '\n' - << " Number of macro definitions: " << LocalNumMacroDefinitions - << '\n'; - dumpLocalRemap("Macro definition ID local -> global map", - MacroDefinitionRemap); - llvm::errs() << " Base type index: " << BaseTypeIndex << '\n' << " Number of types: " << LocalNumTypes << '\n'; dumpLocalRemap("Type index local -> global map", TypeRemap); diff --git a/test/Index/getcursor-pp-pch.c b/test/Index/getcursor-pp-pch.c new file mode 100644 index 0000000000..dc597ffa94 --- /dev/null +++ b/test/Index/getcursor-pp-pch.c @@ -0,0 +1,34 @@ + + + +typedef int T; +void OBSCURE(func)(int x) { + OBSCURE(T) DECORATION value; +} + + +// Without PCH +// RUN: c-index-test -cursor-at=%s.h:1:11 \ +// RUN: -cursor-at=%s.h:2:14 \ +// RUN: -cursor-at=%s.h:4:5 \ +// RUN: -cursor-at=%s:5:7 \ +// RUN: -cursor-at=%s:6:6 \ +// RUN: -cursor-at=%s:6:19 \ +// RUN: -include %s.h %s | FileCheck %s + +// With PCH +// RUN: c-index-test -write-pch %t.h.pch %s.h -Xclang -detailed-preprocessing-record +// RUN: c-index-test -cursor-at=%s.h:1:11 \ +// RUN: -cursor-at=%s.h:2:14 \ +// RUN: -cursor-at=%s.h:4:5 \ +// RUN: -cursor-at=%s:5:7 \ +// RUN: -cursor-at=%s:6:6 \ +// RUN: -cursor-at=%s:6:19 \ +// RUN: -include %t.h %s | FileCheck %s + +// CHECK: macro definition=OBSCURE +// CHECK: macro definition=DECORATION +// CHECK: macro expansion=DECORATION:2:9 +// CHECK: macro expansion=OBSCURE:1:9 +// CHECK: macro expansion=OBSCURE:1:9 +// CHECK: macro expansion=DECORATION:2:9 diff --git a/test/Index/getcursor-pp-pch.c.h b/test/Index/getcursor-pp-pch.c.h new file mode 100644 index 0000000000..184dd8b410 --- /dev/null +++ b/test/Index/getcursor-pp-pch.c.h @@ -0,0 +1,4 @@ +#define OBSCURE(X) X +#define DECORATION + +DECORATION