From f62d43d2afe1960755a1b5813cae1e5983bcac1b Mon Sep 17 00:00:00 2001 From: Douglas Gregor Date: Tue, 19 Jul 2011 16:10:42 +0000 Subject: [PATCH] Revamp the SourceManager to separate the representation of parsed source locations from source locations loaded from an AST/PCH file. Previously, loading an AST/PCH file involved carefully pre-allocating space at the beginning of the source manager for the source locations and FileIDs that correspond to the prefix, and then appending the source locations/FileIDs used for parsing the remaining translation unit. This design forced us into loading PCH files early, as a prefix, whic has become a rather significant limitation. This patch splits the SourceManager space into two parts: for source location "addresses", the lower values (growing upward) are used to describe parsed code, while upper values (growing downward) are used for source locations loaded from AST/PCH files. Similarly, positive FileIDs are used to describe parsed code while negative FileIDs are used to file/macro locations loaded from AST/PCH files. As a result, we can load PCH/AST files even during parsing, making various improvemnts in the future possible, e.g., teaching #include to look for and load if it happens to be already available. This patch was originally written by Sebastian Redl, then brought forward to the modern age by Jonathan Turner, and finally polished/finished by me to be committed. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@135484 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Basic/Diagnostic.h | 5 + include/clang/Basic/SourceLocation.h | 46 ++- include/clang/Basic/SourceManager.h | 211 ++++++---- include/clang/Basic/SourceManagerInternals.h | 12 +- include/clang/Frontend/ASTUnit.h | 17 +- include/clang/Frontend/CompilerInstance.h | 13 +- include/clang/Serialization/ASTBitCodes.h | 13 +- include/clang/Serialization/ASTReader.h | 59 ++- .../clang/Serialization/ContinuousRangeMap.h | 90 ++++ lib/Basic/Diagnostic.cpp | 10 + lib/Basic/SourceManager.cpp | 384 ++++++++++-------- lib/Frontend/ASTUnit.cpp | 114 +++++- lib/Frontend/CompilerInstance.cpp | 3 +- lib/Frontend/FrontendAction.cpp | 5 +- lib/Lex/Lexer.cpp | 6 +- lib/Lex/TokenLexer.cpp | 2 +- lib/Serialization/ASTReader.cpp | 230 +++++++---- lib/Serialization/ASTWriter.cpp | 146 ++++--- test/Index/Inputs/preamble_macro_template.h | 6 + test/Index/preamble_macro_template.cpp | 15 + test/PCH/reinclude.cpp | 2 +- tools/libclang/CIndexInclusionStack.cpp | 17 +- 22 files changed, 927 insertions(+), 479 deletions(-) create mode 100644 include/clang/Serialization/ContinuousRangeMap.h create mode 100644 test/Index/Inputs/preamble_macro_template.h create mode 100644 test/Index/preamble_macro_template.cpp diff --git a/include/clang/Basic/Diagnostic.h b/include/clang/Basic/Diagnostic.h index 6f72976bfc..20eebaff08 100644 --- a/include/clang/Basic/Diagnostic.h +++ b/include/clang/Basic/Diagnostic.h @@ -16,6 +16,7 @@ #include "clang/Basic/DiagnosticIDs.h" #include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/OwningPtr.h" @@ -984,6 +985,10 @@ public: StoredDiagnostic(Diagnostic::Level Level, const DiagnosticInfo &Info); StoredDiagnostic(Diagnostic::Level Level, unsigned ID, llvm::StringRef Message); + StoredDiagnostic(Diagnostic::Level Level, unsigned ID, + llvm::StringRef Message, FullSourceLoc Loc, + llvm::ArrayRef Ranges, + llvm::ArrayRef Fixits); ~StoredDiagnostic(); /// \brief Evaluates true when this object stores a diagnostic. diff --git a/include/clang/Basic/SourceLocation.h b/include/clang/Basic/SourceLocation.h index ee5f96fe93..1cbcd4007f 100644 --- a/include/clang/Basic/SourceLocation.h +++ b/include/clang/Basic/SourceLocation.h @@ -35,8 +35,9 @@ class SourceManager; /// a source file (MemoryBuffer) along with its #include path and #line data. /// class FileID { - /// ID - Opaque identifier, 0 is "invalid". - unsigned ID; + /// ID - Opaque identifier, 0 is "invalid". >0 is this module, <-1 is + /// something loaded from another module. + int ID; public: FileID() : ID(0) {} @@ -49,26 +50,38 @@ public: bool operator>(const FileID &RHS) const { return RHS < *this; } bool operator>=(const FileID &RHS) const { return RHS <= *this; } - static FileID getSentinel() { return get(~0U); } - unsigned getHashValue() const { return ID; } + static FileID getSentinel() { return get(-1); } + unsigned getHashValue() const { return static_cast(ID); } private: friend class SourceManager; friend class ASTWriter; friend class ASTReader; - static FileID get(unsigned V) { + static FileID get(int V) { FileID F; F.ID = V; return F; } - unsigned getOpaqueValue() const { return ID; } + int getOpaqueValue() const { return ID; } }; -/// SourceLocation - This is a carefully crafted 32-bit identifier that encodes -/// a full include stack, line and column number information for a position in -/// an input translation unit. +/// \brief Encodes a location in the source. The SourceManager can decode this +/// to get at the full include stack, line and column information. +/// +/// Technically, a source location is simply an offset into the manager's view +/// of the input source, which is all input buffers (including macro +/// instantiations) concatenated in an effectively arbitrary order. The manager +/// actually maintains two blocks of input buffers. One, starting at offset 0 +/// and growing upwards, contains all buffers from this module. The other, +/// starting at the highest possible offset and growing downwards, contains +/// buffers of loaded modules. +/// +/// In addition, one bit of SourceLocation is used for quick access to the +/// information whether the location is in a file or a macro instantiation. +/// +/// It is important that this type remains small. It is currently 32 bits wide. class SourceLocation { unsigned ID; friend class SourceManager; @@ -77,21 +90,21 @@ class SourceLocation { }; public: - SourceLocation() : ID(0) {} // 0 is an invalid FileID. + SourceLocation() : ID(0) {} bool isFileID() const { return (ID & MacroIDBit) == 0; } bool isMacroID() const { return (ID & MacroIDBit) != 0; } - /// isValid - Return true if this is a valid SourceLocation object. Invalid - /// SourceLocations are often used when events have no corresponding location - /// in the source (e.g. a diagnostic is required for a command line option). + /// \brief Return true if this is a valid SourceLocation object. /// + /// Invalid SourceLocations are often used when events have no corresponding + /// location in the source (e.g. a diagnostic is required for a command line + /// option). bool isValid() const { return ID != 0; } bool isInvalid() const { return ID == 0; } private: - /// getOffset - Return the index for SourceManager's SLocEntryTable table, - /// note that this is not an index *into* it though. + /// \brief Return the offset into the manager's global input view. unsigned getOffset() const { return ID & ~MacroIDBit; } @@ -114,7 +127,8 @@ public: /// getFileLocWithOffset - Return a source location with the specified offset /// from this file SourceLocation. SourceLocation getFileLocWithOffset(int Offset) const { - assert(((getOffset()+Offset) & MacroIDBit) == 0 && "invalid location"); + assert(((getOffset()+Offset) & MacroIDBit) == 0 && + "offset overflow or macro loc"); SourceLocation L; L.ID = ID+Offset; return L; diff --git a/include/clang/Basic/SourceManager.h b/include/clang/Basic/SourceManager.h index 6301f31978..c08e691344 100644 --- a/include/clang/Basic/SourceManager.h +++ b/include/clang/Basic/SourceManager.h @@ -46,7 +46,7 @@ namespace SrcMgr { /// holds normal user code, system code, or system code which is implicitly /// 'extern "C"' in C++ mode. Entire directories can be tagged with this /// (this is maintained by DirectoryLookup and friends) as can specific - /// FileIDInfos when a #pragma system_header is seen or various other cases. + /// FileInfos when a #pragma system_header is seen or various other cases. /// enum CharacteristicKind { C_User, C_System, C_ExternCSystem @@ -356,11 +356,12 @@ class ExternalSLocEntrySource { public: virtual ~ExternalSLocEntrySource(); - /// \brief Read the source location entry with index ID. + /// \brief Read the source location entry with index ID, which will always be + /// less than -1. /// /// \returns true if an error occurred that prevented the source-location /// entry from being loaded. - virtual bool ReadSLocEntry(unsigned ID) = 0; + virtual bool ReadSLocEntry(int ID) = 0; }; @@ -414,8 +415,9 @@ public: }; -/// SourceManager - This file handles loading and caching of source files into -/// memory. This object owns the MemoryBuffer objects for all of the loaded +/// \brief This class handles loading and caching of source files into memory. +/// +/// This object owns the MemoryBuffer objects for all of the loaded /// files and assigns unique FileID's for each unique #include chain. /// /// The SourceManager can be queried for information about SourceLocation @@ -451,16 +453,33 @@ class SourceManager : public llvm::RefCountedBase { /// as they do not refer to a file. std::vector MemBufferInfos; - /// SLocEntryTable - This is an array of SLocEntry's that we have created. - /// FileID is an index into this vector. This array is sorted by the offset. - std::vector SLocEntryTable; - /// NextOffset - This is the next available offset that a new SLocEntry can - /// start at. It is SLocEntryTable.back().getOffset()+size of back() entry. - unsigned NextOffset; + /// \brief The table of SLocEntries that are local to this module. + /// + /// Positive FileIDs are indexes into this table. Entry 0 indicates an invalid + /// instantiation. + std::vector LocalSLocEntryTable; + + /// \brief The table of SLocEntries that are loaded from other modules. + /// + /// Negative FileIDs are indexes into this table. To get from ID to an index, + /// use (-ID - 2). + std::vector LoadedSLocEntryTable; - /// \brief If source location entries are being lazily loaded from - /// an external source, this vector indicates whether the Ith source - /// location entry has already been loaded from the external storage. + /// \brief The starting offset of the next local SLocEntry. + /// + /// This is LocalSLocEntryTable.back().Offset + the size of that entry. + unsigned NextLocalOffset; + + /// \brief The starting offset of the latest batch of loaded SLocEntries. + /// + /// This is LoadedSLocEntryTable.back().Offset, except that that entry might + /// not have been loaded, so that value would be unknown. + unsigned CurrentLoadedOffset; + + /// \brief A bitmap that indicates whether the entries of LoadedSLocEntryTable + /// have already been loaded from the external source. + /// + /// Same indexing as LoadedSLocEntryTable. std::vector SLocEntryLoaded; /// \brief An external source for source location entries. @@ -513,6 +532,15 @@ public: OverridenFilesKeepOriginalName = value; } + /// createMainFileIDForMembuffer - Create the FileID for a memory buffer + /// that will represent the FileID for the main source. One example + /// of when this would be used is when the main source is read from STDIN. + FileID createMainFileIDForMemBuffer(const llvm::MemoryBuffer *Buffer) { + assert(MainFileID.isInvalid() && "MainFileID already set!"); + MainFileID = createFileIDForMemBuffer(Buffer); + return MainFileID; + } + //===--------------------------------------------------------------------===// // MainFileID creation and querying methods. //===--------------------------------------------------------------------===// @@ -541,34 +569,21 @@ public: /// createFileID - Create a new FileID that represents the specified file /// being #included from the specified IncludePosition. This translates NULL /// into standard input. - /// PreallocateID should be non-zero to specify which pre-allocated, - /// lazily computed source location is being filled in by this operation. FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, - unsigned PreallocatedID = 0, - unsigned Offset = 0) { + int LoadedID = 0, unsigned LoadedOffset = 0) { const SrcMgr::ContentCache *IR = getOrCreateContentCache(SourceFile); assert(IR && "getOrCreateContentCache() cannot return NULL"); - return createFileID(IR, IncludePos, FileCharacter, PreallocatedID, Offset); + return createFileID(IR, IncludePos, FileCharacter, LoadedID, LoadedOffset); } /// createFileIDForMemBuffer - Create a new FileID that represents the /// specified memory buffer. This does no caching of the buffer and takes /// ownership of the MemoryBuffer, so only pass a MemoryBuffer to this once. FileID createFileIDForMemBuffer(const llvm::MemoryBuffer *Buffer, - unsigned PreallocatedID = 0, - unsigned Offset = 0) { + int LoadedID = 0, unsigned LoadedOffset = 0) { return createFileID(createMemBufferContentCache(Buffer), SourceLocation(), - SrcMgr::C_User, PreallocatedID, Offset); - } - - /// createMainFileIDForMembuffer - Create the FileID for a memory buffer - /// that will represent the FileID for the main source. One example - /// of when this would be used is when the main source is read from STDIN. - FileID createMainFileIDForMemBuffer(const llvm::MemoryBuffer *Buffer) { - assert(MainFileID.isInvalid() && "MainFileID already set!"); - MainFileID = createFileIDForMemBuffer(Buffer); - return MainFileID; + SrcMgr::C_User, LoadedID, LoadedOffset); } /// createMacroArgInstantiationLoc - Return a new SourceLocation that encodes @@ -586,8 +601,8 @@ public: SourceLocation InstantiationLocStart, SourceLocation InstantiationLocEnd, unsigned TokLength, - unsigned PreallocatedID = 0, - unsigned Offset = 0); + int LoadedID = 0, + unsigned LoadedOffset = 0); /// \brief Retrieve the memory buffer associated with the given file. /// @@ -702,7 +717,6 @@ public: /// getLocForStartOfFile - Return the source location corresponding to the /// first byte of the specified file. SourceLocation getLocForStartOfFile(FileID FID) const { - assert(FID.ID < SLocEntryTable.size() && "FileID out of range"); bool Invalid = false; const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid); if (Invalid || !Entry.isFile()) @@ -900,10 +914,12 @@ public: #ifndef NDEBUG // Make sure offset/length describe a chunk inside the given FileID. unsigned NextOffset; - if (FID.ID+1 == SLocEntryTable.size()) - NextOffset = getNextOffset(); + if (FID.ID == -2) + NextOffset = 1U << 31U; + else if (FID.ID+1 == (int)LocalSLocEntryTable.size()) + NextOffset = getNextLocalOffset(); else - NextOffset = getSLocEntry(FID.ID+1).getOffset(); + NextOffset = getSLocEntryByID(FID.ID+1).getOffset(); assert(start < NextOffset); assert(end < NextOffset); #endif @@ -979,15 +995,23 @@ public: /// \brief Determines the order of 2 source locations in the "source location /// address space". - static bool isBeforeInSourceLocationOffset(SourceLocation LHS, - SourceLocation RHS) { + bool isBeforeInSourceLocationOffset(SourceLocation LHS, + SourceLocation RHS) const { return isBeforeInSourceLocationOffset(LHS, RHS.getOffset()); } /// \brief Determines the order of a source location and a source location /// offset in the "source location address space". - static bool isBeforeInSourceLocationOffset(SourceLocation LHS, unsigned RHS) { - return LHS.getOffset() < RHS; + /// + /// Note that we always consider source locations loaded from + bool isBeforeInSourceLocationOffset(SourceLocation LHS, unsigned RHS) const { + unsigned LHSOffset = LHS.getOffset(); + bool LHSLoaded = LHSOffset >= CurrentLoadedOffset; + bool RHSLoaded = RHS >= CurrentLoadedOffset; + if (LHSLoaded == RHSLoaded) + return LHS.getOffset() < RHS; + + return LHSLoaded; } // Iterators over FileInfos. @@ -1003,53 +1027,70 @@ public: /// void PrintStats() const; - unsigned sloc_entry_size() const { return SLocEntryTable.size(); } - - // FIXME: Exposing this is a little gross; what we want is a good way - // to iterate the entries that were not defined in an AST file (or - // any other external source). - unsigned sloc_loaded_entry_size() const { return SLocEntryLoaded.size(); } - - const SrcMgr::SLocEntry &getSLocEntry(unsigned ID, bool *Invalid = 0) const { - assert(ID < SLocEntryTable.size() && "Invalid id"); - // If we haven't loaded this source-location entry from the external source - // yet, do so now. - if (ExternalSLocEntries && - ID < SLocEntryLoaded.size() && - !SLocEntryLoaded[ID] && - ExternalSLocEntries->ReadSLocEntry(ID) && - Invalid) - *Invalid = true; - - return SLocEntryTable[ID]; + /// \brief Get the number of local SLocEntries we have. + unsigned local_sloc_entry_size() const { return LocalSLocEntryTable.size(); } + + /// \brief Get a local SLocEntry. This is exposed for indexing. + const SrcMgr::SLocEntry &getLocalSLocEntry(unsigned Index, + bool *Invalid = 0) const { + assert(Index < LocalSLocEntryTable.size() && "Invalid index"); + return LocalSLocEntryTable[Index]; } - + + /// \brief Get the number of loaded SLocEntries we have. + unsigned loaded_sloc_entry_size() const { return LoadedSLocEntryTable.size();} + + /// \brief Get a loaded SLocEntry. This is exposed for indexing. + const SrcMgr::SLocEntry &getLoadedSLocEntry(unsigned Index, bool *Invalid=0) const { + assert(Index < LoadedSLocEntryTable.size() && "Invalid index"); + if (!SLocEntryLoaded[Index]) + ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2)); + return LoadedSLocEntryTable[Index]; + } + const SrcMgr::SLocEntry &getSLocEntry(FileID FID, bool *Invalid = 0) const { - return getSLocEntry(FID.ID, Invalid); + return getSLocEntryByID(FID.ID); } - unsigned getNextOffset() const { return NextOffset; } - - /// \brief Preallocate some number of source location entries, which - /// will be loaded as needed from the given external source. - void PreallocateSLocEntries(ExternalSLocEntrySource *Source, - unsigned NumSLocEntries, - unsigned NextOffset); - - /// \brief Clear out any preallocated source location entries that - /// haven't already been loaded. - void ClearPreallocatedSLocEntries(); - + unsigned getNextLocalOffset() const { return NextLocalOffset; } + + void setExternalSLocEntrySource(ExternalSLocEntrySource *Source) { + assert(LoadedSLocEntryTable.empty() && + "Invalidating existing loaded entries"); + ExternalSLocEntries = Source; + } + + /// \brief Allocate a number of loaded SLocEntries, which will be actually + /// loaded on demand from the external source. + /// + /// NumSLocEntries will be allocated, which occupy a total of TotalSize space + /// in the global source view. The lowest ID and the base offset of the + /// entries will be returned. + std::pair + AllocateLoadedSLocEntries(unsigned NumSLocEntries, unsigned TotalSize); + private: const llvm::MemoryBuffer *getFakeBufferForRecovery() const; + /// \brief Get the entry with the given unwrapped FileID. + const SrcMgr::SLocEntry &getSLocEntryByID(int ID) const { + assert(ID != -1 && "Using FileID sentinel value"); + if (ID < 0) + return getLoadedSLocEntryByID(ID); + return getLocalSLocEntry(static_cast(ID)); + } + + const SrcMgr::SLocEntry &getLoadedSLocEntryByID(int ID) const { + return getLoadedSLocEntry(static_cast(-ID - 2)); + } + /// createInstantiationLoc - Implements the common elements of storing an /// instantiation info struct into the SLocEntry table and producing a source /// location that refers to it. SourceLocation createInstantiationLocImpl(const SrcMgr::InstantiationInfo &II, unsigned TokLength, - unsigned PreallocatedID = 0, - unsigned Offset = 0); + int LoadedID = 0, + unsigned LoadedOffset = 0); /// isOffsetInFileID - Return true if the specified FileID contains the /// specified SourceLocation offset. This is a very hot method. @@ -1058,10 +1099,17 @@ private: // If the entry is after the offset, it can't contain it. if (SLocOffset < Entry.getOffset()) return false; - // If this is the last entry than it does. Otherwise, the entry after it - // has to not include it. - if (FID.ID+1 == SLocEntryTable.size()) return true; + // If this is the very last entry then it does. + if (FID.ID == -2) + return true; + + // If it is the last local entry, then it does if the location is local. + if (static_cast(FID.ID+1) == LocalSLocEntryTable.size()) { + return SLocOffset < NextLocalOffset; + } + // Otherwise, the entry after it has to not include it. This works for both + // local and loaded entries. return SLocOffset < getSLocEntry(FileID::get(FID.ID+1)).getOffset(); } @@ -1071,8 +1119,7 @@ private: FileID createFileID(const SrcMgr::ContentCache* File, SourceLocation IncludePos, SrcMgr::CharacteristicKind DirCharacter, - unsigned PreallocatedID = 0, - unsigned Offset = 0); + int LoadedID, unsigned LoadedOffset); const SrcMgr::ContentCache * getOrCreateContentCache(const FileEntry *SourceFile); @@ -1083,6 +1130,8 @@ private: createMemBufferContentCache(const llvm::MemoryBuffer *Buf); FileID getFileIDSlow(unsigned SLocOffset) const; + FileID getFileIDLocal(unsigned SLocOffset) const; + FileID getFileIDLoaded(unsigned SLocOffset) const; SourceLocation getInstantiationLocSlowCase(SourceLocation Loc) const; SourceLocation getSpellingLocSlowCase(SourceLocation Loc) const; diff --git a/include/clang/Basic/SourceManagerInternals.h b/include/clang/Basic/SourceManagerInternals.h index 3f5d1a35e5..6a74d41b54 100644 --- a/include/clang/Basic/SourceManagerInternals.h +++ b/include/clang/Basic/SourceManagerInternals.h @@ -84,7 +84,7 @@ class LineTableInfo { /// LineEntries - This is a map from FileIDs to a list of line entries (sorted /// by the offset they occur in the file. - std::map > LineEntries; + std::map > LineEntries; public: LineTableInfo() { } @@ -104,25 +104,25 @@ public: } unsigned getNumFilenames() const { return FilenamesByID.size(); } - void AddLineNote(unsigned FID, unsigned Offset, + void AddLineNote(int FID, unsigned Offset, unsigned LineNo, int FilenameID); - void AddLineNote(unsigned FID, unsigned Offset, + void AddLineNote(int FID, unsigned Offset, unsigned LineNo, int FilenameID, unsigned EntryExit, SrcMgr::CharacteristicKind FileKind); /// FindNearestLineEntry - Find the line entry nearest to FID that is before /// it. If there is no line entry before Offset in FID, return null. - const LineEntry *FindNearestLineEntry(unsigned FID, unsigned Offset); + const LineEntry *FindNearestLineEntry(int FID, unsigned Offset); // Low-level access - typedef std::map >::iterator iterator; + typedef std::map >::iterator iterator; iterator begin() { return LineEntries.begin(); } iterator end() { return LineEntries.end(); } /// \brief Add a new line entry that has already been encoded into /// the internal representation of the line table. - void AddEntry(unsigned FID, const std::vector &Entries); + void AddEntry(int FID, const std::vector &Entries); }; } // end namespace clang diff --git a/include/clang/Frontend/ASTUnit.h b/include/clang/Frontend/ASTUnit.h index 58a60a1f9d..9fb552131f 100644 --- a/include/clang/Frontend/ASTUnit.h +++ b/include/clang/Frontend/ASTUnit.h @@ -41,6 +41,7 @@ namespace llvm { namespace clang { class ASTContext; +class ASTReader; class CodeCompleteConsumer; class CompilerInvocation; class Decl; @@ -143,6 +144,9 @@ private: // Critical optimization when using clang_getCursor(). ASTLocation LastLoc; + /// \brief The set of diagnostics produced when creating the preamble. + llvm::SmallVector PreambleDiagnostics; + /// \brief The set of diagnostics produced when creating this /// translation unit. llvm::SmallVector StoredDiagnostics; @@ -230,14 +234,6 @@ private: /// when any errors are present. unsigned NumWarningsInPreamble; - /// \brief The number of diagnostics that were stored when parsing - /// the precompiled preamble. - /// - /// This value is used to determine how many of the stored - /// diagnostics should be retained when reparsing in the presence of - /// a precompiled preamble. - unsigned NumStoredDiagnosticsInPreamble; - /// \brief A list of the serialization ID numbers for each of the top-level /// declarations parsed within the precompiled preamble. std::vector TopLevelDeclsInPreamble; @@ -257,6 +253,11 @@ private: const char **ArgBegin, const char **ArgEnd, ASTUnit &AST, bool CaptureDiagnostics); + void TranslateStoredDiagnostics(ASTReader *MMan, llvm::StringRef ModName, + SourceManager &SrcMan, + const llvm::SmallVectorImpl &Diags, + llvm::SmallVectorImpl &Out); + public: /// \brief A cached code-completion result, which may be introduced in one of /// many different contexts. diff --git a/include/clang/Frontend/CompilerInstance.h b/include/clang/Frontend/CompilerInstance.h index 004c8896e2..b2c877ac40 100644 --- a/include/clang/Frontend/CompilerInstance.h +++ b/include/clang/Frontend/CompilerInstance.h @@ -27,13 +27,13 @@ class Timer; namespace clang { class ASTContext; class ASTConsumer; +class ASTReader; class CodeCompleteConsumer; class Diagnostic; class DiagnosticClient; class ExternalASTSource; class FileManager; class FrontendAction; -class ASTReader; class Preprocessor; class Sema; class SourceManager; @@ -88,9 +88,12 @@ class CompilerInstance { /// \brief The semantic analysis object. llvm::OwningPtr TheSema; - /// The frontend timer + /// \brief The frontend timer llvm::OwningPtr FrontendTimer; + /// \brief Non-owning reference to the ASTReader, if one exists. + ASTReader *ModuleManager; + /// \brief Holds information about the output file. /// /// If TempFilename is not empty we must rename it to Filename at the end. @@ -387,6 +390,12 @@ public: Sema *takeSema() { return TheSema.take(); } + /// } + /// @name Module Management + /// { + + ASTReader *getModuleManager() const { return ModuleManager; } + /// } /// @name Code Completion /// { diff --git a/include/clang/Serialization/ASTBitCodes.h b/include/clang/Serialization/ASTBitCodes.h index 11b8bed903..a01bf6235a 100644 --- a/include/clang/Serialization/ASTBitCodes.h +++ b/include/clang/Serialization/ASTBitCodes.h @@ -375,8 +375,14 @@ namespace clang { /// \brief Record code for the set of known namespaces, which are used /// for typo correction. - KNOWN_NAMESPACES = 46 + KNOWN_NAMESPACES = 46, + /// \brief Record code for the source location remapping information. + SOURCE_LOCATION_MAP = 47, + + /// \brief Record code for the source manager line table information, + /// which stores information about #line directives. + SOURCE_MANAGER_LINE_TABLE = 48 }; /// \brief Record types used within a source manager block. @@ -393,10 +399,7 @@ namespace clang { SM_SLOC_BUFFER_BLOB = 3, /// \brief Describes a source location entry (SLocEntry) for a /// macro expansion. - SM_SLOC_EXPANSION_ENTRY = 4, - /// \brief Describes the SourceManager's line table, with - /// information about #line directives. - SM_LINE_TABLE = 5 + SM_SLOC_EXPANSION_ENTRY = 4 }; /// \brief Record types used within a preprocessor block. diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h index 9e210c3db2..a0bc899262 100644 --- a/include/clang/Serialization/ASTReader.h +++ b/include/clang/Serialization/ASTReader.h @@ -15,6 +15,7 @@ #define LLVM_CLANG_FRONTEND_AST_READER_H #include "clang/Serialization/ASTBitCodes.h" +#include "clang/Serialization/ContinuousRangeMap.h" #include "clang/Sema/ExternalSemaSource.h" #include "clang/AST/DeclarationName.h" #include "clang/AST/DeclObjC.h" @@ -49,6 +50,7 @@ class AddrLabelExpr; class ASTConsumer; class ASTContext; class ASTIdentifierIterator; +class ASTUnit; // FIXME: Layering violation and egregious hack. class Attr; class Decl; class DeclContext; @@ -64,6 +66,7 @@ class Preprocessor; class Sema; class SwitchCase; class ASTDeserializationListener; +class ASTWriter; class ASTReader; class ASTDeclReader; class ASTStmtReader; @@ -191,6 +194,8 @@ public: friend class ASTIdentifierIterator; friend class ASTIdentifierLookupTrait; friend class TypeLocReader; + friend class ASTWriter; + friend class ASTUnit; // ASTUnit needs to remap source locations. private: /// \brief The receiver of some callbacks invoked by ASTReader. llvm::OwningPtr Listener; @@ -245,6 +250,12 @@ private: /// \brief The main bitstream cursor for the main block. llvm::BitstreamCursor Stream; + /// \brief The source location where this module was first imported. + SourceLocation ImportLoc; + + /// \brief The first source location in this module. + SourceLocation FirstLoc; + // === Source Locations === /// \brief Cursor used to read source location entries. @@ -253,9 +264,15 @@ private: /// \brief The number of source location entries in this AST file. unsigned LocalNumSLocEntries; + /// \brief The base ID in the source manager's view of this module. + int SLocEntryBaseID; + + /// \brief The base offset in the source manager's view of this module. + unsigned SLocEntryBaseOffset; + /// \brief Offsets for all of the source location entries in the /// AST file. - const uint32_t *SLocOffsets; + const uint32_t *SLocEntryOffsets; /// \brief The number of source location file entries in this AST file. unsigned LocalNumSLocFileEntries; @@ -264,8 +281,8 @@ private: /// AST file. const uint32_t *SLocFileOffsets; - /// \brief The entire size of this module's source location offset range. - unsigned LocalSLocSize; + /// \brief Remapping table for source locations in this module. + ContinuousRangeMap SLocRemap; // === Identifiers === @@ -397,6 +414,9 @@ private: // === Miscellaneous === + /// \brief Diagnostic IDs and their mappings that the user changed. + llvm::SmallVector PragmaDiagMappings; + /// \brief The AST stat cache installed for this file, if any. /// /// The dynamic type of this stat cache is always ASTStatCache @@ -426,7 +446,10 @@ private: llvm::SmallVector Chain; /// \brief SLocEntries that we're going to preload. - llvm::SmallVector PreloadSLocEntries; + llvm::SmallVector PreloadSLocEntries; + + /// \brief A map of negated SLocEntryIDs to the modules containing them. + ContinuousRangeMap GlobalSLocEntryMap; /// \brief Types that have already been loaded from the chain. /// @@ -630,9 +653,6 @@ private: //@} - /// \brief Diagnostic IDs and their mappings that the user changed. - llvm::SmallVector PragmaDiagMappings; - /// \brief The original file name that was used to build the primary AST file, /// which may have been modified for relocatable-pch support. std::string OriginalFileName; @@ -686,9 +706,6 @@ private: /// \brief The number of source location entries in the chain. unsigned TotalNumSLocEntries; - /// \brief The next offset for a SLocEntry after everything in this reader. - unsigned NextSLocOffset; - /// \brief The number of statements (and expressions) de-serialized /// from the chain. unsigned NumStatementsRead; @@ -810,8 +827,8 @@ private: bool CheckPredefinesBuffers(); bool ParseLineTable(PerFileData &F, llvm::SmallVectorImpl &Record); ASTReadResult ReadSourceManagerBlock(PerFileData &F); - ASTReadResult ReadSLocEntryRecord(unsigned ID); - PerFileData *SLocCursorForID(unsigned ID); + ASTReadResult ReadSLocEntryRecord(int ID); + llvm::BitstreamCursor &SLocCursorForID(int ID); SourceLocation getImportLocation(PerFileData *F); bool ParseLanguageOptions(const llvm::SmallVectorImpl &Record); @@ -960,11 +977,6 @@ public: return TotalNumSLocEntries; } - /// \brief Returns the next SLocEntry offset after the chain. - unsigned getNextSLocOffset() const { - return NextSLocOffset; - } - /// \brief Returns the number of identifiers found in the chain. unsigned getTotalNumIdentifiers() const { return static_cast(IdentifiersLoaded.size()); @@ -1158,7 +1170,7 @@ public: } /// \brief Read the source location entry with index ID. - virtual bool ReadSLocEntry(unsigned ID); + virtual bool ReadSLocEntry(int ID); Selector DecodeSelector(unsigned Idx); @@ -1221,8 +1233,15 @@ public: /// \brief Read a source location from raw form. SourceLocation ReadSourceLocation(PerFileData &Module, unsigned Raw) { - (void)Module; // No remapping yet - return SourceLocation::getFromRawEncoding(Raw); + unsigned Flag = Raw & (1U << 31); + unsigned Offset = Raw & ~(1U << 31); + assert(Module.SLocRemap.find(Offset) != Module.SLocRemap.end() && + "Cannot find offset to remap."); + int Remap = Module.SLocRemap.find(Offset)->second; + Offset += Remap; + assert((Offset & (1U << 31)) == 0 && + "Bad offset in reading source location"); + return SourceLocation::getFromRawEncoding(Offset | Flag); } /// \brief Read a source location. diff --git a/include/clang/Serialization/ContinuousRangeMap.h b/include/clang/Serialization/ContinuousRangeMap.h new file mode 100644 index 0000000000..2a27b9115b --- /dev/null +++ b/include/clang/Serialization/ContinuousRangeMap.h @@ -0,0 +1,90 @@ +//===--- ContinuousRangeMap.h - Map with int range as key -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the ContinuousRangeMap class, which is a highly +// specialized container used by serialization. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SERIALIZATION_CONTINUOUS_RANGE_MAP_H +#define LLVM_CLANG_SERIALIZATION_CONTINUOUS_RANGE_MAP_H + +#include "llvm/ADT/SmallVector.h" +#include +#include + +namespace clang { + +/// \brief A map from continuous integer ranges to some value, with a very +/// specialized interface. +/// +/// CRM maps from integer ranges to values. The ranges are continuous, i.e. +/// where one ends, the next one begins. So if the map contains the stops I0-3, +/// the first range is from I0 to I1, the second from I1 to I2, the third from +/// I2 to I3 and the last from I3 to infinity. +/// +/// Ranges must be inserted in order. Inserting a new stop I4 into the map will +/// shrink the fourth range to I3 to I4 and add the new range I4 to inf. +template +class ContinuousRangeMap { +public: + typedef std::pair value_type; + typedef value_type &reference; + typedef const value_type &const_reference; + typedef value_type *pointer; + typedef const value_type *const_pointer; + +private: + typedef llvm::SmallVector Representation; + Representation Rep; + + struct Compare { + bool operator ()(const_reference L, Int R) const { + return L.first < R; + } + bool operator ()(Int L, const_reference R) const { + return L < R.first; + } + }; + +public: + void insert(const value_type &Val) { + assert((Rep.empty() || Rep.back().first < Val.first) && + "Must insert keys in order."); + Rep.push_back(Val); + } + + typedef typename Representation::iterator iterator; + typedef typename Representation::const_iterator const_iterator; + + iterator begin() { return Rep.begin(); } + iterator end() { return Rep.end(); } + const_iterator begin() const { return Rep.begin(); } + const_iterator end() const { return Rep.end(); } + + iterator find(Int K) { + iterator I = std::upper_bound(Rep.begin(), Rep.end(), K, Compare()); + // I points to the first entry with a key > K, which is the range that + // follows the one containing K. + if (I == Rep.begin()) + return Rep.end(); + --I; + return I; + } + const_iterator find(Int K) const { + return const_cast(this)->find(K); + } + + reference back() { return Rep.back(); } + const_reference back() const { return Rep.back(); } +}; + +} + +#endif diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp index ae363a0df0..c12651a3f9 100644 --- a/lib/Basic/Diagnostic.cpp +++ b/lib/Basic/Diagnostic.cpp @@ -726,6 +726,16 @@ StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, FixIts.push_back(Info.getFixItHint(I)); } +StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, unsigned ID, + llvm::StringRef Message, FullSourceLoc Loc, + llvm::ArrayRef Ranges, + llvm::ArrayRef Fixits) + : ID(ID), Level(Level), Loc(Loc), Message(Message) +{ + this->Ranges.assign(Ranges.begin(), Ranges.end()); + this->FixIts.assign(FixIts.begin(), FixIts.end()); +} + StoredDiagnostic::~StoredDiagnostic() { } /// IncludeInDiagnosticCounts - This method (whose default implementation diff --git a/lib/Basic/SourceManager.cpp b/lib/Basic/SourceManager.cpp index 45922c1552..ee13f621b5 100644 --- a/lib/Basic/SourceManager.cpp +++ b/lib/Basic/SourceManager.cpp @@ -186,7 +186,7 @@ unsigned LineTableInfo::getLineTableFilenameID(llvm::StringRef Name) { /// AddLineNote - Add a line note to the line table that indicates that there /// is a #line at the specified FID/Offset location which changes the presumed /// location to LineNo/FilenameID. -void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset, +void LineTableInfo::AddLineNote(int FID, unsigned Offset, unsigned LineNo, int FilenameID) { std::vector &Entries = LineEntries[FID]; @@ -217,7 +217,7 @@ void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset, /// presumed #include stack. If it is 1, this is a file entry, if it is 2 then /// this is a file exit. FileKind specifies whether this is a system header or /// extern C system header. -void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset, +void LineTableInfo::AddLineNote(int FID, unsigned Offset, unsigned LineNo, int FilenameID, unsigned EntryExit, SrcMgr::CharacteristicKind FileKind) { @@ -251,7 +251,7 @@ void LineTableInfo::AddLineNote(unsigned FID, unsigned Offset, /// FindNearestLineEntry - Find the line entry nearest to FID that is before /// it. If there is no line entry before Offset in FID, return null. -const LineEntry *LineTableInfo::FindNearestLineEntry(unsigned FID, +const LineEntry *LineTableInfo::FindNearestLineEntry(int FID, unsigned Offset) { const std::vector &Entries = LineEntries[FID]; assert(!Entries.empty() && "No #line entries for this FID after all!"); @@ -270,7 +270,7 @@ const LineEntry *LineTableInfo::FindNearestLineEntry(unsigned FID, /// \brief Add a new line entry that has already been encoded into /// the internal representation of the line table. -void LineTableInfo::AddEntry(unsigned FID, +void LineTableInfo::AddEntry(int FID, const std::vector &Entries) { LineEntries[FID] = Entries; } @@ -391,7 +391,9 @@ SourceManager::~SourceManager() { void SourceManager::clearIDTables() { MainFileID = FileID(); - SLocEntryTable.clear(); + LocalSLocEntryTable.clear(); + LoadedSLocEntryTable.clear(); + SLocEntryLoaded.clear(); LastLineNoFileIDQuery = FileID(); LastLineNoContentCache = 0; LastFileIDLookup = FileID(); @@ -400,7 +402,10 @@ void SourceManager::clearIDTables() { LineTable->clear(); // Use up FileID #0 as an invalid instantiation. - NextOffset = 0; + NextLocalOffset = 0; + // The highest possible offset is 2^31-1, so CurrentLoadedOffset starts at + // 2^31. + CurrentLoadedOffset = 1U << 31U; createInstantiationLoc(SourceLocation(),SourceLocation(),SourceLocation(), 1); } @@ -452,33 +457,16 @@ SourceManager::createMemBufferContentCache(const MemoryBuffer *Buffer) { return Entry; } -void SourceManager::PreallocateSLocEntries(ExternalSLocEntrySource *Source, - unsigned NumSLocEntries, - unsigned NextOffset) { - ExternalSLocEntries = Source; - this->NextOffset = NextOffset; - unsigned CurPrealloc = SLocEntryLoaded.size(); - // If we've ever preallocated, we must not count the dummy entry. - if (CurPrealloc) --CurPrealloc; - SLocEntryLoaded.resize(NumSLocEntries + 1); - SLocEntryLoaded[0] = true; - SLocEntryTable.resize(SLocEntryTable.size() + NumSLocEntries - CurPrealloc); -} - -void SourceManager::ClearPreallocatedSLocEntries() { - unsigned I = 0; - for (unsigned N = SLocEntryLoaded.size(); I != N; ++I) - if (!SLocEntryLoaded[I]) - break; - - // We've already loaded all preallocated source location entries. - if (I == SLocEntryLoaded.size()) - return; - - // Remove everything from location I onward. - SLocEntryTable.resize(I); - SLocEntryLoaded.clear(); - ExternalSLocEntries = 0; +std::pair +SourceManager::AllocateLoadedSLocEntries(unsigned NumSLocEntries, + unsigned TotalSize) { + assert(ExternalSLocEntries && "Don't have an external sloc source"); + LoadedSLocEntryTable.resize(LoadedSLocEntryTable.size() + NumSLocEntries); + SLocEntryLoaded.resize(LoadedSLocEntryTable.size()); + CurrentLoadedOffset -= TotalSize; + assert(CurrentLoadedOffset >= NextLocalOffset && "Out of source locations"); + int ID = LoadedSLocEntryTable.size(); + return std::make_pair(-ID - 1, CurrentLoadedOffset); } /// \brief As part of recovering from missing or changed content, produce a @@ -501,33 +489,31 @@ const llvm::MemoryBuffer *SourceManager::getFakeBufferForRecovery() const { FileID SourceManager::createFileID(const ContentCache *File, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, - unsigned PreallocatedID, - unsigned Offset) { - if (PreallocatedID) { - // If we're filling in a preallocated ID, just load in the file - // entry and return. - assert(PreallocatedID < SLocEntryLoaded.size() && - "Preallocate ID out-of-range"); - assert(!SLocEntryLoaded[PreallocatedID] && - "Source location entry already loaded"); - assert(Offset && "Preallocate source location cannot have zero offset"); - SLocEntryTable[PreallocatedID] - = SLocEntry::get(Offset, FileInfo::get(IncludePos, File, FileCharacter)); - SLocEntryLoaded[PreallocatedID] = true; - FileID FID = FileID::get(PreallocatedID); - return FID; + int LoadedID, unsigned LoadedOffset) { + if (LoadedID < 0) { + assert(LoadedID != -1 && "Loading sentinel FileID"); + unsigned Index = unsigned(-LoadedID) - 2; + assert(Index < LoadedSLocEntryTable.size() && "FileID out of range"); + assert(!SLocEntryLoaded[Index] && "FileID already loaded"); + LoadedSLocEntryTable[Index] = SLocEntry::get(LoadedOffset, + FileInfo::get(IncludePos, File, FileCharacter)); + SLocEntryLoaded[Index] = true; + return FileID::get(LoadedID); } - - SLocEntryTable.push_back(SLocEntry::get(NextOffset, - FileInfo::get(IncludePos, File, - FileCharacter))); + LocalSLocEntryTable.push_back(SLocEntry::get(NextLocalOffset, + FileInfo::get(IncludePos, File, + FileCharacter))); unsigned FileSize = File->getSize(); - assert(NextOffset+FileSize+1 > NextOffset && "Ran out of source locations!"); - NextOffset += FileSize+1; + assert(NextLocalOffset + FileSize + 1 > NextLocalOffset && + NextLocalOffset + FileSize + 1 <= CurrentLoadedOffset && + "Ran out of source locations!"); + // We do a +1 here because we want a SourceLocation that means "the end of the + // file", e.g. for the "no newline at the end of the file" diagnostic. + NextLocalOffset += FileSize + 1; // Set LastFileIDLookup to the newly created file. The next getFileID call is // almost guaranteed to be from that file. - FileID FID = FileID::get(SLocEntryTable.size()-1); + FileID FID = FileID::get(LocalSLocEntryTable.size()-1); return LastFileIDLookup = FID; } @@ -544,34 +530,34 @@ SourceLocation SourceManager::createInstantiationLoc(SourceLocation SpellingLoc, SourceLocation ILocStart, SourceLocation ILocEnd, unsigned TokLength, - unsigned PreallocatedID, - unsigned Offset) { + int LoadedID, + unsigned LoadedOffset) { InstantiationInfo II = InstantiationInfo::create(SpellingLoc, ILocStart, ILocEnd); - return createInstantiationLocImpl(II, TokLength, PreallocatedID, Offset); + return createInstantiationLocImpl(II, TokLength, LoadedID, LoadedOffset); } SourceLocation SourceManager::createInstantiationLocImpl(const InstantiationInfo &II, unsigned TokLength, - unsigned PreallocatedID, - unsigned Offset) { - if (PreallocatedID) { - // If we're filling in a preallocated ID, just load in the - // instantiation entry and return. - assert(PreallocatedID < SLocEntryLoaded.size() && - "Preallocate ID out-of-range"); - assert(!SLocEntryLoaded[PreallocatedID] && - "Source location entry already loaded"); - assert(Offset && "Preallocate source location cannot have zero offset"); - SLocEntryTable[PreallocatedID] = SLocEntry::get(Offset, II); - SLocEntryLoaded[PreallocatedID] = true; - return SourceLocation::getMacroLoc(Offset); + int LoadedID, + unsigned LoadedOffset) { + if (LoadedID < 0) { + assert(LoadedID != -1 && "Loading sentinel FileID"); + unsigned Index = unsigned(-LoadedID) - 2; + assert(Index < LoadedSLocEntryTable.size() && "FileID out of range"); + assert(!SLocEntryLoaded[Index] && "FileID already loaded"); + LoadedSLocEntryTable[Index] = SLocEntry::get(LoadedOffset, II); + SLocEntryLoaded[Index] = true; + return SourceLocation::getMacroLoc(LoadedOffset); } - SLocEntryTable.push_back(SLocEntry::get(NextOffset, II)); - assert(NextOffset+TokLength+1 > NextOffset && "Ran out of source locations!"); - NextOffset += TokLength+1; - return SourceLocation::getMacroLoc(NextOffset-(TokLength+1)); + LocalSLocEntryTable.push_back(SLocEntry::get(NextLocalOffset, II)); + assert(NextLocalOffset + TokLength + 1 > NextLocalOffset && + NextLocalOffset + TokLength + 1 <= CurrentLoadedOffset && + "Ran out of source locations!"); + // See createFileID for that +1. + NextLocalOffset += TokLength + 1; + return SourceLocation::getMacroLoc(NextLocalOffset - (TokLength + 1)); } const llvm::MemoryBuffer * @@ -604,7 +590,7 @@ void SourceManager::overrideFileContents(const FileEntry *SourceFile, llvm::StringRef SourceManager::getBufferData(FileID FID, bool *Invalid) const { bool MyInvalid = false; - const SLocEntry &SLoc = getSLocEntry(FID.ID, &MyInvalid); + const SLocEntry &SLoc = getSLocEntry(FID, &MyInvalid); if (!SLoc.isFile() || MyInvalid) { if (Invalid) *Invalid = true; @@ -627,15 +613,29 @@ llvm::StringRef SourceManager::getBufferData(FileID FID, bool *Invalid) const { // SourceLocation manipulation methods. //===----------------------------------------------------------------------===// -/// getFileIDSlow - Return the FileID for a SourceLocation. This is a very hot -/// method that is used for all SourceManager queries that start with a -/// SourceLocation object. It is responsible for finding the entry in -/// SLocEntryTable which contains the specified location. +/// \brief Return the FileID for a SourceLocation. /// +/// This is the cache-miss path of getFileID. Not as hot as that function, but +/// still very important. It is responsible for finding the entry in the +/// SLocEntry tables that contains the specified location. FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const { if (!SLocOffset) return FileID::get(0); + // Now it is time to search for the correct file. See where the SLocOffset + // sits in the global view and consult local or loaded buffers for it. + if (SLocOffset < NextLocalOffset) + return getFileIDLocal(SLocOffset); + return getFileIDLoaded(SLocOffset); +} + +/// \brief Return the FileID for a SourceLocation with a low offset. +/// +/// This function knows that the SourceLocation is in a local buffer, not a +/// loaded one. +FileID SourceManager::getFileIDLocal(unsigned SLocOffset) const { + assert(SLocOffset < NextLocalOffset && "Bad function choice"); + // After the first and second level caches, I see two common sorts of // behavior: 1) a lot of searched FileID's are "near" the cached file location // or are "near" the cached instantiation location. 2) others are just @@ -649,12 +649,13 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const { // most newly created FileID. std::vector::const_iterator I; - if (SLocEntryTable[LastFileIDLookup.ID].getOffset() < SLocOffset) { + if (LastFileIDLookup.ID < 0 || + LocalSLocEntryTable[LastFileIDLookup.ID].getOffset() < SLocOffset) { // Neither loc prunes our search. - I = SLocEntryTable.end(); + I = LocalSLocEntryTable.end(); } else { // Perhaps it is near the file point. - I = SLocEntryTable.begin()+LastFileIDLookup.ID; + I = LocalSLocEntryTable.begin()+LastFileIDLookup.ID; } // Find the FileID that contains this. "I" is an iterator that points to a @@ -662,21 +663,8 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const { unsigned NumProbes = 0; while (1) { --I; - if (ExternalSLocEntries) { - bool Invalid = false; - getSLocEntry(FileID::get(I - SLocEntryTable.begin()), &Invalid); - if (Invalid) - return FileID::get(0); - } - if (I->getOffset() <= SLocOffset) { -#if 0 - printf("lin %d -> %d [%s] %d %d\n", SLocOffset, - I-SLocEntryTable.begin(), - I->isInstantiation() ? "inst" : "file", - LastFileIDLookup.ID, int(SLocEntryTable.end()-I)); -#endif - FileID Res = FileID::get(I-SLocEntryTable.begin()); + FileID Res = FileID::get(int(I - LocalSLocEntryTable.begin())); // If this isn't an instantiation, remember it. We have good locality // across FileID lookups. @@ -691,7 +679,7 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const { // Convert "I" back into an index. We know that it is an entry whose index is // larger than the offset we are looking for. - unsigned GreaterIndex = I-SLocEntryTable.begin(); + unsigned GreaterIndex = I - LocalSLocEntryTable.begin(); // LessIndex - This is the lower bound of the range that we're searching. // We know that the offset corresponding to the FileID is is less than // SLocOffset. @@ -700,8 +688,7 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const { while (1) { bool Invalid = false; unsigned MiddleIndex = (GreaterIndex-LessIndex)/2+LessIndex; - unsigned MidOffset = getSLocEntry(FileID::get(MiddleIndex), &Invalid) - .getOffset(); + unsigned MidOffset = getLocalSLocEntry(MiddleIndex, &Invalid).getOffset(); if (Invalid) return FileID::get(0); @@ -715,18 +702,14 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const { } // If the middle index contains the value, succeed and return. + // FIXME: This could be made faster by using a function that's aware of + // being in the local area. if (isOffsetInFileID(FileID::get(MiddleIndex), SLocOffset)) { -#if 0 - printf("bin %d -> %d [%s] %d %d\n", SLocOffset, - I-SLocEntryTable.begin(), - I->isInstantiation() ? "inst" : "file", - LastFileIDLookup.ID, int(SLocEntryTable.end()-I)); -#endif FileID Res = FileID::get(MiddleIndex); // If this isn't an instantiation, remember it. We have good locality // across FileID lookups. - if (!I->isInstantiation()) + if (!LocalSLocEntryTable[MiddleIndex].isInstantiation()) LastFileIDLookup = Res; NumBinaryProbes += NumProbes; return Res; @@ -737,6 +720,68 @@ FileID SourceManager::getFileIDSlow(unsigned SLocOffset) const { } } +/// \brief Return the FileID for a SourceLocation with a high offset. +/// +/// This function knows that the SourceLocation is in a loaded buffer, not a +/// local one. +FileID SourceManager::getFileIDLoaded(unsigned SLocOffset) const { + assert(SLocOffset >= CurrentLoadedOffset && "Bad function choice"); + + // Essentially the same as the local case, but the loaded array is sorted + // in the other direction. + + // First do a linear scan from the last lookup position, if possible. + unsigned I; + int LastID = LastFileIDLookup.ID; + if (LastID >= 0 || getLoadedSLocEntryByID(LastID).getOffset() < SLocOffset) + I = 0; + else + I = (-LastID - 2) + 1; + + unsigned NumProbes; + for (NumProbes = 0; NumProbes < 8; ++NumProbes, ++I) { + // Make sure the entry is loaded! + const SrcMgr::SLocEntry &E = getLoadedSLocEntry(I); + if (E.getOffset() <= SLocOffset) { + FileID Res = FileID::get(-int(I) - 2); + + if (!E.isInstantiation()) + LastFileIDLookup = Res; + NumLinearScans += NumProbes + 1; + return Res; + } + } + + // Linear scan failed. Do the binary search. Note the reverse sorting of the + // table: GreaterIndex is the one where the offset is greater, which is + // actually a lower index! + unsigned GreaterIndex = I; + unsigned LessIndex = LoadedSLocEntryTable.size(); + NumProbes = 0; + while (1) { + ++NumProbes; + unsigned MiddleIndex = (LessIndex - GreaterIndex) / 2 + GreaterIndex; + const SrcMgr::SLocEntry &E = getLoadedSLocEntry(MiddleIndex); + + ++NumProbes; + + if (E.getOffset() > SLocOffset) { + GreaterIndex = MiddleIndex; + continue; + } + + if (isOffsetInFileID(FileID::get(-int(MiddleIndex) - 2), SLocOffset)) { + FileID Res = FileID::get(-int(MiddleIndex) - 2); + if (!E.isInstantiation()) + LastFileIDLookup = Res; + NumBinaryProbes += NumProbes; + return Res; + } + + LessIndex = MiddleIndex; + } +} + SourceLocation SourceManager:: getInstantiationLocSlowCase(SourceLocation Loc) const { do { @@ -1259,7 +1304,7 @@ static llvm::Optional getActualFileInode(const FileEntry *File) { /// \brief Get the source location for the given file:line:col triplet. /// /// If the source file is included multiple times, the source location will -/// be based upon the first inclusion. +/// be based upon an arbitrary inclusion. SourceLocation SourceManager::getLocation(const FileEntry *SourceFile, unsigned Line, unsigned Col) { assert(SourceFile && "Null source file!"); @@ -1308,10 +1353,10 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile, if (FirstFID.isInvalid()) { // The location we're looking for isn't in the main file; look - // through all of the source locations. - for (unsigned I = 0, N = sloc_entry_size(); I != N; ++I) { + // through all of the local source locations. + for (unsigned I = 0, N = local_sloc_entry_size(); I != N; ++I) { bool Invalid = false; - const SLocEntry &SLoc = getSLocEntry(I, &Invalid); + const SLocEntry &SLoc = getLocalSLocEntry(I, &Invalid); if (Invalid) return SourceLocation(); @@ -1322,6 +1367,18 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile, break; } } + // If that still didn't help, try the modules. + if (FirstFID.isInvalid()) { + for (unsigned I = 0, N = loaded_sloc_entry_size(); I != N; ++I) { + const SLocEntry &SLoc = getLoadedSLocEntry(I); + if (SLoc.isFile() && + SLoc.getFile().getContentCache() && + SLoc.getFile().getContentCache()->OrigEntry == SourceFile) { + FirstFID = FileID::get(-int(I) - 2); + break; + } + } + } } // If we haven't found what we want yet, try again, but this time stat() @@ -1333,8 +1390,10 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile, (SourceFileInode || (SourceFileInode = getActualFileInode(SourceFile)))) { bool Invalid = false; - for (unsigned I = 0, N = sloc_entry_size(); I != N; ++I) { - const SLocEntry &SLoc = getSLocEntry(I, &Invalid); + for (unsigned I = 0, N = local_sloc_entry_size(); I != N; ++I) { + FileID IFileID; + IFileID.ID = I; + const SLocEntry &SLoc = getSLocEntry(IFileID, &Invalid); if (Invalid) return SourceLocation(); @@ -1368,7 +1427,7 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile, return SourceLocation(); // If this is the first use of line information for this buffer, compute the - /// SourceLineCache for it on demand. + // SourceLineCache for it on demand. if (Content->SourceLineCache == 0) { bool MyInvalid = false; ComputeLineNumbers(Diag, Content, ContentCacheAlloc, *this, MyInvalid); @@ -1447,39 +1506,25 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS, // Okay, we missed in the cache, start updating the cache for this query. IsBeforeInTUCache.setQueryFIDs(LOffs.first, ROffs.first); - // "Traverse" the include/instantiation stacks of both locations and try to - // find a common "ancestor". FileIDs build a tree-like structure that - // reflects the #include hierarchy, and this algorithm needs to find the - // nearest common ancestor between the two locations. For example, if you - // have a.c that includes b.h and c.h, and are comparing a location in b.h to - // a location in c.h, we need to find that their nearest common ancestor is - // a.c, and compare the locations of the two #includes to find their relative - // ordering. - // - // SourceManager assigns FileIDs in order of parsing. This means that an - // includee always has a larger FileID than an includer. While you might - // think that we could just compare the FileID's here, that doesn't work to - // compare a point at the end of a.c with a point within c.h. Though c.h has - // a larger FileID, we have to compare the include point of c.h to the - // location in a.c. - // - // Despite not being able to directly compare FileID's, we can tell that a - // larger FileID is necessarily more deeply nested than a lower one and use - // this information to walk up the tree to the nearest common ancestor. + // We need to find the common ancestor. The only way of doing this is to + // build the complete include chain for one and then walking up the chain + // of the other looking for a match. + // We use a map from FileID to Offset to store the chain. Easier than writing + // a custom set hash info that only depends on the first part of a pair. + typedef llvm::DenseMap LocSet; + LocSet LChain; do { - // If LOffs is larger than ROffs, then LOffs must be more deeply nested than - // ROffs, walk up the #include chain. - if (LOffs.first.ID > ROffs.first.ID) { - if (MoveUpIncludeHierarchy(LOffs, *this)) - break; // We reached the top. - - } else { - // Otherwise, ROffs is larger than LOffs, so ROffs must be more deeply - // nested than LOffs, walk up the #include chain. - if (MoveUpIncludeHierarchy(ROffs, *this)) - break; // We reached the top. - } - } while (LOffs.first != ROffs.first); + LChain.insert(LOffs); + // We catch the case where LOffs is in a file included by ROffs and + // quit early. The other way round unfortunately remains suboptimal. + } while (LOffs.first != ROffs.first && !MoveUpIncludeHierarchy(LOffs, *this)); + LocSet::iterator I; + while((I = LChain.find(ROffs.first)) == LChain.end()) { + if (MoveUpIncludeHierarchy(ROffs, *this)) + break; // Met at topmost file. + } + if (I != LChain.end()) + LOffs = *I; // If we exited because we found a nearest common ancestor, compare the // locations within the common file and cache them. @@ -1488,26 +1533,21 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS, return IsBeforeInTUCache.getCachedResult(LOffs.second, ROffs.second); } - // There is no common ancestor, most probably because one location is in the - // predefines buffer or an AST file. - // FIXME: We should rearrange the external interface so this simply never - // happens; it can't conceptually happen. Also see PR5662. - IsBeforeInTUCache.setQueryFIDs(FileID(), FileID()); // Don't try caching. - - // Zip both entries up to the top level record. - while (!MoveUpIncludeHierarchy(LOffs, *this)) /*empty*/; - while (!MoveUpIncludeHierarchy(ROffs, *this)) /*empty*/; - - // If exactly one location is a memory buffer, assume it precedes the other. - - // Strip off macro instantation locations, going up to the top-level File - // SLocEntry. - bool LIsMB = getFileEntryForID(LOffs.first) == 0; - bool RIsMB = getFileEntryForID(ROffs.first) == 0; - if (LIsMB != RIsMB) - return LIsMB; - - // Otherwise, just assume FileIDs were created in order. + // This can happen if a location is in a built-ins buffer. + // But see PR5662. + // Clear the lookup cache, it depends on a common location. + IsBeforeInTUCache.setQueryFIDs(FileID(), FileID()); + bool LIsBuiltins = strcmp("", + getBuffer(LOffs.first)->getBufferIdentifier()) == 0; + bool RIsBuiltins = strcmp("", + getBuffer(ROffs.first)->getBufferIdentifier()) == 0; + // built-in is before non-built-in + if (LIsBuiltins != RIsBuiltins) + return LIsBuiltins; + assert(LIsBuiltins && RIsBuiltins && + "Non-built-in locations must be rooted in the main file"); + // Both are in built-in buffers, but from different files. We just claim that + // lower IDs come first. return LOffs.first < ROffs.first; } @@ -1517,11 +1557,15 @@ void SourceManager::PrintStats() const { llvm::errs() << "\n*** Source Manager Stats:\n"; llvm::errs() << FileInfos.size() << " files mapped, " << MemBufferInfos.size() << " mem buffers mapped.\n"; - llvm::errs() << SLocEntryTable.size() << " SLocEntry's allocated (" - << SLocEntryTable.capacity()*sizeof(SrcMgr::SLocEntry) + llvm::errs() << LocalSLocEntryTable.size() << " local SLocEntry's allocated (" + << LocalSLocEntryTable.capacity()*sizeof(SrcMgr::SLocEntry) << " bytes of capacity), " - << NextOffset << "B of Sloc address space used.\n"; - + << NextLocalOffset << "B of Sloc address space used.\n"; + llvm::errs() << LoadedSLocEntryTable.size() + << " loaded SLocEntries allocated, " + << (1U << 31U) - CurrentLoadedOffset + << "B of Sloc address space used.\n"; + unsigned NumLineNumsComputed = 0; unsigned NumFileBytesMapped = 0; for (fileinfo_iterator I = fileinfo_begin(), E = fileinfo_end(); I != E; ++I){ diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp index 7e144ee9a1..f7c6695d71 100644 --- a/lib/Frontend/ASTUnit.cpp +++ b/lib/Frontend/ASTUnit.cpp @@ -976,7 +976,14 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) { if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0].second, Clang->getFrontendOpts().Inputs[0].first)) goto error; - + + if (OverrideMainBuffer) { + std::string ModName = "$" + PreambleFile; + TranslateStoredDiagnostics(Clang->getModuleManager(), ModName, + getSourceManager(), PreambleDiagnostics, + StoredDiagnostics); + } + Act->Execute(); // Steal the created target, context, and preprocessor. @@ -1170,7 +1177,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble( std::pair > NewPreamble = ComputePreamble(*PreambleInvocation, MaxLines, CreatedPreambleBuffer); - // If ComputePreamble() Take ownership of the + // If ComputePreamble() Take ownership of the preamble buffer. llvm::OwningPtr OwnedPreambleBuffer; if (CreatedPreambleBuffer) OwnedPreambleBuffer.reset(NewPreamble.first); @@ -1271,10 +1278,6 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble( ProcessWarningOptions(getDiagnostics(), PreambleInvocation->getDiagnosticOpts()); getDiagnostics().setNumWarnings(NumWarningsInPreamble); - if (StoredDiagnostics.size() > NumStoredDiagnosticsInPreamble) - StoredDiagnostics.erase( - StoredDiagnostics.begin() + NumStoredDiagnosticsInPreamble, - StoredDiagnostics.end()); // Create a version of the main file buffer that is padded to // buffer size we reserved when creating the preamble. @@ -1291,6 +1294,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble( // We can't reuse the previously-computed preamble. Build a new one. Preamble.clear(); + PreambleDiagnostics.clear(); llvm::sys::Path(PreambleFile).eraseFromDisk(); PreambleRebuildCounter = 1; } else if (!AllowRebuild) { @@ -1446,9 +1450,18 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble( return 0; } + // Transfer any diagnostics generated when parsing the preamble into the set + // of preamble diagnostics. + PreambleDiagnostics.clear(); + PreambleDiagnostics.insert(PreambleDiagnostics.end(), + StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver, + StoredDiagnostics.end()); + StoredDiagnostics.erase( + StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver, + StoredDiagnostics.end()); + // Keep track of the preamble we precompiled. PreambleFile = FrontendOpts.OutputFile; - NumStoredDiagnosticsInPreamble = StoredDiagnostics.size(); NumWarningsInPreamble = getDiagnostics().getNumWarnings(); // Keep track of all of the files that the source manager knows about, @@ -1776,6 +1789,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin, llvm::IntrusiveRefCntPtr CI; { + CaptureDroppedDiagnostics Capture(CaptureDiagnostics, *Diags, StoredDiagnostics); @@ -1826,7 +1840,6 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin, AST->CompleteTranslationUnit = CompleteTranslationUnit; AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults; AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size(); - AST->NumStoredDiagnosticsInPreamble = StoredDiagnostics.size(); AST->StoredDiagnostics.swap(StoredDiagnostics); AST->Invocation = CI; AST->NestedMacroExpansions = NestedMacroExpansions; @@ -2272,17 +2285,6 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column, PreprocessorOpts.ImplicitPCHInclude = PreambleFile; PreprocessorOpts.DisablePCHValidation = true; - // The stored diagnostics have the old source manager. Copy them - // to our output set of stored diagnostics, updating the source - // manager to the one we were given. - for (unsigned I = NumStoredDiagnosticsFromDriver, - N = this->StoredDiagnostics.size(); - I < N; ++I) { - StoredDiagnostics.push_back(this->StoredDiagnostics[I]); - FullSourceLoc Loc(StoredDiagnostics[I].getLocation(), SourceMgr); - StoredDiagnostics[I].setLocation(Loc); - } - OwnedBuffers.push_back(OverrideMainBuffer); } else { PreprocessorOpts.PrecompiledPreambleBytes.first = 0; @@ -2296,6 +2298,12 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column, Act.reset(new SyntaxOnlyAction); if (Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0].second, Clang->getFrontendOpts().Inputs[0].first)) { + if (OverrideMainBuffer) { + std::string ModName = "$" + PreambleFile; + TranslateStoredDiagnostics(Clang->getModuleManager(), ModName, + getSourceManager(), PreambleDiagnostics, + StoredDiagnostics); + } Act->Execute(); Act->EndSourceFile(); } @@ -2333,3 +2341,71 @@ bool ASTUnit::serialize(llvm::raw_ostream &OS) { return false; } + +typedef ContinuousRangeMap SLocRemap; + +static void TranslateSLoc(SourceLocation &L, SLocRemap &Remap) { + unsigned Raw = L.getRawEncoding(); + const unsigned MacroBit = 1U << 31; + L = SourceLocation::getFromRawEncoding((Raw & MacroBit) | + ((Raw & ~MacroBit) + Remap.find(Raw & ~MacroBit)->second)); +} + +void ASTUnit::TranslateStoredDiagnostics( + ASTReader *MMan, + llvm::StringRef ModName, + SourceManager &SrcMgr, + const llvm::SmallVectorImpl &Diags, + llvm::SmallVectorImpl &Out) { + // The stored diagnostic has the old source manager in it; update + // the locations to refer into the new source manager. We also need to remap + // all the locations to the new view. This includes the diag location, any + // associated source ranges, and the source ranges of associated fix-its. + // FIXME: There should be a cleaner way to do this. + + llvm::SmallVector Result; + Result.reserve(Diags.size()); + assert(MMan && "Don't have a module manager"); + ASTReader::PerFileData *Mod = MMan->Modules.lookup(ModName); + assert(Mod && "Don't have preamble module"); + SLocRemap &Remap = Mod->SLocRemap; + for (unsigned I = 0, N = Diags.size(); I != N; ++I) { + // Rebuild the StoredDiagnostic. + const StoredDiagnostic &SD = Diags[I]; + SourceLocation L = SD.getLocation(); + TranslateSLoc(L, Remap); + FullSourceLoc Loc(L, SrcMgr); + + llvm::SmallVector Ranges; + Ranges.reserve(SD.range_size()); + for (StoredDiagnostic::range_iterator I = SD.range_begin(), + E = SD.range_end(); + I != E; ++I) { + SourceLocation BL = I->getBegin(); + TranslateSLoc(BL, Remap); + SourceLocation EL = I->getEnd(); + TranslateSLoc(EL, Remap); + Ranges.push_back(CharSourceRange(SourceRange(BL, EL), I->isTokenRange())); + } + + llvm::SmallVector FixIts; + FixIts.reserve(SD.fixit_size()); + for (StoredDiagnostic::fixit_iterator I = SD.fixit_begin(), + E = SD.fixit_end(); + I != E; ++I) { + FixIts.push_back(FixItHint()); + FixItHint &FH = FixIts.back(); + FH.CodeToInsert = I->CodeToInsert; + SourceLocation BL = I->RemoveRange.getBegin(); + TranslateSLoc(BL, Remap); + SourceLocation EL = I->RemoveRange.getEnd(); + TranslateSLoc(EL, Remap); + FH.RemoveRange = CharSourceRange(SourceRange(BL, EL), + I->RemoveRange.isTokenRange()); + } + + Result.push_back(StoredDiagnostic(SD.getLevel(), SD.getID(), + SD.getMessage(), Loc, Ranges, FixIts)); + } + Result.swap(Out); +} diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp index c58e3af508..d9535735b5 100644 --- a/lib/Frontend/CompilerInstance.cpp +++ b/lib/Frontend/CompilerInstance.cpp @@ -42,7 +42,7 @@ using namespace clang; CompilerInstance::CompilerInstance() - : Invocation(new CompilerInvocation()) { + : Invocation(new CompilerInvocation()), ModuleManager(0) { } CompilerInstance::~CompilerInstance() { @@ -275,6 +275,7 @@ void CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path, getPreprocessor(), getASTContext(), DeserializationListener, Preamble)); + ModuleManager = static_cast(Source.get()); getASTContext().setExternalSource(Source); } diff --git a/lib/Frontend/FrontendAction.cpp b/lib/Frontend/FrontendAction.cpp index 0128d6ee05..344c85627f 100644 --- a/lib/Frontend/FrontendAction.cpp +++ b/lib/Frontend/FrontendAction.cpp @@ -224,9 +224,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, } else if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) { // Use PCH. assert(hasPCHSupport() && "This action does not have PCH support!"); - ASTDeserializationListener *DeserialListener - = CI.getInvocation().getFrontendOpts().ChainedPCH ? - Consumer->GetASTDeserializationListener() : 0; + ASTDeserializationListener *DeserialListener = + Consumer->GetASTDeserializationListener(); if (CI.getPreprocessorOpts().DumpDeserializedPCHDecls) DeserialListener = new DeserializedDeclsDumper(DeserialListener); if (!CI.getPreprocessorOpts().DeserializedPCHDeclsToErrorOn.empty()) diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index a28b8f6e7b..21522df75c 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -702,8 +702,8 @@ SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, /// \brief Returns true if the given MacroID location points at the first /// token of the macro expansion. bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, - const SourceManager &SM, - const LangOptions &LangOpts) { + const SourceManager &SM, + const LangOptions &LangOpts) { assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); std::pair infoLoc = SM.getDecomposedLoc(loc); @@ -735,7 +735,7 @@ bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, FileID FID = SM.getFileID(loc); SourceLocation afterLoc = loc.getFileLocWithOffset(tokLen+1); - if (!SM.isBeforeInSourceLocationOffset(afterLoc, SM.getNextOffset())) + if (!SM.isBeforeInSourceLocationOffset(afterLoc, SM.getNextLocalOffset())) return true; // We got past the last FileID, this points to the last token. // FIXME: If the token comes from the macro token paste operator ('##') diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp index 8ff82f1600..c3bf369fb1 100644 --- a/lib/Lex/TokenLexer.cpp +++ b/lib/Lex/TokenLexer.cpp @@ -43,7 +43,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroArgs *Actuals) { MacroExpansionStart = SourceLocation(); SourceManager &SM = PP.getSourceManager(); - MacroStartSLocOffset = SM.getNextOffset(); + MacroStartSLocOffset = SM.getNextLocalOffset(); if (NumTokens > 0) { assert(Tokens[0].getLocation().isValid()); diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp index a4ed5f4da4..61c3a5c2ff 100644 --- a/lib/Serialization/ASTReader.cpp +++ b/lib/Serialization/ASTReader.cpp @@ -49,6 +49,7 @@ #include #include #include +#include using namespace clang; using namespace clang::serialization; @@ -1010,6 +1011,9 @@ bool ASTReader::ParseLineTable(PerFileData &F, std::vector Entries; while (Idx < Record.size()) { int FID = Record[Idx++]; + assert(FID >= 0 && "Serialized line entries for non-local file."); + // Remap FileID from 1-based old view. + FID += F.SLocEntryBaseID - 1; // Extract the line entries unsigned NumEntries = Record[Idx++]; @@ -1188,11 +1192,6 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(PerFileData &F) { default: // Default behavior: ignore. break; - case SM_LINE_TABLE: - if (ParseLineTable(F, Record)) - return Failure; - break; - case SM_SLOC_FILE_ENTRY: case SM_SLOC_BUFFER_ENTRY: case SM_SLOC_EXPANSION_ENTRY: @@ -1235,38 +1234,20 @@ resolveFileRelativeToOriginalDir(const std::string &Filename, return currPCHPath.str(); } -/// \brief Get a cursor that's correctly positioned for reading the source -/// location entry with the given ID. -ASTReader::PerFileData *ASTReader::SLocCursorForID(unsigned ID) { - assert(ID != 0 && ID <= TotalNumSLocEntries && - "SLocCursorForID should only be called for real IDs."); - - ID -= 1; - PerFileData *F = 0; - for (unsigned I = 0, N = Chain.size(); I != N; ++I) { - F = Chain[N - I - 1]; - if (ID < F->LocalNumSLocEntries) - break; - ID -= F->LocalNumSLocEntries; - } - assert(F && F->LocalNumSLocEntries > ID && "Chain corrupted"); - - F->SLocEntryCursor.JumpToBit(F->SLocOffsets[ID]); - return F; -} - /// \brief Read in the source location entry with the given ID. -ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) { +ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) { if (ID == 0) return Success; - if (ID > TotalNumSLocEntries) { + if (unsigned(-ID) - 2 >= TotalNumSLocEntries || ID > 0) { Error("source location entry ID out-of-range for AST file"); return Failure; } - PerFileData *F = SLocCursorForID(ID); + PerFileData *F = GlobalSLocEntryMap.find(-ID)->second; + F->SLocEntryCursor.JumpToBit(F->SLocEntryOffsets[ID - F->SLocEntryBaseID]); llvm::BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor; + unsigned BaseOffset = F->SLocEntryBaseOffset; ++NumSLocEntriesRead; unsigned Code = SLocEntryCursor.ReadCode(); @@ -1326,9 +1307,14 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) { return Failure; } - FileID FID = SourceMgr.createFileID(File, ReadSourceLocation(*F, Record[1]), + SourceLocation IncludeLoc = ReadSourceLocation(*F, Record[1]); + if (IncludeLoc.isInvalid() && F->Type != MainFile) { + // This is the module's main file. + IncludeLoc = getImportLocation(F); + } + FileID FID = SourceMgr.createFileID(File, IncludeLoc, (SrcMgr::CharacteristicKind)Record[2], - ID, Record[0]); + ID, BaseOffset + Record[0]); if (Record[3]) const_cast(SourceMgr.getSLocEntry(FID).getFile()) .setHasLineDirectives(); @@ -1352,7 +1338,8 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) { llvm::MemoryBuffer *Buffer = llvm::MemoryBuffer::getMemBuffer(llvm::StringRef(BlobStart, BlobLen - 1), Name); - FileID BufferID = SourceMgr.createFileIDForMemBuffer(Buffer, ID, Offset); + FileID BufferID = SourceMgr.createFileIDForMemBuffer(Buffer, ID, + BaseOffset + Offset); if (strcmp(Name, "") == 0) { PCHPredefinesBlock Block = { @@ -1372,7 +1359,7 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) { ReadSourceLocation(*F, Record[3]), Record[4], ID, - Record[0]); + BaseOffset + Record[0]); break; } } @@ -1380,6 +1367,23 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) { return Success; } +/// \brief Find the location where the module F is imported. +SourceLocation ASTReader::getImportLocation(PerFileData *F) { + if (F->ImportLoc.isValid()) + return F->ImportLoc; + // Otherwise we have a PCH. It's considered to be "imported" at the first + // location of its includer. + if (F->Loaders.empty() || !F->Loaders[0]) { + // Main file is the importer. We assume that it is the first entry in the + // entry table. We can't ask the manager, because at the time of PCH loading + // the main file entry doesn't exist yet. + // The very first entry is the invalid instantiation loc, which takes up + // offsets 0 and 1. + return SourceLocation::getFromRawEncoding(2U); + } + return F->Loaders[0]->FirstLoc; +} + /// ReadBlockAbbrevs - Enter a subblock of the specified BlockID with the /// specified cursor. Read the abbreviations that are at the top of the block /// and then leave the cursor pointing into the block. @@ -2191,10 +2195,53 @@ ASTReader::ReadASTBlock(PerFileData &F) { Listener->ReadCounter(Record[0]); break; - case SOURCE_LOCATION_OFFSETS: - F.SLocOffsets = (const uint32_t *)BlobStart; + case SOURCE_LOCATION_OFFSETS: { + F.SLocEntryOffsets = (const uint32_t *)BlobStart; F.LocalNumSLocEntries = Record[0]; - F.LocalSLocSize = Record[1]; + llvm::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) = + SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries, Record[1]); + // Make our entry in the range map. BaseID is negative and growing, so + // we invert it. Because we invert it, though, we need the other end of + // the range. + unsigned RangeStart = + unsigned(-F.SLocEntryBaseID) - F.LocalNumSLocEntries + 1; + GlobalSLocEntryMap.insert(std::make_pair(RangeStart, &F)); + F.FirstLoc = SourceLocation::getFromRawEncoding(F.SLocEntryBaseOffset); + + // Initialize the remapping table. + // Invalid stays invalid. + F.SLocRemap.insert(std::make_pair(0U, 0)); + // This module. Base was 2 when being compiled. + F.SLocRemap.insert(std::make_pair(2U, + static_cast(F.SLocEntryBaseOffset - 2))); + break; + } + + case SOURCE_LOCATION_MAP: { + // Additional remapping information. + const unsigned char *Data = (const unsigned char*)BlobStart; + const unsigned char *DataEnd = Data + BlobLen; + while(Data < DataEnd) { + uint32_t Offset = io::ReadUnalignedLE32(Data); + uint16_t Len = io::ReadUnalignedLE16(Data); + llvm::StringRef Name = llvm::StringRef((const char*)Data, Len); + PerFileData *OM = Modules.lookup(Name); + if (!OM) { + Error("SourceLocation remap refers to unknown module"); + return Failure; + } + // My Offset is mapped to OM->SLocEntryBaseOffset. + F.SLocRemap.insert(std::make_pair(Offset, + static_cast(OM->SLocEntryBaseOffset - Offset))); + Data += Len; + } + break; + } + + + case SOURCE_MANAGER_LINE_TABLE: + if (ParseLineTable(F, Record)) + return Failure; break; case FILE_SOURCE_LOCATION_OFFSETS: @@ -2202,13 +2249,14 @@ ASTReader::ReadASTBlock(PerFileData &F) { F.LocalNumSLocFileEntries = Record[0]; break; - case SOURCE_LOCATION_PRELOADS: - if (PreloadSLocEntries.empty()) - PreloadSLocEntries.swap(Record); - else - PreloadSLocEntries.insert(PreloadSLocEntries.end(), - Record.begin(), Record.end()); + case SOURCE_LOCATION_PRELOADS: { + // Need to transform from the local view (1-based IDs) to the global view, + // which is based off F.SLocEntryBaseID. + PreloadSLocEntries.reserve(PreloadSLocEntries.size() + Record.size()); + for (unsigned I = 0, N = Record.size(); I != N; ++I) + PreloadSLocEntries.push_back(int(Record[I] - 1) + F.SLocEntryBaseID); break; + } case STAT_CACHE: { if (!DisableStatCache) { @@ -2326,11 +2374,12 @@ ASTReader::ReadASTBlock(PerFileData &F) { Error("invalid DIAG_USER_MAPPINGS block in AST file"); return Failure; } - if (PragmaDiagMappings.empty()) - PragmaDiagMappings.swap(Record); + + if (F.PragmaDiagMappings.empty()) + F.PragmaDiagMappings.swap(Record); else - PragmaDiagMappings.insert(PragmaDiagMappings.end(), - Record.begin(), Record.end()); + F.PragmaDiagMappings.insert(F.PragmaDiagMappings.end(), + Record.begin(), Record.end()); break; case CUDA_SPECIAL_DECL_REFS: @@ -2478,7 +2527,6 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, TotalNumSelectors = 0; for (unsigned I = 0, N = Chain.size(); I != N; ++I) { TotalNumSLocEntries += Chain[I]->LocalNumSLocEntries; - NextSLocOffset += Chain[I]->LocalSLocSize; TotalNumIdentifiers += Chain[I]->LocalNumIdentifiers; TotalNumTypes += Chain[I]->LocalNumTypes; TotalNumDecls += Chain[I]->LocalNumDecls; @@ -2487,7 +2535,6 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, TotalNumMacroDefs += Chain[I]->LocalNumMacroDefinitions; TotalNumSelectors += Chain[I]->LocalNumSelectors; } - SourceMgr.PreallocateSLocEntries(this, TotalNumSLocEntries, NextSLocOffset); IdentifiersLoaded.resize(TotalNumIdentifiers); TypesLoaded.resize(TotalNumTypes); DeclsLoaded.resize(TotalNumDecls); @@ -2507,7 +2554,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, for (unsigned I = 0, N = PreloadSLocEntries.size(); I != N; ++I) { ASTReadResult Result = ReadSLocEntryRecord(PreloadSLocEntries[I]); if (Result != Success) - return Result; + return Failure; } // Check the predefines buffers. @@ -2572,7 +2619,11 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, if (Loc.isValid()) OriginalFileID = SourceMgr.getDecomposedLoc(Loc).first; } - + else { + OriginalFileID = FileID::get(Chain[0]->SLocEntryBaseID + + OriginalFileID.getOpaqueValue() - 1); + } + if (!OriginalFileID.isInvalid()) SourceMgr.SetPreambleFileID(OriginalFileID); } @@ -2590,6 +2641,8 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName, else FirstInSource = &F; F.Loaders.push_back(Prev); + // A non-module AST file's module name is $filename. + Modules["$" + FileName.str()] = &F; // Set the AST file name. F.FileName = FileName; @@ -2668,9 +2721,8 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName, // AST block, skipping subblocks, to see if there are other // AST blocks elsewhere. - // Clear out any preallocated source location entries, so that - // the source manager does not try to resolve them later. - SourceMgr.ClearPreallocatedSLocEntries(); + // FIXME: We can't clear loaded slocentries anymore. + //SourceMgr.ClearPreallocatedSLocEntries(); // Remove the stat cache. if (F.StatCache) @@ -3062,21 +3114,25 @@ HeaderFileInfo ASTReader::GetHeaderFileInfo(const FileEntry *FE) { } void ASTReader::ReadPragmaDiagnosticMappings(Diagnostic &Diag) { - unsigned Idx = 0; - while (Idx < PragmaDiagMappings.size()) { - SourceLocation - Loc = SourceLocation::getFromRawEncoding(PragmaDiagMappings[Idx++]); - while (1) { - assert(Idx < PragmaDiagMappings.size() && - "Invalid data, didn't find '-1' marking end of diag/map pairs"); - if (Idx >= PragmaDiagMappings.size()) - break; // Something is messed up but at least avoid infinite loop in - // release build. - unsigned DiagID = PragmaDiagMappings[Idx++]; - if (DiagID == (unsigned)-1) - break; // no more diag/map pairs for this location. - diag::Mapping Map = (diag::Mapping)PragmaDiagMappings[Idx++]; - Diag.setDiagnosticMapping(DiagID, Map, Loc); + for (unsigned I = 0, N = Chain.size(); I != N; ++I) { + PerFileData &F = *Chain[I]; + unsigned Idx = 0; + while (Idx < F.PragmaDiagMappings.size()) { + SourceLocation Loc = ReadSourceLocation(F, F.PragmaDiagMappings[Idx++]); + while (1) { + assert(Idx < F.PragmaDiagMappings.size() && + "Invalid data, didn't find '-1' marking end of diag/map pairs"); + if (Idx >= F.PragmaDiagMappings.size()) { + break; // Something is messed up but at least avoid infinite loop in + // release build. + } + unsigned DiagID = F.PragmaDiagMappings[Idx++]; + if (DiagID == (unsigned)-1) { + break; // no more diag/map pairs for this location. + } + diag::Mapping Map = (diag::Mapping)F.PragmaDiagMappings[Idx++]; + Diag.setDiagnosticMapping(DiagID, Map, Loc); + } } } } @@ -4571,7 +4627,7 @@ IdentifierInfo *ASTReader::DecodeIdentifierInfo(unsigned ID) { return IdentifiersLoaded[ID]; } -bool ASTReader::ReadSLocEntry(unsigned ID) { +bool ASTReader::ReadSLocEntry(int ID) { return ReadSLocEntryRecord(ID) != Success; } @@ -5175,9 +5231,10 @@ ASTReader::ASTReader(Preprocessor &PP, ASTContext *Context, : Listener(new PCHValidator(PP, *this)), DeserializationListener(0), SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()), Diags(PP.getDiagnostics()), SemaObj(0), PP(&PP), Context(Context), - Consumer(0), isysroot(isysroot), DisableValidation(DisableValidation), + Consumer(0), FirstInSource(0), RelocatablePCH(false), isysroot(isysroot), + DisableValidation(DisableValidation), DisableStatCache(DisableStatCache), NumStatHits(0), NumStatMisses(0), - NumSLocEntriesRead(0), TotalNumSLocEntries(0), NextSLocOffset(0), + NumSLocEntriesRead(0), TotalNumSLocEntries(0), NumStatementsRead(0), TotalNumStatements(0), NumMacrosRead(0), TotalNumMacros(0), NumSelectorsRead(0), NumMethodPoolEntriesRead(0), NumMethodPoolMisses(0), TotalNumMethodPoolEntries(0), @@ -5185,24 +5242,25 @@ ASTReader::ASTReader(Preprocessor &PP, ASTContext *Context, NumVisibleDeclContextsRead(0), TotalVisibleDeclContexts(0), NumCurrentElementsDeserializing(0) { - RelocatablePCH = false; + SourceMgr.setExternalSLocEntrySource(this); } ASTReader::ASTReader(SourceManager &SourceMgr, FileManager &FileMgr, Diagnostic &Diags, const char *isysroot, bool DisableValidation, bool DisableStatCache) : DeserializationListener(0), SourceMgr(SourceMgr), FileMgr(FileMgr), - Diags(Diags), SemaObj(0), PP(0), Context(0), Consumer(0), - isysroot(isysroot), DisableValidation(DisableValidation), - DisableStatCache(DisableStatCache), NumStatHits(0), NumStatMisses(0), - NumSLocEntriesRead(0), TotalNumSLocEntries(0), - NextSLocOffset(0), NumStatementsRead(0), TotalNumStatements(0), - NumMacrosRead(0), TotalNumMacros(0), NumSelectorsRead(0), - NumMethodPoolEntriesRead(0), NumMethodPoolMisses(0), + Diags(Diags), SemaObj(0), PP(0), Context(0), Consumer(0), FirstInSource(0), + RelocatablePCH(false), isysroot(isysroot), + DisableValidation(DisableValidation), DisableStatCache(DisableStatCache), + NumStatHits(0), NumStatMisses(0), NumSLocEntriesRead(0), + TotalNumSLocEntries(0), NumStatementsRead(0), + TotalNumStatements(0), NumMacrosRead(0), TotalNumMacros(0), + NumSelectorsRead(0), NumMethodPoolEntriesRead(0), NumMethodPoolMisses(0), TotalNumMethodPoolEntries(0), NumLexicalDeclContextsRead(0), TotalLexicalDeclContexts(0), NumVisibleDeclContextsRead(0), - TotalVisibleDeclContexts(0), NumCurrentElementsDeserializing(0) { - RelocatablePCH = false; + TotalVisibleDeclContexts(0), NumCurrentElementsDeserializing(0) +{ + SourceMgr.setExternalSLocEntrySource(this); } ASTReader::~ASTReader() { @@ -5231,13 +5289,13 @@ ASTReader::~ASTReader() { } ASTReader::PerFileData::PerFileData(ASTFileType Ty) - : Type(Ty), SizeInBits(0), LocalNumSLocEntries(0), SLocOffsets(0), - SLocFileOffsets(0), LocalSLocSize(0), - LocalNumIdentifiers(0), IdentifierOffsets(0), IdentifierTableData(0), + : Type(Ty), SizeInBits(0), LocalNumSLocEntries(0), SLocEntryBaseID(0), + SLocEntryBaseOffset(0), SLocEntryOffsets(0), + SLocFileOffsets(0), LocalNumIdentifiers(0), + IdentifierOffsets(0), IdentifierTableData(0), IdentifierLookupTable(0), LocalNumMacroDefinitions(0), - MacroDefinitionOffsets(0), - LocalNumHeaderFileInfos(0), HeaderFileInfoTableData(0), - HeaderFileInfoTable(0), + MacroDefinitionOffsets(0), LocalNumHeaderFileInfos(0), + HeaderFileInfoTableData(0), HeaderFileInfoTable(0), LocalNumSelectors(0), SelectorOffsets(0), SelectorLookupTableData(0), SelectorLookupTable(0), LocalNumDecls(0), DeclOffsets(0), LocalNumCXXBaseSpecifiers(0), CXXBaseSpecifiersOffsets(0), diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp index 39f2fbddf8..a3a03a5b1f 100644 --- a/lib/Serialization/ASTWriter.cpp +++ b/lib/Serialization/ASTWriter.cpp @@ -45,8 +45,10 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" +#include #include #include +#include using namespace clang; using namespace clang::serialization; @@ -790,7 +792,6 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(SM_SLOC_BUFFER_ENTRY); RECORD(SM_SLOC_BUFFER_BLOB); RECORD(SM_SLOC_EXPANSION_ENTRY); - RECORD(SM_LINE_TABLE); // Preprocessor Block. BLOCK(PREPROCESSOR_BLOCK); @@ -1416,43 +1417,6 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, unsigned SLocBufferBlobAbbrv = CreateSLocBufferBlobAbbrev(Stream); unsigned SLocExpansionAbbrv = CreateSLocExpansionAbbrev(Stream); - // Write the line table. - if (SourceMgr.hasLineTable()) { - LineTableInfo &LineTable = SourceMgr.getLineTable(); - - // Emit the file names - Record.push_back(LineTable.getNumFilenames()); - for (unsigned I = 0, N = LineTable.getNumFilenames(); I != N; ++I) { - // Emit the file name - const char *Filename = LineTable.getFilename(I); - Filename = adjustFilenameForRelocatablePCH(Filename, isysroot); - unsigned FilenameLen = Filename? strlen(Filename) : 0; - Record.push_back(FilenameLen); - if (FilenameLen) - Record.insert(Record.end(), Filename, Filename + FilenameLen); - } - - // Emit the line entries - for (LineTableInfo::iterator L = LineTable.begin(), LEnd = LineTable.end(); - L != LEnd; ++L) { - // Emit the file ID - Record.push_back(L->first); - - // Emit the line entries - Record.push_back(L->second.size()); - for (std::vector::iterator LE = L->second.begin(), - LEEnd = L->second.end(); - LE != LEEnd; ++LE) { - Record.push_back(LE->FileOffset); - Record.push_back(LE->LineNo); - Record.push_back(LE->FilenameID); - Record.push_back((unsigned)LE->FileKind); - Record.push_back(LE->IncludeOffset); - } - } - Stream.EmitRecord(SM_LINE_TABLE, Record); - } - // Write out the source location entry table. We skip the first // entry, which is always the same dummy entry. std::vector SLocEntryOffsets; @@ -1460,12 +1424,11 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, // We will go through them in ASTReader::validateFileEntries(). std::vector SLocFileEntryOffsets; RecordData PreloadSLocs; - unsigned BaseSLocID = Chain ? Chain->getTotalNumSLocs() : 0; - SLocEntryOffsets.reserve(SourceMgr.sloc_entry_size() - 1 - BaseSLocID); - for (unsigned I = BaseSLocID + 1, N = SourceMgr.sloc_entry_size(); + SLocEntryOffsets.reserve(SourceMgr.local_sloc_entry_size() - 1); + for (unsigned I = 1, N = SourceMgr.local_sloc_entry_size(); I != N; ++I) { // Get this source location entry. - const SrcMgr::SLocEntry *SLoc = &SourceMgr.getSLocEntry(I); + const SrcMgr::SLocEntry *SLoc = &SourceMgr.getLocalSLocEntry(I); // Record the offset of this source-location entry. SLocEntryOffsets.push_back(Stream.GetCurrentBitNo()); @@ -1483,7 +1446,8 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, Record.clear(); Record.push_back(Code); - Record.push_back(SLoc->getOffset()); + // Starting offset of this entry within this module, so skip the dummy. + Record.push_back(SLoc->getOffset() - 2); if (SLoc->isFile()) { const SrcMgr::FileInfo &File = SLoc->getFile(); Record.push_back(File.getIncludeLoc().getRawEncoding()); @@ -1535,8 +1499,9 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, llvm::StringRef(Buffer->getBufferStart(), Buffer->getBufferSize() + 1)); - if (strcmp(Name, "") == 0) - PreloadSLocs.push_back(BaseSLocID + SLocEntryOffsets.size()); + if (strcmp(Name, "") == 0) { + PreloadSLocs.push_back(SLocEntryOffsets.size()); + } } } else { // The source location entry is a macro expansion. @@ -1546,9 +1511,9 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, Record.push_back(Inst.getInstantiationLocEnd().getRawEncoding()); // Compute the token length for this macro expansion. - unsigned NextOffset = SourceMgr.getNextOffset(); + unsigned NextOffset = SourceMgr.getNextLocalOffset(); if (I + 1 != N) - NextOffset = SourceMgr.getSLocEntry(I + 1).getOffset(); + NextOffset = SourceMgr.getLocalSLocEntry(I + 1).getOffset(); Record.push_back(NextOffset - SLoc->getOffset() - 1); Stream.EmitRecordWithAbbrev(SLocExpansionAbbrv, Record); } @@ -1565,17 +1530,54 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, BitCodeAbbrev *Abbrev = new BitCodeAbbrev(); Abbrev->Add(BitCodeAbbrevOp(SOURCE_LOCATION_OFFSETS)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // # of slocs - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // next offset + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // total size Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // offsets unsigned SLocOffsetsAbbrev = Stream.EmitAbbrev(Abbrev); Record.clear(); Record.push_back(SOURCE_LOCATION_OFFSETS); Record.push_back(SLocEntryOffsets.size()); - unsigned BaseOffset = Chain ? Chain->getNextSLocOffset() : 0; - Record.push_back(SourceMgr.getNextOffset() - BaseOffset); + Record.push_back(SourceMgr.getNextLocalOffset() - 1); // skip dummy Stream.EmitRecordWithBlob(SLocOffsetsAbbrev, Record, data(SLocEntryOffsets)); + // If we have module dependencies, write the mapping from source locations to + // their containing modules, so that the reader can build the remapping. + if (Chain) { + // The map consists solely of a blob with the following format: + // *(offset:i32 len:i16 name:len*i8) + // Sorted by offset. + typedef std::pair ModuleOffset; + llvm::SmallVector Modules; + Modules.reserve(Chain->Modules.size()); + for (llvm::StringMap::const_iterator + I = Chain->Modules.begin(), E = Chain->Modules.end(); + I != E; ++I) { + Modules.push_back(ModuleOffset(I->getValue()->SLocEntryBaseOffset, + I->getKey())); + } + std::sort(Modules.begin(), Modules.end()); + + Abbrev = new BitCodeAbbrev(); + Abbrev->Add(BitCodeAbbrevOp(SOURCE_LOCATION_MAP)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned SLocMapAbbrev = Stream.EmitAbbrev(Abbrev); + llvm::SmallString<2048> Buffer; + { + llvm::raw_svector_ostream Out(Buffer); + for (llvm::SmallVector::iterator I = Modules.begin(), + E = Modules.end(); + I != E; ++I) { + io::Emit32(Out, I->first); + io::Emit16(Out, I->second.size()); + Out.write(I->second.data(), I->second.size()); + } + } + Record.clear(); + Record.push_back(SOURCE_LOCATION_MAP); + Stream.EmitRecordWithBlob(SLocMapAbbrev, Record, + Buffer.data(), Buffer.size()); + } + Abbrev = new BitCodeAbbrev(); Abbrev->Add(BitCodeAbbrevOp(FILE_SOURCE_LOCATION_OFFSETS)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // # of slocs @@ -1591,6 +1593,49 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, // Write the source location entry preloads array, telling the AST // reader which source locations entries it should load eagerly. Stream.EmitRecord(SOURCE_LOCATION_PRELOADS, PreloadSLocs); + + // Write the line table. It depends on remapping working, so it must come + // after the source location offsets. + if (SourceMgr.hasLineTable()) { + LineTableInfo &LineTable = SourceMgr.getLineTable(); + + Record.clear(); + // Emit the file names + Record.push_back(LineTable.getNumFilenames()); + for (unsigned I = 0, N = LineTable.getNumFilenames(); I != N; ++I) { + // Emit the file name + const char *Filename = LineTable.getFilename(I); + Filename = adjustFilenameForRelocatablePCH(Filename, isysroot); + unsigned FilenameLen = Filename? strlen(Filename) : 0; + Record.push_back(FilenameLen); + if (FilenameLen) + Record.insert(Record.end(), Filename, Filename + FilenameLen); + } + + // Emit the line entries + for (LineTableInfo::iterator L = LineTable.begin(), LEnd = LineTable.end(); + L != LEnd; ++L) { + // Only emit entries for local files. + if (L->first < 0) + continue; + + // Emit the file ID + Record.push_back(L->first); + + // Emit the line entries + Record.push_back(L->second.size()); + for (std::vector::iterator LE = L->second.begin(), + LEEnd = L->second.end(); + LE != LEEnd; ++LE) { + Record.push_back(LE->FileOffset); + Record.push_back(LE->LineNo); + Record.push_back(LE->FilenameID); + Record.push_back((unsigned)LE->FileKind); + Record.push_back(LE->IncludeOffset); + } + } + Stream.EmitRecord(SOURCE_MANAGER_LINE_TABLE, Record); + } } //===----------------------------------------------------------------------===// @@ -3926,6 +3971,7 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) { FirstMacroID == NextMacroID && FirstCXXBaseSpecifiersID == NextCXXBaseSpecifiersID && "Setting chain after writing has started."); + Chain = Reader; FirstDeclID += Chain->getTotalNumDecls(); diff --git a/test/Index/Inputs/preamble_macro_template.h b/test/Index/Inputs/preamble_macro_template.h new file mode 100644 index 0000000000..18b076d95c --- /dev/null +++ b/test/Index/Inputs/preamble_macro_template.h @@ -0,0 +1,6 @@ +#define STATIC_CAST static_cast + +template +void foo(T *p) { + (void)STATIC_CAST(0); +} diff --git a/test/Index/preamble_macro_template.cpp b/test/Index/preamble_macro_template.cpp new file mode 100644 index 0000000000..dc98927101 --- /dev/null +++ b/test/Index/preamble_macro_template.cpp @@ -0,0 +1,15 @@ +template void foo(int *); + +int main() { } + +// RUN: c-index-test -write-pch %t.pch -x c++-header %S/Inputs/preamble_macro_template.h +// RUN: env CINDEXTEST_EDITING=1 c-index-test -test-load-source-reparse 5 local -I %S/Inputs -include %t %s | FileCheck %s +// CHECK: preamble_macro_template.h:4:6: FunctionDecl=foo:4:6 (Definition) [Specialization of foo:4:6] Extent=[4:1 - 6:2] +// CHECK: preamble_macro_template.h:4:13: ParmDecl=p:4:13 (Definition) Extent=[4:10 - 4:14] +// CHECK: preamble_macro_template.h:4:16: UnexposedStmt= Extent=[4:16 - 6:2] +// CHECK: preamble_macro_template.h:5:3: UnexposedExpr= Extent=[5:3 - 5:27] +// CHECK: preamble_macro_template.h:1:21: UnexposedExpr= Extent=[1:21 - 5:27] +// CHECK: preamble_macro_template.h:5:25: UnexposedExpr= Extent=[5:25 - 5:26] +// CHECK: preamble_macro_template.h:5:25: UnexposedExpr= Extent=[5:25 - 5:26] +// CHECK: preamble_macro_template.cpp:3:5: FunctionDecl=main:3:5 (Definition) Extent=[3:1 - 3:15] +// CHECK: preamble_macro_template.cpp:3:12: UnexposedStmt= Extent=[3:12 - 3:15] diff --git a/test/PCH/reinclude.cpp b/test/PCH/reinclude.cpp index 71f90282ba..97e22cf9d6 100644 --- a/test/PCH/reinclude.cpp +++ b/test/PCH/reinclude.cpp @@ -4,7 +4,7 @@ // RUN: %clang_cc1 -x c++-header %S/reinclude1.h -emit-pch -o %t1 // RUN: %clang_cc1 -x c++-header %S/reinclude2.h -include-pch %t1 -emit-pch -o %t2 // RUN: %clang_cc1 %s -include-pch %t2 -fsyntax-only -verify -// RUN: %clang_cc1 -x c++-header %S/reinclude2.h -include-pch %t1 -emit-pch -o %t2 -chained-pch +// RUN: %clang_cc1 -x c++-header %S/reinclude2.h -include-pch %t1 -emit-pch -o %t2 // RUN: %clang_cc1 %s -include-pch %t2 -fsyntax-only -verify int q2 = A::y; diff --git a/tools/libclang/CIndexInclusionStack.cpp b/tools/libclang/CIndexInclusionStack.cpp index 6bc4f2e776..b78a0a061a 100644 --- a/tools/libclang/CIndexInclusionStack.cpp +++ b/tools/libclang/CIndexInclusionStack.cpp @@ -30,18 +30,21 @@ void clang_getInclusions(CXTranslationUnit TU, CXInclusionVisitor CB, ASTContext &Ctx = CXXUnit->getASTContext(); llvm::SmallVector InclusionStack; - unsigned i = SM.sloc_loaded_entry_size(); - unsigned n = SM.sloc_entry_size(); + unsigned n = SM.local_sloc_entry_size(); // In the case where all the SLocEntries are in an external source, traverse // those SLocEntries as well. This is the case where we are looking // at the inclusion stack of an AST/PCH file. - if (i >= n) - i = 0; - - for ( ; i < n ; ++i) { + const SrcMgr::SLocEntry &(SourceManager::*Getter)(unsigned, bool*) const; + if (n == 1) { + Getter = &SourceManager::getLoadedSLocEntry; + n = SM.loaded_sloc_entry_size(); + } else + Getter = &SourceManager::getLocalSLocEntry; + + for (unsigned i = 0 ; i < n ; ++i) { bool Invalid = false; - const SrcMgr::SLocEntry &SL = SM.getSLocEntry(i, &Invalid); + const SrcMgr::SLocEntry &SL = (SM.*Getter)(i, &Invalid); if (!SL.isFile() || Invalid) continue; -- 2.40.0