From: Ted Kremenek Date: Tue, 2 Dec 2008 19:45:05 +0000 (+0000) Subject: Added PTHManager, a utility class that will be used by Preprocessor to lazily create... X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=be1ee79d20db9cec7ef81d7a22ba2eaddc9c95b5;p=clang Added PTHManager, a utility class that will be used by Preprocessor to lazily create PTHLexer objects for pre-tokenized files. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@60436 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/clang/Lex/PTHManager.h b/include/clang/Lex/PTHManager.h new file mode 100644 index 0000000000..fc37a8a151 --- /dev/null +++ b/include/clang/Lex/PTHManager.h @@ -0,0 +1,87 @@ +//===--- PTHManager.h - Manager object for PTH processing -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the PTHManager interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_PTHMANAGER_H +#define LLVM_CLANG_PTHMANAGER_H + +#include "clang/Lex/PTHLexer.h" +#include "clang/Basic/LangOptions.h" +#include + +namespace llvm { + class MemoryBuffer; +} + +namespace clang { + +class FileEntry; +class IdentifierInfo; +class IdentifierTable; + +class PTHManager { + /// The memory mapped PTH file. + const llvm::MemoryBuffer* Buf; + + /// IdMap - A lazily generated cache mapping from persistent identifiers to + /// IdentifierInfo*. + void* PersistentIDCache; + + /// FileLookup - Abstract data structure used for mapping between files + /// and token data in the PTH file. + void* FileLookup; + + /// IdDataTable - Array representing the mapping from persistent IDs to the + /// data offset within the PTH file containing the information to + /// reconsitute an IdentifierInfo. + const char* IdDataTable; + + /// ITable - The IdentifierTable used for the translation unit being lexed. + IdentifierTable& ITable; + + /// PP - The Preprocessor object that will use this PTHManager to create + /// PTHLexer objects. + Preprocessor& PP; + + /// This constructor is intended to only be called by the static 'Create' + /// method. + PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup, + const char* idDataTable, Preprocessor& pp); + + // Do not implement. + PTHManager(); + void operator=(const PTHManager&); + + /// ReadIdentifierInfo - Used by PTHManager to reconstruct IdentifierInfo + /// objects from the PTH file. + IdentifierInfo* ReadIdentifierInfo(const char*& D); + + /// ReadToken - Used by PTHManager to read tokens from the PTH file. + void ReadToken(const char*& D, unsigned FileID, Token& T); + +public: + + ~PTHManager(); + + /// Create - This method creates PTHManager objects. The 'file' argument + /// is the name of the PTH file. This method returns NULL upon failure. + static PTHManager* Create(const std::string& file, Preprocessor& PP); + + /// CreateLexer - Return a PTHLexer that "lexes" the cached tokens for the + /// specified file. This method returns NULL if no cached tokens exist. + /// It is the responsibility of the caller to 'delete' the returned object. + PTHLexer* CreateLexer(unsigned FileID, const FileEntry* FE); +}; + +} // end namespace clang + +#endif diff --git a/lib/Lex/PTHManager.cpp b/lib/Lex/PTHManager.cpp new file mode 100644 index 0000000000..594f55d454 --- /dev/null +++ b/lib/Lex/PTHManager.cpp @@ -0,0 +1,248 @@ +//===--- PTHManager.cpp - Manager object for PTH processing -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the PTHManager interface. +// +//===----------------------------------------------------------------------===// + +#include "clang/Lex/PTHManager.h" +#include "clang/Lex/Token.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Basic/FileManager.h" +#include "clang/Basic/IdentifierTable.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/OwningPtr.h" +#include "llvm/ADT/DenseMap.h" + +using namespace clang; + +//===----------------------------------------------------------------------===// +// Utility methods for reading from the mmap'ed PTH file. +//===----------------------------------------------------------------------===// + +static uint8_t Read8(const char*& data) { + return (uint8_t) *(data++); +} + +static uint32_t Read32(const char*& data) { + uint32_t V = (uint32_t) Read8(data); + V |= (((uint32_t) Read8(data)) << 8); + V |= (((uint32_t) Read8(data)) << 16); + V |= (((uint32_t) Read8(data)) << 24); + return V; +} + +//===----------------------------------------------------------------------===// +// Internal Data Structures. +//===----------------------------------------------------------------------===// + +typedef llvm::DenseMap IDCache; + +/// PTHFileLookup - This internal data structure is used by the PTHManager +/// to map from FileEntry objects managed by FileManager to offsets within +/// the PTH file. +namespace { +class VISIBILITY_HIDDEN PTHFileLookup { +public: + class Val { + uint32_t v; + + public: + Val() : v(~0) {} + Val(uint32_t x) : v(x) {} + + operator uint32_t() const { + assert(v != ~((uint32_t)0) && "PTHFileLookup entry initialized."); + return v; + } + + Val& operator=(uint32_t x) { v = x; return *this; } + bool isValid() const { return v != ~((uint32_t)0); } + }; + +private: + llvm::StringMap FileMap; + +public: + PTHFileLookup() {}; + + Val Lookup(const FileEntry* FE) { + const char* s = FE->getName(); + unsigned size = strlen(s); + return FileMap.GetOrCreateValue(s, s+size).getValue(); + } + + void ReadTable(const char* D) { + uint32_t N = Read32(D); // Read the length of the table. + + for ( ; N > 0; --N) { // The rest of the data is the table itself. + uint32_t len = Read32(D); + const char* s = D; + D += len; + FileMap.GetOrCreateValue(s, s+len).getValue() = Read32(D); + } + } +}; +} // end anonymous namespace + +//===----------------------------------------------------------------------===// +// PTHManager methods. +//===----------------------------------------------------------------------===// + +PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup, + const char* idDataTable, Preprocessor& pp) + : Buf(buf), PersistentIDCache(0), FileLookup(fileLookup), + IdDataTable(idDataTable), ITable(pp.getIdentifierTable()), PP(pp) {} + +PTHManager::~PTHManager() { + delete Buf; + delete (PTHFileLookup*) FileLookup; + delete (IDCache*) PersistentIDCache; +} + +PTHManager* PTHManager::Create(const std::string& file, Preprocessor& PP) { + + // Memory map the PTH file. + llvm::OwningPtr + File(llvm::MemoryBuffer::getFile(file.c_str())); + + if (!File) + return 0; + + // Get the buffer ranges and check if there are at least three 32-bit + // words at the end of the file. + const char* BufBeg = File->getBufferStart(); + const char* BufEnd = File->getBufferEnd(); + + if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) { + assert(false && "Invalid PTH file."); + return 0; // FIXME: Proper error diagnostic? + } + + // Compute the address of the index table at the end of the PTH file. + // This table contains the offset of the file lookup table, the + // persistent ID -> identifer data table. + const char* EndTable = BufEnd - sizeof(uint32_t)*3; + + // Construct the file lookup table. This will be used for mapping from + // FileEntry*'s to cached tokens. + const char* FileTableOffset = EndTable + sizeof(uint32_t)*2; + const char* FileTable = BufBeg + Read32(FileTableOffset); + + if (!(FileTable > BufBeg && FileTable < BufEnd)) { + assert(false && "Invalid PTH file."); + return 0; // FIXME: Proper error diagnostic? + } + + llvm::OwningPtr FL(new PTHFileLookup()); + FL->ReadTable(FileTable); + + // Get the location of the table mapping from persistent ids to the + // data needed to reconstruct identifiers. + const char* IDTableOffset = EndTable + sizeof(uint32_t)*1; + const char* IData = BufBeg + Read32(IDTableOffset); + if (!(IData > BufBeg && IData < BufEnd)) { + assert(false && "Invalid PTH file."); + return 0; // FIXME: Proper error diagnostic? + } + + return new PTHManager(File.take(), FL.take(), IData, PP); +} + +IdentifierInfo* PTHManager::ReadIdentifierInfo(const char*& D) { + // Read the persistent ID from the PTH file. + uint32_t persistentID = Read32(D); + + // A persistent ID of '0' always maps to NULL. + if (!persistentID) + return 0; + + // Adjust the persistent ID by subtracting '1' so that it can be used + // as an index within a table in the PTH file. + --persistentID; + + // Check if the IdentifierInfo has already been resolved. + if (!PersistentIDCache) + PersistentIDCache = new IDCache(); + + // FIXME: We can make this an array, but what is the performance tradeoff? + IdentifierInfo*& II = (*((IDCache*) PersistentIDCache))[persistentID]; + if (II) return II; + + // Look in the PTH file for the string data for the IdentifierInfo object. + const char* TableEntry = IdDataTable + sizeof(uint32_t) * persistentID; + const char* IDData = Buf->getBufferStart() + Read32(TableEntry); + assert(IDData < Buf->getBufferEnd()); + + // Read the length of the string. + uint32_t len = Read32(IDData); + + // Get the IdentifierInfo* with the specified string. + II = &ITable.get(IDData, IDData+len); + return II; +} + +void PTHManager::ReadToken(const char*& D, unsigned FileID, Token& T) { + // Clear the token. + // FIXME: Setting the flags directly should obviate this step. + T.startToken(); + + // Read the type of the token. + T.setKind((tok::TokenKind) Read8(D)); + + // Set flags. This is gross, since we are really setting multiple flags. + T.setFlag((Token::TokenFlags) Read8(D)); + + // Set the IdentifierInfo* (if any). + T.setIdentifierInfo(ReadIdentifierInfo(D)); + + // Set the SourceLocation. Since all tokens are constructed using a + // raw lexer, they will all be offseted from the same FileID. + T.setLocation(SourceLocation::getFileLoc(FileID, Read32(D))); + + // Finally, read and set the length of the token. + T.setLength(Read32(D)); +} + +PTHLexer* PTHManager::CreateLexer(unsigned FileID, const FileEntry* FE) { + + if (!FE) + return 0; + + // Lookup the FileEntry object in our file lookup data structure. It will + // return a variant that indicates whether or not there is an offset within + // the PTH file that contains cached tokens. + PTHFileLookup::Val Off = ((PTHFileLookup*) FileLookup)->Lookup(FE); + + if (!Off.isValid()) // No tokens available. + return 0; + + // Compute the offset of the token data within the buffer. + const char* data = Buf->getBufferStart() + Off; + assert(data < Buf->getBufferEnd()); + + // First cut: read the tokens from the file into a vector. + // Later, stream them. + SourceLocation Loc = SourceLocation::getFileLoc(FileID, 0); + llvm::OwningPtr L(new PTHLexer(PP, Loc)); + std::vector& Tokens = L->getTokens(); + + Token T; + do { + ReadToken(data, FileID, T); + Tokens.push_back(T); + } + while (T.isNot(tok::eof)); + + // Return the lexer to the client. The client assumes ownership of this + // PTHLexer object. + return L.take(); +}