From: Ted Kremenek Date: Thu, 12 Feb 2009 03:26:59 +0000 (+0000) Subject: PTH: Cache stat information for files in the PTH file. Hook up FileManager X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=337edcdbec05316b407d0d64865c88ff8597d910;p=clang PTH: Cache stat information for files in the PTH file. Hook up FileManager to use this stat information in the PTH file using a 'StatSysCallCache' object. Performance impact (Cocoa.h, PTH): - number of stat calls reduces from 1230 to 425 - fsyntax-only: time improves by 4.2% We can reduce the number of stat calls to almost zero by caching negative stat calls and directory stat calls in the PTH file as well. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@64353 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/Driver/CacheTokens.cpp b/Driver/CacheTokens.cpp index 6b9f96e981..52ce082d13 100644 --- a/Driver/CacheTokens.cpp +++ b/Driver/CacheTokens.cpp @@ -47,6 +47,17 @@ static void Emit32(llvm::raw_ostream& Out, uint32_t V) { Out << (unsigned char)(V >> 24); } +static void Emit64(llvm::raw_ostream& Out, uint64_t V) { + Out << (unsigned char)(V); + Out << (unsigned char)(V >> 8); + Out << (unsigned char)(V >> 16); + Out << (unsigned char)(V >> 24); + Out << (unsigned char)(V >> 32); + Out << (unsigned char)(V >> 40); + Out << (unsigned char)(V >> 48); + Out << (unsigned char)(V >> 56); +} + static void Pad(llvm::raw_fd_ostream& Out, unsigned A) { Offset off = (Offset) Out.tell(); uint32_t n = ((uintptr_t)(off+A-1) & ~(uintptr_t)(A-1)) - off; @@ -149,7 +160,7 @@ public: const std::pair& Len = Info::EmitKeyDataLength(out, I->key, I->data); Info::EmitKey(out, I->key, Len.first); - Info::EmitData(out, I->data, Len.second); + Info::EmitData(out, I->key, I->data, Len.second); } } @@ -212,16 +223,23 @@ public: unsigned n = strlen(FE->getName()) + 1; ::Emit16(Out, n); - return std::make_pair(n, 8); + return std::make_pair(n,(4*2)+(4+4+2+8+8)); } static void EmitKey(llvm::raw_ostream& Out, const FileEntry* FE, unsigned n) { Out.write(FE->getName(), n); } - static void EmitData(llvm::raw_ostream& Out, const PCHEntry& E, unsigned) { + static void EmitData(llvm::raw_ostream& Out, const FileEntry* FE, + const PCHEntry& E, unsigned) { ::Emit32(Out, E.getTokenOffset()); ::Emit32(Out, E.getPPCondTableOffset()); + // Emit stat information. + ::Emit32(Out, FE->getInode()); + ::Emit32(Out, FE->getDevice()); + ::Emit16(Out, FE->getFileMode()); + ::Emit64(Out, FE->getModificationTime()); + ::Emit64(Out, FE->getSize()); } }; @@ -537,8 +555,6 @@ void PTHWriter::GeneratePTH() { if (!P.isAbsolute()) continue; - // assert(!PM.count(FE) && "fileinfo's are not uniqued on FileEntry?"); - const llvm::MemoryBuffer *B = C.getBuffer(); if (!B) continue; @@ -620,7 +636,8 @@ public: Out.write(key->II->getName(), n); } - static void EmitData(llvm::raw_ostream& Out, uint32_t pID, unsigned) { + static void EmitData(llvm::raw_ostream& Out, PCHIdKey*, uint32_t pID, + unsigned) { ::Emit32(Out, pID); } }; diff --git a/include/clang/Lex/PTHManager.h b/include/clang/Lex/PTHManager.h index 049c0d2f0d..4d0ecf2134 100644 --- a/include/clang/Lex/PTHManager.h +++ b/include/clang/Lex/PTHManager.h @@ -30,6 +30,7 @@ namespace clang { class FileEntry; class PTHLexer; class Diagnostic; +class StatSysCallCache; class PTHManager : public IdentifierInfoLookup { friend class PTHLexer; @@ -95,7 +96,7 @@ class PTHManager : public IdentifierInfoLookup { public: // The current PTH version. - enum { Version = 6 }; + enum { Version = 7 }; ~PTHManager(); @@ -115,6 +116,12 @@ public: /// specified file. This method returns NULL if no cached tokens exist. /// It is the responsibility of the caller to 'delete' the returned object. PTHLexer *CreateLexer(FileID FID); + + /// createStatCache - Returns a StatSysCallCache object for use with + /// FileManager objects. These objects use the PTH data to speed up + /// calls to stat by memoizing their results from when the PTH file + /// was generated. + StatSysCallCache *createStatCache(); }; } // end namespace clang diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h index 94a7ef45b3..5200725ba1 100644 --- a/include/clang/Lex/Preprocessor.h +++ b/include/clang/Lex/Preprocessor.h @@ -206,7 +206,7 @@ public: IdentifierTable &getIdentifierTable() { return Identifiers; } SelectorTable &getSelectorTable() { return Selectors; } - void setPTHManager(PTHManager* pm) { PTH.reset(pm); } + void setPTHManager(PTHManager* pm); /// SetCommentRetentionState - Control whether or not the preprocessor retains /// comments in output. diff --git a/lib/Lex/PTHLexer.cpp b/lib/Lex/PTHLexer.cpp index 9f878b630d..9a973c9793 100644 --- a/lib/Lex/PTHLexer.cpp +++ b/lib/Lex/PTHLexer.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/System/Host.h" +#include using namespace clang; #define DISK_TOKEN_SIZE (1+1+2+4+4) @@ -49,6 +50,19 @@ static inline uint32_t ReadUnalignedLE32(const unsigned char *&Data) { return V; } +static inline uint64_t ReadUnalignedLE64(const unsigned char *&Data) { + uint64_t V = ((uint64_t)Data[0]) | + ((uint64_t)Data[1] << 8) | + ((uint64_t)Data[2] << 16) | + ((uint64_t)Data[3] << 24) | + ((uint64_t)Data[1] << 32) | + ((uint64_t)Data[2] << 40) | + ((uint64_t)Data[3] << 48) | + ((uint64_t)Data[3] << 56); + Data += 8; + return V; +} + static inline uint32_t ReadLE32(const unsigned char *&Data) { // Hosts that directly support little-endian 32-bit loads can just // use them. Big-endian hosts need a bswap. @@ -353,7 +367,11 @@ public: "'buckets' must have a 4-byte alignment"); } - + unsigned getNumBuckets() const { return NumBuckets; } + unsigned getNumEntries() const { return NumEntries; } + const unsigned char* const getBase() const { return Base; } + const unsigned char* const getBuckets() const { return Buckets; } + bool isEmpty() const { return NumEntries == 0; } class iterator { @@ -451,32 +469,37 @@ public: uint32_t getPPCondOffset() const { return PPCondOff; } }; -class VISIBILITY_HIDDEN PTHFileLookupTrait { + +class VISIBILITY_HIDDEN PTHFileLookupCommonTrait { public: - typedef PTHFileData data_type; - typedef const FileEntry* external_key_type; typedef const char* internal_key_type; static bool EqualKey(const char* a, const char* b) { return strcmp(a, b) == 0; } - + static unsigned ComputeHash(const char* x) { return BernsteinHash(x); } - - static const char* GetInternalKey(const FileEntry* FE) { - return FE->getName(); - } static std::pair ReadKeyDataLength(const unsigned char*& d) { - return std::make_pair((unsigned) ReadUnalignedLE16(d), 8U); + return std::make_pair((unsigned) ReadUnalignedLE16(d), 8U + (4+4+2+8+8)); } static const char* ReadKey(const unsigned char* d, unsigned) { return (const char*) d; } +}; + +class VISIBILITY_HIDDEN PTHFileLookupTrait : public PTHFileLookupCommonTrait { +public: + typedef const FileEntry* external_key_type; + typedef PTHFileData data_type; + + static const char* GetInternalKey(const FileEntry* FE) { + return FE->getName(); + } static PTHFileData ReadData(const unsigned char* d, unsigned) { uint32_t x = ::ReadUnalignedLE32(d); @@ -737,3 +760,70 @@ PTHLexer *PTHManager::CreateLexer(FileID FID) { assert(PP && "No preprocessor set yet!"); return new PTHLexer(*PP, FID, data, ppcond, *this); } + +//===----------------------------------------------------------------------===// +// 'stat' caching. +//===----------------------------------------------------------------------===// + +namespace { +class VISIBILITY_HIDDEN PTHStatData { +public: + const ino_t ino; + const dev_t dev; + const mode_t mode; + const time_t mtime; + const off_t size; + + PTHStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s) + : ino(i), dev(d), mode(mo), mtime(m), size(s) {} +}; + +class VISIBILITY_HIDDEN PTHStatLookupTrait : public PTHFileLookupCommonTrait { +public: + typedef internal_key_type external_key_type; // const char* + typedef PTHStatData data_type; + + static const char* GetInternalKey(external_key_type x) { return x; } + + static data_type ReadData(const unsigned char* d, unsigned) { + d += 4 * 2; // Skip the first 2 words. + ino_t ino = (ino_t) ReadUnalignedLE32(d); + dev_t dev = (dev_t) ReadUnalignedLE32(d); + mode_t mode = (mode_t) ReadUnalignedLE16(d); + time_t mtime = (time_t) ReadUnalignedLE64(d); + return data_type(ino, dev, mode, mtime, (off_t) ReadUnalignedLE64(d)); + } +}; +} + +class VISIBILITY_HIDDEN PTHStatCache : public StatSysCallCache { + typedef OnDiskChainedHashTable CacheTy; + CacheTy Cache; + +public: + PTHStatCache(PTHFileLookup &FL) : + Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(), + FL.getBase()) {} + + ~PTHStatCache() {} + + int stat(const char *path, struct stat *buf) { + // Do the lookup for the file's data in the PTH file. + CacheTy::iterator I = Cache.find(path); + + // If we don't get a hit in the PTH file just forward to 'stat'. + if (I == Cache.end()) return ::stat(path, buf); + + const PTHStatData& Data = *I; + buf->st_ino = Data.ino; + buf->st_dev = Data.dev; + buf->st_mtime = Data.mtime; + buf->st_mode = Data.mode; + buf->st_size = Data.size; + return 0; + } +}; + +StatSysCallCache *PTHManager::createStatCache() { + return new PTHStatCache(*((PTHFileLookup*) FileLookup)); +} diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index db5ca2bd7f..69aa7cb51a 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -32,6 +32,7 @@ #include "clang/Lex/ScratchBuffer.h" #include "clang/Lex/LexDiagnostic.h" #include "clang/Basic/SourceManager.h" +#include "clang/Basic/FileManager.h" #include "clang/Basic/TargetInfo.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/SmallVector.h" @@ -117,6 +118,11 @@ Preprocessor::~Preprocessor() { delete Callbacks; } +void Preprocessor::setPTHManager(PTHManager* pm) { + PTH.reset(pm); + FileMgr.setStatCache(PTH->createStatCache()); +} + void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { llvm::cerr << tok::getTokenName(Tok.getKind()) << " '" << getSpelling(Tok) << "'";