From 31309ab547b5e9a050ee8532a061fe5f26e99c7c Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Wed, 26 Nov 2008 02:18:33 +0000 Subject: [PATCH] Revert 60071, depends on uncommitted LLVM changes. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@60077 91177308-0d34-0410-b5e6-96231b3b80d8 --- Driver/CacheTokens.cpp | 135 ++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 91 deletions(-) diff --git a/Driver/CacheTokens.cpp b/Driver/CacheTokens.cpp index d27073b4dc..6b37830c00 100644 --- a/Driver/CacheTokens.cpp +++ b/Driver/CacheTokens.cpp @@ -28,19 +28,10 @@ typedef llvm::DenseMap PCHMap; typedef llvm::DenseMap IDMap; static void Emit32(llvm::raw_ostream& Out, uint32_t V) { -#if 0 Out << (unsigned char)(V); Out << (unsigned char)(V >> 8); Out << (unsigned char)(V >> 16); Out << (unsigned char)(V >> 24); -#else - Out << V; -#endif -} - -static void EmitOffset(llvm::raw_ostream& Out, uint64_t V) { - assert(((uint32_t) V) == V && "Offset exceeds 32 bits."); - Emit32(Out, (uint32_t) V); } static void Emit8(llvm::raw_ostream& Out, uint32_t V) { @@ -84,11 +75,8 @@ static void EmitIdentifier(llvm::raw_ostream& Out, const IdentifierInfo& II) { Emit32(Out, X); } -static uint64_t EmitIdentifierTable(llvm::raw_fd_ostream& Out, - const IdentifierTable& T, const IDMap& IM) { - - // Record the location within the PTH file. - uint64_t Off = Out.tell(); +static void EmitIdentifierTable(llvm::raw_ostream& Out, + const IdentifierTable& T, const IDMap& IM) { for (IdentifierTable::const_iterator I=T.begin(), E=T.end(); I!=E; ++I) { const IdentifierInfo& II = I->getValue(); @@ -103,68 +91,10 @@ static uint64_t EmitIdentifierTable(llvm::raw_fd_ostream& Out, unsigned len = I->getKeyLength(); Emit32(Out, len); const char* buf = I->getKeyData(); - EmitBuf(Out, buf, buf+len); + EmitBuf(Out, buf, buf+len); } - - return Off; } -static uint64_t EmitFileTable(llvm::raw_fd_ostream& Out, SourceManager& SM, - PCHMap& PM) { - - uint64_t off = Out.tell(); - assert (0 && "Write out the table."); - return off; -} - -static uint64_t LexTokens(llvm::raw_fd_ostream& Out, Lexer& L, Preprocessor& PP, - uint32_t& idcount, IDMap& IM) { - - // Record the location within the token file. - uint64_t off = Out.tell(); - - Token Tok; - - do { - L.LexFromRawLexer(Tok); - - if (Tok.is(tok::identifier)) { - Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok)); - } - else if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) { - // Special processing for #include. Store the '#' token and lex - // the next token. - EmitToken(Out, Tok, idcount, IM); - L.LexFromRawLexer(Tok); - - // Did we see 'include'/'import'/'include_next'? - if (!Tok.is(tok::identifier)) - continue; - - IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok); - Tok.setIdentifierInfo(II); - tok::PPKeywordKind K = II->getPPKeywordID(); - - if (K == tok::pp_include || K == tok::pp_import || - K == tok::pp_include_next) { - - // Save the 'include' token. - EmitToken(Out, Tok, idcount, IM); - - // Lex the next token as an include string. - L.setParsingPreprocessorDirective(true); - L.LexIncludeFilename(Tok); - L.setParsingPreprocessorDirective(false); - - if (Tok.is(tok::identifier)) - Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok)); - } - } - } - while (EmitToken(Out, Tok, idcount, IM), Tok.isNot(tok::eof)); - - return off; -} void clang::CacheTokens(Preprocessor& PP, const std::string& OutFile) { // Lex through the entire file. This will populate SourceManager with @@ -181,6 +111,7 @@ void clang::CacheTokens(Preprocessor& PP, const std::string& OutFile) { PCHMap PM; IDMap IM; + uint64_t tokIdx = 0; uint32_t idcount = 0; std::string ErrMsg; @@ -195,30 +126,52 @@ void clang::CacheTokens(Preprocessor& PP, const std::string& OutFile) { I!=E; ++I) { const SrcMgr::ContentCache* C = I.getFileIDInfo().getContentCache(); - if (!C) continue; - - const FileEntry* FE = C->Entry; // Does this entry correspond to a file? - if (!FE) continue; - PCHMap::iterator PI = PM.find(FE); // Have we already processed this file? - if (PI != PM.end()) continue; + if (!C) + continue; + + const FileEntry* FE = C->Entry; + + if (!FE) + continue; + + PCHMap::iterator PI = PM.find(FE); + if (PI != PM.end()) continue; + PM[FE] = tokIdx; - const llvm::MemoryBuffer* B = C->Buffer; - if (!B) continue; + // os << "Processing: " << FE->getName() << " : " << tokIdx << "\n"; + const llvm::MemoryBuffer* B = C->Buffer; + + if (!B) + continue; + + // Create a raw lexer. Lexer L(SourceLocation::getFileLoc(I.getFileID(), 0), LOpts, B->getBufferStart(), B->getBufferEnd(), B); + + // Ignore whitespace. + L.SetKeepWhitespaceMode(false); + L.SetCommentRetentionState(false); - PM[FE] = LexTokens(Out, L, PP, idcount, IM); - } + // Lex the file, populating our data structures. + Token Tok; + L.LexFromRawLexer(Tok); - // Write out the identifier table. - uint64_t IdTableOff = EmitIdentifierTable(Out, PP.getIdentifierTable(), IM); - - // Write out the file table. - uint64_t FileTableOff = EmitFileTable(Out, SM, PM); + while (Tok.isNot(tok::eof)) { + ++tokIdx; + + if (Tok.is(tok::identifier)) + Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok)); + + // Write the token to disk. + EmitToken(Out, Tok, idcount, IM); + + // Lex the next token. + L.LexFromRawLexer(Tok); + } + } - // Finally, write out the offset table at the end. - EmitOffset(Out, IdTableOff); - EmitOffset(Out, FileTableOff); + // Now, write out the identifier table. + EmitIdentifierTable(Out, PP.getIdentifierTable(), IM); } -- 2.40.0