#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/OwningPtr.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/Support/Allocator.h"
#include <vector>
unsigned NumCachedTokenLexers;
TokenLexer *TokenLexerCache[TokenLexerCacheSize];
+ /// \brief Keeps macro expanded tokens for TokenLexers.
+ //
+ /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+ /// going to lex in the cache and when it finishes the tokens are removed
+ /// from the end of the cache.
+ llvm::SmallVector<Token, 16> MacroExpandedTokens;
+ std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
+
/// \brief A record of the macro definitions and instantiations that
/// occurred during preprocessing.
///
/// the macro should not be expanded return true, otherwise return false.
bool HandleMacroExpandedIdentifier(Token &Tok, MacroInfo *MI);
+ /// \brief Cache macro expanded tokens for TokenLexers.
+ //
+ /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+ /// going to lex in the cache and when it finishes the tokens are removed
+ /// from the end of the cache.
+ Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
+ llvm::ArrayRef<Token> tokens);
+ void removeCachedMacroExpandedTokensOfLastLexer();
+ friend void TokenLexer::ExpandFunctionArguments();
+
/// isNextPPTokenLParen - Determine whether the next preprocessor token to be
/// lexed is a '('. If so, consume the token and return true, if not, this
/// method should have no observable side-effect on the lexed tokens.
/// Tokens - This is the pointer to an array of tokens that the macro is
/// defined to, with arguments expanded for function-like macros. If this is
/// a token stream, these are the tokens we are returning. This points into
- /// the macro definition we are lexing from, a scratch buffer allocated from
- /// the preprocessor's bump pointer allocator, or some other buffer that we
- /// may or may not own (depending on OwnsTokens).
+ /// the macro definition we are lexing from, a cache buffer that is owned by
+ /// the preprocessor, or some other buffer that we may or may not own
+ /// (depending on OwnsTokens).
+ /// Note that if it points into Preprocessor's cache buffer, the Preprocessor
+ /// may update the pointer as needed.
const Token *Tokens;
+ friend class Preprocessor;
/// NumTokens - This is the length of the Tokens array.
///
assert(CurTokenLexer && !CurPPLexer &&
"Ending a macro when currently in a #include file!");
+ if (!MacroExpandingLexersStack.empty() &&
+ MacroExpandingLexersStack.back().first == CurTokenLexer.get())
+ removeCachedMacroExpandedTokensOfLastLexer();
+
// Delete or cache the now-dead macro expander.
if (NumCachedTokenLexers == TokenLexerCacheSize)
CurTokenLexer.reset();
#include "clang/Lex/CodeCompletionHandler.h"
#include "clang/Lex/ExternalPreprocessorSource.h"
#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/Config/config.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
isVarargsElided, *this);
}
+/// \brief Keeps macro expanded tokens for TokenLexers.
+//
+/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+/// going to lex in the cache and when it finishes the tokens are removed
+/// from the end of the cache.
+Token *Preprocessor::cacheMacroExpandedTokens(TokenLexer *tokLexer,
+ llvm::ArrayRef<Token> tokens) {
+ assert(tokLexer);
+ if (tokens.empty())
+ return 0;
+
+ size_t newIndex = MacroExpandedTokens.size();
+ bool cacheNeedsToGrow = tokens.size() >
+ MacroExpandedTokens.capacity()-MacroExpandedTokens.size();
+ MacroExpandedTokens.append(tokens.begin(), tokens.end());
+
+ if (cacheNeedsToGrow) {
+ // Go through all the TokenLexers whose 'Tokens' pointer points in the
+ // buffer and update the pointers to the (potential) new buffer array.
+ for (unsigned i = 0, e = MacroExpandingLexersStack.size(); i != e; ++i) {
+ TokenLexer *prevLexer;
+ size_t tokIndex;
+ llvm::tie(prevLexer, tokIndex) = MacroExpandingLexersStack[i];
+ prevLexer->Tokens = MacroExpandedTokens.data() + tokIndex;
+ }
+ }
+
+ MacroExpandingLexersStack.push_back(std::make_pair(tokLexer, newIndex));
+ return MacroExpandedTokens.data() + newIndex;
+}
+
+void Preprocessor::removeCachedMacroExpandedTokensOfLastLexer() {
+ assert(!MacroExpandingLexersStack.empty());
+ size_t tokIndex = MacroExpandingLexersStack.back().second;
+ assert(tokIndex < MacroExpandedTokens.size());
+ // Pop the cached macro expanded tokens from the end.
+ MacroExpandedTokens.resize(tokIndex);
+ MacroExpandingLexersStack.pop_back();
+}
+
/// ComputeDATE_TIME - Compute the current time, enter it into the specified
/// scratch buffer, then return DATELoc/TIMELoc locations with the position of
/// the identifier tokens inserted.
Preprocessor::~Preprocessor() {
assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
+ assert(MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty() &&
+ "Preprocessor::HandleEndOfTokenLexer should have cleared those");
while (!IncludeMacroStack.empty()) {
delete IncludeMacroStack.back().TheLexer;
}
size_t Preprocessor::getTotalMemory() const {
- return BP.getTotalMemory();
+ return BP.getTotalMemory() + MacroExpandedTokens.capacity()*sizeof(Token);
}
Preprocessor::macro_iterator
assert(!OwnsTokens && "This would leak if we already own the token list");
// This is deleted in the dtor.
NumTokens = ResultToks.size();
- llvm::BumpPtrAllocator &Alloc = PP.getPreprocessorAllocator();
- Token *Res =
- static_cast<Token *>(Alloc.Allocate(sizeof(Token)*ResultToks.size(),
- llvm::alignOf<Token>()));
- if (NumTokens)
- memcpy(Res, &ResultToks[0], NumTokens*sizeof(Token));
- Tokens = Res;
-
- // The preprocessor bump pointer owns these tokens, not us.
+ // The tokens will be added to Preprocessor's cache and will be removed
+ // when this TokenLexer finishes lexing them.
+ Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
+
+ // The preprocessor cache of macro expanded tokens owns these tokens,not us.
OwnsTokens = false;
}
}
projects / clang / commitdiff