From: Richard Smith <richard-llvm@metafoo.co.uk>
Date: Tue, 23 Sep 2014 21:05:52 +0000 (+0000)
Subject: PR18793: If we try to EnterTokenStream when our current lexer is a caching
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=f7a830a6abe71a6dc118fd88f2fa9e71395e1bcc;p=clang

PR18793: If we try to EnterTokenStream when our current lexer is a caching
lexer, add the token buffer underneath the caching lexer where possible and
push the tokens directly into the caching lexer otherwise. We previously
put the lexer into a corrupted state where we could not guarantee to provide
the tokens in the right order and would sometimes assert.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@218333 91177308-0d34-0410-b5e6-96231b3b80d8
---

diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index e08c07bd20..4d2994feb3 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -1361,6 +1361,7 @@ public:
 private:
 
   void PushIncludeMacroStack() {
+    assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
     IncludeMacroStack.push_back(IncludeStackInfo(
         CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer),
         CurPPLexer, std::move(CurTokenLexer), CurDirLookup));
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index 1ce3796aa7..f0d3d67aca 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -190,6 +190,25 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
                                     bool DisableMacroExpansion,
                                     bool OwnsTokens) {
+  if (CurLexerKind == CLK_CachingLexer) {
+    if (CachedLexPos < CachedTokens.size()) {
+      // We're entering tokens into the middle of our cached token stream. We
+      // can't represent that, so just insert the tokens into the buffer.
+      CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
+                          Toks, Toks + NumToks);
+      if (OwnsTokens)
+        delete [] Toks;
+      return;
+    }
+
+    // New tokens are at the end of the cached token sequnece; insert the
+    // token stream underneath the caching lexer.
+    ExitCachingLexMode();
+    EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+    EnterCachingLexMode();
+    return;
+  }
+
   // Create a macro expander to expand from the specified token stream.
   std::unique_ptr<TokenLexer> TokLexer;
   if (NumCachedTokenLexers == 0) {
diff --git a/test/Parser/cxx-template-argument.cpp b/test/Parser/cxx-template-argument.cpp
index bbd53b2bdd..c9cc6b8079 100644
--- a/test/Parser/cxx-template-argument.cpp
+++ b/test/Parser/cxx-template-argument.cpp
@@ -106,3 +106,8 @@ namespace pr16225add {
   { };
 
 }
+
+namespace PR18793 {
+  template<typename T, T> struct S {};
+  template<typename T> int g(S<T, (T())> *);
+}