PR18793: If we try to EnterTokenStream when our current lexer is a caching

lexer, add the token buffer underneath the caching lexer where possible and
push the tokens directly into the caching lexer otherwise. We previously
put the lexer into a corrupted state where we could not guarantee to provide
the tokens in the right order and would sometimes assert.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@218333 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index e08c07bd20d81de54818b35341a370842422e7b1..4d2994feb3312af65967973f1696b5740d5ee9fb 100644 (file)
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -1361,6 +1361,7 @@ public:
 private:
 
   void PushIncludeMacroStack() {
+    assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
     IncludeMacroStack.push_back(IncludeStackInfo(
         CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer),
         CurPPLexer, std::move(CurTokenLexer), CurDirLookup));

diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index 1ce3796aa7922dff8f578a003ae7fff6d55983f5..f0d3d67acae360f2b0c09c3d32122c8fe742cdb2 100644 (file)
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -190,6 +190,25 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
                                     bool DisableMacroExpansion,
                                     bool OwnsTokens) {
+  if (CurLexerKind == CLK_CachingLexer) {
+    if (CachedLexPos < CachedTokens.size()) {
+      // We're entering tokens into the middle of our cached token stream. We
+      // can't represent that, so just insert the tokens into the buffer.
+      CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
+                          Toks, Toks + NumToks);
+      if (OwnsTokens)
+        delete [] Toks;
+      return;
+    }
+
+    // New tokens are at the end of the cached token sequnece; insert the
+    // token stream underneath the caching lexer.
+    ExitCachingLexMode();
+    EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+    EnterCachingLexMode();
+    return;
+  }
+
   // Create a macro expander to expand from the specified token stream.
   std::unique_ptr<TokenLexer> TokLexer;
   if (NumCachedTokenLexers == 0) {

diff --git a/test/Parser/cxx-template-argument.cpp b/test/Parser/cxx-template-argument.cpp
index bbd53b2bdd6911218b6e04fba9a5d1968ac39660..c9cc6b8079028ed36cd2b1d570df8eeb5b87afbf 100644 (file)
--- a/test/Parser/cxx-template-argument.cpp
+++ b/test/Parser/cxx-template-argument.cpp
@@ -106,3 +106,8 @@ namespace pr16225add {
   { };
 
 }
+
+namespace PR18793 {
+  template<typename T, T> struct S {};
+  template<typename T> int g(S<T, (T())> *);
+}