Clang-format: extracted FormatTokenSource from UnwrappedLineParser.

Summary: FormatTokenLexer is here, FormatTokenBuffer is on the way. This will allow to re-parse unwrapped lines when needed.

Reviewers: djasper, klimek

Reviewed By: klimek

CC: cfe-commits
Differential Revision: http://llvm-reviews.chandlerc.com/D186

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@169605 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Format/Format.cpp b/lib/Format/Format.cpp
index 7085aaf8b770cb41717305c5264ffacfb5a643b8..3bedc6a4cd5f7941a506c855a9572025d93b0540 100644 (file)
--- a/lib/Format/Format.cpp
+++ b/lib/Format/Format.cpp
@@ -751,6 +751,67 @@ private:
   std::vector<TokenAnnotation> Annotations;
 };
 
+class LexerBasedFormatTokenSource : public FormatTokenSource {
+public:
+  LexerBasedFormatTokenSource(Lexer &Lex, SourceManager &SourceMgr)
+      : GreaterStashed(false),
+        Lex(Lex),
+        SourceMgr(SourceMgr),
+        IdentTable(Lex.getLangOpts()) {
+    Lex.SetKeepWhitespaceMode(true);
+  }
+
+  virtual FormatToken getNextToken() {
+    if (GreaterStashed) {
+      FormatTok.NewlinesBefore = 0;
+      FormatTok.WhiteSpaceStart =
+          FormatTok.Tok.getLocation().getLocWithOffset(1);
+      FormatTok.WhiteSpaceLength = 0;
+      GreaterStashed = false;
+      return FormatTok;
+    }
+
+    FormatTok = FormatToken();
+    Lex.LexFromRawLexer(FormatTok.Tok);
+    FormatTok.WhiteSpaceStart = FormatTok.Tok.getLocation();
+
+    // Consume and record whitespace until we find a significant token.
+    while (FormatTok.Tok.is(tok::unknown)) {
+      FormatTok.NewlinesBefore += tokenText(FormatTok.Tok).count('\n');
+      FormatTok.WhiteSpaceLength += FormatTok.Tok.getLength();
+
+      if (FormatTok.Tok.is(tok::eof))
+        return FormatTok;
+      Lex.LexFromRawLexer(FormatTok.Tok);
+    }
+
+    if (FormatTok.Tok.is(tok::raw_identifier)) {
+      const IdentifierInfo &Info = IdentTable.get(tokenText(FormatTok.Tok));
+      FormatTok.Tok.setKind(Info.getTokenID());
+    }
+
+    if (FormatTok.Tok.is(tok::greatergreater)) {
+      FormatTok.Tok.setKind(tok::greater);
+      GreaterStashed = true;
+    }
+
+    return FormatTok;
+  }
+
+private:
+  FormatToken FormatTok;
+  bool GreaterStashed;
+  Lexer &Lex;
+  SourceManager &SourceMgr;
+  IdentifierTable IdentTable;
+
+  /// Returns the text of \c FormatTok.
+  StringRef tokenText(Token &Tok) {
+    return StringRef(SourceMgr.getCharacterData(Tok.getLocation()),
+                     Tok.getLength());
+  }
+};
+
 class Formatter : public UnwrappedLineConsumer {
 public:
   Formatter(const FormatStyle &Style, Lexer &Lex, SourceManager &SourceMgr,
@@ -766,7 +827,8 @@ public:
   }
 
   tooling::Replacements format() {
-    UnwrappedLineParser Parser(Style, Lex, SourceMgr, *this);
+    LexerBasedFormatTokenSource Tokens(Lex, SourceMgr);
+    UnwrappedLineParser Parser(Style, Tokens, *this);
     StructuralError = Parser.parse();
     for (std::vector<UnwrappedLine>::iterator I = UnwrappedLines.begin(),
                                               E = UnwrappedLines.end();

diff --git a/lib/Format/UnwrappedLineParser.cpp b/lib/Format/UnwrappedLineParser.cpp
index ebebece7eb435622fb580beefdb2b26ec9764572..a225f3b3e78f06eef80a983c2b8b18b59df2a885 100644 (file)
--- a/lib/Format/UnwrappedLineParser.cpp
+++ b/lib/Format/UnwrappedLineParser.cpp
@@ -22,20 +22,16 @@
 namespace clang {
 namespace format {
 
-UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style, Lexer &Lex,
-                                         SourceManager &SourceMgr,
+UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
+                                         FormatTokenSource &Tokens,
                                          UnwrappedLineConsumer &Callback)
-    : GreaterStashed(false),
-      Style(Style),
-      Lex(Lex),
-      SourceMgr(SourceMgr),
-      IdentTable(Lex.getLangOpts()),
+    : Style(Style),
+      Tokens(Tokens),
       Callback(Callback) {
-  Lex.SetKeepWhitespaceMode(true);
 }
 
 bool UnwrappedLineParser::parse() {
-  parseToken();
+  FormatTok = Tokens.getNextToken();
   return parseLevel();
 }
 
@@ -371,47 +367,7 @@ void UnwrappedLineParser::nextToken() {
   if (eof())
     return;
   Line.Tokens.push_back(FormatTok);
-  parseToken();
-}
-
-void UnwrappedLineParser::parseToken() {
-  if (GreaterStashed) {
-    FormatTok.NewlinesBefore = 0;
-    FormatTok.WhiteSpaceStart = FormatTok.Tok.getLocation().getLocWithOffset(1);
-    FormatTok.WhiteSpaceLength = 0;
-    GreaterStashed = false;
-    return;
-  }
-
-  FormatTok = FormatToken();
-  Lex.LexFromRawLexer(FormatTok.Tok);
-  FormatTok.WhiteSpaceStart = FormatTok.Tok.getLocation();
-
-  // Consume and record whitespace until we find a significant token.
-  while (FormatTok.Tok.is(tok::unknown)) {
-    FormatTok.NewlinesBefore += tokenText().count('\n');
-    FormatTok.WhiteSpaceLength += FormatTok.Tok.getLength();
-
-    if (eof())
-      return;
-    Lex.LexFromRawLexer(FormatTok.Tok);
-  }
-
-  if (FormatTok.Tok.is(tok::raw_identifier)) {
-    const IdentifierInfo &Info = IdentTable.get(tokenText());
-    FormatTok.Tok.setKind(Info.getTokenID());
-  }
-
-  if (FormatTok.Tok.is(tok::greatergreater)) {
-    FormatTok.Tok.setKind(tok::greater);
-    GreaterStashed = true;
-  }
-}
-
-StringRef UnwrappedLineParser::tokenText() {
-  StringRef Data(SourceMgr.getCharacterData(FormatTok.Tok.getLocation()),
-                 FormatTok.Tok.getLength());
-  return Data;
+  FormatTok = Tokens.getNextToken();
 }
 
 }  // end namespace format

diff --git a/lib/Format/UnwrappedLineParser.h b/lib/Format/UnwrappedLineParser.h
index 03dda9957da12f44e24ab4a605c8b21d12b4fe61..92a51abf10ee7472f10282ed573e80b0ecb5d110 100644 (file)
--- a/lib/Format/UnwrappedLineParser.h
+++ b/lib/Format/UnwrappedLineParser.h
@@ -77,10 +77,14 @@ public:
   virtual void consumeUnwrappedLine(const UnwrappedLine &Line) = 0;
 };
 
+class FormatTokenSource {
+public:
+  virtual FormatToken getNextToken() = 0;
+};
+
 class UnwrappedLineParser {
 public:
-  UnwrappedLineParser(const FormatStyle &Style, Lexer &Lex,
-                      SourceManager &SourceMgr,
+  UnwrappedLineParser(const FormatStyle &Style, FormatTokenSource &Tokens,
                       UnwrappedLineConsumer &Callback);
 
   /// Returns true in case of a structural error.
@@ -105,19 +109,12 @@ private:
   void addUnwrappedLine();
   bool eof() const;
   void nextToken();
-  void parseToken();
-
-  /// Returns the text of \c FormatTok.
-  StringRef tokenText();
 
   UnwrappedLine Line;
   FormatToken FormatTok;
-  bool GreaterStashed;
 
   const FormatStyle &Style;
-  Lexer &Lex;
-  SourceManager &SourceMgr;
-  IdentifierTable IdentTable;
+  FormatTokenSource &Tokens;
   UnwrappedLineConsumer &Callback;
 };