Add a new -dump-raw-tokens option, which allows us to see raw tokens.

Rename -dumptokens to -dump-tokens.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@57405 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/Driver/clang.cpp b/Driver/clang.cpp
index f415de11ceb39e42c2ee71958a258097e79afe62..91d521246ab3e464f5a8078f2239420f6dce2bfa 100644 (file)
--- a/Driver/clang.cpp
+++ b/Driver/clang.cpp
@@ -80,7 +80,8 @@ enum ProgActions {
   ParseNoop,                    // Parse with noop callbacks.
   RunPreprocessorOnly,          // Just lex, no output.
   PrintPreprocessedInput,       // -E mode.
-  DumpTokens,                   // Token dump mode.
+  DumpTokens,                   // Dump out preprocessed tokens.
+  DumpRawTokens,                // Dump out raw tokens.
   RunAnalysis                   // Run one or more source code analyses. 
 };
 
@@ -92,7 +93,9 @@ ProgAction(llvm::cl::desc("Choose output type:"), llvm::cl::ZeroOrMore,
                         "Just run preprocessor, no output (for timings)"),
              clEnumValN(PrintPreprocessedInput, "E",
                         "Run preprocessor, emit preprocessed file"),
-             clEnumValN(DumpTokens, "dumptokens",
+             clEnumValN(DumpRawTokens, "dump-raw-tokens",
+                        "Lex file in raw mode and dump raw tokens"),
+             clEnumValN(DumpTokens, "dump-tokens",
                         "Run preprocessor, dump internal rep of tokens"),
              clEnumValN(ParseNoop, "parse-noop",
                         "Run parser with noop callbacks (for timings)"),
@@ -1104,9 +1107,29 @@ static void ProcessInputFile(Preprocessor &PP, PreprocessorFactory &PPF,
 
     break;
       
+  case DumpRawTokens: {
+    SourceManager &SM = PP.getSourceManager();
+    std::pair<const char*,const char*> File =
+      SM.getBufferData(SM.getMainFileID());
+    // Start lexing the specified input file.
+    Lexer RawLex(SourceLocation::getFileLoc(SM.getMainFileID(), 0),
+                 PP.getLangOptions(), File.first, File.second);
+    RawLex.SetKeepWhitespaceMode(true);
+
+    Token RawTok;
+
+    RawLex.LexFromRawLexer(RawTok);
+    while (RawTok.isNot(tok::eof)) {
+      PP.DumpToken(RawTok, true);
+      fprintf(stderr, "\n");
+      RawLex.LexFromRawLexer(RawTok);
+    }
+    ClearSourceMgr = true;
+    break;
+  }
   case DumpTokens: {                 // Token dump mode.
     Token Tok;
-    // Start parsing the specified input file.
+    // Start preprocessing the specified input file.
     PP.EnterMainSourceFile();
     do {
       PP.Lex(Tok);

diff --git a/test/Preprocessor/dumptokens_phyloc.c b/test/Preprocessor/dumptokens_phyloc.c
index ea468728bb4fe0d495ef1ee475bfb2db66db49c6..b5c4d7bae3a85f67529e8eb2a65e7a8ed7c4182e 100644 (file)
--- a/test/Preprocessor/dumptokens_phyloc.c
+++ b/test/Preprocessor/dumptokens_phyloc.c
@@ -1,4 +1,4 @@
-// RUN: clang -dumptokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
+// RUN: clang -dump-tokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
 
 #define TESTPHYLOC 10