From: Ted Kremenek <kremenek@apple.com>
Date: Wed, 12 Nov 2008 21:37:15 +0000 (+0000)
Subject: Add skeleton for PTH lexer.
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=274b20863a728cc6a31ee75c670e3733600c1531;p=clang

Add skeleton for PTH lexer.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@59169 91177308-0d34-0410-b5e6-96231b3b80d8
---

diff --git a/include/clang/Lex/PTHLexer.h b/include/clang/Lex/PTHLexer.h
new file mode 100644
index 0000000000..cdca3419a2
--- /dev/null
+++ b/include/clang/Lex/PTHLexer.h
@@ -0,0 +1,67 @@
+//===--- PTHLexer.h - Lexer based on Pre-tokenized input --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PTHLexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_PTHLexer_H
+#define LLVM_CLANG_PTHLexer_H
+
+#include "clang/Lex/PreprocessorLexer.h"
+
+namespace clang {
+  
+class PTHLexer : public PreprocessorLexer {
+
+  /// PP - Preprocessor.
+  Preprocessor& PP;
+  
+  /// FileLoc - Location for the start of the file.
+  ///
+  SourceLocation FileLoc;
+  
+  /// Tokens - This is the pointer to an array of tokens that the macro is
+  /// defined to, with arguments expanded for function-like macros.  If this is
+  /// a token stream, these are the tokens we are returning.
+  const Token *Tokens;
+  
+  /// NumTokens - This is the length of the Tokens array.
+  ///
+  unsigned NumTokens;
+  
+  /// CurToken - This is the next token that Lex will return.
+  ///
+  unsigned CurToken;
+        
+  PTHLexer(const PTHLexer&);  // DO NOT IMPLEMENT
+  void operator=(const PTHLexer&); // DO NOT IMPLEMENT
+
+public:
+
+  /// Create a PTHLexer for the specified token stream.
+  PTHLexer(Preprocessor& pp, SourceLocation fileloc,
+           const Token *TokArray, unsigned NumToks);
+  ~PTHLexer() {}
+    
+  /// Lex - Return the next token.
+  void Lex(Token &Tok);
+  
+  void setEOF(Token &Tok);
+  
+  /// getFileLoc - Return the File Location for the file we are lexing out of.
+  /// The physical location encodes the location where the characters come from,
+  /// the virtual location encodes where we should *claim* the characters came
+  /// from.  Currently this is only used by _Pragma handling.
+  SourceLocation getFileLoc() const { return FileLoc; }
+};
+
+}  // end namespace clang
+
+#endif
diff --git a/lib/Lex/PTHLexer.cpp b/lib/Lex/PTHLexer.cpp
new file mode 100644
index 0000000000..e1b392068c
--- /dev/null
+++ b/lib/Lex/PTHLexer.cpp
@@ -0,0 +1,79 @@
+//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the PTHLexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/PTHLexer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/Token.h"
+#include "clang/Basic/TokenKinds.h"
+
+using namespace clang;
+
+PTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc,
+                   const Token *TokArray, unsigned NumToks)
+  : PP(pp), FileLoc(fileloc), Tokens(TokArray), NumTokens(NumToks), CurToken(0){
+
+  assert (Tokens[NumTokens-1].is(tok::eof));
+  --NumTokens;
+    
+  LexingRawMode = false;
+  ParsingPreprocessorDirective = false;
+  ParsingFilename = false;
+}
+
+void PTHLexer::Lex(Token& Tok) {
+
+  if (CurToken == NumTokens) {    
+    // If we hit the end of the file while parsing a preprocessor directive,
+    // end the preprocessor directive first.  The next token returned will
+    // then be the end of file.
+    //   OR
+    // If we are in raw mode, return this event as an EOF token.  Let the caller
+    // that put us in raw mode handle the event.
+    if (ParsingPreprocessorDirective || LexingRawMode) {
+      // Done parsing the "line".
+      ParsingPreprocessorDirective = false;
+      Tok = Tokens[CurToken]; // not an out-of-bound access
+      // FIXME: eom handling?
+    }
+    else
+      PP.HandleEndOfFile(Tok, false);
+    
+    return;
+  }
+
+  Tok = Tokens[CurToken];
+  
+  if (ParsingPreprocessorDirective && Tok.isAtStartOfLine()) {
+    ParsingPreprocessorDirective = false; // Done parsing the "line".
+    MIOpt.ReadToken();
+    // FIXME:  Need to replicate:
+    // FormTokenWithChars(Tok, CurPtr, tok::eom);
+    Tok.setKind(tok::eom);    
+    return;
+  }
+  else // Otherwise, advance to the next token.
+    ++CurToken;
+
+  if (Tok.isAtStartOfLine() && Tok.is(tok::hash) && !LexingRawMode) {
+    PP.HandleDirective(Tok);
+    PP.Lex(Tok);
+    return;
+  }
+    
+  MIOpt.ReadToken();
+}
+
+void PTHLexer::setEOF(Token& Tok) {
+  Tok = Tokens[NumTokens]; // NumTokens is already adjusted, so this isn't
+                           // an overflow.
+}