// line" flag set on it.
bool IsAtStartOfLine;
+ bool IsAtPhysicalStartOfLine;
+
+ bool HasLeadingSpace;
+
+ bool HasLeadingEmptyMacro;
+
// CurrentConflictMarkerState - The kind of conflict marker we are handling.
ConflictMarkerKind CurrentConflictMarkerState;
/// from. Currently this is only used by _Pragma handling.
SourceLocation getFileLoc() const { return FileLoc; }
+private:
/// Lex - Return the next token in the file. If this is the end of file, it
/// return the tok::eof token. This implicitly involves the preprocessor.
- void Lex(Token &Result) {
- // Start a new token.
- Result.startToken();
-
- // NOTE, any changes here should also change code after calls to
- // Preprocessor::HandleDirective
- if (IsAtStartOfLine) {
- Result.setFlag(Token::StartOfLine);
- IsAtStartOfLine = false;
- }
-
- // Get a token. Note that this may delete the current lexer if the end of
- // file is reached.
- LexTokenInternal(Result);
- }
+ bool Lex(Token &Result);
+public:
/// isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
bool isPragmaLexer() const { return Is_PragmaLexer; }
+private:
/// IndirectLex - An indirect call to 'Lex' that can be invoked via
/// the PreprocessorLexer interface.
void IndirectLex(Token &Result) { Lex(Result); }
+public:
/// LexFromRawLexer - Lex a token from a designated raw lexer (one with no
/// associated preprocessor object. Return true if the 'next character to
/// read' pointer points at the end of the lexer buffer, false otherwise.
/// LexTokenInternal - Internal interface to lex a preprocessing token. Called
/// by Lex.
///
- void LexTokenInternal(Token &Result);
+ bool LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine);
+
+ bool CheckUnicodeWhitespace(Token &Result, uint32_t C, const char *CurPtr);
/// Given that a token begins with the Unicode character \p C, figure out
/// what kind of token it is and dispatch to the appropriate lexing helper
/// function.
- void LexUnicode(Token &Result, uint32_t C, const char *CurPtr);
+ bool LexUnicode(Token &Result, uint32_t C, const char *CurPtr);
/// FormTokenWithChars - When we lex a token, we have identified a span
/// starting at BufferPtr, going to TokEnd that forms the token. This method
void SkipBytes(unsigned Bytes, bool StartOfLine);
+ void PropagateLineStartLeadingSpaceInfo(Token &Result);
+
const char *LexUDSuffix(Token &Result, const char *CurPtr,
bool IsStringLiteral);
// Helper functions to lex the remainder of a token of the specific type.
- void LexIdentifier (Token &Result, const char *CurPtr);
- void LexNumericConstant (Token &Result, const char *CurPtr);
- void LexStringLiteral (Token &Result, const char *CurPtr,
+ bool LexIdentifier (Token &Result, const char *CurPtr);
+ bool LexNumericConstant (Token &Result, const char *CurPtr);
+ bool LexStringLiteral (Token &Result, const char *CurPtr,
tok::TokenKind Kind);
- void LexRawStringLiteral (Token &Result, const char *CurPtr,
+ bool LexRawStringLiteral (Token &Result, const char *CurPtr,
tok::TokenKind Kind);
- void LexAngledStringLiteral(Token &Result, const char *CurPtr);
- void LexCharConstant (Token &Result, const char *CurPtr,
+ bool LexAngledStringLiteral(Token &Result, const char *CurPtr);
+ bool LexCharConstant (Token &Result, const char *CurPtr,
tok::TokenKind Kind);
bool LexEndOfFile (Token &Result, const char *CurPtr);
-
- bool SkipWhitespace (Token &Result, const char *CurPtr);
- bool SkipLineComment (Token &Result, const char *CurPtr);
- bool SkipBlockComment (Token &Result, const char *CurPtr);
+ bool SkipWhitespace (Token &Result, const char *CurPtr,
+ bool &TokAtPhysicalStartOfLine);
+ bool SkipLineComment (Token &Result, const char *CurPtr,
+ bool &TokAtPhysicalStartOfLine);
+ bool SkipBlockComment (Token &Result, const char *CurPtr,
+ bool &TokAtPhysicalStartOfLine);
bool SaveLineComment (Token &Result, const char *CurPtr);
bool IsStartOfConflictMarker(const char *CurPtr);
// Start of the file is a start of line.
IsAtStartOfLine = true;
+ IsAtPhysicalStartOfLine = true;
+
+ HasLeadingSpace = false;
+ HasLeadingEmptyMacro = false;
// We are not after parsing a #.
ParsingPreprocessorDirective = false;
BufferPtr += Bytes;
if (BufferPtr > BufferEnd)
BufferPtr = BufferEnd;
+ // FIXME: What exactly does the StartOfLine bit mean? There are two
+ // possible meanings for the "start" of the line: the first token on the
+ // unexpanded line, or the first token on the expanded line.
IsAtStartOfLine = StartOfLine;
+ IsAtPhysicalStartOfLine = StartOfLine;
}
static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) {
}
}
-void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
+bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
// Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
unsigned Size;
unsigned char C = *CurPtr++;
// If we are in raw mode, return this identifier raw. There is no need to
// look up identifier information or attempt to macro expand it.
if (LexingRawMode)
- return;
+ return true;
// Fill in Result.IdentifierInfo and update the token kind,
// looking up the identifier in the identifier table.
// Finally, now that we know we have an identifier, pass this off to the
// preprocessor, which may macro expand it or something.
if (II->isHandleIdentifierCase())
- PP->HandleIdentifier(Result);
+ return PP->HandleIdentifier(Result);
- return;
+ return true;
}
// Otherwise, $,\,? in identifier found. Enter slower path.
/// LexNumericConstant - Lex the remainder of a integer or floating point
/// constant. From[-1] is the first character lexed. Return the end of the
/// constant.
-void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
+bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
unsigned Size;
char C = getCharAndSize(CurPtr, Size);
char PrevCh = 0;
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
Result.setLiteralData(TokStart);
+ return true;
}
/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
/// either " or L" or u8" or u" or U".
-void Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
+bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
tok::TokenKind Kind) {
const char *NulCharacter = 0; // Does this string contain the \0 character?
if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
Diag(BufferPtr, diag::ext_unterminated_string);
FormTokenWithChars(Result, CurPtr-1, tok::unknown);
- return;
+ return true;
}
if (C == 0) {
if (isCodeCompletionPoint(CurPtr-1)) {
PP->CodeCompleteNaturalLanguage();
FormTokenWithChars(Result, CurPtr-1, tok::unknown);
- return cutOffLexing();
+ cutOffLexing();
+ return true;
}
NulCharacter = CurPtr-1;
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, Kind);
Result.setLiteralData(TokStart);
+ return true;
}
/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
/// having lexed R", LR", u8R", uR", or UR".
-void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
+bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
tok::TokenKind Kind) {
// This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
// Between the initial and final double quote characters of the raw string,
}
FormTokenWithChars(Result, CurPtr, tok::unknown);
- return;
+ return true;
}
// Save prefix and move CurPtr past it
Diag(BufferPtr, diag::err_unterminated_raw_string)
<< StringRef(Prefix, PrefixLen);
FormTokenWithChars(Result, CurPtr-1, tok::unknown);
- return;
+ return true;
}
}
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, Kind);
Result.setLiteralData(TokStart);
+ return true;
}
/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
/// after having lexed the '<' character. This is used for #include filenames.
-void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
+bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
const char *NulCharacter = 0; // Does this string contain the \0 character?
const char *AfterLessPos = CurPtr;
char C = getAndAdvanceChar(CurPtr, Result);
// If the filename is unterminated, then it must just be a lone <
// character. Return this as such.
FormTokenWithChars(Result, AfterLessPos, tok::less);
- return;
+ return true;
} else if (C == 0) {
NulCharacter = CurPtr-1;
}
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
Result.setLiteralData(TokStart);
+ return true;
}
/// LexCharConstant - Lex the remainder of a character constant, after having
/// lexed either ' or L' or u' or U'.
-void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
+bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
tok::TokenKind Kind) {
const char *NulCharacter = 0; // Does this character contain the \0 character?
if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
Diag(BufferPtr, diag::ext_empty_character);
FormTokenWithChars(Result, CurPtr, tok::unknown);
- return;
+ return true;
}
while (C != '\'') {
if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
Diag(BufferPtr, diag::ext_unterminated_char);
FormTokenWithChars(Result, CurPtr-1, tok::unknown);
- return;
+ return true;
}
if (C == 0) {
if (isCodeCompletionPoint(CurPtr-1)) {
PP->CodeCompleteNaturalLanguage();
FormTokenWithChars(Result, CurPtr-1, tok::unknown);
- return cutOffLexing();
+ cutOffLexing();
+ return true;
}
NulCharacter = CurPtr-1;
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, Kind);
Result.setLiteralData(TokStart);
+ return true;
}
/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
///
/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
///
-bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
+bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr,
+ bool &TokAtPhysicalStartOfLine) {
// Whitespace - Skip it, then return the token after the whitespace.
bool SawNewline = isVerticalWhitespace(CurPtr[-1]);
// If the client wants us to return whitespace, return it now.
if (isKeepWhitespaceMode()) {
FormTokenWithChars(Result, CurPtr, tok::unknown);
- if (SawNewline)
+ if (SawNewline) {
IsAtStartOfLine = true;
+ IsAtPhysicalStartOfLine = true;
+ }
// FIXME: The next token will not have LeadingSpace set.
return true;
}
bool HasLeadingSpace = !isVerticalWhitespace(PrevChar);
Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
- if (SawNewline)
+ if (SawNewline) {
Result.setFlag(Token::StartOfLine);
+ TokAtPhysicalStartOfLine = true;
+ }
BufferPtr = CurPtr;
return false;
///
/// If we're in KeepCommentMode or any CommentHandler has inserted
/// some tokens, this will store the first token and return true.
-bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) {
+bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
+ bool &TokAtPhysicalStartOfLine) {
// If Line comments aren't explicitly enabled for this language, emit an
// extension warning.
if (!LangOpts.LineComment && !isLexingRawMode()) {
// The next returned token is at the start of the line.
Result.setFlag(Token::StartOfLine);
+ TokAtPhysicalStartOfLine = true;
// No leading whitespace seen so far.
Result.clearFlag(Token::LeadingSpace);
BufferPtr = CurPtr;
///
/// If we're in KeepCommentMode or any CommentHandler has inserted
/// some tokens, this will store the first token and return true.
-bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
+bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr,
+ bool &TokAtPhysicalStartOfLine) {
// Scan one character past where we should, looking for a '/' character. Once
// we find it, check to see if it was preceded by a *. This common
// optimization helps people who like to put a lot of * characters in their
// efficiently now. This is safe even in KeepWhitespaceMode because we would
// have already returned above with the comment as a token.
if (isHorizontalWhitespace(*CurPtr)) {
- SkipWhitespace(Result, CurPtr+1);
+ SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine);
return false;
}
// Save state that can be changed while lexing so that we can restore it.
const char *TmpBufferPtr = BufferPtr;
bool inPPDirectiveMode = ParsingPreprocessorDirective;
+ bool atStartOfLine = IsAtStartOfLine;
+ bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
+ bool leadingSpace = HasLeadingSpace;
Token Tok;
- Tok.startToken();
- LexTokenInternal(Tok);
+ Lex(Tok);
// Restore state that may have changed.
BufferPtr = TmpBufferPtr;
ParsingPreprocessorDirective = inPPDirectiveMode;
+ HasLeadingSpace = leadingSpace;
+ IsAtStartOfLine = atStartOfLine;
+ IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
// Restore the lexer back to non-skipping mode.
LexingRawMode = false;
return CodePoint;
}
-void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
+bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
+ const char *CurPtr) {
static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars(
UnicodeWhitespaceCharRanges);
if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
<< makeCharRange(*this, BufferPtr, CurPtr);
Result.setFlag(Token::LeadingSpace);
- if (SkipWhitespace(Result, CurPtr))
- return; // KeepWhitespaceMode
-
- return LexTokenInternal(Result);
+ return true;
}
+ return false;
+}
+bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) {
if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
!PP->isPreprocessedOutput()) {
<< FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr));
BufferPtr = CurPtr;
- return LexTokenInternal(Result);
+ return false;
}
// Otherwise, we have an explicit UCN or a character that's unlikely to show
// up by accident.
MIOpt.ReadToken();
FormTokenWithChars(Result, CurPtr, tok::unknown);
+ return true;
+}
+
+void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
+ IsAtStartOfLine = Result.isAtStartOfLine();
+ HasLeadingSpace = Result.hasLeadingSpace();
+ HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro();
+ // Note that this doesn't affect IsAtPhysicalStartOfLine.
}
+bool Lexer::Lex(Token &Result) {
+ // Start a new token.
+ Result.startToken();
+
+ // Set up misc whitespace flags for LexTokenInternal.
+ if (IsAtStartOfLine) {
+ Result.setFlag(Token::StartOfLine);
+ IsAtStartOfLine = false;
+ }
+
+ if (HasLeadingSpace) {
+ Result.setFlag(Token::LeadingSpace);
+ HasLeadingSpace = false;
+ }
+
+ if (HasLeadingEmptyMacro) {
+ Result.setFlag(Token::LeadingEmptyMacro);
+ HasLeadingEmptyMacro = false;
+ }
+
+ bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
+ IsAtPhysicalStartOfLine = false;
+ bool result = LexTokenInternal(Result, atPhysicalStartOfLine);
+ assert((result || !isLexingRawMode()) && "Raw lex must succeed");
+ return result;
+}
/// LexTokenInternal - This implements a simple C family lexer. It is an
/// extremely performance critical piece of code. This assumes that the buffer
/// has a null character at the end of the file. This returns a preprocessing
/// token, not a normal token, as such, it is an internal interface. It assumes
/// that the Flags of result have been cleared before calling this.
-void Lexer::LexTokenInternal(Token &Result) {
+bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
LexNextToken:
// New token, can't need cleaning yet.
Result.clearFlag(Token::NeedsCleaning);
if (isKeepWhitespaceMode()) {
FormTokenWithChars(Result, CurPtr, tok::unknown);
// FIXME: The next token will not have LeadingSpace set.
- return;
+ return true;
}
BufferPtr = CurPtr;
switch (Char) {
case 0: // Null.
// Found end of file?
- if (CurPtr-1 == BufferEnd) {
- // Read the PP instance variable into an automatic variable, because
- // LexEndOfFile will often delete 'this'.
- Preprocessor *PPCache = PP;
- if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file.
- return; // Got a token to return.
- assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
- return PPCache->Lex(Result);
- }
+ if (CurPtr-1 == BufferEnd)
+ return LexEndOfFile(Result, CurPtr-1);
// Check if we are performing code completion.
if (isCodeCompletionPoint(CurPtr-1)) {
// Return the code-completion token.
Result.startToken();
FormTokenWithChars(Result, CurPtr, tok::code_completion);
- return;
+ return true;
}
if (!isLexingRawMode())
Diag(CurPtr-1, diag::null_in_file);
Result.setFlag(Token::LeadingSpace);
- if (SkipWhitespace(Result, CurPtr))
- return; // KeepWhitespaceMode
+ if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
+ return true; // KeepWhitespaceMode
- goto LexNextToken; // GCC isn't tail call eliminating.
+ // We know the lexer hasn't changed, so just try again with this lexer.
+ // (We manually eliminate the tail call to avoid recursion.)
+ goto LexNextToken;
case 26: // DOS & CP/M EOF: "^Z".
// If we're in Microsoft extensions mode, treat this as end of file.
- if (LangOpts.MicrosoftExt) {
- // Read the PP instance variable into an automatic variable, because
- // LexEndOfFile will often delete 'this'.
- Preprocessor *PPCache = PP;
- if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file.
- return; // Got a token to return.
- assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
- return PPCache->Lex(Result);
- }
+ if (LangOpts.MicrosoftExt)
+ return LexEndOfFile(Result, CurPtr-1);
+
// If Microsoft extensions are disabled, this is just random garbage.
Kind = tok::unknown;
break;
// Since we consumed a newline, we are back at the start of a line.
IsAtStartOfLine = true;
+ IsAtPhysicalStartOfLine = true;
Kind = tok::eod;
break;
// No leading whitespace seen so far.
Result.clearFlag(Token::LeadingSpace);
- if (SkipWhitespace(Result, CurPtr))
- return; // KeepWhitespaceMode
- goto LexNextToken; // GCC isn't tail call eliminating.
+ if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
+ return true; // KeepWhitespaceMode
+
+ // We only saw whitespace, so just try again with this lexer.
+ // (We manually eliminate the tail call to avoid recursion.)
+ goto LexNextToken;
case ' ':
case '\t':
case '\f':
case '\v':
SkipHorizontalWhitespace:
Result.setFlag(Token::LeadingSpace);
- if (SkipWhitespace(Result, CurPtr))
- return; // KeepWhitespaceMode
+ if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
+ return true; // KeepWhitespaceMode
SkipIgnoredUnits:
CurPtr = BufferPtr;
if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
LangOpts.LineComment &&
(LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) {
- if (SkipLineComment(Result, CurPtr+2))
- return; // There is a token to return.
+ if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine))
+ return true; // There is a token to return.
goto SkipIgnoredUnits;
} else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
- if (SkipBlockComment(Result, CurPtr+2))
- return; // There is a token to return.
+ if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine))
+ return true; // There is a token to return.
goto SkipIgnoredUnits;
} else if (isHorizontalWhitespace(*CurPtr)) {
goto SkipHorizontalWhitespace;
}
- goto LexNextToken; // GCC isn't tail call eliminating.
+ // We only saw whitespace, so just try again with this lexer.
+ // (We manually eliminate the tail call to avoid recursion.)
+ goto LexNextToken;
// C99 6.4.4.1: Integer Constants.
// C99 6.4.4.2: Floating Constants.
TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*';
if (TreatAsComment) {
- if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
- return; // There is a token to return.
+ if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ TokAtPhysicalStartOfLine))
+ return true; // There is a token to return.
// It is common for the tokens immediately after a // comment to be
// whitespace (indentation for the next line). Instead of going through
}
if (Char == '*') { // /**/ comment.
- if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
- return; // There is a token to return.
- goto LexNextToken; // GCC isn't tail call eliminating.
+ if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+ TokAtPhysicalStartOfLine))
+ return true; // There is a token to return.
+
+ // We only saw whitespace, so just try again with this lexer.
+ // (We manually eliminate the tail call to avoid recursion.)
+ goto LexNextToken;
}
if (Char == '=') {
// it's actually the start of a preprocessing directive. Callback to
// the preprocessor to handle it.
// FIXME: -fpreprocessed mode??
- if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer)
+ if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer)
goto HandleDirective;
Kind = tok::hash;
// it's actually the start of a preprocessing directive. Callback to
// the preprocessor to handle it.
// FIXME: -fpreprocessed mode??
- if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer)
+ if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer)
goto HandleDirective;
Kind = tok::hash;
// UCNs (C99 6.4.3, C++11 [lex.charset]p2)
case '\\':
- if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result))
+ if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) {
+ if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) {
+ if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
+ return true; // KeepWhitespaceMode
+
+ // We only saw whitespace, so just try again with this lexer.
+ // (We manually eliminate the tail call to avoid recursion.)
+ goto LexNextToken;
+ }
+
return LexUnicode(Result, CodePoint, CurPtr);
+ }
Kind = tok::unknown;
break;
(const UTF8 *)BufferEnd,
&CodePoint,
strictConversion);
- if (Status == conversionOK)
+ if (Status == conversionOK) {
+ if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) {
+ if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
+ return true; // KeepWhitespaceMode
+
+ // We only saw whitespace, so just try again with this lexer.
+ // (We manually eliminate the tail call to avoid recursion.)
+ goto LexNextToken;
+ }
return LexUnicode(Result, CodePoint, CurPtr);
+ }
if (isLexingRawMode() || ParsingPreprocessorDirective ||
PP->isPreprocessedOutput()) {
Diag(CurPtr, diag::err_invalid_utf8);
BufferPtr = CurPtr+1;
+ // We're pretending the character didn't exist, so just try again with
+ // this lexer.
+ // (We manually eliminate the tail call to avoid recursion.)
goto LexNextToken;
}
}
// Update the location of token as well as BufferPtr.
FormTokenWithChars(Result, CurPtr, Kind);
- return;
+ return true;
HandleDirective:
// We parsed a # character and it's the start of a preprocessing directive.
if (PP->hadModuleLoaderFatalFailure()) {
// With a fatal failure in the module loader, we abort parsing.
assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof");
- return;
+ return true;
}
- // As an optimization, if the preprocessor didn't switch lexers, tail
- // recurse.
- if (PP->isCurrentLexer(this)) {
- // Start a new token. If this is a #include or something, the PP may
- // want us starting at the beginning of the line again. If so, set
- // the StartOfLine flag and clear LeadingSpace.
- if (IsAtStartOfLine) {
- Result.setFlag(Token::StartOfLine);
- Result.clearFlag(Token::LeadingSpace);
- IsAtStartOfLine = false;
- }
- goto LexNextToken; // GCC isn't tail call eliminating.
- }
- return PP->Lex(Result);
+ // We parsed the directive; lex a token with the new state.
+ return false;
}
projects / clang / commitdiff