From: Richard Smith Date: Tue, 19 Mar 2019 22:09:55 +0000 (+0000) Subject: Replace tok::angle_string_literal with new tok::header_name. X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b995249659f360b78c8e40b6c544a68a4f0e6ef8;p=clang Replace tok::angle_string_literal with new tok::header_name. Use the new kind for both angled header-name tokens and for double-quoted header-name tokens. This is in preparation for C++20's context-sensitive header-name token formation rules. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@356530 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/clang/Basic/TokenKinds.def b/include/clang/Basic/TokenKinds.def index 93d741221d..ee0a782046 100644 --- a/include/clang/Basic/TokenKinds.def +++ b/include/clang/Basic/TokenKinds.def @@ -154,7 +154,9 @@ TOK(utf32_char_constant) // U'a' // C99 6.4.5: String Literals. TOK(string_literal) // "foo" TOK(wide_string_literal) // L"foo" -TOK(angle_string_literal)// + +// C11 6.4.7: Header Names +TOK(header_name) // , or "foo" lexed as a header-name // C++11 String Literals. TOK(utf8_string_literal) // u8"foo" diff --git a/include/clang/Basic/TokenKinds.h b/include/clang/Basic/TokenKinds.h index 264a8e2375..1d5be5f915 100644 --- a/include/clang/Basic/TokenKinds.h +++ b/include/clang/Basic/TokenKinds.h @@ -86,7 +86,7 @@ inline bool isLiteral(TokenKind K) { return K == tok::numeric_constant || K == tok::char_constant || K == tok::wide_char_constant || K == tok::utf8_char_constant || K == tok::utf16_char_constant || K == tok::utf32_char_constant || - isStringLiteral(K) || K == tok::angle_string_literal; + isStringLiteral(K) || K == tok::header_name; } /// Return true if this is any of tok::annot_* kinds. diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h index 9e4d01f178..6f7ab0b01f 100644 --- a/include/clang/Lex/Preprocessor.h +++ b/include/clang/Lex/Preprocessor.h @@ -1274,7 +1274,7 @@ public: void Lex(Token &Result); /// Lex a token, forming a header-name token if possible. - bool LexHeaderName(Token &Result, bool AllowConcatenation = true); + bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true); void LexAfterModuleImport(Token &Result); diff --git a/include/clang/Lex/PreprocessorLexer.h b/include/clang/Lex/PreprocessorLexer.h index 29b3e4426e..03b1cc2c10 100644 --- a/include/clang/Lex/PreprocessorLexer.h +++ b/include/clang/Lex/PreprocessorLexer.h @@ -48,8 +48,7 @@ protected: /// True when parsing \#XXX; turns '\\n' into a tok::eod token. bool ParsingPreprocessorDirective = false; - /// True after \#include; turns \ into a tok::angle_string_literal - /// token. + /// True after \#include; turns \ or "xxx" into a tok::header_name token. bool ParsingFilename = false; /// True if in raw mode. diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp index 41003baced..40c6387466 100644 --- a/lib/Lex/Lexer.cpp +++ b/lib/Lex/Lexer.cpp @@ -2072,7 +2072,7 @@ bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { // Update the location of token as well as BufferPtr. const char *TokStart = BufferPtr; - FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); + FormTokenWithChars(Result, CurPtr, tok::header_name); Result.setLiteralData(TokStart); return true; } @@ -3465,7 +3465,9 @@ LexNextToken: case '"': // Notify MIOpt that we read a non-whitespace/non-comment token. MIOpt.ReadToken(); - return LexStringLiteral(Result, CurPtr, tok::string_literal); + return LexStringLiteral(Result, CurPtr, + ParsingFilename ? tok::header_name + : tok::string_literal); // C99 6.4.6: Punctuators. case '?': diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp index 40eb3af257..8d4b9f0656 100644 --- a/lib/Lex/PPDirectives.cpp +++ b/lib/Lex/PPDirectives.cpp @@ -1446,6 +1446,14 @@ bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc, // Get the text form of the filename. assert(!Buffer.empty() && "Can't have tokens with empty spellings!"); + // FIXME: Consider warning on some of the cases described in C11 6.4.7/3 and + // C++20 [lex.header]/2: + // + // If `"`, `'`, `\`, `/*`, or `//` appears in a header-name, then + // in C: behavior is undefined + // in C++: program is conditionally-supported with implementation-defined + // semantics + // Make sure the filename is or "x". bool isAngled; if (Buffer[0] == '<') { @@ -1613,7 +1621,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, if (LexHeaderName(FilenameTok)) return; - if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) { + if (FilenameTok.isNot(tok::header_name)) { Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename); if (FilenameTok.isNot(tok::eod)) DiscardUntilEndOfDirective(); diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp index 751f99950d..d698f0e657 100644 --- a/lib/Lex/PPMacroExpansion.cpp +++ b/lib/Lex/PPMacroExpansion.cpp @@ -1166,7 +1166,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok, PP.Diag(LParenLoc, diag::err_pp_expected_after) << II << tok::l_paren; // If the next token looks like a filename or the start of one, // assume it is and process it as such. - if (!Tok.is(tok::angle_string_literal) && !Tok.is(tok::string_literal)) + if (Tok.isNot(tok::header_name)) return false; } else { // Save '(' location for possible missing ')' message. @@ -1175,7 +1175,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok, return false; } - if (!Tok.isOneOf(tok::angle_string_literal, tok::string_literal)) { + if (Tok.isNot(tok::header_name)) { PP.Diag(Tok.getLocation(), diag::err_pp_expects_filename); return false; } diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp index 369fd92dd6..286b863b35 100644 --- a/lib/Lex/Pragma.cpp +++ b/lib/Lex/Pragma.cpp @@ -486,7 +486,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) { return; // If the next token wasn't a header-name, diagnose the error. - if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) { + if (FilenameTok.isNot(tok::header_name)) { Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename); return; } @@ -670,8 +670,7 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) { StringRef SourceFileName; SmallString<128> FileNameBuffer; - if (SourceFilenameTok.is(tok::string_literal) || - SourceFilenameTok.is(tok::angle_string_literal)) { + if (SourceFilenameTok.is(tok::header_name)) { SourceFileName = getSpelling(SourceFilenameTok, FileNameBuffer); } else { Diag(Tok, diag::warn_pragma_include_alias_expected_filename); @@ -691,8 +690,7 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) { return; StringRef ReplaceFileName; - if (ReplaceFilenameTok.is(tok::string_literal) || - ReplaceFilenameTok.is(tok::angle_string_literal)) { + if (ReplaceFilenameTok.is(tok::header_name)) { ReplaceFileName = getSpelling(ReplaceFilenameTok, FileNameBuffer); } else { Diag(Tok, diag::warn_pragma_include_alias_expected_filename); diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index ebe3797717..832a70620e 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -901,11 +901,12 @@ void Preprocessor::Lex(Token &Result) { /// \param FilenameTok Filled in with the next token. On success, this will /// be either an angle_header_name or a string_literal token. On /// failure, it will be whatever other token was found instead. -/// \param AllowConcatenation If \c true, allow a < token, followed by other -/// tokens and finally a > token, to form a single header-name token. +/// \param AllowMacroExpansion If \c true, allow the header name to be formed +/// by macro expansion (concatenating tokens as necessary if the first +/// token is a '<'). /// \return \c true if we reached EOD or EOF while looking for a > token in /// a concatenated header name and diagnosed it. \c false otherwise. -bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) { +bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { // Lex using header-name tokenization rules if tokens are being lexed from // a file. Just grab a token normally if we're in a macro expansion. if (CurPPLexer) @@ -915,13 +916,16 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) { // This could be a file coming from a macro expansion. In this // case, glue the tokens together into an angle_string_literal token. - if (FilenameTok.is(tok::less) && AllowConcatenation) { - SmallString<128> FilenameBuffer; + SmallString<128> FilenameBuffer; + if (FilenameTok.is(tok::less) && AllowMacroExpansion) { SourceLocation Start = FilenameTok.getLocation(); SourceLocation End; FilenameBuffer.push_back('<'); // Consume tokens until we find a '>'. + // FIXME: A header-name could be formed starting or ending with an + // alternative token. It's not clear whether that's ill-formed in all + // cases. while (FilenameTok.isNot(tok::greater)) { Lex(FilenameTok); if (FilenameTok.isOneOf(tok::eod, tok::eof)) { @@ -962,8 +966,22 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) { } FilenameTok.startToken(); - FilenameTok.setKind(tok::angle_string_literal); + FilenameTok.setKind(tok::header_name); CreateString(FilenameBuffer, FilenameTok, Start, End); + } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) { + // Convert a string-literal token of the form " h-char-sequence " + // (produced by macro expansion) into a header-name token. + // + // The rules for header-names don't quite match the rules for + // string-literals, but all the places where they differ result in + // undefined behavior, so we can and do treat them the same. + // + // A string-literal with a prefix or suffix is not translated into a + // header-name. This could theoretically be observable via the C++20 + // context-sensitive header-name formation rules. + StringRef Str = getSpelling(FilenameTok, FilenameBuffer); + if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"') + FilenameTok.setKind(tok::header_name); } return false; diff --git a/test/Preprocessor/_Pragma-dependency.c b/test/Preprocessor/_Pragma-dependency.c index 4534cc2e4f..0b785ac87d 100644 --- a/test/Preprocessor/_Pragma-dependency.c +++ b/test/Preprocessor/_Pragma-dependency.c @@ -1,5 +1,11 @@ // RUN: %clang_cc1 -E -verify %s +#pragma GCC dependency "./_Pragma-dependency.c" + +#define self "./_Pragma-dependency.c" +// expected-error@+1 {{expected "FILENAME" or }} +#pragma GCC dependency self + #define DO_PRAGMA _Pragma #define STR "GCC dependency \"parse.y\"") // expected-error@+1 {{'parse.y' file not found}}