Replace tok::angle_string_literal with new tok::header_name.

Use the new kind for both angled header-name tokens and for
double-quoted header-name tokens.

This is in preparation for C++20's context-sensitive header-name token
formation rules.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@356530 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/clang/Basic/TokenKinds.def b/include/clang/Basic/TokenKinds.def
index 93d741221d7d540a77ea007a37182f5962ae346a..ee0a78204669f68f27dfbcb96c44beca8604581b 100644 (file)
--- a/include/clang/Basic/TokenKinds.def
+++ b/include/clang/Basic/TokenKinds.def
@@ -154,7 +154,9 @@ TOK(utf32_char_constant) // U'a'
 // C99 6.4.5: String Literals.
 TOK(string_literal)      // "foo"
 TOK(wide_string_literal) // L"foo"
-TOK(angle_string_literal)// <foo>
+
+// C11 6.4.7: Header Names
+TOK(header_name)         // <foo>, or "foo" lexed as a header-name
 
 // C++11 String Literals.
 TOK(utf8_string_literal) // u8"foo"

diff --git a/include/clang/Basic/TokenKinds.h b/include/clang/Basic/TokenKinds.h
index 264a8e2375f9d3e3b5ff14e4d46e787d804bdc98..1d5be5f9152ebb9dfdd50671d3acde70fc85486b 100644 (file)
--- a/include/clang/Basic/TokenKinds.h
+++ b/include/clang/Basic/TokenKinds.h
@@ -86,7 +86,7 @@ inline bool isLiteral(TokenKind K) {
   return K == tok::numeric_constant || K == tok::char_constant ||
          K == tok::wide_char_constant || K == tok::utf8_char_constant ||
          K == tok::utf16_char_constant || K == tok::utf32_char_constant ||
-         isStringLiteral(K) || K == tok::angle_string_literal;
+         isStringLiteral(K) || K == tok::header_name;
 }
 
 /// Return true if this is any of tok::annot_* kinds.

diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index 9e4d01f178d481e3713cd94f2b0b33c66eabf199..6f7ab0b01fd216deaaba7481118ad0bbe3c047e0 100644 (file)
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -1274,7 +1274,7 @@ public:
   void Lex(Token &Result);
 
   /// Lex a token, forming a header-name token if possible.
-  bool LexHeaderName(Token &Result, bool AllowConcatenation = true);
+  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
 
   void LexAfterModuleImport(Token &Result);
 

diff --git a/include/clang/Lex/PreprocessorLexer.h b/include/clang/Lex/PreprocessorLexer.h
index 29b3e4426e84bb2df3c7af7fd09ae012921953a7..03b1cc2c10e2c0f4907b83a4911d47a47caaa7b3 100644 (file)
--- a/include/clang/Lex/PreprocessorLexer.h
+++ b/include/clang/Lex/PreprocessorLexer.h
@@ -48,8 +48,7 @@ protected:
   /// True when parsing \#XXX; turns '\\n' into a tok::eod token.
   bool ParsingPreprocessorDirective = false;
 
-  /// True after \#include; turns \<xx> into a tok::angle_string_literal
-  /// token.
+  /// True after \#include; turns \<xx> or "xxx" into a tok::header_name token.
   bool ParsingFilename = false;
 
   /// True if in raw mode.

diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 41003baced634c8ca93dd35d5fb8f774ad312714..40c63874661118c64e27c5a04ea28c1793f53f29 100644 (file)
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -2072,7 +2072,7 @@ bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = BufferPtr;
-  FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
+  FormTokenWithChars(Result, CurPtr, tok::header_name);
   Result.setLiteralData(TokStart);
   return true;
 }
@@ -3465,7 +3465,9 @@ LexNextToken:
   case '"':
     // Notify MIOpt that we read a non-whitespace/non-comment token.
     MIOpt.ReadToken();
-    return LexStringLiteral(Result, CurPtr, tok::string_literal);
+    return LexStringLiteral(Result, CurPtr,
+                            ParsingFilename ? tok::header_name
+                                            : tok::string_literal);
 
   // C99 6.4.6: Punctuators.
   case '?':

diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 40eb3af2573697ded3beb83470df46e8a9a7d244..8d4b9f065695b7b431261552c5cd9e88bb4e995e 100644 (file)
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -1446,6 +1446,14 @@ bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
   // Get the text form of the filename.
   assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
 
+  // FIXME: Consider warning on some of the cases described in C11 6.4.7/3 and
+  // C++20 [lex.header]/2:
+  //
+  // If `"`, `'`, `\`, `/*`, or `//` appears in a header-name, then
+  //   in C: behavior is undefined
+  //   in C++: program is conditionally-supported with implementation-defined
+  //           semantics
+
   // Make sure the filename is <x> or "x".
   bool isAngled;
   if (Buffer[0] == '<') {
@@ -1613,7 +1621,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   if (LexHeaderName(FilenameTok))
     return;
 
-  if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+  if (FilenameTok.isNot(tok::header_name)) {
     Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
     if (FilenameTok.isNot(tok::eod))
       DiscardUntilEndOfDirective();

diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp
index 751f99950de1662d7f05d056c0fa1fb4b5d8bc24..d698f0e6578c25b14821233390f2bf8590346369 100644 (file)
--- a/lib/Lex/PPMacroExpansion.cpp
+++ b/lib/Lex/PPMacroExpansion.cpp
@@ -1166,7 +1166,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
     PP.Diag(LParenLoc, diag::err_pp_expected_after) << II << tok::l_paren;
     // If the next token looks like a filename or the start of one,
     // assume it is and process it as such.
-    if (!Tok.is(tok::angle_string_literal) && !Tok.is(tok::string_literal))
+    if (Tok.isNot(tok::header_name))
       return false;
   } else {
     // Save '(' location for possible missing ')' message.
@@ -1175,7 +1175,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
       return false;
   }
 
-  if (!Tok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+  if (Tok.isNot(tok::header_name)) {
     PP.Diag(Tok.getLocation(), diag::err_pp_expects_filename);
     return false;
   }

diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp
index 369fd92dd6116f3971fef9a9a60949563b280bd0..286b863b3547a181fec25733e181db29d2761a56 100644 (file)
--- a/lib/Lex/Pragma.cpp
+++ b/lib/Lex/Pragma.cpp
@@ -486,7 +486,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
     return;
 
   // If the next token wasn't a header-name, diagnose the error.
-  if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
+  if (FilenameTok.isNot(tok::header_name)) {
     Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
     return;
   }
@@ -670,8 +670,7 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) {
 
   StringRef SourceFileName;
   SmallString<128> FileNameBuffer;
-  if (SourceFilenameTok.is(tok::string_literal) ||
-      SourceFilenameTok.is(tok::angle_string_literal)) {
+  if (SourceFilenameTok.is(tok::header_name)) {
     SourceFileName = getSpelling(SourceFilenameTok, FileNameBuffer);
   } else {
     Diag(Tok, diag::warn_pragma_include_alias_expected_filename);
@@ -691,8 +690,7 @@ void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) {
     return;
 
   StringRef ReplaceFileName;
-  if (ReplaceFilenameTok.is(tok::string_literal) ||
-      ReplaceFilenameTok.is(tok::angle_string_literal)) {
+  if (ReplaceFilenameTok.is(tok::header_name)) {
     ReplaceFileName = getSpelling(ReplaceFilenameTok, FileNameBuffer);
   } else {
     Diag(Tok, diag::warn_pragma_include_alias_expected_filename);

diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index ebe3797717cd451d05968628a511ec0f3044ca18..832a70620e87cb1ba541864209663a2b5848d16c 100644 (file)
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -901,11 +901,12 @@ void Preprocessor::Lex(Token &Result) {
 /// \param FilenameTok Filled in with the next token. On success, this will
 ///        be either an angle_header_name or a string_literal token. On
 ///        failure, it will be whatever other token was found instead.
-/// \param AllowConcatenation If \c true, allow a < token, followed by other
-///        tokens and finally a > token, to form a single header-name token.
+/// \param AllowMacroExpansion If \c true, allow the header name to be formed
+///        by macro expansion (concatenating tokens as necessary if the first
+///        token is a '<').
 /// \return \c true if we reached EOD or EOF while looking for a > token in
 ///         a concatenated header name and diagnosed it. \c false otherwise.
-bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) {
+bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
   // Lex using header-name tokenization rules if tokens are being lexed from
   // a file. Just grab a token normally if we're in a macro expansion.
   if (CurPPLexer)
@@ -915,13 +916,16 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) {
 
   // This could be a <foo/bar.h> file coming from a macro expansion.  In this
   // case, glue the tokens together into an angle_string_literal token.
-  if (FilenameTok.is(tok::less) && AllowConcatenation) {
-    SmallString<128> FilenameBuffer;
+  SmallString<128> FilenameBuffer;
+  if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
     SourceLocation Start = FilenameTok.getLocation();
     SourceLocation End;
     FilenameBuffer.push_back('<');
 
     // Consume tokens until we find a '>'.
+    // FIXME: A header-name could be formed starting or ending with an
+    // alternative token. It's not clear whether that's ill-formed in all
+    // cases.
     while (FilenameTok.isNot(tok::greater)) {
       Lex(FilenameTok);
       if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
@@ -962,8 +966,22 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) {
     }
 
     FilenameTok.startToken();
-    FilenameTok.setKind(tok::angle_string_literal);
+    FilenameTok.setKind(tok::header_name);
     CreateString(FilenameBuffer, FilenameTok, Start, End);
+  } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
+    // Convert a string-literal token of the form " h-char-sequence "
+    // (produced by macro expansion) into a header-name token.
+    //
+    // The rules for header-names don't quite match the rules for
+    // string-literals, but all the places where they differ result in
+    // undefined behavior, so we can and do treat them the same.
+    //
+    // A string-literal with a prefix or suffix is not translated into a
+    // header-name. This could theoretically be observable via the C++20
+    // context-sensitive header-name formation rules.
+    StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
+    if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
+      FilenameTok.setKind(tok::header_name);
   }
 
   return false;

diff --git a/test/Preprocessor/_Pragma-dependency.c b/test/Preprocessor/_Pragma-dependency.c
index 4534cc2e4f352331b2b44c0671ec28f7f151e6e4..0b785ac87daaf4a090af08fa00d3025c21ae7f4f 100644 (file)
--- a/test/Preprocessor/_Pragma-dependency.c
+++ b/test/Preprocessor/_Pragma-dependency.c
@@ -1,5 +1,11 @@
 // RUN: %clang_cc1 -E -verify %s
 
+#pragma GCC dependency "./_Pragma-dependency.c"
+
+#define self "./_Pragma-dependency.c"
+// expected-error@+1 {{expected "FILENAME" or <FILENAME>}}
+#pragma GCC dependency self
+
 #define DO_PRAGMA _Pragma 
 #define STR "GCC dependency \"parse.y\"")
 // expected-error@+1 {{'parse.y' file not found}}