[Syntax] Added function to get macro expansion tokens to TokenBuffer.

Summary:
Returns the first token in every mapping where the token is an identifier.
This API is required to be able to highlight macro expansions in clangd.

Reviewers: hokein, ilya-biryukov

Subscribers: kadircet, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D66470

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@369385 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/clang/Tooling/Syntax/Tokens.h b/include/clang/Tooling/Syntax/Tokens.h
index 4640ccb2d30aaff0b6aa7a8f2aabe1a7afb0e24b..301432d3888b302efb8fc07355111447c9b32e1f 100644 (file)
--- a/include/clang/Tooling/Syntax/Tokens.h
+++ b/include/clang/Tooling/Syntax/Tokens.h
@@ -236,6 +236,16 @@ public:
   ///        #pragma, etc.
   llvm::ArrayRef<syntax::Token> spelledTokens(FileID FID) const;
 
+  /// Get all tokens that expand a macro in \p FID. For the following input
+  ///     #define FOO B
+  ///     #define FOO2(X) int X
+  ///     FOO2(XY)
+  ///     int B;
+  ///     FOO;
+  /// macroExpansions() returns {"FOO2", "FOO"} (from line 3 and 5
+  /// respecitvely).
+  std::vector<const syntax::Token *> macroExpansions(FileID FID) const;
+
   const SourceManager &sourceManager() const { return *SourceMgr; }
 
   std::string dumpForTests() const;

diff --git a/lib/Tooling/Syntax/Tokens.cpp b/lib/Tooling/Syntax/Tokens.cpp
index 3852e71c37ffea29594165afddea36ca87893120..a2c3bc137d6ba7a5b47242ccb20cc1d781ad0479 100644 (file)
--- a/lib/Tooling/Syntax/Tokens.cpp
+++ b/lib/Tooling/Syntax/Tokens.cpp
@@ -232,6 +232,21 @@ TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const {
   return E;
 }
 
+std::vector<const syntax::Token *>
+TokenBuffer::macroExpansions(FileID FID) const {
+  auto FileIt = Files.find(FID);
+  assert(FileIt != Files.end() && "file not tracked by token buffer");
+  auto &File = FileIt->second;
+  std::vector<const syntax::Token *> Expansions;
+  auto &Spelled = File.SpelledTokens;
+  for (auto Mapping : File.Mappings) {
+    const syntax::Token *Token = &Spelled[Mapping.BeginSpelled];
+    if (Token->kind() == tok::TokenKind::identifier)
+      Expansions.push_back(Token);
+  }
+  return Expansions;
+}
+
 std::vector<syntax::Token> syntax::tokenize(FileID FID, const SourceManager &SM,
                                             const LangOptions &LO) {
   std::vector<syntax::Token> Tokens;

diff --git a/unittests/Tooling/Syntax/TokensTest.cpp b/unittests/Tooling/Syntax/TokensTest.cpp
index 78f0d2fdd87461d40f69295246f5c93866dfe701..6ffe2c43dd0ff75bfead6b41b5654ffae6edb5ed 100644 (file)
--- a/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/unittests/Tooling/Syntax/TokensTest.cpp
@@ -755,4 +755,27 @@ TEST_F(TokenBufferTest, TokensToFileRange) {
   // We don't test assertion failures because death tests are slow.
 }
 
+TEST_F(TokenBufferTest, macroExpansions) {
+  llvm::Annotations Code(R"cpp(
+    #define FOO B
+    #define FOO2 BA
+    #define CALL(X) int X
+    #define G CALL(FOO2)
+    int B;
+    $macro[[FOO]];
+    $macro[[CALL]](A);
+    $macro[[G]];
+  )cpp");
+  recordTokens(Code.code());
+  auto &SM = *SourceMgr;
+  auto Expansions = Buffer.macroExpansions(SM.getMainFileID());
+  std::vector<FileRange> ExpectedMacroRanges;
+  for (auto Range : Code.ranges("macro"))
+    ExpectedMacroRanges.push_back(
+        FileRange(SM.getMainFileID(), Range.Begin, Range.End));
+  std::vector<FileRange> ActualMacroRanges;
+  for (auto Expansion : Expansions)
+    ActualMacroRanges.push_back(Expansion->range(SM));
+  EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges);
+}
 } // namespace