Improve the implementation of libclang's token-annotation logic for

entities in the preprocessing record. Previously, we would only end up
getting the first token of a preprocessing record annotated
correctly. For example, given

  #include "foo.h"

we would only get the '#' annotated as an inclusion directive; the
'include' and '"foo.h"' tokens would be given the general 'processing
directive' annotation.

Now, we get proper annotations for entities in the preprocessing
record.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@117001 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/test/Index/annotate-tokens-include.c b/test/Index/annotate-tokens-include.c
index b60c1af181685d316e70b8fc9f75084cea03f6ef..2190125c098fe15d6c70423b11154af567fcd061 100644 (file)
--- a/test/Index/annotate-tokens-include.c
+++ b/test/Index/annotate-tokens-include.c
@@ -2,6 +2,6 @@
 
 // RUN: c-index-test -test-annotate-tokens=%s:1:1:2:1 %s | FileCheck %s
 // CHECK: Punctuation: "#" [1:1 - 1:2] inclusion directive=annotate-tokens-include.h
-// CHECK: Identifier: "include" [1:2 - 1:9] preprocessing directive=
-// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] preprocessing directive=
+// CHECK: Identifier: "include" [1:2 - 1:9] inclusion directive=annotate-tokens-include.h
+// CHECK: Literal: ""annotate-tokens-include.h"" [1:10 - 1:37] inclusion directive=annotate-tokens-include.h
 

diff --git a/test/Index/annotate-tokens-pp.c b/test/Index/annotate-tokens-pp.c
index 631caab839c6b6ac71004067e393e4a6e3df92ee..a6e7fb99d60f678b145dcfc27920be8267cda297 100644 (file)
--- a/test/Index/annotate-tokens-pp.c
+++ b/test/Index/annotate-tokens-pp.c
@@ -29,27 +29,27 @@ void test() {
 // CHECK: Punctuation: "#" [2:1 - 2:2] preprocessing directive=
 // CHECK: Identifier: "define" [2:2 - 2:8] preprocessing directive=
 // CHECK: Identifier: "STILL_NOTHING" [2:9 - 2:22] macro definition=STILL_NOTHING
-// CHECK: Identifier: "NOTHING" [2:23 - 2:30] preprocessing directive=
-// CHECK: Punctuation: "(" [2:30 - 2:31] preprocessing directive=
-// CHECK: Identifier: "honk" [2:31 - 2:35] preprocessing directive=
-// CHECK: Punctuation: "," [2:35 - 2:36] preprocessing directive=
-// CHECK: Identifier: "warble" [2:36 - 2:42] preprocessing directive=
-// CHECK: Punctuation: ")" [2:42 - 2:43] preprocessing directive=
+// CHECK: Identifier: "NOTHING" [2:23 - 2:30] macro definition=STILL_NOTHING
+// CHECK: Punctuation: "(" [2:30 - 2:31] macro definition=STILL_NOTHING
+// CHECK: Identifier: "honk" [2:31 - 2:35] macro definition=STILL_NOTHING
+// CHECK: Punctuation: "," [2:35 - 2:36] macro definition=STILL_NOTHING
+// CHECK: Identifier: "warble" [2:36 - 2:42] macro definition=STILL_NOTHING
+// CHECK: Punctuation: ")" [2:42 - 2:43] macro definition=STILL_NOTHING
 // CHECK: Punctuation: "#" [3:1 - 3:2] preprocessing directive=
 // CHECK: Identifier: "define" [3:2 - 3:8] preprocessing directive=
 // CHECK: Identifier: "BAR" [3:9 - 3:12] macro definition=BAR
-// CHECK: Identifier: "baz" [3:13 - 3:16] preprocessing directive=
+// CHECK: Identifier: "baz" [3:13 - 3:16] macro definition=BAR
 // CHECK: Punctuation: "#" [4:1 - 4:2] preprocessing directive=
 // CHECK: Identifier: "define" [4:2 - 4:8] preprocessing directive=
 // CHECK: Identifier: "WIBBLE" [4:9 - 4:15] macro definition=WIBBLE
-// CHECK: Punctuation: "(" [4:15 - 4:16] preprocessing directive=
-// CHECK: Identifier: "X" [4:16 - 4:17] preprocessing directive=
-// CHECK: Punctuation: "," [4:17 - 4:18] preprocessing directive=
-// CHECK: Identifier: "Y" [4:19 - 4:20] preprocessing directive=
-// CHECK: Punctuation: ")" [4:20 - 4:21] preprocessing directive=
-// CHECK: Identifier: "X" [4:22 - 4:23] preprocessing directive=
-// CHECK: Punctuation: "##" [4:23 - 4:25] preprocessing directive=
-// CHECK: Identifier: "Y" [4:25 - 4:26] preprocessing directive=
+// CHECK: Punctuation: "(" [4:15 - 4:16] macro definition=WIBBLE
+// CHECK: Identifier: "X" [4:16 - 4:17] macro definition=WIBBLE
+// CHECK: Punctuation: "," [4:17 - 4:18] macro definition=WIBBLE
+// CHECK: Identifier: "Y" [4:19 - 4:20] macro definition=WIBBLE
+// CHECK: Punctuation: ")" [4:20 - 4:21] macro definition=WIBBLE
+// CHECK: Identifier: "X" [4:22 - 4:23] macro definition=WIBBLE
+// CHECK: Punctuation: "##" [4:23 - 4:25] macro definition=WIBBLE
+// CHECK: Identifier: "Y" [4:25 - 4:26] macro definition=WIBBLE
 // CHECK: Identifier: "NOTHING" [5:1 - 5:8] macro instantiation=NOTHING:1:9
 // CHECK: Punctuation: "(" [5:8 - 5:9]
 // CHECK: Identifier: "more" [5:9 - 5:13]
@@ -69,31 +69,31 @@ void test() {
 // CHECK: Identifier: "STILL_NOTHING" [6:9 - 6:22] macro instantiation=STILL_NOTHING:2:9
 // CHECK: Punctuation: ";" [6:22 - 6:23]
 // CHECK: Punctuation: "#" [7:1 - 7:2] inclusion directive=foo.h
-// CHECK: Identifier: "include" [7:2 - 7:9] preprocessing directive=
-// CHECK: Literal: ""foo.h"" [7:10 - 7:17] preprocessing directive=
+// CHECK: Identifier: "include" [7:2 - 7:9] inclusion directive=foo.h
+// CHECK: Literal: ""foo.h"" [7:10 - 7:17] inclusion directive=foo.h
 // CHECK: Punctuation: "#" [8:1 - 8:2] preprocessing directive=
 // CHECK: Identifier: "undef" [8:2 - 8:7] preprocessing directive=
 // CHECK: Identifier: "BAR" [8:8 - 8:11] preprocessing directive=
 // CHECK: Punctuation: "#" [10:1 - 10:2] preprocessing directive=
 // CHECK: Identifier: "define" [10:2 - 10:8] preprocessing directive=
 // CHECK: Identifier: "REVERSE_MACRO" [10:9 - 10:22] macro definition=REVERSE_MACRO
-// CHECK: Punctuation: "(" [10:22 - 10:23] preprocessing directive=
-// CHECK: Identifier: "x" [10:23 - 10:24] preprocessing directive=
-// CHECK: Punctuation: "," [10:24 - 10:25] preprocessing directive=
-// CHECK: Identifier: "y" [10:25 - 10:26] preprocessing directive=
-// CHECK: Punctuation: ")" [10:26 - 10:27] preprocessing directive=
-// CHECK: Identifier: "y" [10:28 - 10:29] preprocessing directive=
-// CHECK: Punctuation: "+" [10:30 - 10:31] preprocessing directive=
-// CHECK: Identifier: "x" [10:32 - 10:33] preprocessing directive=
+// CHECK: Punctuation: "(" [10:22 - 10:23] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "x" [10:23 - 10:24] macro definition=REVERSE_MACRO
+// CHECK: Punctuation: "," [10:24 - 10:25] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "y" [10:25 - 10:26] macro definition=REVERSE_MACRO
+// CHECK: Punctuation: ")" [10:26 - 10:27] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "y" [10:28 - 10:29] macro definition=REVERSE_MACRO
+// CHECK: Punctuation: "+" [10:30 - 10:31] macro definition=REVERSE_MACRO
+// CHECK: Identifier: "x" [10:32 - 10:33] macro definition=REVERSE_MACRO
 // CHECK: Punctuation: "#" [11:1 - 11:2] preprocessing directive=
 // CHECK: Identifier: "define" [11:2 - 11:8] preprocessing directive=
 // CHECK: Identifier: "TWICE_MACRO" [11:9 - 11:20] macro definition=TWICE_MACRO
-// CHECK: Punctuation: "(" [11:20 - 11:21] preprocessing directive=
-// CHECK: Identifier: "y" [11:21 - 11:22] preprocessing directive=
-// CHECK: Punctuation: ")" [11:22 - 11:23] preprocessing directive=
-// CHECK: Identifier: "y" [11:24 - 11:25] preprocessing directive=
-// CHECK: Punctuation: "+" [11:26 - 11:27] preprocessing directive=
-// CHECK: Identifier: "y" [11:28 - 11:29] preprocessing directive=
+// CHECK: Punctuation: "(" [11:20 - 11:21] macro definition=TWICE_MACRO
+// CHECK: Identifier: "y" [11:21 - 11:22] macro definition=TWICE_MACRO
+// CHECK: Punctuation: ")" [11:22 - 11:23] macro definition=TWICE_MACRO
+// CHECK: Identifier: "y" [11:24 - 11:25] macro definition=TWICE_MACRO
+// CHECK: Punctuation: "+" [11:26 - 11:27] macro definition=TWICE_MACRO
+// CHECK: Identifier: "y" [11:28 - 11:29] macro definition=TWICE_MACRO
 // CHECK: Keyword: "void" [13:1 - 13:5] FunctionDecl=test_macro_args:13:6 (Definition)
 // CHECK: Identifier: "test_macro_args" [13:6 - 13:21] FunctionDecl=test_macro_args:13:6 (Definition)
 // CHECK: Punctuation: "(" [13:21 - 13:22] FunctionDecl=test_macro_args:13:6 (Definition)
@@ -140,23 +140,23 @@ void test() {
 // CHECK: Punctuation: "#" [21:1 - 21:2] preprocessing directive=
 // CHECK: Identifier: "define" [21:2 - 21:8] preprocessing directive=
 // CHECK: Identifier: "fun_with_macro_bodies" [21:9 - 21:30] macro definition=fun_with_macro_bodies
-// CHECK: Punctuation: "(" [21:30 - 21:31] preprocessing directive=
-// CHECK: Identifier: "x" [21:31 - 21:32] preprocessing directive=
-// CHECK: Punctuation: "," [21:32 - 21:33] preprocessing directive=
-// CHECK: Identifier: "y" [21:34 - 21:35] preprocessing directive=
-// CHECK: Punctuation: ")" [21:35 - 21:36] preprocessing directive=
-// CHECK: Keyword: "do" [21:37 - 21:39] preprocessing directive=
-// CHECK: Punctuation: "{" [21:40 - 21:41] preprocessing directive=
-// CHECK: Keyword: "if" [21:42 - 21:44] preprocessing directive=
-// CHECK: Punctuation: "(" [21:45 - 21:46] preprocessing directive=
-// CHECK: Identifier: "x" [21:46 - 21:47] preprocessing directive=
-// CHECK: Punctuation: ")" [21:47 - 21:48] preprocessing directive=
-// CHECK: Identifier: "y" [21:49 - 21:50] preprocessing directive=
-// CHECK: Punctuation: "}" [21:51 - 21:52] preprocessing directive=
-// CHECK: Keyword: "while" [21:53 - 21:58] preprocessing directive=
-// CHECK: Punctuation: "(" [21:59 - 21:60] preprocessing directive=
-// CHECK: Literal: "0" [21:60 - 21:61] preprocessing directive=
-// CHECK: Punctuation: ")" [21:61 - 21:62] preprocessing directive=
+// CHECK: Punctuation: "(" [21:30 - 21:31] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "x" [21:31 - 21:32] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "," [21:32 - 21:33] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "y" [21:34 - 21:35] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: ")" [21:35 - 21:36] macro definition=fun_with_macro_bodies
+// CHECK: Keyword: "do" [21:37 - 21:39] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "{" [21:40 - 21:41] macro definition=fun_with_macro_bodies
+// CHECK: Keyword: "if" [21:42 - 21:44] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "(" [21:45 - 21:46] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "x" [21:46 - 21:47] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: ")" [21:47 - 21:48] macro definition=fun_with_macro_bodies
+// CHECK: Identifier: "y" [21:49 - 21:50] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "}" [21:51 - 21:52] macro definition=fun_with_macro_bodies
+// CHECK: Keyword: "while" [21:53 - 21:58] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: "(" [21:59 - 21:60] macro definition=fun_with_macro_bodies
+// CHECK: Literal: "0" [21:60 - 21:61] macro definition=fun_with_macro_bodies
+// CHECK: Punctuation: ")" [21:61 - 21:62] macro definition=fun_with_macro_bodies
 // CHECK: Keyword: "void" [23:1 - 23:5] FunctionDecl=test:23:6 (Definition)
 // CHECK: Identifier: "test" [23:6 - 23:10] FunctionDecl=test:23:6 (Definition)
 // CHECK: Punctuation: "(" [23:10 - 23:11] FunctionDecl=test:23:6 (Definition)

diff --git a/test/Index/annotate-tokens.m b/test/Index/annotate-tokens.m
index ec919fa16470b93ec05a3d8cf179a8f696def7c4..6b8a39c55d53b727b524e6002c908953f5845f03 100644 (file)
--- a/test/Index/annotate-tokens.m
+++ b/test/Index/annotate-tokens.m
@@ -285,7 +285,7 @@ void f() {
 // CHECK: Punctuation: "#" [63:1 - 63:2] preprocessing directive=
 // CHECK: Identifier: "define" [63:2 - 63:8] preprocessing directive=
 // CHECK: Identifier: "VAL" [63:9 - 63:12] macro definition=VAL
-// CHECK: Literal: "0" [63:13 - 63:14] preprocessing directive=
+// CHECK: Literal: "0" [63:13 - 63:14] macro definition=VAL
 // CHECK: Punctuation: "@" [65:1 - 65:2] ObjCInterfaceDecl=R7974151:65:12
 // CHECK: Keyword: "interface" [65:2 - 65:11] ObjCInterfaceDecl=R7974151:65:12
 // CHECK: Identifier: "R7974151" [65:12 - 65:20] ObjCInterfaceDecl=R7974151:65:12

diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp
index c944883fa4d5dab0dffe960f858fcd36639db309..b9147fa1a1f95b5ad958be7142b603c042ac178c 100644 (file)
--- a/tools/libclang/CIndex.cpp
+++ b/tools/libclang/CIndex.cpp
@@ -3657,6 +3657,7 @@ class AnnotateTokensWorker {
   CXCursor *Cursors;
   unsigned NumTokens;
   unsigned TokIdx;
+  unsigned PreprocessingTokIdx;
   CursorVisitor AnnotateVis;
   SourceManager &SrcMgr;
 
@@ -3672,7 +3673,7 @@ public:
                        CXToken *tokens, CXCursor *cursors, unsigned numTokens,
                        ASTUnit *CXXUnit, SourceRange RegionOfInterest)
     : Annotated(annotated), Tokens(tokens), Cursors(cursors),
-      NumTokens(numTokens), TokIdx(0),
+      NumTokens(numTokens), TokIdx(0), PreprocessingTokIdx(0),
       AnnotateVis(CXXUnit, AnnotateTokensVisitor, this,
                   Decl::MaxPCHLevel, RegionOfInterest),
       SrcMgr(CXXUnit->getSourceManager()) {}
@@ -3690,7 +3691,9 @@ void AnnotateTokensWorker::AnnotateTokens(CXCursor parent) {
 
   for (unsigned I = 0 ; I < TokIdx ; ++I) {
     AnnotateTokensData::iterator Pos = Annotated.find(Tokens[I].int_data[1]);
-    if (Pos != Annotated.end())
+    if (Pos != Annotated.end() && 
+        (clang_isInvalid(Cursors[I].kind) ||
+         Pos->second.kind != CXCursor_PreprocessingDirective))
       Cursors[I] = Pos->second;
   }
 
@@ -3706,16 +3709,66 @@ void AnnotateTokensWorker::AnnotateTokens(CXCursor parent) {
 }
 
 enum CXChildVisitResult
-AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
+AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {  
   CXSourceLocation Loc = clang_getCursorLocation(cursor);
-  // We can always annotate a preprocessing directive/macro instantiation.
-  if (clang_isPreprocessing(cursor.kind)) {
-    Annotated[Loc.int_data] = cursor;
+  SourceRange cursorRange = getRawCursorExtent(cursor);
+  
+  if (clang_isPreprocessing(cursor.kind)) {    
+    // For macro instantiations, just note where the beginning of the macro
+    // instantiation occurs.
+    if (cursor.kind == CXCursor_MacroInstantiation) {
+      Annotated[Loc.int_data] = cursor;
+      return CXChildVisit_Recurse;
+    }
+    
+    if (cursorRange.isInvalid())
+      return CXChildVisit_Continue;
+    
+    // Items in the preprocessing record are kept separate from items in
+    // declarations, so we keep a separate token index.
+    unsigned SavedTokIdx = TokIdx;
+    TokIdx = PreprocessingTokIdx;
+
+    // Skip tokens up until we catch up to the beginning of the preprocessing
+    // entry.
+    while (MoreTokens()) {
+      const unsigned I = NextToken();
+      SourceLocation TokLoc = GetTokenLoc(I);
+      switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) {
+      case RangeBefore:
+        AdvanceToken();
+        continue;
+      case RangeAfter:
+      case RangeOverlap:
+        break;
+      }
+      break;
+    }
+    
+    // Look at all of the tokens within this range.
+    while (MoreTokens()) {
+      const unsigned I = NextToken();
+      SourceLocation TokLoc = GetTokenLoc(I);
+      switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) {
+      case RangeBefore:
+        assert(0 && "Infeasible");
+      case RangeAfter:
+        break;
+      case RangeOverlap:
+        Cursors[I] = cursor;
+        AdvanceToken();
+        continue;
+      }
+      break;
+    }
+
+    // Save the preprocessing token index; restore the non-preprocessing
+    // token index.
+    PreprocessingTokIdx = TokIdx;
+    TokIdx = SavedTokIdx;
     return CXChildVisit_Recurse;
   }
 
-  SourceRange cursorRange = getRawCursorExtent(cursor);
-  
   if (cursorRange.isInvalid())
     return CXChildVisit_Continue;
   
@@ -3753,7 +3806,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
     // directive.  Here we assume that the default construction of CXCursor
     // results in CXCursor.kind being an initialized value (i.e., 0).  If
     // this isn't the case, we can fix by doing lookup + insertion.
-
+    
     CXCursor &oldC = Annotated[rawEncoding];
     if (!clang_isPreprocessing(oldC.kind))
       oldC = cursor;
@@ -3815,6 +3868,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
   for (unsigned I = BeforeChildren; I != AfterChildren; ++I) {
     if (!clang_isInvalid(clang_getCursorKind(Cursors[I])))
       break;
+    
     Cursors[I] = cursor;
   }
   // Scan the tokens that are at the end of the cursor, but are not captured
@@ -3841,14 +3895,14 @@ void clang_annotateTokens(CXTranslationUnit TU,
   if (NumTokens == 0 || !Tokens || !Cursors)
     return;
 
+  // Any token we don't specifically annotate will have a NULL cursor.
+  CXCursor C = clang_getNullCursor();
+  for (unsigned I = 0; I != NumTokens; ++I)
+    Cursors[I] = C;
+
   ASTUnit *CXXUnit = static_cast<ASTUnit *>(TU);
-  if (!CXXUnit) {
-    // Any token we don't specifically annotate will have a NULL cursor.
-    const CXCursor &C = clang_getNullCursor();
-    for (unsigned I = 0; I != NumTokens; ++I)
-      Cursors[I] = C;
+  if (!CXXUnit)
     return;
-  }
 
   ASTUnit::ConcurrencyCheck Check(*CXXUnit);