From: Argyrios Kyrtzidis Date: Thu, 18 Aug 2011 18:03:34 +0000 (+0000) Subject: [libclang] Annotate correctly macro argument tokens. X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=a676379b26edc959193f9f919ba9c6d296a57824;p=clang [libclang] Annotate correctly macro argument tokens. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@137961 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/test/Index/annotate-macro-args.h b/test/Index/annotate-macro-args.h new file mode 100644 index 0000000000..40ec8dc0b8 --- /dev/null +++ b/test/Index/annotate-macro-args.h @@ -0,0 +1,16 @@ +@interface MyClass ++(void)meth; +@end + +#define MACRO2(x) x +#define MACRO(x) MACRO2(x) + +void test() { + MACRO([MyClass meth]); +} + +#define INVOKE(METHOD, CLASS) [CLASS METHOD] + +void test2() { + INVOKE(meth, MyClass); +} diff --git a/test/Index/annotate-macro-args.m b/test/Index/annotate-macro-args.m new file mode 100644 index 0000000000..6adfb8ded1 --- /dev/null +++ b/test/Index/annotate-macro-args.m @@ -0,0 +1,23 @@ +// Test without PCH +// RUN: c-index-test -test-annotate-tokens=%S/annotate-macro-args.h:9:1:10:1 %s -include annotate-macro-args.h | FileCheck -check-prefix=CHECK1 %s +// RUN: c-index-test -test-annotate-tokens=%S/annotate-macro-args.h:15:1:16:1 %s -include annotate-macro-args.h | FileCheck -check-prefix=CHECK2 %s + +// Test with PCH +// RUN: c-index-test -write-pch %t.pch -x objective-c-header %S/annotate-macro-args.h -Xclang -detailed-preprocessing-record +// RUN: c-index-test -test-annotate-tokens=%S/annotate-macro-args.h:9:1:10:1 %s -include-pch %t.pch | FileCheck -check-prefix=CHECK1 %s +// RUN: c-index-test -test-annotate-tokens=%S/annotate-macro-args.h:15:1:16:1 %s -include-pch %t.pch | FileCheck -check-prefix=CHECK2 %s + +// CHECK1: Identifier: "MACRO" [9:3 - 9:8] macro expansion=MACRO:6:9 +// CHECK1: Punctuation: "(" [9:8 - 9:9] +// CHECK1: Punctuation: "[" [9:9 - 9:10] ObjCMessageExpr=meth:2:1 +// CHECK1: Identifier: "MyClass" [9:10 - 9:17] ObjCClassRef=MyClass:1:12 +// CHECK1: Identifier: "meth" [9:18 - 9:22] ObjCMessageExpr=meth:2:1 +// CHECK1: Punctuation: "]" [9:22 - 9:23] ObjCMessageExpr=meth:2:1 +// CHECK1: Punctuation: ")" [9:23 - 9:24] + +// CHECK2: Identifier: "INVOKE" [15:3 - 15:9] macro expansion=INVOKE:12:9 +// CHECK2: Punctuation: "(" [15:9 - 15:10] +// CHECK2: Identifier: "meth" [15:10 - 15:14] ObjCMessageExpr=meth:2:1 +// CHECK2: Punctuation: "," [15:14 - 15:15] +// CHECK2: Identifier: "MyClass" [15:16 - 15:23] ObjCClassRef=MyClass:1:12 +// CHECK2: Punctuation: ")" [15:23 - 15:24] diff --git a/test/Index/annotate-tokens-pp.c b/test/Index/annotate-tokens-pp.c index e6bb0869d0..a8ef8a7bdc 100644 --- a/test/Index/annotate-tokens-pp.c +++ b/test/Index/annotate-tokens-pp.c @@ -116,9 +116,9 @@ void test() { // CHECK: Identifier: "k" [16:7 - 16:8] VarDecl=k:16:7 (Definition) // CHECK: Punctuation: "=" [16:9 - 16:10] VarDecl=k:16:7 (Definition) // CHECK: Identifier: "REVERSE_MACRO" [16:11 - 16:24] macro expansion=REVERSE_MACRO:10:9 -// CHECK: Punctuation: "(" [16:24 - 16:25] UnexposedStmt= +// CHECK: Punctuation: "(" [16:24 - 16:25] // CHECK: Identifier: "t" [16:25 - 16:26] DeclRefExpr=t:15:7 -// CHECK: Punctuation: "," [16:26 - 16:27] UnexposedStmt= +// CHECK: Punctuation: "," [16:26 - 16:27] // CHECK: Identifier: "z" [16:27 - 16:28] DeclRefExpr=z:14:7 // CHECK: Punctuation: ")" [16:28 - 16:29] UnexposedStmt= // CHECK: Punctuation: ";" [16:29 - 16:30] UnexposedStmt= @@ -126,9 +126,9 @@ void test() { // CHECK: Identifier: "j" [17:7 - 17:8] VarDecl=j:17:7 (Definition) // CHECK: Punctuation: "=" [17:9 - 17:10] VarDecl=j:17:7 (Definition) // CHECK: Identifier: "TWICE_MACRO" [17:11 - 17:22] macro expansion=TWICE_MACRO:11:9 -// CHECK: Punctuation: "(" [17:22 - 17:23] UnexposedStmt= +// CHECK: Punctuation: "(" [17:22 - 17:23] // CHECK: Identifier: "k" [17:23 - 17:24] DeclRefExpr=k:16:7 -// CHECK: Punctuation: "+" [17:25 - 17:26] UnexposedStmt= +// CHECK: Punctuation: "+" [17:25 - 17:26] UnexposedExpr= // CHECK: Identifier: "k" [17:27 - 17:28] DeclRefExpr=k:16:7 // CHECK: Punctuation: ")" [17:28 - 17:29] UnexposedStmt= // CHECK: Punctuation: ";" [17:29 - 17:30] UnexposedStmt= @@ -173,11 +173,11 @@ void test() { // CHECK: Identifier: "fun_with_macro_bodies" [25:3 - 25:24] macro expansion=fun_with_macro_bodies:21:9 // CHECK: Punctuation: "(" [25:24 - 25:25] UnexposedStmt= // CHECK: Identifier: "x" [25:25 - 25:26] DeclRefExpr=x:24:7 -// CHECK: Punctuation: "," [25:26 - 25:27] UnexposedStmt= +// CHECK: Punctuation: "," [25:26 - 25:27] // CHECK: Punctuation: "{" [25:28 - 25:29] UnexposedStmt= // CHECK: Keyword: "int" [25:30 - 25:33] UnexposedStmt= // CHECK: Identifier: "z" [25:34 - 25:35] VarDecl=z:25:34 (Definition) -// CHECK: Punctuation: "=" [25:36 - 25:37] UnexposedStmt= +// CHECK: Punctuation: "=" [25:36 - 25:37] VarDecl=z:25:34 (Definition) // CHECK: Identifier: "x" [25:38 - 25:39] DeclRefExpr=x:24:7 // CHECK: Punctuation: ";" [25:39 - 25:40] UnexposedStmt= // CHECK: Punctuation: "++" [25:41 - 25:43] UnexposedExpr= diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp index 173aa23786..61cc113bf5 100644 --- a/tools/libclang/CIndex.cpp +++ b/tools/libclang/CIndex.cpp @@ -4424,7 +4424,7 @@ CXString clang_getTokenSpelling(CXTranslationUnit TU, CXToken CXTok) { SourceLocation Loc = SourceLocation::getFromRawEncoding(CXTok.int_data[1]); std::pair LocInfo - = CXXUnit->getSourceManager().getDecomposedLoc(Loc); + = CXXUnit->getSourceManager().getDecomposedSpellingLoc(Loc); bool Invalid = false; StringRef Buffer = CXXUnit->getSourceManager().getBufferData(LocInfo.first, &Invalid); @@ -4580,6 +4580,16 @@ class AnnotateTokensWorker { SourceLocation GetTokenLoc(unsigned tokI) { return SourceLocation::getFromRawEncoding(Tokens[tokI].int_data[1]); } + bool isMacroArgToken(unsigned tokI) const { + return Tokens[tokI].int_data[3] != 0; + } + SourceLocation getMacroArgLoc(unsigned tokI) const { + return SourceLocation::getFromRawEncoding(Tokens[tokI].int_data[3]); + } + + void annotateAndAdvanceTokens(CXCursor, RangeComparisonResult, SourceRange); + void annotateAndAdvanceMacroArgTokens(CXCursor, RangeComparisonResult, + SourceRange); public: AnnotateTokensWorker(AnnotateTokensData &annotated, @@ -4632,6 +4642,63 @@ void AnnotateTokensWorker::AnnotateTokens(CXCursor parent) { } } +/// \brief It annotates and advances tokens with a cursor until the comparison +//// between the cursor location and the source range is the same as +/// \arg compResult. +/// +/// Pass RangeBefore to annotate tokens with a cursor until a range is reached. +/// Pass RangeOverlap to annotate tokens inside a range. +void AnnotateTokensWorker::annotateAndAdvanceTokens(CXCursor updateC, + RangeComparisonResult compResult, + SourceRange range) { + while (MoreTokens()) { + const unsigned I = NextToken(); + if (isMacroArgToken(I)) + return annotateAndAdvanceMacroArgTokens(updateC, compResult, range); + + SourceLocation TokLoc = GetTokenLoc(I); + if (LocationCompare(SrcMgr, TokLoc, range) == compResult) { + Cursors[I] = updateC; + AdvanceToken(); + continue; + } + break; + } +} + +/// \brief Special annotation handling for macro argument tokens. +void AnnotateTokensWorker::annotateAndAdvanceMacroArgTokens(CXCursor updateC, + RangeComparisonResult compResult, + SourceRange range) { + assert(isMacroArgToken(NextToken()) && + "Should be called only for macro arg tokens"); + + // This works differently than annotateAndAdvanceTokens; because expanded + // macro arguments can have arbitrary translation-unit source order, we do not + // advance the token index one by one until a token fails the range test. + // We only advance once past all of the macro arg tokens if all of them + // pass the range test. If one of them fails we keep the token index pointing + // at the start of the macro arg tokens so that the failing token will be + // annotated by a subsequent annotation try. + + bool atLeastOneCompFail = false; + + unsigned I = NextToken(); + for (; isMacroArgToken(I); ++I) { + SourceLocation TokLoc = getMacroArgLoc(I); + if (TokLoc.isFileID()) + continue; // not macro arg token, it's parens or comma. + if (LocationCompare(SrcMgr, TokLoc, range) == compResult) { + if (clang_isInvalid(clang_getCursorKind(Cursors[I]))) + Cursors[I] = updateC; + } else + atLeastOneCompFail = true; + } + + if (!atLeastOneCompFail) + TokIdx = I; // All of the tokens were handled, advance beyond all of them. +} + enum CXChildVisitResult AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) { CXSourceLocation Loc = clang_getCursorLocation(cursor); @@ -4783,20 +4850,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) { (clang_isInvalid(K) || K == CXCursor_TranslationUnit) ? clang_getNullCursor() : parent; - while (MoreTokens()) { - const unsigned I = NextToken(); - SourceLocation TokLoc = GetTokenLoc(I); - switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) { - case RangeBefore: - Cursors[I] = updateC; - AdvanceToken(); - continue; - case RangeAfter: - case RangeOverlap: - break; - } - break; - } + annotateAndAdvanceTokens(updateC, RangeBefore, cursorRange); // Avoid having the cursor of an expression "overwrite" the annotation of the // variable declaration that it belongs to. @@ -4821,46 +4875,19 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) { VisitChildren(cursor); const unsigned AfterChildren = NextToken(); - // Adjust 'Last' to the last token within the extent of the cursor. - while (MoreTokens()) { - const unsigned I = NextToken(); - SourceLocation TokLoc = GetTokenLoc(I); - switch (LocationCompare(SrcMgr, TokLoc, cursorRange)) { - case RangeBefore: - assert(0 && "Infeasible"); - case RangeAfter: - break; - case RangeOverlap: - Cursors[I] = updateC; - AdvanceToken(); - continue; - } - break; - } - const unsigned Last = NextToken(); + // Scan the tokens that are at the end of the cursor, but are not captured + // but the child cursors. + annotateAndAdvanceTokens(cursor, RangeOverlap, cursorRange); // Scan the tokens that are at the beginning of the cursor, but are not // capture by the child cursors. - - // For AST elements within macros, rely on a post-annotate pass to - // to correctly annotate the tokens with cursors. Otherwise we can - // get confusing results of having tokens that map to cursors that really - // are expanded by an instantiation. - if (L.isMacroID()) - cursor = clang_getNullCursor(); - for (unsigned I = BeforeChildren; I != AfterChildren; ++I) { if (!clang_isInvalid(clang_getCursorKind(Cursors[I]))) break; Cursors[I] = cursor; } - // Scan the tokens that are at the end of the cursor, but are not captured - // but the child cursors. - for (unsigned I = AfterChildren; I != Last; ++I) - Cursors[I] = cursor; - TokIdx = Last; return CXChildVisit_Continue; } @@ -4870,6 +4897,74 @@ static enum CXChildVisitResult AnnotateTokensVisitor(CXCursor cursor, return static_cast(client_data)->Visit(cursor, parent); } +namespace { + +/// \brief Uses the macro expansions in the preprocessing record to find +/// and mark tokens that are macro arguments. This info is used by the +/// AnnotateTokensWorker. +class MarkMacroArgTokensVisitor { + SourceManager &SM; + CXToken *Tokens; + unsigned NumTokens; + unsigned CurIdx; + +public: + MarkMacroArgTokensVisitor(SourceManager &SM, + CXToken *tokens, unsigned numTokens) + : SM(SM), Tokens(tokens), NumTokens(numTokens), CurIdx(0) { } + + CXChildVisitResult visit(CXCursor cursor, CXCursor parent) { + if (cursor.kind != CXCursor_MacroExpansion) + return CXChildVisit_Continue; + + SourceRange macroRange = getCursorMacroExpansion(cursor)->getSourceRange(); + if (macroRange.getBegin() == macroRange.getEnd()) + return CXChildVisit_Continue; // it's not a function macro. + + for (; CurIdx < NumTokens; ++CurIdx) { + if (!SM.isBeforeInTranslationUnit(getTokenLoc(CurIdx), + macroRange.getBegin())) + break; + } + + if (CurIdx == NumTokens) + return CXChildVisit_Break; + + for (; CurIdx < NumTokens; ++CurIdx) { + SourceLocation tokLoc = getTokenLoc(CurIdx); + if (!SM.isBeforeInTranslationUnit(tokLoc, macroRange.getEnd())) + break; + + setMacroArgExpandedLoc(CurIdx, SM.getMacroArgExpandedLocation(tokLoc)); + } + + if (CurIdx == NumTokens) + return CXChildVisit_Break; + + return CXChildVisit_Continue; + } + +private: + SourceLocation getTokenLoc(unsigned tokI) { + return SourceLocation::getFromRawEncoding(Tokens[tokI].int_data[1]); + } + + void setMacroArgExpandedLoc(unsigned tokI, SourceLocation loc) { + // The third field is reserved and currently not used. Use it here + // to mark macro arg expanded tokens with their expanded locations. + Tokens[tokI].int_data[3] = loc.getRawEncoding(); + } +}; + +} // end anonymous namespace + +static CXChildVisitResult +MarkMacroArgTokensVisitorDelegate(CXCursor cursor, CXCursor parent, + CXClientData client_data) { + return static_cast(client_data)->visit(cursor, + parent); +} + namespace { struct clang_annotateTokens_Data { CXTranslationUnit TU; @@ -4960,6 +5055,16 @@ static void clang_annotateTokensImpl(void *UserData) { } } + if (CXXUnit->getPreprocessor().getPreprocessingRecord()) { + // Search and mark tokens that are macro argument expansions. + MarkMacroArgTokensVisitor Visitor(CXXUnit->getSourceManager(), + Tokens, NumTokens); + CursorVisitor MacroArgMarker(TU, + MarkMacroArgTokensVisitorDelegate, &Visitor, + Decl::MaxPCHLevel, true, RegionOfInterest); + MacroArgMarker.visitPreprocessedEntitiesInRegion(); + } + // Annotate all of the source locations in the region of interest that map to // a specific cursor. AnnotateTokensWorker W(Annotated, Tokens, Cursors, NumTokens,