From: Argyrios Kyrtzidis Date: Sat, 8 Nov 2008 16:17:04 +0000 (+0000) Subject: Introduce annotation tokens, a special kind of token, created and used only by the... X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=3604e3895ecd850291b518e5a82246c888ce9d0f;p=clang Introduce annotation tokens, a special kind of token, created and used only by the parser to replace a group of tokens with a single token encoding semantic information. Will be fully utilized later for C++ nested-name-specifiers. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@58911 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/clang/Basic/TokenKinds.def b/include/clang/Basic/TokenKinds.def index a557d5b9b9..706c5fb85f 100644 --- a/include/clang/Basic/TokenKinds.def +++ b/include/clang/Basic/TokenKinds.def @@ -35,6 +35,9 @@ #ifndef OBJC2_AT_KEYWORD #define OBJC2_AT_KEYWORD(X) #endif +#ifndef ANNOTATION +#define ANNOTATION(X) TOK(annot_ ## X) +#endif //===----------------------------------------------------------------------===// // Preprocessor keywords. @@ -365,6 +368,11 @@ OBJC2_AT_KEYWORD(dynamic) // TODO: What to do about context-sensitive keywords like: // bycopy/byref/in/inout/oneway/out? +ANNOTATION(cxxscope) // annotation for a C++ scope spec, e.g. "::foo::bar::" +ANNOTATION(qualtypename) // annotation for a C typedef name, or a C++ (possibly + // qualified) typename, e.g. "foo::MyClass" + +#undef ANNOTATION #undef OBJC2_AT_KEYWORD #undef OBJC1_AT_KEYWORD #undef CXX_KEYWORD_OPERATOR diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h index 4c8fa67e94..c0caee660e 100644 --- a/include/clang/Lex/Preprocessor.h +++ b/include/clang/Lex/Preprocessor.h @@ -352,6 +352,28 @@ public: else return PeekAhead(N+1); } + + /// EnterToken - Enters a token in the token stream to be lexed next. If + /// BackTrack() is called afterwards, the token will remain at the insertion + /// point. + void EnterToken(const Token &Tok) { + EnterCachingLexMode(); + CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok); + } + + /// AnnotateCachedTokens - We notify the Preprocessor that if it is caching + /// tokens (because backtrack is enabled) it should replace the most recent + /// cached tokens with the given annotation token. This function has no effect + /// if backtracking is not enabled. + /// + /// Note that the use of this function is just for optimization; so that the + /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is + /// invoked. + void AnnotateCachedTokens(const Token &Tok) { + assert(Tok.isAnnotationToken() && "Expected annotation token"); + if (CachedLexPos != 0 && InCachingLexMode()) + AnnotatePreviousCachedTokens(Tok); + } /// Diag - Forwarding function for diagnostics. This emits a diagnostic at /// the specified Token's location, translating the token's start @@ -559,6 +581,7 @@ private: RemoveTopOfLexerStack(); } const Token &PeekAhead(unsigned N); + void AnnotatePreviousCachedTokens(const Token &Tok); //===--------------------------------------------------------------------===// /// Handle*Directive - implement the various preprocessor directives. These diff --git a/include/clang/Lex/Token.h b/include/clang/Lex/Token.h index 45d8f2b5d4..1dd6d7bef3 100644 --- a/include/clang/Lex/Token.h +++ b/include/clang/Lex/Token.h @@ -25,14 +25,31 @@ class IdentifierInfo; /// It is not intended to be space efficient, it is intended to return as much /// information as possible about each returned token. This is expected to be /// compressed into a smaller form if memory footprint is important. +/// +/// The parser can create a special "annotation token" representing a stream of +/// tokens that were parsed and semantically resolved, e.g.: "foo::MyClass" +/// can be represented by a single typename annotation token that carries +/// information about the SourceRange of the tokens and the type object. class Token { - /// The location and length of the token text itself. + /// The location of the token. SourceLocation Loc; - unsigned Length; + + union { + /// The end of the SourceRange of an annotation token. + unsigned AnnotEndLocID; + + /// The length of the token text itself. + unsigned Length; + }; - /// IdentifierInfo - If this was an identifier, this points to the uniqued - /// information about this identifier. - IdentifierInfo *IdentInfo; + union { + /// IdentifierInfo - If this was an identifier, this points to the uniqued + /// information about this identifier. + IdentifierInfo *IdentInfo; + + /// AnnotVal - Information specific to an annotation token. + void *AnnotVal; + }; /// Kind - The actual flavor of token this is. /// @@ -60,13 +77,39 @@ public: bool is(tok::TokenKind K) const { return Kind == (unsigned) K; } bool isNot(tok::TokenKind K) const { return Kind != (unsigned) K; } + bool isAnnotationToken() const { + return is(tok::annot_qualtypename) || is(tok::annot_cxxscope); + } + /// getLocation - Return a source location identifier for the specified /// offset in the current file. SourceLocation getLocation() const { return Loc; } - unsigned getLength() const { return Length; } + unsigned getLength() const { + assert(!isAnnotationToken() && "Used Length on annotation token"); + return Length; + } void setLocation(SourceLocation L) { Loc = L; } void setLength(unsigned Len) { Length = Len; } + + SourceLocation getAnnotationEndLoc() const { + assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token"); + return SourceLocation::getFromRawEncoding(AnnotEndLocID); + } + void setAnnotationEndLoc(SourceLocation L) { + assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token"); + AnnotEndLocID = L.getRawEncoding(); + } + + /// getAnnotationRange - SourceRange of the group of tokens that this + /// annotation token represents. + SourceRange getAnnotationRange() const { + return SourceRange(getLocation(), getAnnotationEndLoc()); + } + void setAnnotationRange(SourceRange R) { + setLocation(R.getBegin()); + setAnnotationEndLoc(R.getEnd()); + } const char *getName() const { return tok::getTokenName( (tok::TokenKind) Kind); @@ -80,10 +123,22 @@ public: Loc = SourceLocation(); } - IdentifierInfo *getIdentifierInfo() const { return IdentInfo; } + IdentifierInfo *getIdentifierInfo() const { + assert(!isAnnotationToken() && "Used IdentInfo on annotation token"); + return IdentInfo; + } void setIdentifierInfo(IdentifierInfo *II) { IdentInfo = II; } + + void *getAnnotationValue() const { + assert(isAnnotationToken() && "Used AnnotVal on non-annotation token"); + return AnnotVal; + } + void setAnnotationValue(void *val) { + assert(isAnnotationToken() && "Used AnnotVal on non-annotation token"); + AnnotVal = val; + } /// setFlag - Set the specified flag. void setFlag(TokenFlags Flag) { diff --git a/lib/Lex/PPCaching.cpp b/lib/Lex/PPCaching.cpp index 822b207cdc..1af79b4ba3 100644 --- a/lib/Lex/PPCaching.cpp +++ b/lib/Lex/PPCaching.cpp @@ -93,3 +93,27 @@ const Token &Preprocessor::PeekAhead(unsigned N) { EnterCachingLexMode(); return CachedTokens.back(); } + +void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) { + assert(Tok.isAnnotationToken() && "Expected annotation token"); + assert(CachedLexPos != 0 && "Expected to have some cached tokens"); + assert(CachedTokens[CachedLexPos-1].getLocation() == Tok.getAnnotationEndLoc() + && "The annotation should be until the most recent cached token"); + + // Start from the end of the cached tokens list and look for the token + // that is the beginning of the annotation token. + for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) { + CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1; + if (AnnotBegin->getLocation() == Tok.getLocation()) { + assert((BacktrackPositions.empty() || BacktrackPositions.back() < i) && + "The backtrack pos points inside the annotated tokens!"); + // Replace the cached tokens with the single annotation token. + CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos); + *AnnotBegin = Tok; + CachedLexPos = i; + return; + } + } + + assert(0&&"Didn't find the first token represented by the annotation token!"); +}