From 05db4278ecd193edcff63fb8c54818226fceaad2 Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Fri, 13 Feb 2009 19:33:24 +0000 Subject: [PATCH] Fix rdar://6562329, a static analyzer crash Ted noticed on wine sources. This was happening because HighlightMacros was calling EnterMainFile multiple times on the same preprocessor object and getting an assert due to the new #line stuff (the file in question was bison output with #line directives). The fix for this is to not reenter the file. Instead, relex the tokens in raw mode, swizzle them a bit and repreprocess the token stream. An added bonus of this is that rewrite macros will now hilight the macro definition as well as its uses. Woo. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@64480 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Lex/Preprocessor.cpp | 5 +++- lib/Rewrite/HTMLRewrite.cpp | 58 ++++++++++++++++++++++++++++--------- test/Misc/emit-html.c | 5 ++++ 3 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 test/Misc/emit-html.c diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp index 69aa7cb51a..f16d83c5a2 100644 --- a/lib/Lex/Preprocessor.cpp +++ b/lib/Lex/Preprocessor.cpp @@ -626,7 +626,10 @@ static void InitializePredefinedMacros(Preprocessor &PP, /// EnterMainSourceFile - Enter the specified FileID as the main source file, /// which implicitly adds the builtin defines etc. void Preprocessor::EnterMainSourceFile() { - + // We do not allow the preprocessor to reenter the main file. Doing so will + // cause FileID's to accumulate information from both runs (e.g. #line + // information) and predefined macros aren't guaranteed to be set properly. + assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); FileID MainFileID = SourceMgr.getMainFileID(); // Enter the main file source buffer. diff --git a/lib/Rewrite/HTMLRewrite.cpp b/lib/Rewrite/HTMLRewrite.cpp index c0f3937600..a0f289b2b4 100644 --- a/lib/Rewrite/HTMLRewrite.cpp +++ b/lib/Rewrite/HTMLRewrite.cpp @@ -344,8 +344,8 @@ void html::AddHeaderFooterInternalBuiltinCSS(Rewriter& R, FileID FID, void html::SyntaxHighlight(Rewriter &R, FileID FID, Preprocessor &PP) { RewriteBuffer &RB = R.getEditBuffer(FID); - const SourceManager &SourceMgr = PP.getSourceManager(); - Lexer L(FID, SourceMgr, PP.getLangOptions()); + const SourceManager &SM = PP.getSourceManager(); + Lexer L(FID, SM, PP.getLangOptions()); const char *BufferStart = L.getBufferStart(); // Inform the preprocessor that we want to retain comments as tokens, so we @@ -360,7 +360,7 @@ void html::SyntaxHighlight(Rewriter &R, FileID FID, Preprocessor &PP) { while (Tok.isNot(tok::eof)) { // Since we are lexing unexpanded tokens, all tokens are from the main // FileID. - unsigned TokOffs = SourceMgr.getFileOffset(Tok.getLocation()); + unsigned TokOffs = SM.getFileOffset(Tok.getLocation()); unsigned TokLen = Tok.getLength(); switch (Tok.getKind()) { default: break; @@ -398,7 +398,7 @@ void html::SyntaxHighlight(Rewriter &R, FileID FID, Preprocessor &PP) { unsigned TokEnd = TokOffs+TokLen; L.LexFromRawLexer(Tok); while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) { - TokEnd = SourceMgr.getFileOffset(Tok.getLocation())+Tok.getLength(); + TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength(); L.LexFromRawLexer(Tok); } @@ -416,23 +416,55 @@ void html::SyntaxHighlight(Rewriter &R, FileID FID, Preprocessor &PP) { } /// HighlightMacros - This uses the macro table state from the end of the -/// file, to reexpand macros and insert (into the HTML) information about the +/// file, to re-expand macros and insert (into the HTML) information about the /// macro expansions. This won't be perfectly perfect, but it will be /// reasonably close. void html::HighlightMacros(Rewriter &R, FileID FID, Preprocessor& PP) { RewriteBuffer &RB = R.getEditBuffer(FID); + // Re-lex the raw token stream into a token buffer. + const SourceManager &SM = PP.getSourceManager(); + std::vector TokenStream; + + Lexer L(FID, SM, PP.getLangOptions()); + + // Lex all the tokens in raw mode, to avoid entering #includes or expanding + // macros. + while (1) { + Token Tok; + L.LexFromRawLexer(Tok); + + // If this is a # at the start of a line, discard it from the token stream. + // We don't want the re-preprocess step to see #defines, #includes or other + // preprocessor directives. + if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) + continue; + + // If this raw token is an identifier, the raw lexer won't have looked up + // the corresponding identifier info for it. Do this now so that it will be + // macro expanded when we re-preprocess it. + if (Tok.is(tok::identifier)) { + // Change the kind of this identifier to the appropriate token kind, e.g. + // turning "for" into a keyword. + Tok.setKind(PP.LookUpIdentifierInfo(Tok)->getTokenID()); + } + + TokenStream.push_back(Tok); + + if (Tok.is(tok::eof)) break; + } + // Inform the preprocessor that we don't want comments. PP.SetCommentRetentionState(false, false); - - // Start parsing the specified input file. - PP.EnterMainSourceFile(); + + // Enter the tokens we just lexed. This will cause them to be macro expanded + // but won't enter sub-files (because we removed #'s). + PP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false); TokenConcatenation ConcatInfo(PP); // Lex all the tokens. - const SourceManager &SourceMgr = PP.getSourceManager(); Token Tok; PP.Lex(Tok); while (Tok.isNot(tok::eof)) { @@ -443,8 +475,8 @@ void html::HighlightMacros(Rewriter &R, FileID FID, Preprocessor& PP) { } // Ignore tokens whose instantiation location was not the main file. - SourceLocation LLoc = SourceMgr.getInstantiationLoc(Tok.getLocation()); - std::pair LLocInfo = SourceMgr.getDecomposedLoc(LLoc); + SourceLocation LLoc = SM.getInstantiationLoc(Tok.getLocation()); + std::pair LLocInfo = SM.getDecomposedLoc(LLoc); if (LLocInfo.first != FID) { PP.Lex(Tok); @@ -457,7 +489,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, Preprocessor& PP) { // Get the size of current macro call itself. // FIXME: This should highlight the args of a function-like // macro, using a heuristic. - unsigned TokLen = Lexer::MeasureTokenLength(LLoc, SourceMgr); + unsigned TokLen = Lexer::MeasureTokenLength(LLoc, SM); unsigned TokOffs = LLocInfo.second; // Highlight the macro invocation itself. @@ -476,7 +508,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, Preprocessor& PP) { // instantiation. It would be really nice to pop up a window with all the // spelling of the tokens or something. while (!Tok.is(tok::eof) && - SourceMgr.getInstantiationLoc(Tok.getLocation()) == LLoc) { + SM.getInstantiationLoc(Tok.getLocation()) == LLoc) { // Insert a newline if the macro expansion is getting large. if (LineLen > 60) { Expansion += "
"; diff --git a/test/Misc/emit-html.c b/test/Misc/emit-html.c new file mode 100644 index 0000000000..1bcab12dbc --- /dev/null +++ b/test/Misc/emit-html.c @@ -0,0 +1,5 @@ +// RUN: clang %s -emit-html -o - + +// rdar://6562329 +#line 42 "foo.c" + -- 2.40.0