From: Dmitri Gribenko Date: Wed, 18 Jul 2012 23:01:58 +0000 (+0000) Subject: Comment parsing: don't parse whitespace before \endverbatim as a separate line of... X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=64da4e55c111f4733135e1780216609569767351;p=clang Comment parsing: don't parse whitespace before \endverbatim as a separate line of whitespace. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@160464 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/AST/CommentLexer.cpp b/lib/AST/CommentLexer.cpp index 5b411ca9cc..31468321cf 100644 --- a/lib/AST/CommentLexer.cpp +++ b/lib/AST/CommentLexer.cpp @@ -201,6 +201,10 @@ const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) { return BufferEnd; } +bool isWhitespace(const char *BufferPtr, const char *BufferEnd) { + return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd; +} + bool isCommandNameCharacter(char C) { return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || @@ -429,6 +433,7 @@ void Lexer::setupAndLexVerbatimBlock(Token &T, } void Lexer::lexVerbatimBlockFirstLine(Token &T) { +again: assert(BufferPtr < CommentEnd); // FIXME: It would be better to scan the text once, finding either the block @@ -458,6 +463,11 @@ void Lexer::lexVerbatimBlockFirstLine(Token &T) { // There is some text, followed by end command. Extract text first. TextEnd = BufferPtr + Pos; NextLine = TextEnd; + // If there is only whitespace before end command, skip whitespace. + if (isWhitespace(BufferPtr, TextEnd)) { + BufferPtr = TextEnd; + goto again; + } } StringRef Text(BufferPtr, TextEnd - BufferPtr); diff --git a/unittests/AST/CommentLexer.cpp b/unittests/AST/CommentLexer.cpp index 6be5909f8a..dd92df421f 100644 --- a/unittests/AST/CommentLexer.cpp +++ b/unittests/AST/CommentLexer.cpp @@ -514,7 +514,7 @@ TEST_F(CommentLexerTest, VerbatimBlock6) { lexString(Source, Toks); - ASSERT_EQ(11U, Toks.size()); + ASSERT_EQ(10U, Toks.size()); ASSERT_EQ(tok::text, Toks[0].getKind()); ASSERT_EQ(StringRef(" "), Toks[0].getText()); @@ -536,13 +536,10 @@ TEST_F(CommentLexerTest, VerbatimBlock6) { ASSERT_EQ(tok::newline, Toks[7].getKind()); - ASSERT_EQ(tok::verbatim_block_line, Toks[8].getKind()); - ASSERT_EQ(StringRef(" "), Toks[8].getVerbatimBlockText()); + ASSERT_EQ(tok::verbatim_block_end, Toks[8].getKind()); + ASSERT_EQ(StringRef("endverbatim"), Toks[8].getVerbatimBlockName()); - ASSERT_EQ(tok::verbatim_block_end, Toks[9].getKind()); - ASSERT_EQ(StringRef("endverbatim"), Toks[9].getVerbatimBlockName()); - - ASSERT_EQ(tok::newline, Toks[10].getKind()); + ASSERT_EQ(tok::newline, Toks[9].getKind()); } TEST_F(CommentLexerTest, VerbatimBlock7) { @@ -558,7 +555,7 @@ TEST_F(CommentLexerTest, VerbatimBlock7) { lexString(Source, Toks); - ASSERT_EQ(11U, Toks.size()); + ASSERT_EQ(10U, Toks.size()); ASSERT_EQ(tok::text, Toks[0].getKind()); ASSERT_EQ(StringRef(" "), Toks[0].getText()); @@ -575,19 +572,16 @@ TEST_F(CommentLexerTest, VerbatimBlock7) { ASSERT_EQ(tok::verbatim_block_line, Toks[4].getKind()); ASSERT_EQ(StringRef(" Bbb"), Toks[4].getVerbatimBlockText()); - ASSERT_EQ(tok::verbatim_block_line, Toks[5].getKind()); - ASSERT_EQ(StringRef(" "), Toks[5].getVerbatimBlockText()); - - ASSERT_EQ(tok::verbatim_block_end, Toks[6].getKind()); - ASSERT_EQ(StringRef("endverbatim"), Toks[6].getVerbatimBlockName()); + ASSERT_EQ(tok::verbatim_block_end, Toks[5].getKind()); + ASSERT_EQ(StringRef("endverbatim"), Toks[5].getVerbatimBlockName()); - ASSERT_EQ(tok::newline, Toks[7].getKind()); + ASSERT_EQ(tok::newline, Toks[6].getKind()); - ASSERT_EQ(tok::text, Toks[8].getKind()); - ASSERT_EQ(StringRef(" "), Toks[8].getText()); + ASSERT_EQ(tok::text, Toks[7].getKind()); + ASSERT_EQ(StringRef(" "), Toks[7].getText()); + ASSERT_EQ(tok::newline, Toks[8].getKind()); ASSERT_EQ(tok::newline, Toks[9].getKind()); - ASSERT_EQ(tok::newline, Toks[10].getKind()); } // Complex test for verbatim blocks. diff --git a/unittests/AST/CommentParser.cpp b/unittests/AST/CommentParser.cpp index cc9ed9c4cc..5f91947f0b 100644 --- a/unittests/AST/CommentParser.cpp +++ b/unittests/AST/CommentParser.cpp @@ -1077,12 +1077,43 @@ TEST_F(CommentParserTest, VerbatimBlock5) { } { VerbatimBlockComment *VBC; - ASSERT_TRUE(HasVerbatimBlockAt(FC, 1, VBC, "verbatim", " Aaa", " ")); + ASSERT_TRUE(HasVerbatimBlockAt(FC, 1, VBC, "verbatim", " Aaa")); } } } TEST_F(CommentParserTest, VerbatimBlock6) { + const char *Sources[] = { + "// \\verbatim\n" + "// Aaa\n" + "// Bbb\n" + "// \\endverbatim\n", + + "/* \\verbatim\n" + " * Aaa\n" + " * Bbb\n" + " * \\endverbatim*/" + }; + + for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) { + FullComment *FC = parseString(Sources[i]); + ASSERT_TRUE(HasChildCount(FC, 2)); + + { + ParagraphComment *PC; + ASSERT_TRUE(GetChildAt(FC, 0, PC)); + + ASSERT_TRUE(HasChildCount(PC, 1)); + ASSERT_TRUE(HasTextAt(PC, 0, " ")); + } + { + VerbatimBlockComment *VBC; + ASSERT_TRUE(HasVerbatimBlockAt(FC, 1, VBC, "verbatim", " Aaa", " Bbb")); + } + } +} + +TEST_F(CommentParserTest, VerbatimBlock7) { const char *Sources[] = { "// \\verbatim\n" "// Aaa\n" @@ -1097,25 +1128,24 @@ TEST_F(CommentParserTest, VerbatimBlock6) { " * \\endverbatim*/" }; for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) { - FullComment *FC = parseString(Sources[i]); - ASSERT_TRUE(HasChildCount(FC, 2)); + FullComment *FC = parseString(Sources[i]); + ASSERT_TRUE(HasChildCount(FC, 2)); - { - ParagraphComment *PC; - ASSERT_TRUE(GetChildAt(FC, 0, PC)); + { + ParagraphComment *PC; + ASSERT_TRUE(GetChildAt(FC, 0, PC)); - ASSERT_TRUE(HasChildCount(PC, 1)); - ASSERT_TRUE(HasTextAt(PC, 0, " ")); - } - { - VerbatimBlockComment *VBC; - ASSERT_TRUE(HasVerbatimBlockAt(FC, 1, VBC, "verbatim")); - ASSERT_EQ(4U, VBC->getNumLines()); - ASSERT_EQ(" Aaa", VBC->getText(0)); - ASSERT_EQ("", VBC->getText(1)); - ASSERT_EQ(" Bbb", VBC->getText(2)); - ASSERT_EQ(" ", VBC->getText(3)); - } + ASSERT_TRUE(HasChildCount(PC, 1)); + ASSERT_TRUE(HasTextAt(PC, 0, " ")); + } + { + VerbatimBlockComment *VBC; + ASSERT_TRUE(HasVerbatimBlockAt(FC, 1, VBC, "verbatim")); + ASSERT_EQ(3U, VBC->getNumLines()); + ASSERT_EQ(" Aaa", VBC->getText(0)); + ASSERT_EQ("", VBC->getText(1)); + ASSERT_EQ(" Bbb", VBC->getText(2)); + } } }