From 5168003cf2e3a55da294e51947c20f472ccfcc19 Mon Sep 17 00:00:00 2001 From: Ulya Trofimovich Date: Thu, 22 Dec 2016 11:58:48 +0000 Subject: [PATCH] Simplified line tracking in lexer. --- re2c/bootstrap/src/parse/lex.cc | 58 ++++++++++++++++----------------- re2c/src/ir/regexp/regexp.cc | 4 +-- re2c/src/parse/lex.re | 16 ++++----- re2c/src/parse/parser.ypp | 6 ---- re2c/src/parse/scanner.cc | 11 +------ re2c/src/parse/scanner.h | 9 ----- re2c/test/bug1529351.c | 2 +- 7 files changed, 41 insertions(+), 65 deletions(-) diff --git a/re2c/bootstrap/src/parse/lex.cc b/re2c/bootstrap/src/parse/lex.cc index bd972f45..aedd5ad8 100644 --- a/re2c/bootstrap/src/parse/lex.cc +++ b/re2c/bootstrap/src/parse/lex.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.16 on Thu Dec 22 09:52:03 2016 */ +/* Generated by re2c 0.16 on Thu Dec 22 11:52:24 2016 */ #line 1 "../src/parse/lex.re" #include "src/util/c99_stdint.h" #include @@ -875,11 +875,9 @@ yy152: int Scanner::scan() { - uint32_t depth; - + uint32_t depth, code_line; scan: tchar = cur - pos; - tline = cline; tok = cur; switch (lexer_state) { @@ -889,7 +887,7 @@ scan: start: -#line 893 "src/parse/lex.cc" +#line 891 "src/parse/lex.cc" { YYCTYPE yych; unsigned int yyaccept = 0; @@ -999,7 +997,7 @@ yy160: fatalf("unexpected character: '%c'", *tok); goto scan; } -#line 1003 "src/parse/lex.cc" +#line 1001 "src/parse/lex.cc" yy161: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); @@ -1011,7 +1009,7 @@ yy161: { goto scan; } -#line 1015 "src/parse/lex.cc" +#line 1013 "src/parse/lex.cc" yy164: yyaccept = 0; yych = (YYCTYPE)*(YYMARKER = ++YYCURSOR); @@ -1029,7 +1027,7 @@ yy165: cline++; goto scan; } -#line 1033 "src/parse/lex.cc" +#line 1031 "src/parse/lex.cc" yy166: yych = (YYCTYPE)*++YYCURSOR; if (yych == '\n') goto yy164; @@ -1041,12 +1039,12 @@ yy168: { return *tok; } -#line 1045 "src/parse/lex.cc" +#line 1043 "src/parse/lex.cc" yy169: ++YYCURSOR; #line 262 "../src/parse/lex.re" { yylval.regexp = lex_str('"', opts->bCaseInsensitive || opts->bCaseInverted); return TOKEN_REGEXP; } -#line 1050 "src/parse/lex.cc" +#line 1048 "src/parse/lex.cc" yy171: yych = (YYCTYPE)*++YYCURSOR; if (yych == '}') goto yy195; @@ -1055,7 +1053,7 @@ yy172: ++YYCURSOR; #line 261 "../src/parse/lex.re" { yylval.regexp = lex_str('\'', opts->bCaseInsensitive || !opts->bCaseInverted); return TOKEN_REGEXP; } -#line 1059 "src/parse/lex.cc" +#line 1057 "src/parse/lex.cc" yy174: yych = (YYCTYPE)*++YYCURSOR; if (yych == '/') goto yy195; @@ -1064,7 +1062,7 @@ yy174: yylval.op = *tok; return TOKEN_STAR; } -#line 1068 "src/parse/lex.cc" +#line 1066 "src/parse/lex.cc" yy176: ++YYCURSOR; #line 283 "../src/parse/lex.re" @@ -1072,7 +1070,7 @@ yy176: yylval.op = *tok; return TOKEN_CLOSE; } -#line 1076 "src/parse/lex.cc" +#line 1074 "src/parse/lex.cc" yy178: ++YYCURSOR; #line 367 "../src/parse/lex.re" @@ -1080,7 +1078,7 @@ yy178: yylval.regexp = mkDot(); return TOKEN_REGEXP; } -#line 1084 "src/parse/lex.cc" +#line 1082 "src/parse/lex.cc" yy180: yych = (YYCTYPE)*++YYCURSOR; if (yych == '*') goto yy197; @@ -1142,7 +1140,7 @@ yy185: if (yych == '^') goto yy212; #line 263 "../src/parse/lex.re" { yylval.regexp = lex_cls(false); return TOKEN_REGEXP; } -#line 1146 "src/parse/lex.cc" +#line 1144 "src/parse/lex.cc" yy187: yych = (YYCTYPE)*++YYCURSOR; if (yych == 'e') goto yy214; @@ -1165,12 +1163,13 @@ yy188: } } yy189: -#line 232 "../src/parse/lex.re" +#line 230 "../src/parse/lex.re" { depth = 1; + code_line = cline; goto code; } -#line 1174 "src/parse/lex.cc" +#line 1173 "src/parse/lex.cc" yy190: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); @@ -1211,7 +1210,7 @@ yy195: tok = cur; return 0; } -#line 1215 "src/parse/lex.cc" +#line 1214 "src/parse/lex.cc" yy197: ++YYCURSOR; #line 250 "../src/parse/lex.re" @@ -1219,14 +1218,14 @@ yy197: depth = 1; goto comment; } -#line 1223 "src/parse/lex.cc" +#line 1222 "src/parse/lex.cc" yy199: ++YYCURSOR; #line 247 "../src/parse/lex.re" { goto nextLine; } -#line 1230 "src/parse/lex.cc" +#line 1229 "src/parse/lex.cc" yy201: ++YYCURSOR; YYCURSOR -= 1; @@ -1246,7 +1245,7 @@ yy201: return TOKEN_REGEXP; } } -#line 1250 "src/parse/lex.cc" +#line 1249 "src/parse/lex.cc" yy203: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); @@ -1271,14 +1270,15 @@ yy205: yylval.str = new std::string (tok, tok_len ()); return TOKEN_ID; } -#line 1275 "src/parse/lex.cc" +#line 1274 "src/parse/lex.cc" yy207: yych = (YYCTYPE)*++YYCURSOR; if (yych == '>') goto yy224; -#line 241 "../src/parse/lex.re" +#line 240 "../src/parse/lex.re" { tok += 2; /* skip ":=" */ depth = 0; + code_line = cline; goto code; } #line 1285 "src/parse/lex.cc" @@ -1395,7 +1395,7 @@ yy222: yy224: ++YYCURSOR; YYCURSOR -= 2; -#line 237 "../src/parse/lex.re" +#line 236 "../src/parse/lex.re" { return *tok; } @@ -1720,7 +1720,7 @@ yy271: { --cur; } - yylval.code = new Code (tok, tok_len (), get_fname (), tline); + yylval.code = new Code (tok, tok_len (), get_fname (), code_line); return TOKEN_CODE; } else if (cur == eof) @@ -1767,7 +1767,7 @@ yy276: } else if (--depth == 0) { - yylval.code = new Code (tok, tok_len (), get_fname (), tline); + yylval.code = new Code (tok, tok_len (), get_fname (), code_line); return TOKEN_CODE; } goto code; @@ -2276,7 +2276,7 @@ yy361: { u = lex_cls_chr(); if (l > u) { - warn.swapped_range(get_line(), l, u); + warn.swapped_range(cline, l, u); std::swap(l, u); } goto add; @@ -2374,7 +2374,7 @@ yy371: ++YYCURSOR; #line 615 "../src/parse/lex.re" { - warn.useless_escape(tline, tok - pos, tok[1]); + warn.useless_escape(cline, tok - pos, tok[1]); return static_cast(tok[1]); } #line 2381 "src/parse/lex.cc" @@ -2667,7 +2667,7 @@ yy421: #line 646 "../src/parse/lex.re" { if (tok[1] != quote) { - warn.useless_escape(tline, tok - pos, tok[1]); + warn.useless_escape(cline, tok - pos, tok[1]); } return static_cast(tok[1]); } diff --git a/re2c/src/ir/regexp/regexp.cc b/re2c/src/ir/regexp/regexp.cc index 4dcf482e..d8992985 100644 --- a/re2c/src/ir/regexp/regexp.cc +++ b/re2c/src/ir/regexp/regexp.cc @@ -107,10 +107,10 @@ const RegExp *Scanner::cls(const Range *r) const if (!r) { switch (opts->empty_class_policy) { case EMPTY_CLASS_MATCH_EMPTY: - warn.empty_class(get_line()); + warn.empty_class(cline); return RegExp::make_nil(); case EMPTY_CLASS_MATCH_NONE: - warn.empty_class(get_line()); + warn.empty_class(cline); break; case EMPTY_CLASS_ERROR: fatal("empty character class"); diff --git a/re2c/src/parse/lex.re b/re2c/src/parse/lex.re index 8d72dc36..d9a2cfa1 100644 --- a/re2c/src/parse/lex.re +++ b/re2c/src/parse/lex.re @@ -215,11 +215,9 @@ void Scanner::lex_tags() int Scanner::scan() { - uint32_t depth; - + uint32_t depth, code_line; scan: tchar = cur - pos; - tline = cline; tok = cur; switch (lexer_state) { @@ -231,6 +229,7 @@ start: /*!re2c "{" { depth = 1; + code_line = cline; goto code; } @@ -241,6 +240,7 @@ start: ":=" { tok += 2; /* skip ":=" */ depth = 0; + code_line = cline; goto code; } @@ -415,7 +415,7 @@ code: } else if (--depth == 0) { - yylval.code = new Code (tok, tok_len (), get_fname (), tline); + yylval.code = new Code (tok, tok_len (), get_fname (), code_line); return TOKEN_CODE; } goto code; @@ -456,7 +456,7 @@ code: { --cur; } - yylval.code = new Code (tok, tok_len (), get_fname (), tline); + yylval.code = new Code (tok, tok_len (), get_fname (), code_line); return TOKEN_CODE; } else if (cur == eof) @@ -571,7 +571,7 @@ snd: "-" / [^\]] { u = lex_cls_chr(); if (l > u) { - warn.swapped_range(get_line(), l, u); + warn.swapped_range(cline, l, u); std::swap(l, u); } goto add; @@ -613,7 +613,7 @@ uint32_t Scanner::lex_cls_chr() esc "-" { return static_cast('-'); } esc "]" { return static_cast(']'); } esc . { - warn.useless_escape(tline, tok - pos, tok[1]); + warn.useless_escape(cline, tok - pos, tok[1]); return static_cast(tok[1]); } */ @@ -645,7 +645,7 @@ uint32_t Scanner::lex_str_chr(char quote, bool &end) esc "\\" { return static_cast('\\'); } esc . { if (tok[1] != quote) { - warn.useless_escape(tline, tok - pos, tok[1]); + warn.useless_escape(cline, tok - pos, tok[1]); } return static_cast(tok[1]); } diff --git a/re2c/src/parse/parser.ypp b/re2c/src/parse/parser.ypp index d8036ab3..750e0f63 100644 --- a/re2c/src/parse/parser.ypp +++ b/re2c/src/parse/parser.ypp @@ -465,9 +465,7 @@ void parse(Scanner &in, Output & o) dfa_map.clear(); } specMap.clear(); - in.set_in_parse(true); yyparse(in); - in.set_in_parse(false); if (opts->rFlag && mode == Scanner::Reuse) { if (!specMap.empty() || opts->encoding != encodingOld) { // Re-parse rules @@ -477,16 +475,12 @@ void parse(Scanner &in, Output & o) dfa_map.clear(); parse_cleanup(); specMap.clear(); - in.set_in_parse(true); yyparse(in); - in.set_in_parse(false); // Now append potential new rules in.restore_state(curr_state); mode = Scanner::Parse; - in.set_in_parse(true); yyparse(in); - in.set_in_parse(false); } encodingOld = opts->encoding; } diff --git a/re2c/src/parse/scanner.cc b/re2c/src/parse/scanner.cc index b4fd63dc..32462573 100644 --- a/re2c/src/parse/scanner.cc +++ b/re2c/src/parse/scanner.cc @@ -30,9 +30,7 @@ ScannerState::ScannerState () , top (NULL) , eof (NULL) , tchar (0) - , tline (0) , cline (1) - , in_parse (false) , lexer_state (LEX_NORMAL) {} @@ -47,9 +45,7 @@ ScannerState::ScannerState (const ScannerState & s) , top (s.top) , eof (s.eof) , tchar (s.tchar) - , tline (s.tline) , cline (s.cline) - , in_parse (s.in_parse) , lexer_state (s.lexer_state) {} @@ -124,11 +120,6 @@ void Scanner::fill (uint32_t need) } } -void Scanner::set_in_parse(bool new_in_parse) -{ - in_parse = new_in_parse; -} - void Scanner::fatal_at(uint32_t line, ptrdiff_t ofs, const char *msg) const { std::cerr << "re2c: error: " @@ -139,7 +130,7 @@ void Scanner::fatal_at(uint32_t line, ptrdiff_t ofs, const char *msg) const void Scanner::fatal(ptrdiff_t ofs, const char *msg) const { - fatal_at(in_parse ? tline : cline, ofs, msg); + fatal_at(cline, ofs, msg); } void Scanner::fatalf_at(uint32_t line, const char* fmt, ...) const diff --git a/re2c/src/parse/scanner.h b/re2c/src/parse/scanner.h index 87f5e145..c6ca91eb 100644 --- a/re2c/src/parse/scanner.h +++ b/re2c/src/parse/scanner.h @@ -40,10 +40,8 @@ struct ScannerState char * eof; ptrdiff_t tchar; - uint32_t tline; uint32_t cline; - bool in_parse; lexer_state_t lexer_state; ScannerState (); @@ -107,9 +105,7 @@ public: void restore_state(const ScannerState&); uint32_t get_cline() const; - uint32_t get_line() const; const std::string & get_fname () const; - void set_in_parse(bool new_in_parse); void fatal_at(uint32_t line, ptrdiff_t ofs, const char *msg) const; void fatalf_at(uint32_t line, const char*, ...) const RE2C_GXX_ATTRIBUTE ((format (printf, 3, 4))); void fatalf(const char*, ...) const RE2C_GXX_ATTRIBUTE ((format (printf, 2, 3))); @@ -139,11 +135,6 @@ inline uint32_t Scanner::get_cline() const return cline; } -inline uint32_t Scanner::get_line() const -{ - return in_parse ? tline : cline; -} - inline void Scanner::save_state(ScannerState& state) const { state = *this; diff --git a/re2c/test/bug1529351.c b/re2c/test/bug1529351.c index 1a61908e..b4038d2a 100644 --- a/re2c/test/bug1529351.c +++ b/re2c/test/bug1529351.c @@ -1 +1 @@ -re2c: error: line 3, column 9: missing '}' +re2c: error: line 5, column 9: missing '}' -- 2.40.0