From e02b761a748833293b39fd69766f8073c0e6fe29 Mon Sep 17 00:00:00 2001 From: Ulya Trofimovich Date: Thu, 30 Jul 2015 15:11:03 +0100 Subject: [PATCH] Fixed bug #115 "flex-style named definitions cause ambiguity in re2c grammar". This commit removes 10 shift/reduce conflicts in bison grammar for re2c. These conflicts are caused by allowing flex-style named definitions name regular-expression to contain newlines and to be mixed with rules. It's not just some conflicts in LALR(1) grammar, it is genuine ambiguity as can be observed from the following example: /*!re2c name "a" "b" "c" {} */ which can be parsed in two ways: definition -> name "a" rule -> "b" "c" {} and definition -> name "a" "b" rule -> "c" {} , both ways being perfectly valid. This commit resolves ambiguity by forbidding newlines in flex-style named definitions (conforming to flex syntax). Newline in these definitions is treated in a special way: lexer emits token 'FID_END', which marks the end of flex-style named definition in parser. --- re2c/bootstrap/src/conf/parse_opts.cc | 2 +- re2c/bootstrap/src/parse/scanner_lex.cc | 670 +++++++++++++----------- re2c/src/parse/parser.ypp | 9 +- re2c/src/parse/scanner.cc | 19 +- re2c/src/parse/scanner.h | 10 +- re2c/src/parse/scanner_lex.re | 46 +- 6 files changed, 406 insertions(+), 350 deletions(-) diff --git a/re2c/bootstrap/src/conf/parse_opts.cc b/re2c/bootstrap/src/conf/parse_opts.cc index d5852e14..ce4c30ce 100644 --- a/re2c/bootstrap/src/conf/parse_opts.cc +++ b/re2c/bootstrap/src/conf/parse_opts.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.14.3 on Wed Jul 29 14:24:46 2015 */ +/* Generated by re2c 0.14.3 on Thu Jul 30 14:53:05 2015 */ #include #include "src/conf/msg.h" diff --git a/re2c/bootstrap/src/parse/scanner_lex.cc b/re2c/bootstrap/src/parse/scanner_lex.cc index f771f7cb..8cb9dfc4 100644 --- a/re2c/bootstrap/src/parse/scanner_lex.cc +++ b/re2c/bootstrap/src/parse/scanner_lex.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.14.3 on Sun Jul 19 18:08:49 2015 */ +/* Generated by re2c 0.14.3 on Thu Jul 30 14:53:06 2015 */ #include #include #include @@ -550,15 +550,16 @@ scan: tchar = cursor - pos; tline = cline; tok = cursor; - if (iscfg == 1) + switch (lexer_state) { - goto config; - } - else if (iscfg == 2) - { - goto value; + case LEX_NORMAL: goto start; + case LEX_CONFIG: goto config; + case LEX_CONFIG_VALUE: goto value; + case LEX_FLEX_NAME: goto flex_name; } +start: + { YYCTYPE yych; unsigned int yyaccept = 0; @@ -1054,7 +1055,7 @@ yy176: { cur = cursor; tok += 5; /* skip "re2c:" */ - iscfg = 1; + lexer_state = LEX_CONFIG; yylval.str = new std::string (tok, cur - tok); return CONFIG; } @@ -1119,7 +1120,15 @@ yy181: { cur = ptr > tok ? ptr - 1 : cursor; yylval.str = new std::string (tok, cur - tok); - return FFlag ? FID : ID; + if (FFlag) + { + lexer_state = LEX_FLEX_NAME; + return FID; + } + else + { + return ID; + } } yy183: ++YYCURSOR; @@ -1455,6 +1464,37 @@ yy248: } +flex_name: + +{ + YYCTYPE yych; + if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2); + yych = *YYCURSOR; + if (yych == '\n') goto yy253; + if (yych == '\r') goto yy255; + ++YYCURSOR; +yy252: + { + YYCURSOR = tok; + goto start; + } +yy253: + ++YYCURSOR; +yy254: + { + YYCURSOR = tok; + lexer_state = LEX_NORMAL; + return FID_END; + } +yy255: + yych = *++YYCURSOR; + if (yych != '\n') goto yy252; + ++YYCURSOR; + yych = *YYCURSOR; + goto yy254; +} + + code: { @@ -1498,24 +1538,24 @@ code: yych = *YYCURSOR; if (yych <= '&') { if (yych <= '\n') { - if (yych <= 0x00) goto yy251; - if (yych <= '\t') goto yy253; - goto yy255; + if (yych <= 0x00) goto yy259; + if (yych <= '\t') goto yy261; + goto yy263; } else { - if (yych == '"') goto yy257; - goto yy253; + if (yych == '"') goto yy265; + goto yy261; } } else { if (yych <= '{') { - if (yych <= '\'') goto yy258; - if (yych <= 'z') goto yy253; - goto yy259; + if (yych <= '\'') goto yy266; + if (yych <= 'z') goto yy261; + goto yy267; } else { - if (yych == '}') goto yy261; - goto yy253; + if (yych == '}') goto yy269; + goto yy261; } } -yy251: +yy259: ++YYCURSOR; { if (cursor == eof) @@ -1528,31 +1568,31 @@ yy251: } goto code; } -yy253: +yy261: ++YYCURSOR; -yy254: +yy262: { goto code; } -yy255: +yy263: YYCTXMARKER = YYCURSOR + 1; yyaccept = 0; yych = *(YYMARKER = ++YYCURSOR); if (yych <= '\r') { if (yych <= '\t') { - if (yych >= '\t') goto yy272; + if (yych >= '\t') goto yy280; } else { - if (yych <= '\n') goto yy274; - if (yych >= '\r') goto yy274; + if (yych <= '\n') goto yy282; + if (yych >= '\r') goto yy282; } } else { if (yych <= ' ') { - if (yych >= ' ') goto yy272; + if (yych >= ' ') goto yy280; } else { - if (yych == '#') goto yy275; + if (yych == '#') goto yy283; } } -yy256: +yy264: { if (depth == 0) { @@ -1573,17 +1613,17 @@ yy256: cline++; goto code; } -yy257: +yy265: yyaccept = 1; yych = *(YYMARKER = ++YYCURSOR); - if (yych == '\n') goto yy254; - goto yy270; -yy258: + if (yych == '\n') goto yy262; + goto yy278; +yy266: yyaccept = 1; yych = *(YYMARKER = ++YYCURSOR); - if (yych == '\n') goto yy254; - goto yy264; -yy259: + if (yych == '\n') goto yy262; + goto yy272; +yy267: ++YYCURSOR; { if (depth == 0) @@ -1596,7 +1636,7 @@ yy259: } goto code; } -yy261: +yy269: ++YYCURSOR; { if (depth == 0) @@ -1611,64 +1651,64 @@ yy261: } goto code; } -yy263: +yy271: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy264: +yy272: if (yybm[0+yych] & 16) { - goto yy263; + goto yy271; } - if (yych <= '\n') goto yy265; - if (yych <= '\'') goto yy267; - goto yy266; -yy265: + if (yych <= '\n') goto yy273; + if (yych <= '\'') goto yy275; + goto yy274; +yy273: YYCURSOR = YYMARKER; if (yyaccept <= 1) { if (yyaccept == 0) { - goto yy256; + goto yy264; } else { - goto yy254; + goto yy262; } } else { - goto yy273; + goto yy281; } -yy266: +yy274: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy265; - goto yy263; -yy267: + if (yych == '\n') goto yy273; + goto yy271; +yy275: ++YYCURSOR; { goto code; } -yy269: +yy277: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy270: +yy278: if (yybm[0+yych] & 32) { - goto yy269; + goto yy277; } - if (yych <= '\n') goto yy265; - if (yych <= '"') goto yy267; + if (yych <= '\n') goto yy273; + if (yych <= '"') goto yy275; ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy265; - goto yy269; -yy272: + if (yych == '\n') goto yy273; + goto yy277; +yy280: yyaccept = 2; yych = *(YYMARKER = ++YYCURSOR); if (yych <= 0x1F) { - if (yych == '\t') goto yy294; + if (yych == '\t') goto yy302; } else { - if (yych <= ' ') goto yy294; - if (yych == '#') goto yy275; + if (yych <= ' ') goto yy302; + if (yych == '#') goto yy283; } -yy273: +yy281: YYCURSOR = YYCTXMARKER; { if (depth == 0) @@ -1683,112 +1723,112 @@ yy273: cline++; goto code; } -yy274: +yy282: yych = *++YYCURSOR; - goto yy273; -yy275: + goto yy281; +yy283: ++YYCURSOR; if ((YYLIMIT - YYCURSOR) < 5) YYFILL(5); yych = *YYCURSOR; if (yybm[0+yych] & 64) { - goto yy275; + goto yy283; } - if (yych != 'l') goto yy265; + if (yych != 'l') goto yy273; yych = *++YYCURSOR; - if (yych != 'i') goto yy265; + if (yych != 'i') goto yy273; yych = *++YYCURSOR; - if (yych != 'n') goto yy265; + if (yych != 'n') goto yy273; yych = *++YYCURSOR; - if (yych != 'e') goto yy265; + if (yych != 'e') goto yy273; yych = *++YYCURSOR; - if (yych <= '0') goto yy282; - if (yych <= '9') goto yy265; - goto yy282; -yy281: + if (yych <= '0') goto yy290; + if (yych <= '9') goto yy273; + goto yy290; +yy289: YYCTXMARKER = YYCURSOR + 1; ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy282: +yy290: if (yych <= 0x1F) { - if (yych == '\t') goto yy281; - goto yy265; + if (yych == '\t') goto yy289; + goto yy273; } else { - if (yych <= ' ') goto yy281; - if (yych <= '0') goto yy265; - if (yych >= ':') goto yy265; + if (yych <= ' ') goto yy289; + if (yych <= '0') goto yy273; + if (yych >= ':') goto yy273; } -yy283: +yy291: ++YYCURSOR; if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2); yych = *YYCURSOR; if (yybm[0+yych] & 128) { - goto yy283; + goto yy291; } if (yych <= '\f') { - if (yych <= 0x08) goto yy265; - if (yych <= '\t') goto yy285; - if (yych <= '\n') goto yy287; - goto yy265; + if (yych <= 0x08) goto yy273; + if (yych <= '\t') goto yy293; + if (yych <= '\n') goto yy295; + goto yy273; } else { - if (yych <= '\r') goto yy289; - if (yych != ' ') goto yy265; + if (yych <= '\r') goto yy297; + if (yych != ' ') goto yy273; } -yy285: +yy293: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yych <= 0x1F) { - if (yych == '\t') goto yy285; - goto yy265; + if (yych == '\t') goto yy293; + goto yy273; } else { - if (yych <= ' ') goto yy285; - if (yych == '"') goto yy290; - goto yy265; + if (yych <= ' ') goto yy293; + if (yych == '"') goto yy298; + goto yy273; } -yy287: +yy295: ++YYCURSOR; YYCURSOR = YYCTXMARKER; { set_sourceline(cursor); goto code; } -yy289: +yy297: yych = *++YYCURSOR; - if (yych == '\n') goto yy287; - goto yy265; -yy290: + if (yych == '\n') goto yy295; + goto yy273; +yy298: ++YYCURSOR; if ((YYLIMIT - YYCURSOR) < 3) YYFILL(3); yych = *YYCURSOR; if (yych <= '!') { - if (yych == '\n') goto yy265; - goto yy290; + if (yych == '\n') goto yy273; + goto yy298; } else { - if (yych <= '"') goto yy293; - if (yych != '\\') goto yy290; + if (yych <= '"') goto yy301; + if (yych != '\\') goto yy298; } ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy265; - goto yy290; -yy293: + if (yych == '\n') goto yy273; + goto yy298; +yy301: yych = *++YYCURSOR; - if (yych == '\n') goto yy287; - if (yych == '\r') goto yy289; - goto yy265; -yy294: + if (yych == '\n') goto yy295; + if (yych == '\r') goto yy297; + goto yy273; +yy302: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yych <= 0x1F) { - if (yych == '\t') goto yy294; - goto yy265; + if (yych == '\t') goto yy302; + goto yy273; } else { - if (yych <= ' ') goto yy294; - if (yych == '#') goto yy275; - goto yy265; + if (yych <= ' ') goto yy302; + if (yych == '#') goto yy283; + goto yy273; } } @@ -1834,13 +1874,13 @@ comment: if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2); yych = *YYCURSOR; if (yych <= ')') { - if (yych == '\n') goto yy300; + if (yych == '\n') goto yy308; } else { - if (yych <= '*') goto yy302; - if (yych == '/') goto yy303; + if (yych <= '*') goto yy310; + if (yych == '/') goto yy311; } ++YYCURSOR; -yy299: +yy307: { if (cursor == eof) { @@ -1848,13 +1888,13 @@ yy299: } goto comment; } -yy300: +yy308: yych = *(YYMARKER = ++YYCURSOR); if (yybm[0+yych] & 32) { - goto yy308; + goto yy316; } - if (yych == '#') goto yy311; -yy301: + if (yych == '#') goto yy319; +yy309: { if (cursor == eof) { @@ -1864,20 +1904,20 @@ yy301: cline++; goto comment; } -yy302: +yy310: yych = *++YYCURSOR; - if (yych == '/') goto yy306; - goto yy299; -yy303: + if (yych == '/') goto yy314; + goto yy307; +yy311: yych = *++YYCURSOR; - if (yych != '*') goto yy299; + if (yych != '*') goto yy307; ++YYCURSOR; { ++depth; fatal("ambiguous /* found"); goto comment; } -yy306: +yy314: ++YYCURSOR; { if (--depth == 0) @@ -1889,110 +1929,110 @@ yy306: goto comment; } } -yy308: +yy316: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yybm[0+yych] & 32) { - goto yy308; + goto yy316; } - if (yych == '#') goto yy311; -yy310: + if (yych == '#') goto yy319; +yy318: YYCURSOR = YYMARKER; - goto yy301; -yy311: + goto yy309; +yy319: ++YYCURSOR; if ((YYLIMIT - YYCURSOR) < 5) YYFILL(5); yych = *YYCURSOR; if (yych <= 0x1F) { - if (yych == '\t') goto yy311; - goto yy310; + if (yych == '\t') goto yy319; + goto yy318; } else { - if (yych <= ' ') goto yy311; - if (yych != 'l') goto yy310; + if (yych <= ' ') goto yy319; + if (yych != 'l') goto yy318; } yych = *++YYCURSOR; - if (yych != 'i') goto yy310; + if (yych != 'i') goto yy318; yych = *++YYCURSOR; - if (yych != 'n') goto yy310; + if (yych != 'n') goto yy318; yych = *++YYCURSOR; - if (yych != 'e') goto yy310; + if (yych != 'e') goto yy318; yych = *++YYCURSOR; - if (yych <= '0') goto yy318; - if (yych <= '9') goto yy310; - goto yy318; -yy317: + if (yych <= '0') goto yy326; + if (yych <= '9') goto yy318; + goto yy326; +yy325: YYCTXMARKER = YYCURSOR + 1; ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy318: +yy326: if (yych <= 0x1F) { - if (yych == '\t') goto yy317; - goto yy310; + if (yych == '\t') goto yy325; + goto yy318; } else { - if (yych <= ' ') goto yy317; - if (yych <= '0') goto yy310; - if (yych >= ':') goto yy310; + if (yych <= ' ') goto yy325; + if (yych <= '0') goto yy318; + if (yych >= ':') goto yy318; } -yy319: +yy327: ++YYCURSOR; if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2); yych = *YYCURSOR; if (yybm[0+yych] & 64) { - goto yy319; + goto yy327; } if (yych <= '\f') { - if (yych <= 0x08) goto yy310; - if (yych <= '\t') goto yy321; - if (yych <= '\n') goto yy323; - goto yy310; + if (yych <= 0x08) goto yy318; + if (yych <= '\t') goto yy329; + if (yych <= '\n') goto yy331; + goto yy318; } else { - if (yych <= '\r') goto yy325; - if (yych != ' ') goto yy310; + if (yych <= '\r') goto yy333; + if (yych != ' ') goto yy318; } -yy321: +yy329: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yych <= 0x1F) { - if (yych == '\t') goto yy321; - goto yy310; + if (yych == '\t') goto yy329; + goto yy318; } else { - if (yych <= ' ') goto yy321; - if (yych == '"') goto yy326; - goto yy310; + if (yych <= ' ') goto yy329; + if (yych == '"') goto yy334; + goto yy318; } -yy323: +yy331: ++YYCURSOR; YYCURSOR = YYCTXMARKER; { set_sourceline(cursor); goto comment; } -yy325: +yy333: yych = *++YYCURSOR; - if (yych == '\n') goto yy323; - goto yy310; -yy326: + if (yych == '\n') goto yy331; + goto yy318; +yy334: ++YYCURSOR; if ((YYLIMIT - YYCURSOR) < 3) YYFILL(3); yych = *YYCURSOR; if (yybm[0+yych] & 128) { - goto yy326; + goto yy334; } - if (yych <= '\n') goto yy310; - if (yych <= '"') goto yy329; + if (yych <= '\n') goto yy318; + if (yych <= '"') goto yy337; ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy310; - goto yy326; -yy329: + if (yych == '\n') goto yy318; + goto yy334; +yy337: ++YYCURSOR; - if ((yych = *YYCURSOR) == '\n') goto yy323; - if (yych == '\r') goto yy325; - goto yy310; + if ((yych = *YYCURSOR) == '\n') goto yy331; + if (yych == '\r') goto yy333; + goto yy318; } @@ -2002,14 +2042,14 @@ nextLine: YYCTYPE yych; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy334; + if (yych == '\n') goto yy342; ++YYCURSOR; { if(cursor == eof) { RETURN(0); } goto nextLine; } -yy334: +yy342: ++YYCURSOR; { if(cursor == eof) { RETURN(0); @@ -2062,50 +2102,50 @@ config: if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2); yych = *YYCURSOR; if (yych <= 0x1F) { - if (yych == '\t') goto yy340; + if (yych == '\t') goto yy348; } else { - if (yych <= ' ') goto yy340; - if (yych == '=') goto yy342; + if (yych <= ' ') goto yy348; + if (yych == '=') goto yy350; } ++YYCURSOR; { fatal("missing '='"); } -yy340: +yy348: ++YYCURSOR; yych = *YYCURSOR; - goto yy347; -yy341: + goto yy355; +yy349: { goto config; } -yy342: +yy350: ++YYCURSOR; yych = *YYCURSOR; - goto yy345; -yy343: + goto yy353; +yy351: { - iscfg = 2; + lexer_state = LEX_CONFIG_VALUE; cur = cursor; RETURN('='); } -yy344: +yy352: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy345: +yy353: if (yybm[0+yych] & 128) { - goto yy344; + goto yy352; } - goto yy343; -yy346: + goto yy351; +yy354: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy347: - if (yych == '\t') goto yy346; - if (yych == ' ') goto yy346; - goto yy341; +yy355: + if (yych == '\t') goto yy354; + if (yych == ' ') goto yy354; + goto yy349; } @@ -2151,191 +2191,191 @@ value: yych = *YYCURSOR; if (yych <= '&') { if (yych <= '\r') { - if (yych <= 0x08) goto yy356; - if (yych <= '\n') goto yy350; - if (yych <= '\f') goto yy356; + if (yych <= 0x08) goto yy364; + if (yych <= '\n') goto yy358; + if (yych <= '\f') goto yy364; } else { if (yych <= ' ') { - if (yych <= 0x1F) goto yy356; + if (yych <= 0x1F) goto yy364; } else { - if (yych == '"') goto yy358; - goto yy356; + if (yych == '"') goto yy366; + goto yy364; } } } else { if (yych <= '/') { - if (yych <= '\'') goto yy360; - if (yych == '-') goto yy353; - goto yy356; + if (yych <= '\'') goto yy368; + if (yych == '-') goto yy361; + goto yy364; } else { if (yych <= '9') { - if (yych <= '0') goto yy351; - goto yy354; + if (yych <= '0') goto yy359; + goto yy362; } else { - if (yych != ';') goto yy356; + if (yych != ';') goto yy364; } } } -yy350: +yy358: { cur = cursor; yylval.str = new std::string (tok, cur - tok); - iscfg = 0; + lexer_state = LEX_NORMAL; return VALUE; } -yy351: +yy359: ++YYCURSOR; if (yybm[0+(yych = *YYCURSOR)] & 8) { - goto yy356; + goto yy364; } -yy352: +yy360: { cur = cursor; yylval.number = atoi(std::string (tok, cur - tok).c_str()); - iscfg = 0; + lexer_state = LEX_NORMAL; return NUMBER; } -yy353: +yy361: yych = *++YYCURSOR; - if (yych <= '0') goto yy357; - if (yych >= ':') goto yy357; -yy354: + if (yych <= '0') goto yy365; + if (yych >= ':') goto yy365; +yy362: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yybm[0+yych] & 4) { - goto yy354; + goto yy362; } if (yych <= '\r') { - if (yych <= 0x08) goto yy356; - if (yych <= '\n') goto yy352; - if (yych >= '\r') goto yy352; + if (yych <= 0x08) goto yy364; + if (yych <= '\n') goto yy360; + if (yych >= '\r') goto yy360; } else { if (yych <= ' ') { - if (yych >= ' ') goto yy352; + if (yych >= ' ') goto yy360; } else { - if (yych == ';') goto yy352; + if (yych == ';') goto yy360; } } -yy356: +yy364: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy357: +yy365: if (yybm[0+yych] & 8) { - goto yy356; + goto yy364; } - goto yy350; -yy358: + goto yy358; +yy366: YYMARKER = ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yybm[0+yych] & 16) { - goto yy358; + goto yy366; } if (yych <= ' ') { - if (yych == '\n') goto yy350; - goto yy368; + if (yych == '\n') goto yy358; + goto yy376; } else { - if (yych <= '"') goto yy356; - if (yych <= ';') goto yy368; - goto yy370; + if (yych <= '"') goto yy364; + if (yych <= ';') goto yy376; + goto yy378; } -yy360: +yy368: YYMARKER = ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yybm[0+yych] & 32) { - goto yy360; + goto yy368; } if (yych <= ' ') { - if (yych == '\n') goto yy350; + if (yych == '\n') goto yy358; } else { - if (yych <= '\'') goto yy356; - if (yych >= '<') goto yy365; + if (yych <= '\'') goto yy364; + if (yych >= '<') goto yy373; } -yy362: +yy370: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yybm[0+yych] & 64) { - goto yy362; + goto yy370; } - if (yych <= '\n') goto yy364; - if (yych <= '\'') goto yy366; - goto yy367; -yy364: + if (yych <= '\n') goto yy372; + if (yych <= '\'') goto yy374; + goto yy375; +yy372: YYCURSOR = YYMARKER; - goto yy350; -yy365: + goto yy358; +yy373: YYMARKER = ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yych <= '\r') { if (yych <= '\t') { - if (yych <= 0x08) goto yy360; - goto yy362; + if (yych <= 0x08) goto yy368; + goto yy370; } else { - if (yych <= '\n') goto yy350; - if (yych <= '\f') goto yy360; - goto yy362; + if (yych <= '\n') goto yy358; + if (yych <= '\f') goto yy368; + goto yy370; } } else { if (yych <= ' ') { - if (yych <= 0x1F) goto yy360; - goto yy362; + if (yych <= 0x1F) goto yy368; + goto yy370; } else { - if (yych == ';') goto yy362; - goto yy360; + if (yych == ';') goto yy370; + goto yy368; } } -yy366: +yy374: yych = *++YYCURSOR; - goto yy350; -yy367: + goto yy358; +yy375: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy364; - goto yy362; -yy368: + if (yych == '\n') goto yy372; + goto yy370; +yy376: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yybm[0+yych] & 128) { - goto yy368; + goto yy376; } - if (yych <= '\n') goto yy364; - if (yych <= '"') goto yy366; - goto yy371; -yy370: + if (yych <= '\n') goto yy372; + if (yych <= '"') goto yy374; + goto yy379; +yy378: YYMARKER = ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; if (yych <= '\r') { if (yych <= '\t') { - if (yych <= 0x08) goto yy358; - goto yy368; + if (yych <= 0x08) goto yy366; + goto yy376; } else { - if (yych <= '\n') goto yy350; - if (yych <= '\f') goto yy358; - goto yy368; + if (yych <= '\n') goto yy358; + if (yych <= '\f') goto yy366; + goto yy376; } } else { if (yych <= ' ') { - if (yych <= 0x1F) goto yy358; - goto yy368; + if (yych <= 0x1F) goto yy366; + goto yy376; } else { - if (yych == ';') goto yy368; - goto yy358; + if (yych == ';') goto yy376; + goto yy366; } } -yy371: +yy379: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy364; - goto yy368; + if (yych == '\n') goto yy372; + goto yy376; } } @@ -2398,19 +2438,19 @@ sourceline: if ((YYLIMIT - YYCURSOR) < 2) YYFILL(2); yych = *YYCURSOR; if (yych <= '!') { - if (yych == '\n') goto yy376; + if (yych == '\n') goto yy384; } else { - if (yych <= '"') goto yy378; - if (yych <= '0') goto yy374; - if (yych <= '9') goto yy379; + if (yych <= '"') goto yy386; + if (yych <= '0') goto yy382; + if (yych <= '9') goto yy387; } -yy374: +yy382: ++YYCURSOR; -yy375: +yy383: { goto sourceline; } -yy376: +yy384: ++YYCURSOR; { if (cursor == eof) @@ -2424,50 +2464,50 @@ yy376: tok = cursor; return; } -yy378: +yy386: yych = *(YYMARKER = ++YYCURSOR); - if (yych == '\n') goto yy375; - goto yy384; -yy379: + if (yych == '\n') goto yy383; + goto yy392; +yy387: ++YYCURSOR; yych = *YYCURSOR; - goto yy382; -yy380: + goto yy390; +yy388: { cur = cursor; cline = atoi(std::string (tok, cur - tok).c_str()); goto sourceline; } -yy381: +yy389: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy382: +yy390: if (yybm[0+yych] & 64) { - goto yy381; + goto yy389; } - goto yy380; -yy383: + goto yy388; +yy391: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; -yy384: +yy392: if (yybm[0+yych] & 128) { - goto yy383; + goto yy391; } - if (yych <= '\n') goto yy385; - if (yych <= '"') goto yy387; - goto yy386; -yy385: + if (yych <= '\n') goto yy393; + if (yych <= '"') goto yy395; + goto yy394; +yy393: YYCURSOR = YYMARKER; - goto yy375; -yy386: + goto yy383; +yy394: ++YYCURSOR; if (YYLIMIT <= YYCURSOR) YYFILL(1); yych = *YYCURSOR; - if (yych == '\n') goto yy385; - goto yy383; -yy387: + if (yych == '\n') goto yy393; + goto yy391; +yy395: ++YYCURSOR; { cur = cursor; diff --git a/re2c/src/parse/parser.ypp b/re2c/src/parse/parser.ypp index 7d6b8ca3..bcb64402 100644 --- a/re2c/src/parse/parser.ypp +++ b/re2c/src/parse/parser.ypp @@ -161,9 +161,9 @@ void default_rule(CondList *clist, Token *code) }; %token CLOSESIZE CLOSE STAR NOCOND ID CODE RANGE STRING -%token CONFIG VALUE NUMBER SETUP FID +%token CONFIG VALUE NUMBER SETUP FID FID_END -%type CLOSE STAR SETUP +%type CLOSE STAR SETUP FID_END %type close %type CLOSESIZE %type CODE @@ -196,10 +196,7 @@ decl: delete $1; $3->ins_access = RegExp::PRIVATE; } - // all the conflicts are caused by the following production - // flex-style named definitions - // see https://github.com/skvadrik/re2c/issues/115 - | FID expr + | FID expr FID_END { if (!symbol_table.insert (std::make_pair (* $1, $2)).second) { diff --git a/re2c/src/parse/scanner.cc b/re2c/src/parse/scanner.cc index d8985c9b..f4efa780 100644 --- a/re2c/src/parse/scanner.cc +++ b/re2c/src/parse/scanner.cc @@ -288,8 +288,8 @@ ScannerState::ScannerState () , tchar (0) , tline (0) , cline (1) - , iscfg (0) , in_parse (false) + , lexer_state (LEX_NORMAL) {} ScannerState::ScannerState (const ScannerState & s) @@ -305,26 +305,13 @@ ScannerState::ScannerState (const ScannerState & s) , tchar (s.tchar) , tline (s.tline) , cline (s.cline) - , iscfg (s.iscfg) , in_parse (s.in_parse) + , lexer_state (s.lexer_state) {} ScannerState & ScannerState::operator = (const ScannerState & s) { - tok = s.tok; - ptr = s.ptr; - cur = s.cur; - pos = s.pos; - ctx = s.ctx; - bot = s.bot; - lim = s.lim; - top = s.top; - eof = s.eof; - tchar = s.tchar; - tline = s.tline; - cline = s.cline; - iscfg = s.iscfg; - in_parse = s.in_parse; + new (this) ScannerState (s); return * this; } diff --git a/re2c/src/parse/scanner.h b/re2c/src/parse/scanner.h index 8ec6e0a8..bd97177f 100644 --- a/re2c/src/parse/scanner.h +++ b/re2c/src/parse/scanner.h @@ -17,6 +17,14 @@ namespace re2c struct ScannerState { + enum lexer_state_t + { + LEX_NORMAL, + LEX_CONFIG, + LEX_CONFIG_VALUE, + LEX_FLEX_NAME + }; + // positioning char * tok; char * ptr; @@ -33,9 +41,9 @@ struct ScannerState uint32_t tchar; uint32_t tline; uint32_t cline; - uint32_t iscfg; bool in_parse; + lexer_state_t lexer_state; ScannerState (); ScannerState (const ScannerState &); diff --git a/re2c/src/parse/scanner_lex.re b/re2c/src/parse/scanner_lex.re index 1eca7603..ad64e543 100644 --- a/re2c/src/parse/scanner_lex.re +++ b/re2c/src/parse/scanner_lex.re @@ -214,14 +214,15 @@ scan: tchar = cursor - pos; tline = cline; tok = cursor; - if (iscfg == 1) + switch (lexer_state) { - goto config; - } - else if (iscfg == 2) - { - goto value; + case LEX_NORMAL: goto start; + case LEX_CONFIG: goto config; + case LEX_CONFIG_VALUE: goto value; + case LEX_FLEX_NAME: goto flex_name; } + +start: /*!re2c "{" { depth = 1; @@ -364,7 +365,7 @@ scan: config { cur = cursor; tok += 5; /* skip "re2c:" */ - iscfg = 1; + lexer_state = LEX_CONFIG; yylval.str = new std::string (tok, cur - tok); return CONFIG; } @@ -372,7 +373,15 @@ scan: name / (space+ [^=>,]) { cur = ptr > tok ? ptr - 1 : cursor; yylval.str = new std::string (tok, cur - tok); - return FFlag ? FID : ID; + if (FFlag) + { + lexer_state = LEX_FLEX_NAME; + return FID; + } + else + { + return ID; + } } name / (space* [=>,]) { @@ -430,6 +439,21 @@ scan: } */ +flex_name: +/*!re2c + eol + { + YYCURSOR = tok; + lexer_state = LEX_NORMAL; + return FID_END; + } + * + { + YYCURSOR = tok; + goto start; + } +*/ + code: /*!re2c "}" { @@ -573,7 +597,7 @@ config: goto config; } "=" space* { - iscfg = 2; + lexer_state = LEX_CONFIG_VALUE; cur = cursor; RETURN('='); } @@ -587,13 +611,13 @@ value: number { cur = cursor; yylval.number = atoi(std::string (tok, cur - tok).c_str()); - iscfg = 0; + lexer_state = LEX_NORMAL; return NUMBER; } value { cur = cursor; yylval.str = new std::string (tok, cur - tok); - iscfg = 0; + lexer_state = LEX_NORMAL; return VALUE; } */ -- 2.40.0