From c37efa83ec92a7ccd3634834e154b0e9c13db144 Mon Sep 17 00:00:00 2001 From: helly Date: Sat, 8 Apr 2006 19:23:06 +0000 Subject: [PATCH] - Add tests for -f switch --- test/push.f.c | 1212 +++++++++++++++++++++++++++++++++++++++++++++++ test/push.f.re | 340 +++++++++++++ test/push.fb.c | 650 +++++++++++++++++++++++++ test/push.fb.re | 340 +++++++++++++ test/push.fs.c | 408 ++++++++++++++++ test/push.fs.re | 54 +++ 6 files changed, 3004 insertions(+) create mode 100755 test/push.f.c create mode 100755 test/push.f.re create mode 100755 test/push.fb.c create mode 100755 test/push.fb.re create mode 100755 test/push.fs.c create mode 100755 test/push.fs.re diff --git a/test/push.f.c b/test/push.f.c new file mode 100755 index 00000000..ce2cdbbf --- /dev/null +++ b/test/push.f.c @@ -0,0 +1,1212 @@ +/* Generated by re2c */ +#line 1 "push.f.re" +/* + * A push-model scanner example for re2c -f + * Written Mon Apr 11 2005 by mgix@mgix.com + * This file is in the public domain. + * + */ + +// ---------------------------------------------------------------------- + +#include +#include +#include +#include +#include + +#if defined(WIN32) + + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +#else + + #include + #include + + #ifndef O_BINARY + #define O_BINARY 0 + #endif + +#endif + +// ---------------------------------------------------------------------- +#define TOKENS \ + \ + TOK(kEOF) \ + TOK(kEOL) \ + TOK(kUnknown) \ + TOK(kIdentifier) \ + TOK(kDecimalConstant) \ + \ + TOK(kEqual) \ + TOK(kLeftParen) \ + TOK(kRightParen) \ + TOK(kMinus) \ + TOK(kPlus) \ + TOK(kStar) \ + TOK(kSlash) \ + \ + TOK(kIf) \ + TOK(kFor) \ + TOK(kElse) \ + TOK(kGoto) \ + TOK(kBreak) \ + TOK(kWhile) \ + TOK(kReturn) \ + + +// ---------------------------------------------------------------------- +static const char *tokenNames[] = +{ + #define TOK(x) #x, + TOKENS + #undef TOK +}; + +// ---------------------------------------------------------------------- +class PushScanner +{ +public: + + enum Token + { + #define TOK(x) x, + TOKENS + #undef TOK + }; + +private: + + bool eof; + int32_t state; + + uint8_t *limit; + uint8_t *start; + uint8_t *cursor; + uint8_t *marker; + + uint8_t *buffer; + uint8_t *bufferEnd; + + uint8_t yych; + uint32_t yyaccept; + +public: + + // ---------------------------------------------------------------------- + PushScanner() + { + limit = 0; + start = 0; + state = -1; + cursor = 0; + marker = 0; + buffer = 0; + eof = false; + bufferEnd = 0; + } + + // ---------------------------------------------------------------------- + ~PushScanner() + { + } + + // ---------------------------------------------------------------------- + void send( + Token token + ) + { + size_t tokenSize = cursor-start; + const char *tokenName = tokenNames[token]; + printf( + "scanner is pushing out a token of type %d (%s)", + token, + tokenName + ); + + if(token==kEOF) putchar('\n'); + else + { + size_t tokenNameSize = strlen(tokenNames[token]); + size_t padSize = 20-(20"); + + fwrite( + start, + tokenSize, + 1, + stdout + ); + + printf("<----\n"); + } + } + + // ---------------------------------------------------------------------- + uint32_t push( + const void *input, + ssize_t inputSize + ) + { + printf( + "scanner is receiving a new data batch of length %d\n" + "scanner continues with saved state = %d\n", + inputSize, + state + ); + + /* + * Data source is signaling end of file when batch size + * is less than maxFill. This is slightly annoying because + * maxFill is a value that can only be known after re2c does + * its thing. Practically though, maxFill is never bigger than + * the longest keyword, so given our grammar, 32 is a safe bet. + */ + uint8_t null[64]; + const ssize_t maxFill = 32; + if(inputSize" +{ + + switch(YYGETSTATE()) + { + case -1: goto yy0; + case 0: goto yyFillLabel0; + case 1: goto yyFillLabel1; + case 2: goto yyFillLabel2; + default: /* abort() */; + } +yyNext: +yy0: + YYSETSTATE(0); + if((YYLIMIT - YYCURSOR) < 7) YYFILL(7); +yyFillLabel0: + yych = *YYCURSOR; + switch(yych){ + case 0x00: goto yy32; + case 0x09: + case 0x0B: + case 0x0C: + case 0x0D: + case ' ': goto yy30; + case 0x0A: goto yy28; + case '(': goto yy16; + case ')': goto yy18; + case '*': goto yy24; + case '+': goto yy22; + case '-': goto yy20; + case '/': goto yy26; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': goto yy12; + case '=': goto yy14; + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'c': + case 'd': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 's': + case 't': + case 'u': + case 'v': + case 'x': + case 'y': + case 'z': goto yy10; + case 'b': goto yy7; + case 'e': goto yy5; + case 'f': goto yy4; + case 'g': goto yy6; + case 'h': goto yy11; + case 'i': goto yy2; + case 'r': goto yy9; + case 'w': goto yy8; + default: goto yy34; + } +yy2: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case 'f': goto yy67; + default: goto yy39; + } +yy3: +#line 246 "push.f.re" + { SEND(kIdentifier); } +#line 339 "" +yy4: + yych = *++YYCURSOR; + switch(yych){ + case 'o': goto yy64; + default: goto yy39; + } +yy5: + yych = *++YYCURSOR; + switch(yych){ + case 'l': goto yy60; + default: goto yy39; + } +yy6: + yych = *++YYCURSOR; + switch(yych){ + case 'o': goto yy56; + default: goto yy39; + } +yy7: + yych = *++YYCURSOR; + switch(yych){ + case 'r': goto yy51; + default: goto yy39; + } +yy8: + yych = *++YYCURSOR; + switch(yych){ + case 'h': goto yy46; + default: goto yy39; + } +yy9: + yych = *++YYCURSOR; + switch(yych){ + case 'e': goto yy40; + default: goto yy39; + } +yy10: + yych = *++YYCURSOR; + goto yy39; +yy11: + yych = *++YYCURSOR; + goto yy39; +yy12: + ++YYCURSOR; + yych = *YYCURSOR; + goto yy37; +yy13: +#line 247 "push.f.re" + { SEND(kDecimalConstant);} +#line 389 "" +yy14: + ++YYCURSOR; +#line 249 "push.f.re" + { SEND(kEqual); } +#line 394 "" +yy16: + ++YYCURSOR; +#line 250 "push.f.re" + { SEND(kLeftParen); } +#line 399 "" +yy18: + ++YYCURSOR; +#line 251 "push.f.re" + { SEND(kRightParen); } +#line 404 "" +yy20: + ++YYCURSOR; +#line 252 "push.f.re" + { SEND(kMinus); } +#line 409 "" +yy22: + ++YYCURSOR; +#line 253 "push.f.re" + { SEND(kPlus); } +#line 414 "" +yy24: + ++YYCURSOR; +#line 254 "push.f.re" + { SEND(kStar); } +#line 419 "" +yy26: + ++YYCURSOR; +#line 255 "push.f.re" + { SEND(kSlash); } +#line 424 "" +yy28: + ++YYCURSOR; +#line 257 "push.f.re" + { SKIP(); } +#line 429 "" +yy30: + ++YYCURSOR; +#line 258 "push.f.re" + { SKIP(); } +#line 434 "" +yy32: + ++YYCURSOR; +#line 259 "push.f.re" + { send(kEOF); return 1; } +#line 439 "" +yy34: + ++YYCURSOR; +#line 260 "push.f.re" + { SEND(kUnknown); } +#line 444 "" +yy36: + ++YYCURSOR; + YYSETSTATE(1); + if(YYLIMIT == YYCURSOR) YYFILL(1); +yyFillLabel1: + yych = *YYCURSOR; +yy37: + switch(yych){ + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': goto yy36; + default: goto yy13; + } +yy38: + ++YYCURSOR; + YYSETSTATE(2); + if(YYLIMIT == YYCURSOR) YYFILL(1); +yyFillLabel2: + yych = *YYCURSOR; +yy39: + switch(yych){ + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy3; + } +yy40: + yych = *++YYCURSOR; + switch(yych){ + case 't': goto yy41; + default: goto yy39; + } +yy41: + yych = *++YYCURSOR; + switch(yych){ + case 'u': goto yy42; + default: goto yy39; + } +yy42: + yych = *++YYCURSOR; + switch(yych){ + case 'r': goto yy43; + default: goto yy39; + } +yy43: + yych = *++YYCURSOR; + switch(yych){ + case 'n': goto yy44; + default: goto yy39; + } +yy44: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy45; + } +yy45: +#line 245 "push.f.re" + { SEND(kReturn); } +#line 633 "" +yy46: + yych = *++YYCURSOR; + switch(yych){ + case 'i': goto yy47; + default: goto yy39; + } +yy47: + yych = *++YYCURSOR; + switch(yych){ + case 'l': goto yy48; + default: goto yy39; + } +yy48: + yych = *++YYCURSOR; + switch(yych){ + case 'e': goto yy49; + default: goto yy39; + } +yy49: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy50; + } +yy50: +#line 244 "push.f.re" + { SEND(kWhile); } +#line 723 "" +yy51: + yych = *++YYCURSOR; + switch(yych){ + case 'e': goto yy52; + default: goto yy39; + } +yy52: + yych = *++YYCURSOR; + switch(yych){ + case 'a': goto yy53; + default: goto yy39; + } +yy53: + yych = *++YYCURSOR; + switch(yych){ + case 'k': goto yy54; + default: goto yy39; + } +yy54: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy55; + } +yy55: +#line 243 "push.f.re" + { SEND(kBreak); } +#line 813 "" +yy56: + yych = *++YYCURSOR; + switch(yych){ + case 't': goto yy57; + default: goto yy39; + } +yy57: + yych = *++YYCURSOR; + switch(yych){ + case 'o': goto yy58; + default: goto yy39; + } +yy58: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy59; + } +yy59: +#line 242 "push.f.re" + { SEND(kGoto); } +#line 897 "" +yy60: + yych = *++YYCURSOR; + switch(yych){ + case 's': goto yy61; + default: goto yy39; + } +yy61: + yych = *++YYCURSOR; + switch(yych){ + case 'e': goto yy62; + default: goto yy39; + } +yy62: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy63; + } +yy63: +#line 241 "push.f.re" + { SEND(kElse); } +#line 981 "" +yy64: + yych = *++YYCURSOR; + switch(yych){ + case 'r': goto yy65; + default: goto yy39; + } +yy65: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy66; + } +yy66: +#line 240 "push.f.re" + { SEND(kFor); } +#line 1059 "" +yy67: + ++YYCURSOR; + switch((yych = *YYCURSOR)) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': goto yy38; + default: goto yy68; + } +yy68: +#line 239 "push.f.re" + { SEND(kIf); } +#line 1131 "" +} +#line 261 "push.f.re" + + + fill: + ssize_t unfinishedSize = cursor-start; + printf( + "scanner needs a refill. Exiting for now with:\n" + " saved fill state = %d\n" + " unfinished token size = %d\n", + state, + unfinishedSize + ); + + if(0 +#include +#include +#include +#include + +#if defined(WIN32) + + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +#else + + #include + #include + + #ifndef O_BINARY + #define O_BINARY 0 + #endif + +#endif + +// ---------------------------------------------------------------------- +#define TOKENS \ + \ + TOK(kEOF) \ + TOK(kEOL) \ + TOK(kUnknown) \ + TOK(kIdentifier) \ + TOK(kDecimalConstant) \ + \ + TOK(kEqual) \ + TOK(kLeftParen) \ + TOK(kRightParen) \ + TOK(kMinus) \ + TOK(kPlus) \ + TOK(kStar) \ + TOK(kSlash) \ + \ + TOK(kIf) \ + TOK(kFor) \ + TOK(kElse) \ + TOK(kGoto) \ + TOK(kBreak) \ + TOK(kWhile) \ + TOK(kReturn) \ + + +// ---------------------------------------------------------------------- +static const char *tokenNames[] = +{ + #define TOK(x) #x, + TOKENS + #undef TOK +}; + +// ---------------------------------------------------------------------- +class PushScanner +{ +public: + + enum Token + { + #define TOK(x) x, + TOKENS + #undef TOK + }; + +private: + + bool eof; + int32_t state; + + uint8_t *limit; + uint8_t *start; + uint8_t *cursor; + uint8_t *marker; + + uint8_t *buffer; + uint8_t *bufferEnd; + + uint8_t yych; + uint32_t yyaccept; + +public: + + // ---------------------------------------------------------------------- + PushScanner() + { + limit = 0; + start = 0; + state = -1; + cursor = 0; + marker = 0; + buffer = 0; + eof = false; + bufferEnd = 0; + } + + // ---------------------------------------------------------------------- + ~PushScanner() + { + } + + // ---------------------------------------------------------------------- + void send( + Token token + ) + { + size_t tokenSize = cursor-start; + const char *tokenName = tokenNames[token]; + printf( + "scanner is pushing out a token of type %d (%s)", + token, + tokenName + ); + + if(token==kEOF) putchar('\n'); + else + { + size_t tokenNameSize = strlen(tokenNames[token]); + size_t padSize = 20-(20"); + + fwrite( + start, + tokenSize, + 1, + stdout + ); + + printf("<----\n"); + } + } + + // ---------------------------------------------------------------------- + uint32_t push( + const void *input, + ssize_t inputSize + ) + { + printf( + "scanner is receiving a new data batch of length %d\n" + "scanner continues with saved state = %d\n", + inputSize, + state + ); + + /* + * Data source is signaling end of file when batch size + * is less than maxFill. This is slightly annoying because + * maxFill is a value that can only be known after re2c does + * its thing. Practically though, maxFill is never bigger than + * the longest keyword, so given our grammar, 32 is a safe bet. + */ + uint8_t null[64]; + const ssize_t maxFill = 32; + if(inputSize +#include +#include +#include +#include + +#if defined(WIN32) + + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +#else + + #include + #include + + #ifndef O_BINARY + #define O_BINARY 0 + #endif + +#endif + +// ---------------------------------------------------------------------- +#define TOKENS \ + \ + TOK(kEOF) \ + TOK(kEOL) \ + TOK(kUnknown) \ + TOK(kIdentifier) \ + TOK(kDecimalConstant) \ + \ + TOK(kEqual) \ + TOK(kLeftParen) \ + TOK(kRightParen) \ + TOK(kMinus) \ + TOK(kPlus) \ + TOK(kStar) \ + TOK(kSlash) \ + \ + TOK(kIf) \ + TOK(kFor) \ + TOK(kElse) \ + TOK(kGoto) \ + TOK(kBreak) \ + TOK(kWhile) \ + TOK(kReturn) \ + + +// ---------------------------------------------------------------------- +static const char *tokenNames[] = +{ + #define TOK(x) #x, + TOKENS + #undef TOK +}; + +// ---------------------------------------------------------------------- +class PushScanner +{ +public: + + enum Token + { + #define TOK(x) x, + TOKENS + #undef TOK + }; + +private: + + bool eof; + int32_t state; + + uint8_t *limit; + uint8_t *start; + uint8_t *cursor; + uint8_t *marker; + + uint8_t *buffer; + uint8_t *bufferEnd; + + uint8_t yych; + uint32_t yyaccept; + +public: + + // ---------------------------------------------------------------------- + PushScanner() + { + limit = 0; + start = 0; + state = -1; + cursor = 0; + marker = 0; + buffer = 0; + eof = false; + bufferEnd = 0; + } + + // ---------------------------------------------------------------------- + ~PushScanner() + { + } + + // ---------------------------------------------------------------------- + void send( + Token token + ) + { + size_t tokenSize = cursor-start; + const char *tokenName = tokenNames[token]; + printf( + "scanner is pushing out a token of type %d (%s)", + token, + tokenName + ); + + if(token==kEOF) putchar('\n'); + else + { + size_t tokenNameSize = strlen(tokenNames[token]); + size_t padSize = 20-(20"); + + fwrite( + start, + tokenSize, + 1, + stdout + ); + + printf("<----\n"); + } + } + + // ---------------------------------------------------------------------- + uint32_t push( + const void *input, + ssize_t inputSize + ) + { + printf( + "scanner is receiving a new data batch of length %d\n" + "scanner continues with saved state = %d\n", + inputSize, + state + ); + + /* + * Data source is signaling end of file when batch size + * is less than maxFill. This is slightly annoying because + * maxFill is a value that can only be known after re2c does + * its thing. Practically though, maxFill is never bigger than + * the longest keyword, so given our grammar, 32 is a safe bet. + */ + uint8_t null[64]; + const ssize_t maxFill = 32; + if(inputSize" + { + + switch(YYGETSTATE()) + { + case -1: goto yy0; + case 0: goto yyFillLabel0; + case 1: goto yyFillLabel1; + case 2: goto yyFillLabel2; + default: /* abort() */; + } +yyNext: +yy0: + YYSETSTATE(0); + if((YYLIMIT - YYCURSOR) < 7) YYFILL(7); +yyFillLabel0: + yych = *YYCURSOR; + if(yych <= '=') { + if(yych <= '(') { + if(yych <= 0x0A) { + if(yych <= 0x00) goto yy32; + if(yych <= 0x08) goto yy34; + if(yych <= 0x09) goto yy30; + goto yy28; + } else { + if(yych <= 0x1F) { + if(yych <= 0x0D) goto yy30; + goto yy34; + } else { + if(yych <= ' ') goto yy30; + if(yych <= '\'') goto yy34; + goto yy16; + } + } + } else { + if(yych <= '-') { + if(yych <= '*') { + if(yych <= ')') goto yy18; + goto yy24; + } else { + if(yych <= '+') goto yy22; + if(yych <= ',') goto yy34; + goto yy20; + } + } else { + if(yych <= '/') { + if(yych <= '.') goto yy34; + goto yy26; + } else { + if(yych <= '9') goto yy12; + if(yych <= '<') goto yy34; + goto yy14; + } + } + } + } else { + if(yych <= 'e') { + if(yych <= '_') { + if(yych <= '@') goto yy34; + if(yych <= 'Z') goto yy10; + if(yych <= '^') goto yy34; + goto yy10; + } else { + if(yych <= 'a') { + if(yych <= '`') goto yy34; + goto yy10; + } else { + if(yych <= 'b') goto yy7; + if(yych <= 'd') goto yy10; + goto yy5; + } + } + } else { + if(yych <= 'q') { + if(yych <= 'g') { + if(yych <= 'f') goto yy4; + goto yy6; + } else { + if(yych <= 'h') goto yy11; + if(yych >= 'j') goto yy10; + } + } else { + if(yych <= 'v') { + if(yych <= 'r') goto yy9; + goto yy10; + } else { + if(yych <= 'w') goto yy8; + if(yych <= 'z') goto yy10; + goto yy34; + } + } + } + } + ++YYCURSOR; + if((yych = *YYCURSOR) == 'f') goto yy67; + goto yy39; +yy3: +#line 246 "push.fb.re" + { SEND(kIdentifier); } +#line 367 "" +yy4: + yych = *++YYCURSOR; + if(yych == 'o') goto yy64; + goto yy39; +yy5: + yych = *++YYCURSOR; + if(yych == 'l') goto yy60; + goto yy39; +yy6: + yych = *++YYCURSOR; + if(yych == 'o') goto yy56; + goto yy39; +yy7: + yych = *++YYCURSOR; + if(yych == 'r') goto yy51; + goto yy39; +yy8: + yych = *++YYCURSOR; + if(yych == 'h') goto yy46; + goto yy39; +yy9: + yych = *++YYCURSOR; + if(yych == 'e') goto yy40; + goto yy39; +yy10: + yych = *++YYCURSOR; + goto yy39; +yy11: + yych = *++YYCURSOR; + goto yy39; +yy12: + ++YYCURSOR; + yych = *YYCURSOR; + goto yy37; +yy13: +#line 247 "push.fb.re" + { SEND(kDecimalConstant);} +#line 405 "" +yy14: + ++YYCURSOR; +#line 249 "push.fb.re" + { SEND(kEqual); } +#line 410 "" +yy16: + ++YYCURSOR; +#line 250 "push.fb.re" + { SEND(kLeftParen); } +#line 415 "" +yy18: + ++YYCURSOR; +#line 251 "push.fb.re" + { SEND(kRightParen); } +#line 420 "" +yy20: + ++YYCURSOR; +#line 252 "push.fb.re" + { SEND(kMinus); } +#line 425 "" +yy22: + ++YYCURSOR; +#line 253 "push.fb.re" + { SEND(kPlus); } +#line 430 "" +yy24: + ++YYCURSOR; +#line 254 "push.fb.re" + { SEND(kStar); } +#line 435 "" +yy26: + ++YYCURSOR; +#line 255 "push.fb.re" + { SEND(kSlash); } +#line 440 "" +yy28: + ++YYCURSOR; +#line 257 "push.fb.re" + { SKIP(); } +#line 445 "" +yy30: + ++YYCURSOR; +#line 258 "push.fb.re" + { SKIP(); } +#line 450 "" +yy32: + ++YYCURSOR; +#line 259 "push.fb.re" + { send(kEOF); return 1; } +#line 455 "" +yy34: + ++YYCURSOR; +#line 260 "push.fb.re" + { SEND(kUnknown); } +#line 460 "" +yy36: + ++YYCURSOR; + YYSETSTATE(1); + if(YYLIMIT == YYCURSOR) YYFILL(1); +yyFillLabel1: + yych = *YYCURSOR; +yy37: + if(yybm[0+yych] & 64) { + goto yy36; + } + goto yy13; +yy38: + ++YYCURSOR; + YYSETSTATE(2); + if(YYLIMIT == YYCURSOR) YYFILL(1); +yyFillLabel2: + yych = *YYCURSOR; +yy39: + if(yybm[0+yych] & 128) { + goto yy38; + } + goto yy3; +yy40: + yych = *++YYCURSOR; + if(yych != 't') goto yy39; + yych = *++YYCURSOR; + if(yych != 'u') goto yy39; + yych = *++YYCURSOR; + if(yych != 'r') goto yy39; + yych = *++YYCURSOR; + if(yych != 'n') goto yy39; + ++YYCURSOR; + if(yybm[0+(yych = *YYCURSOR)] & 128) { + goto yy38; + } +#line 245 "push.fb.re" + { SEND(kReturn); } +#line 498 "" +yy46: + yych = *++YYCURSOR; + if(yych != 'i') goto yy39; + yych = *++YYCURSOR; + if(yych != 'l') goto yy39; + yych = *++YYCURSOR; + if(yych != 'e') goto yy39; + ++YYCURSOR; + if(yybm[0+(yych = *YYCURSOR)] & 128) { + goto yy38; + } +#line 244 "push.fb.re" + { SEND(kWhile); } +#line 512 "" +yy51: + yych = *++YYCURSOR; + if(yych != 'e') goto yy39; + yych = *++YYCURSOR; + if(yych != 'a') goto yy39; + yych = *++YYCURSOR; + if(yych != 'k') goto yy39; + ++YYCURSOR; + if(yybm[0+(yych = *YYCURSOR)] & 128) { + goto yy38; + } +#line 243 "push.fb.re" + { SEND(kBreak); } +#line 526 "" +yy56: + yych = *++YYCURSOR; + if(yych != 't') goto yy39; + yych = *++YYCURSOR; + if(yych != 'o') goto yy39; + ++YYCURSOR; + if(yybm[0+(yych = *YYCURSOR)] & 128) { + goto yy38; + } +#line 242 "push.fb.re" + { SEND(kGoto); } +#line 538 "" +yy60: + yych = *++YYCURSOR; + if(yych != 's') goto yy39; + yych = *++YYCURSOR; + if(yych != 'e') goto yy39; + ++YYCURSOR; + if(yybm[0+(yych = *YYCURSOR)] & 128) { + goto yy38; + } +#line 241 "push.fb.re" + { SEND(kElse); } +#line 550 "" +yy64: + yych = *++YYCURSOR; + if(yych != 'r') goto yy39; + ++YYCURSOR; + if(yybm[0+(yych = *YYCURSOR)] & 128) { + goto yy38; + } +#line 240 "push.fb.re" + { SEND(kFor); } +#line 560 "" +yy67: + ++YYCURSOR; + if(yybm[0+(yych = *YYCURSOR)] & 128) { + goto yy38; + } +#line 239 "push.fb.re" + { SEND(kIf); } +#line 568 "" + } +} +#line 261 "push.fb.re" + + + fill: + ssize_t unfinishedSize = cursor-start; + printf( + "scanner needs a refill. Exiting for now with:\n" + " saved fill state = %d\n" + " unfinished token size = %d\n", + state, + unfinishedSize + ); + + if(0 +#include +#include +#include +#include + +#if defined(WIN32) + + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +#else + + #include + #include + + #ifndef O_BINARY + #define O_BINARY 0 + #endif + +#endif + +// ---------------------------------------------------------------------- +#define TOKENS \ + \ + TOK(kEOF) \ + TOK(kEOL) \ + TOK(kUnknown) \ + TOK(kIdentifier) \ + TOK(kDecimalConstant) \ + \ + TOK(kEqual) \ + TOK(kLeftParen) \ + TOK(kRightParen) \ + TOK(kMinus) \ + TOK(kPlus) \ + TOK(kStar) \ + TOK(kSlash) \ + \ + TOK(kIf) \ + TOK(kFor) \ + TOK(kElse) \ + TOK(kGoto) \ + TOK(kBreak) \ + TOK(kWhile) \ + TOK(kReturn) \ + + +// ---------------------------------------------------------------------- +static const char *tokenNames[] = +{ + #define TOK(x) #x, + TOKENS + #undef TOK +}; + +// ---------------------------------------------------------------------- +class PushScanner +{ +public: + + enum Token + { + #define TOK(x) x, + TOKENS + #undef TOK + }; + +private: + + bool eof; + int32_t state; + + uint8_t *limit; + uint8_t *start; + uint8_t *cursor; + uint8_t *marker; + + uint8_t *buffer; + uint8_t *bufferEnd; + + uint8_t yych; + uint32_t yyaccept; + +public: + + // ---------------------------------------------------------------------- + PushScanner() + { + limit = 0; + start = 0; + state = -1; + cursor = 0; + marker = 0; + buffer = 0; + eof = false; + bufferEnd = 0; + } + + // ---------------------------------------------------------------------- + ~PushScanner() + { + } + + // ---------------------------------------------------------------------- + void send( + Token token + ) + { + size_t tokenSize = cursor-start; + const char *tokenName = tokenNames[token]; + printf( + "scanner is pushing out a token of type %d (%s)", + token, + tokenName + ); + + if(token==kEOF) putchar('\n'); + else + { + size_t tokenNameSize = strlen(tokenNames[token]); + size_t padSize = 20-(20"); + + fwrite( + start, + tokenSize, + 1, + stdout + ); + + printf("<----\n"); + } + } + + // ---------------------------------------------------------------------- + uint32_t push( + const void *input, + ssize_t inputSize + ) + { + printf( + "scanner is receiving a new data batch of length %d\n" + "scanner continues with saved state = %d\n", + inputSize, + state + ); + + /* + * Data source is signaling end of file when batch size + * is less than maxFill. This is slightly annoying because + * maxFill is a value that can only be known after re2c does + * its thing. Practically though, maxFill is never bigger than + * the longest keyword, so given our grammar, 32 is a safe bet. + */ + uint8_t null[64]; + const ssize_t maxFill = 32; + if(inputSize" +{ + + switch(YYGETSTATE()) + { + case -1: goto yy0; + case 0: goto yyFillLabel0; + case 1: goto yyFillLabel1; + case 2: goto yyFillLabel2; + default: /* abort() */; + } +yyNext: +yy0: + YYSETSTATE(0); + if((YYLIMIT - YYCURSOR) < 7) YYFILL(7); +yyFillLabel0: + yych = *YYCURSOR; + if(yych <= '=') { + if(yych <= '(') { + if(yych <= 0x0A) { + if(yych <= 0x00) goto yy32; + if(yych <= 0x08) goto yy34; + if(yych <= 0x09) goto yy30; + goto yy28; + } else { + if(yych <= 0x1F) { + if(yych <= 0x0D) goto yy30; + goto yy34; + } else { + if(yych <= ' ') goto yy30; + if(yych <= '\'') goto yy34; + goto yy16; + } + } + } else { + if(yych <= '-') { + if(yych <= '*') { + if(yych <= ')') goto yy18; + goto yy24; + } else { + if(yych <= '+') goto yy22; + if(yych <= ',') goto yy34; + goto yy20; + } + } else { + if(yych <= '/') { + if(yych <= '.') goto yy34; + goto yy26; + } else { + if(yych <= '9') goto yy12; + if(yych <= '<') goto yy34; + goto yy14; + } + } + } + } else { + if(yych <= 'e') { + if(yych <= '_') { + if(yych <= '@') goto yy34; + if(yych <= 'Z') goto yy10; + if(yych <= '^') goto yy34; + goto yy10; + } else { + if(yych <= 'a') { + if(yych <= '`') goto yy34; + goto yy10; + } else { + if(yych <= 'b') goto yy7; + if(yych <= 'd') goto yy10; + goto yy5; + } + } + } else { + if(yych <= 'q') { + if(yych <= 'g') { + if(yych <= 'f') goto yy4; + goto yy6; + } else { + if(yych <= 'h') goto yy11; + if(yych >= 'j') goto yy10; + } + } else { + if(yych <= 'v') { + if(yych <= 'r') goto yy9; + goto yy10; + } else { + if(yych <= 'w') goto yy8; + if(yych <= 'z') goto yy10; + goto yy34; + } + } + } + } + ++YYCURSOR; + if((yych = *YYCURSOR) == 'f') goto yy67; + goto yy39; +yy3: +#line 35 "push.fs.re" + { SEND(kIdentifier); } +#line 121 "" +yy4: + yych = *++YYCURSOR; + if(yych == 'o') goto yy64; + goto yy39; +yy5: + yych = *++YYCURSOR; + if(yych == 'l') goto yy60; + goto yy39; +yy6: + yych = *++YYCURSOR; + if(yych == 'o') goto yy56; + goto yy39; +yy7: + yych = *++YYCURSOR; + if(yych == 'r') goto yy51; + goto yy39; +yy8: + yych = *++YYCURSOR; + if(yych == 'h') goto yy46; + goto yy39; +yy9: + yych = *++YYCURSOR; + if(yych == 'e') goto yy40; + goto yy39; +yy10: + yych = *++YYCURSOR; + goto yy39; +yy11: + yych = *++YYCURSOR; + goto yy39; +yy12: + ++YYCURSOR; + yych = *YYCURSOR; + goto yy37; +yy13: +#line 36 "push.fs.re" + { SEND(kDecimalConstant);} +#line 159 "" +yy14: + ++YYCURSOR; +#line 38 "push.fs.re" + { SEND(kEqual); } +#line 164 "" +yy16: + ++YYCURSOR; +#line 39 "push.fs.re" + { SEND(kLeftParen); } +#line 169 "" +yy18: + ++YYCURSOR; +#line 40 "push.fs.re" + { SEND(kRightParen); } +#line 174 "" +yy20: + ++YYCURSOR; +#line 41 "push.fs.re" + { SEND(kMinus); } +#line 179 "" +yy22: + ++YYCURSOR; +#line 42 "push.fs.re" + { SEND(kPlus); } +#line 184 "" +yy24: + ++YYCURSOR; +#line 43 "push.fs.re" + { SEND(kStar); } +#line 189 "" +yy26: + ++YYCURSOR; +#line 44 "push.fs.re" + { SEND(kSlash); } +#line 194 "" +yy28: + ++YYCURSOR; +#line 46 "push.fs.re" + { SKIP(); } +#line 199 "" +yy30: + ++YYCURSOR; +#line 47 "push.fs.re" + { SKIP(); } +#line 204 "" +yy32: + ++YYCURSOR; +#line 48 "push.fs.re" + { send(kEOF); return 1; } +#line 209 "" +yy34: + ++YYCURSOR; +#line 49 "push.fs.re" + { SEND(kUnknown); } +#line 214 "" +yy36: + ++YYCURSOR; + YYSETSTATE(1); + if(YYLIMIT == YYCURSOR) YYFILL(1); +yyFillLabel1: + yych = *YYCURSOR; +yy37: + if(yych <= '/') goto yy13; + if(yych <= '9') goto yy36; + goto yy13; +yy38: + ++YYCURSOR; + YYSETSTATE(2); + if(YYLIMIT == YYCURSOR) YYFILL(1); +yyFillLabel2: + yych = *YYCURSOR; +yy39: + if(yych <= 'Z') { + if(yych <= '/') goto yy3; + if(yych <= '9') goto yy38; + if(yych <= '@') goto yy3; + goto yy38; + } else { + if(yych <= '_') { + if(yych <= '^') goto yy3; + goto yy38; + } else { + if(yych <= '`') goto yy3; + if(yych <= 'z') goto yy38; + goto yy3; + } + } +yy40: + yych = *++YYCURSOR; + if(yych != 't') goto yy39; + yych = *++YYCURSOR; + if(yych != 'u') goto yy39; + yych = *++YYCURSOR; + if(yych != 'r') goto yy39; + yych = *++YYCURSOR; + if(yych != 'n') goto yy39; + ++YYCURSOR; + if((yych = *YYCURSOR) <= 'Z') { + if(yych <= '/') goto yy45; + if(yych <= '9') goto yy38; + if(yych >= 'A') goto yy38; + } else { + if(yych <= '_') { + if(yych >= '_') goto yy38; + } else { + if(yych <= '`') goto yy45; + if(yych <= 'z') goto yy38; + } + } +yy45: +#line 34 "push.fs.re" + { SEND(kReturn); } +#line 272 "" +yy46: + yych = *++YYCURSOR; + if(yych != 'i') goto yy39; + yych = *++YYCURSOR; + if(yych != 'l') goto yy39; + yych = *++YYCURSOR; + if(yych != 'e') goto yy39; + ++YYCURSOR; + if((yych = *YYCURSOR) <= 'Z') { + if(yych <= '/') goto yy50; + if(yych <= '9') goto yy38; + if(yych >= 'A') goto yy38; + } else { + if(yych <= '_') { + if(yych >= '_') goto yy38; + } else { + if(yych <= '`') goto yy50; + if(yych <= 'z') goto yy38; + } + } +yy50: +#line 33 "push.fs.re" + { SEND(kWhile); } +#line 296 "" +yy51: + yych = *++YYCURSOR; + if(yych != 'e') goto yy39; + yych = *++YYCURSOR; + if(yych != 'a') goto yy39; + yych = *++YYCURSOR; + if(yych != 'k') goto yy39; + ++YYCURSOR; + if((yych = *YYCURSOR) <= 'Z') { + if(yych <= '/') goto yy55; + if(yych <= '9') goto yy38; + if(yych >= 'A') goto yy38; + } else { + if(yych <= '_') { + if(yych >= '_') goto yy38; + } else { + if(yych <= '`') goto yy55; + if(yych <= 'z') goto yy38; + } + } +yy55: +#line 32 "push.fs.re" + { SEND(kBreak); } +#line 320 "" +yy56: + yych = *++YYCURSOR; + if(yych != 't') goto yy39; + yych = *++YYCURSOR; + if(yych != 'o') goto yy39; + ++YYCURSOR; + if((yych = *YYCURSOR) <= 'Z') { + if(yych <= '/') goto yy59; + if(yych <= '9') goto yy38; + if(yych >= 'A') goto yy38; + } else { + if(yych <= '_') { + if(yych >= '_') goto yy38; + } else { + if(yych <= '`') goto yy59; + if(yych <= 'z') goto yy38; + } + } +yy59: +#line 31 "push.fs.re" + { SEND(kGoto); } +#line 342 "" +yy60: + yych = *++YYCURSOR; + if(yych != 's') goto yy39; + yych = *++YYCURSOR; + if(yych != 'e') goto yy39; + ++YYCURSOR; + if((yych = *YYCURSOR) <= 'Z') { + if(yych <= '/') goto yy63; + if(yych <= '9') goto yy38; + if(yych >= 'A') goto yy38; + } else { + if(yych <= '_') { + if(yych >= '_') goto yy38; + } else { + if(yych <= '`') goto yy63; + if(yych <= 'z') goto yy38; + } + } +yy63: +#line 30 "push.fs.re" + { SEND(kElse); } +#line 364 "" +yy64: + yych = *++YYCURSOR; + if(yych != 'r') goto yy39; + ++YYCURSOR; + if((yych = *YYCURSOR) <= 'Z') { + if(yych <= '/') goto yy66; + if(yych <= '9') goto yy38; + if(yych >= 'A') goto yy38; + } else { + if(yych <= '_') { + if(yych >= '_') goto yy38; + } else { + if(yych <= '`') goto yy66; + if(yych <= 'z') goto yy38; + } + } +yy66: +#line 29 "push.fs.re" + { SEND(kFor); } +#line 384 "" +yy67: + ++YYCURSOR; + if((yych = *YYCURSOR) <= 'Z') { + if(yych <= '/') goto yy68; + if(yych <= '9') goto yy38; + if(yych >= 'A') goto yy38; + } else { + if(yych <= '_') { + if(yych >= '_') goto yy38; + } else { + if(yych <= '`') goto yy68; + if(yych <= 'z') goto yy38; + } + } +yy68: +#line 28 "push.fs.re" + { SEND(kIf); } +#line 402 "" +} +#line 50 "push.fs.re" + + +fill: + return 0; +} diff --git a/test/push.fs.re b/test/push.fs.re new file mode 100755 index 00000000..4a22b96e --- /dev/null +++ b/test/push.fs.re @@ -0,0 +1,54 @@ +uint32_t push() +{ + // The scanner starts here + #define YYLIMIT limit + #define YYCURSOR cursor + #define YYMARKER marker + #define YYCTYPE uint8_t + + #define SKIP(x) { start = cursor; goto yy0; } + #define SEND(x) { send(x); SKIP(); } + #define YYFILL(n) { goto fill; } + + #define YYGETSTATE() state + #define YYSETSTATE(x) { state = (x); } + +start: + + /*!re2c + re2c:startlabel = 1; + eol = "\n"; + eof = "\000"; + digit = [0-9]; + integer = digit+; + alpha = [A-Za-z_]; + any = [\000-\377]; + space = [ \h\t\v\f\r]; + + "if" { SEND(kIf); } + "for" { SEND(kFor); } + "else" { SEND(kElse); } + "goto" { SEND(kGoto); } + "break" { SEND(kBreak); } + "while" { SEND(kWhile); } + "return" { SEND(kReturn); } + alpha (alpha|digit)* { SEND(kIdentifier); } + integer { SEND(kDecimalConstant);} + + "=" { SEND(kEqual); } + "(" { SEND(kLeftParen); } + ")" { SEND(kRightParen); } + "-" { SEND(kMinus); } + "+" { SEND(kPlus); } + "*" { SEND(kStar); } + "/" { SEND(kSlash); } + + eol { SKIP(); } + space { SKIP(); } + eof { send(kEOF); return 1; } + any { SEND(kUnknown); } + */ + +fill: + return 0; +} -- 2.40.0