Fix tokenizer_data_gen.sh for non-posix bison

And run it to update tokenizer_data.c after recent changes in
zend_language_parser.y that reordered some tokens

diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index 3c047ddf1fb946659e7bb8d06fb408b7bac000fb..8b5c6be94542fdff6787917bd1611ecd6b0f2e72 100644 (file)
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -29,7 +29,6 @@
 void tokenizer_register_constants(INIT_FUNC_ARGS) {
        REGISTER_LONG_CONSTANT("T_INCLUDE", T_INCLUDE, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_INCLUDE_ONCE", T_INCLUDE_ONCE, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_REQUIRE", T_REQUIRE, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_REQUIRE_ONCE", T_REQUIRE_ONCE, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_LOGICAL_OR", T_LOGICAL_OR, CONST_CS | CONST_PERSISTENT);
@@ -79,13 +78,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
        REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_ELSEIF", T_ELSEIF, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_ELSE", T_ELSE, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
-       REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
@@ -95,8 +87,10 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
        REGISTER_LONG_CONSTANT("T_CONSTANT_ENCAPSED_STRING", T_CONSTANT_ENCAPSED_STRING, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_WHILE", T_WHILE, CONST_CS | CONST_PERSISTENT);
@@ -125,6 +119,12 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
        REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
+       REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_VAR", T_VAR, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT);
        REGISTER_LONG_CONSTANT("T_ISSET", T_ISSET, CONST_CS | CONST_PERSISTENT);
@@ -170,7 +170,6 @@ char *get_token_type_name(int token_type)
 
                case T_INCLUDE: return "T_INCLUDE";
                case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE";
-               case T_EVAL: return "T_EVAL";
                case T_REQUIRE: return "T_REQUIRE";
                case T_REQUIRE_ONCE: return "T_REQUIRE_ONCE";
                case T_LOGICAL_OR: return "T_LOGICAL_OR";
@@ -220,13 +219,6 @@ char *get_token_type_name(int token_type)
                case T_CLONE: return "T_CLONE";
                case T_ELSEIF: return "T_ELSEIF";
                case T_ELSE: return "T_ELSE";
-               case T_ENDIF: return "T_ENDIF";
-               case T_STATIC: return "T_STATIC";
-               case T_ABSTRACT: return "T_ABSTRACT";
-               case T_FINAL: return "T_FINAL";
-               case T_PRIVATE: return "T_PRIVATE";
-               case T_PROTECTED: return "T_PROTECTED";
-               case T_PUBLIC: return "T_PUBLIC";
                case T_LNUMBER: return "T_LNUMBER";
                case T_DNUMBER: return "T_DNUMBER";
                case T_STRING: return "T_STRING";
@@ -236,8 +228,10 @@ char *get_token_type_name(int token_type)
                case T_CONSTANT_ENCAPSED_STRING: return "T_CONSTANT_ENCAPSED_STRING";
                case T_STRING_VARNAME: return "T_STRING_VARNAME";
                case T_NUM_STRING: return "T_NUM_STRING";
+               case T_EVAL: return "T_EVAL";
                case T_EXIT: return "T_EXIT";
                case T_IF: return "T_IF";
+               case T_ENDIF: return "T_ENDIF";
                case T_ECHO: return "T_ECHO";
                case T_DO: return "T_DO";
                case T_WHILE: return "T_WHILE";
@@ -266,6 +260,12 @@ char *get_token_type_name(int token_type)
                case T_USE: return "T_USE";
                case T_INSTEADOF: return "T_INSTEADOF";
                case T_GLOBAL: return "T_GLOBAL";
+               case T_STATIC: return "T_STATIC";
+               case T_ABSTRACT: return "T_ABSTRACT";
+               case T_FINAL: return "T_FINAL";
+               case T_PRIVATE: return "T_PRIVATE";
+               case T_PROTECTED: return "T_PROTECTED";
+               case T_PUBLIC: return "T_PUBLIC";
                case T_VAR: return "T_VAR";
                case T_UNSET: return "T_UNSET";
                case T_ISSET: return "T_ISSET";

diff --git a/ext/tokenizer/tokenizer_data_gen.sh b/ext/tokenizer/tokenizer_data_gen.sh
index 38d6b0f31d174f20e883b5b45d69fe60dca2b1c0..d545f041d3aec329b149367c900d76e1178b4e0d 100755 (executable)
--- a/ext/tokenizer/tokenizer_data_gen.sh
+++ b/ext/tokenizer/tokenizer_data_gen.sh
@@ -44,8 +44,8 @@ echo '/*
 
 echo 'void tokenizer_register_constants(INIT_FUNC_ARGS) {' >> $OUTFILE
 $AWK '
-       /^#define T_(NOELSE|ERROR)/ { next }
-       /^#define T_/  { print "        REGISTER_LONG_CONSTANT(\"" $2 "\", " $2 ", CONST_CS | CONST_PERSISTENT);" }
+       /^    T_(NOELSE|ERROR)/ { next }
+       /^    T_/  { print "    REGISTER_LONG_CONSTANT(\"" $1 "\", " $1 ", CONST_CS | CONST_PERSISTENT);" }
 ' < $INFILE >> $OUTFILE
 echo ' REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);' >> $OUTFILE
 echo '}' >> $OUTFILE
@@ -58,13 +58,13 @@ char *get_token_type_name(int token_type)
 ' >> $OUTFILE
 
 $AWK '
-       /^#define T_PAAMAYIM_NEKUDOTAYIM/ {
+       /^    T_PAAMAYIM_NEKUDOTAYIM/ {
                print "         case T_PAAMAYIM_NEKUDOTAYIM: return \"T_DOUBLE_COLON\";"
                next
        }
-       /^#define T_(NOELSE|ERROR)/ { next }
-       /^#define T_/ {
-               print "         case " $2 ": return \"" $2 "\";"
+       /^    T_(NOELSE|ERROR)/ { next }
+       /^    T_/ {
+               print "         case " $1 ": return \"" $1 "\";"
        }
 ' < $INFILE >> $OUTFILE