Fixed bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise close tag)

diff --git a/NEWS b/NEWS
index d3d142df4e1bcb84f1b314783ccf38ff783d302c..70b13aae907a05967e73361937013813e4fce358 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -27,6 +27,8 @@ PHP                                                                        NEWS
     `!(zval_gc_flags((str)->gc)). (Nikita, Laruence)
 
 - Tokenizer:
+  . Fixed bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise
+    close tag). (Laruence)
   . Fixed bug #75218 (Change remaining uncatchable fatal errors for parsing
     into ParseError). (Nikita)
 

diff --git a/ext/tokenizer/tests/bug76437.phpt b/ext/tokenizer/tests/bug76437.phpt
new file mode 100644 (file)
index 0000000..4a979db
--- /dev/null
+++ b/ext/tokenizer/tests/bug76437.phpt
@@ -0,0 +1,34 @@
+--TEST--
+Bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise close tag)
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$open_tag1 = token_get_all('<?=$a?>')[0];
+$open_tag2 = token_get_all('<?=$a?>', TOKEN_PARSE)[0];
+var_dump($open_tag1);
+var_dump($open_tag1 === $open_tag2);
+$open_tag1 = token_get_all('<?php echo 2; ?>')[6];
+$open_tag2 = token_get_all('<?php echo 2; ?>', TOKEN_PARSE)[6];
+var_dump($open_tag1);
+var_dump($open_tag1 === $open_tag2);
+?>
+--EXPECT--
+array(3) {
+  [0]=>
+  int(380)
+  [1]=>
+  string(3) "<?="
+  [2]=>
+  int(1)
+}
+bool(true)
+array(3) {
+  [0]=>
+  int(381)
+  [1]=>
+  string(2) "?>"
+  [2]=>
+  int(1)
+}
+bool(true)

diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index ef9d1362b757b03e944a2fee165029362e5c35ea..cb68fd408790fb95281d878b16417189e90fe109 100644 (file)
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -191,8 +191,16 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
 
        switch (event) {
                case ON_TOKEN:
-                       if (token == END) break;
-                       add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
+                       {
+                               if (token == END) break;
+                               /* Specical cases */
+                               if (token == ';' && LANG_SCNG(yy_leng) == sizeof("?>") - 1) {
+                                       token = T_CLOSE_TAG;
+                               } else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
+                                       token = T_OPEN_TAG_WITH_ECHO;
+                               }
+                               add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
+                       }
                        break;
                case ON_FEEDBACK:
                        tokens_ht = Z_ARRVAL_P(token_stream);