From: Xinchen Hui <laruence@gmail.com>
Date: Mon, 18 Jun 2018 03:33:48 +0000 (+0800)
Subject: Fixed bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise close tag)
X-Git-Tag: php-7.3.0alpha2~7
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=4d69bbeee79c6a94348935ad071b6c7c05dd8eae;p=php

Fixed bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise close tag)
---

diff --git a/NEWS b/NEWS
index d3d142df4e..70b13aae90 100644
--- a/NEWS
+++ b/NEWS
@@ -27,6 +27,8 @@ PHP                                                                        NEWS
     `!(zval_gc_flags((str)->gc)). (Nikita, Laruence)
 
 - Tokenizer:
+  . Fixed bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise
+    close tag). (Laruence)
   . Fixed bug #75218 (Change remaining uncatchable fatal errors for parsing
     into ParseError). (Nikita)
 
diff --git a/ext/tokenizer/tests/bug76437.phpt b/ext/tokenizer/tests/bug76437.phpt
new file mode 100644
index 0000000000..4a979db710
--- /dev/null
+++ b/ext/tokenizer/tests/bug76437.phpt
@@ -0,0 +1,34 @@
+--TEST--
+Bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise close tag)
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$open_tag1 = token_get_all('<?=$a?>')[0];
+$open_tag2 = token_get_all('<?=$a?>', TOKEN_PARSE)[0];
+var_dump($open_tag1);
+var_dump($open_tag1 === $open_tag2);
+$open_tag1 = token_get_all('<?php echo 2; ?>')[6];
+$open_tag2 = token_get_all('<?php echo 2; ?>', TOKEN_PARSE)[6];
+var_dump($open_tag1);
+var_dump($open_tag1 === $open_tag2);
+?>
+--EXPECT--
+array(3) {
+  [0]=>
+  int(380)
+  [1]=>
+  string(3) "<?="
+  [2]=>
+  int(1)
+}
+bool(true)
+array(3) {
+  [0]=>
+  int(381)
+  [1]=>
+  string(2) "?>"
+  [2]=>
+  int(1)
+}
+bool(true)
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index ef9d1362b7..cb68fd4087 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -191,8 +191,16 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
 
 	switch (event) {
 		case ON_TOKEN:
-			if (token == END) break;
-			add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
+			{
+				if (token == END) break;
+				/* Specical cases */
+				if (token == ';' && LANG_SCNG(yy_leng) == sizeof("?>") - 1) {
+					token = T_CLOSE_TAG;
+				} else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
+					token = T_OPEN_TAG_WITH_ECHO;
+				}
+				add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
+			}
 			break;
 		case ON_FEEDBACK:
 			tokens_ht = Z_ARRVAL_P(token_stream);