- MFH: Added linenumbers to array returned by token_get_all()

diff --git a/NEWS b/NEWS
index 259e773a4e385584070dd353f1360094c69e1e72..969f1362b57db3fc46b14d167fdd04c4819e2621 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -38,6 +38,7 @@ PHP                                                                        NEWS
 - Added php_pdo_sqlite_external.dll, a version of the PDO SQLite driver that
   links against an external sqlite3.dll.  This provides Windows users to upgrade
   their sqlite3 version outside of the PHP release cycle.  (Wez, Edin)
+- Added linenumbers to array returned by token_get_all(). (Johannes)
 - Fixed possible multi bytes issues in openssl csr parser (Pierre)
 - Fixed shmop_open() with IPC_CREAT|IPC_EXCL flags on Windows. 
   (Vladimir Kamaev, Tony).

diff --git a/ext/tokenizer/tests/002.phpt b/ext/tokenizer/tests/002.phpt
index fad8f2ae75e27d53c841ff3f7579cbc3ee8df1b0..2236f08b8580d4e27063860f6e15578ebdfc843d 100644 (file)
--- a/ext/tokenizer/tests/002.phpt
+++ b/ext/tokenizer/tests/002.phpt
@@ -2,8 +2,6 @@
 token_get_all()
 --SKIPIF--
 <?php if (!extension_loaded("tokenizer")) print "skip"; ?>
---INI--
-short_open_tag=1
 --FILE--
 <?php
 
@@ -21,768 +19,962 @@ foreach ($strings as $s) {
 
 echo "Done\n";
 ?>
---EXPECTF--    
+--EXPECTF--
 array(49) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "echo"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [5]=>
   string(1) ";"
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
   string(1) "("
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "isset"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) "("
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [13]=>
   string(1) ")"
   [14]=>
   string(1) ")"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "print"
+    [2]=>
+    int(1)
   }
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
   string(1) "+"
   [20]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [21]=>
   string(1) ";"
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "++"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) ";"
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [27]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "--"
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) ";"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "=="
+    [2]=>
+    int(1)
   }
   [34]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(3) "==="
+    [2]=>
+    int(1)
   }
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) ";"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [45]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "endif"
+    [2]=>
+    int(1)
   }
   [46]=>
   string(1) ";"
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [48]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(37) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "switch"
+    [2]=>
+    int(1)
   }
   [2]=>
   string(1) "("
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [4]=>
   string(1) ")"
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "{"
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "case"
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) ":"
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) ";"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(7) "default"
+    [2]=>
+    int(1)
   }
   [17]=>
   string(1) ":"
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) ";"
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
   string(1) "}"
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "while"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) "("
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ")"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) "{"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "exit"
+    [2]=>
+    int(1)
   }
   [32]=>
   string(1) ";"
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "}"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [36]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(48) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "/* comment */"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "("
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "||"
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [12]=>
   string(1) ")"
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) "{"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
   string(1) "}"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) "="
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
   string(1) "|"
   [25]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ";"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$b"
+    [2]=>
+    int(1)
   }
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
   string(1) "="
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "3"
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "^"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$c"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
   string(1) "="
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "4"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) "&"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [45]=>
   string(1) ";"
   [46]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(17) "wrong syntax here"
+    [2]=>
+    int(1)
   }
 }
 Done

diff --git a/ext/tokenizer/tests/003.phpt b/ext/tokenizer/tests/003.phpt
index 66d088eec7506f2cf0aa05157eadfc58d1c906ac..fdcf7748a8440cd7b9caf452d2b9d94756b5ddc0 100644 (file)
--- a/ext/tokenizer/tests/003.phpt
+++ b/ext/tokenizer/tests/003.phpt
@@ -23,20 +23,24 @@ array(0) {
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "0"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "-1"
+    [2]=>
+    int(1)
   }
 }
 Done

diff --git a/ext/tokenizer/tests/bug26463.phpt b/ext/tokenizer/tests/bug26463.phpt
index c5d2148ea6b243537c3170cc2f603f6c1737459d..d07476b2046c806bd93e86ba192eace6163dc4c7 100644 (file)
--- a/ext/tokenizer/tests/bug26463.phpt
+++ b/ext/tokenizer/tests/bug26463.phpt
@@ -18,119 +18,147 @@ var_dump(token_get_all($str));
 --EXPECTF--
 array(19) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php
 "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$x"
+    [2]=>
+    int(2)
   }
   [2]=>
   string(1) "="
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<<<DD
 "
+    [2]=>
+    int(2)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(3)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "DD"
+    [2]=>
+    int(4)
   }
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(4)
   }
   [7]=>
   string(1) "."
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) """"
+    [2]=>
+    int(5)
   }
   [9]=>
   string(1) ";"
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(5)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(6)
   }
   [12]=>
   string(1) "="
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(8) "<<<DDDD
 "
+    [2]=>
+    int(6)
   }
   [14]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(7)
   }
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "DDDD"
+    [2]=>
+    int(8)
   }
   [16]=>
   string(1) ";"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(8)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(9)
   }
 }

diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index a500f88712c7e88e86a6303a50e8e0040ba4a766..7df3439b1ef7ad43dcea064e19c1bb9e65c34dab 100644 (file)
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -272,6 +272,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
        zval *keyword;
        int token_type;
        zend_bool destroy;
+       int token_line = 1;
 
        array_init(return_value);
 
@@ -301,6 +302,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
                        } else {
                                add_next_index_stringl(keyword, zendtext, zendleng, 1);
                        }
+                       add_next_index_long(keyword, token_line);
                        add_next_index_zval(return_value, keyword);
                } else {
                        add_next_index_stringl(return_value, zendtext, zendleng, 1);
@@ -309,6 +311,8 @@ static void tokenize(zval *return_value TSRMLS_DC)
                        zval_dtor(&token);
                }
                ZVAL_NULL(&token);
+
+               token_line = CG(zend_lineno);
        }
 }