From: Johannes Schlüter <johannes@php.net>
Date: Sun, 8 Apr 2007 00:18:38 +0000 (+0000)
Subject: - MFH: Added linenumbers to array returned by token_get_all()
X-Git-Tag: php-5.2.2RC1~24
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8591b20d4ee289d99cb8caba41e6a1beff7a237d;p=php

- MFH: Added linenumbers to array returned by token_get_all()
---

diff --git a/NEWS b/NEWS
index 259e773a4e..969f1362b5 100644
--- a/NEWS
+++ b/NEWS
@@ -38,6 +38,7 @@ PHP                                                                        NEWS
 - Added php_pdo_sqlite_external.dll, a version of the PDO SQLite driver that
   links against an external sqlite3.dll.  This provides Windows users to upgrade
   their sqlite3 version outside of the PHP release cycle.  (Wez, Edin)
+- Added linenumbers to array returned by token_get_all(). (Johannes)
 - Fixed possible multi bytes issues in openssl csr parser (Pierre)
 - Fixed shmop_open() with IPC_CREAT|IPC_EXCL flags on Windows. 
   (Vladimir Kamaev, Tony).
diff --git a/ext/tokenizer/tests/002.phpt b/ext/tokenizer/tests/002.phpt
index fad8f2ae75..2236f08b85 100644
--- a/ext/tokenizer/tests/002.phpt
+++ b/ext/tokenizer/tests/002.phpt
@@ -2,8 +2,6 @@
 token_get_all()
 --SKIPIF--
 <?php if (!extension_loaded("tokenizer")) print "skip"; ?>
---INI--
-short_open_tag=1
 --FILE--
 <?php
 
@@ -21,768 +19,962 @@ foreach ($strings as $s) {
 
 echo "Done\n";
 ?>
---EXPECTF--	
+--EXPECTF--
 array(49) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "echo"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [5]=>
   string(1) ";"
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
   string(1) "("
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "isset"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) "("
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [13]=>
   string(1) ")"
   [14]=>
   string(1) ")"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "print"
+    [2]=>
+    int(1)
   }
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
   string(1) "+"
   [20]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [21]=>
   string(1) ";"
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "++"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) ";"
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [27]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "--"
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) ";"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "=="
+    [2]=>
+    int(1)
   }
   [34]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(3) "==="
+    [2]=>
+    int(1)
   }
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) ";"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [45]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "endif"
+    [2]=>
+    int(1)
   }
   [46]=>
   string(1) ";"
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [48]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(37) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "switch"
+    [2]=>
+    int(1)
   }
   [2]=>
   string(1) "("
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [4]=>
   string(1) ")"
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "{"
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "case"
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) ":"
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) ";"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(7) "default"
+    [2]=>
+    int(1)
   }
   [17]=>
   string(1) ":"
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) ";"
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
   string(1) "}"
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "while"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) "("
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ")"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) "{"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "exit"
+    [2]=>
+    int(1)
   }
   [32]=>
   string(1) ";"
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "}"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [36]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(48) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "/* comment */"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "("
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "||"
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [12]=>
   string(1) ")"
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) "{"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
   string(1) "}"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) "="
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
   string(1) "|"
   [25]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ";"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$b"
+    [2]=>
+    int(1)
   }
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
   string(1) "="
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "3"
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "^"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$c"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
   string(1) "="
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "4"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) "&"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [45]=>
   string(1) ";"
   [46]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(17) "wrong syntax here"
+    [2]=>
+    int(1)
   }
 }
 Done
diff --git a/ext/tokenizer/tests/003.phpt b/ext/tokenizer/tests/003.phpt
index 66d088eec7..fdcf7748a8 100644
--- a/ext/tokenizer/tests/003.phpt
+++ b/ext/tokenizer/tests/003.phpt
@@ -23,20 +23,24 @@ array(0) {
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "0"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "-1"
+    [2]=>
+    int(1)
   }
 }
 Done
diff --git a/ext/tokenizer/tests/bug26463.phpt b/ext/tokenizer/tests/bug26463.phpt
index c5d2148ea6..d07476b204 100644
--- a/ext/tokenizer/tests/bug26463.phpt
+++ b/ext/tokenizer/tests/bug26463.phpt
@@ -18,119 +18,147 @@ var_dump(token_get_all($str));
 --EXPECTF--
 array(19) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php
 "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$x"
+    [2]=>
+    int(2)
   }
   [2]=>
   string(1) "="
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<<<DD
 "
+    [2]=>
+    int(2)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(3)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "DD"
+    [2]=>
+    int(4)
   }
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(4)
   }
   [7]=>
   string(1) "."
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) """"
+    [2]=>
+    int(5)
   }
   [9]=>
   string(1) ";"
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(5)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(6)
   }
   [12]=>
   string(1) "="
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(8) "<<<DDDD
 "
+    [2]=>
+    int(6)
   }
   [14]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(7)
   }
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "DDDD"
+    [2]=>
+    int(8)
   }
   [16]=>
   string(1) ";"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(8)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(9)
   }
 }
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index a500f88712..7df3439b1e 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -272,6 +272,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
 	zval *keyword;
 	int token_type;
 	zend_bool destroy;
+	int token_line = 1;
 
 	array_init(return_value);
 
@@ -301,6 +302,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			} else {
 				add_next_index_stringl(keyword, zendtext, zendleng, 1);
 			}
+			add_next_index_long(keyword, token_line);
 			add_next_index_zval(return_value, keyword);
 		} else {
 			add_next_index_stringl(return_value, zendtext, zendleng, 1);
@@ -309,6 +311,8 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			zval_dtor(&token);
 		}
 		ZVAL_NULL(&token);
+
+		token_line = CG(zend_lineno);
 	}
 }