Reduce memory used by token_get_all()

Around a quarter of all strings in array tokens would have a string that's one
character long (e.g. ` `, `\`, `1`)

For parsing a large number of php files,
The memory increase dropped from 378374248 to 369535688 (2.5%)

Closes GH-4753.

diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index 91ace6f701770235c728332403b6e19d90cfb8a1..3d343fec4d687681e6e9093fa139c9bb32f1fb16 100644 (file)
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -110,7 +110,11 @@ static void add_token(zval *return_value, int token_type,
                zval keyword;
                array_init(&keyword);
                add_next_index_long(&keyword, token_type);
-               add_next_index_stringl(&keyword, (char *) text, leng);
+               if (leng == 1) {
+                       add_next_index_str(&keyword, ZSTR_CHAR(text[0]));
+               } else {
+                       add_next_index_stringl(&keyword, (char *) text, leng);
+               }
                add_next_index_long(&keyword, lineno);
                add_next_index_zval(return_value, &keyword);
        } else {