Use ZSTR_CHAR in token_get_all()

diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index f530c555a9895ef28597977b0031c772f221505f..bb5f30320fac736c0cfebadd7049ca57d16ee8c5 100644 (file)
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -106,12 +106,29 @@ PHP_MINFO_FUNCTION(tokenizer)
 }
 /* }}} */
 
+static void add_token(zval *return_value, int token_type,
+               unsigned char *text, size_t leng, int lineno) {
+       if (token_type >= 256) {
+               zval keyword;
+               array_init(&keyword);
+               add_next_index_long(&keyword, token_type);
+               add_next_index_stringl(&keyword, (char *) text, leng);
+               add_next_index_long(&keyword, lineno);
+               add_next_index_zval(return_value, &keyword);
+       } else {
+               if (leng == 1) {
+                       add_next_index_str(return_value, ZSTR_CHAR(text[0]));
+               } else {
+                       add_next_index_stringl(return_value, (char *) text, leng);
+               }
+       }
+}
+
 static zend_bool tokenize(zval *return_value, zend_string *source)
 {
        zval source_zval;
        zend_lex_state original_lex_state;
        zval token;
-       zval keyword;
        int token_type;
        int token_line = 1;
        int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
@@ -129,15 +146,7 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 
        ZVAL_UNDEF(&token);
        while ((token_type = lex_scan(&token))) {
-               if (token_type >= 256) {
-                       array_init(&keyword);
-                       add_next_index_long(&keyword, token_type);
-                       add_next_index_stringl(&keyword, (char *)zendtext, zendleng);
-                       add_next_index_long(&keyword, token_line);
-                       add_next_index_zval(return_value, &keyword);
-               } else {
-                       add_next_index_stringl(return_value, (char *)zendtext, zendleng);
-               }
+               add_token(return_value, token_type, zendtext, zendleng, token_line);
 
                if (Z_TYPE(token) != IS_UNDEF) {
                        zval_dtor(&token);
@@ -152,11 +161,8 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
                        ) {
                                /* fetch the rest into a T_INLINE_HTML */
                                if (zendcursor != zendlimit) {
-                                       array_init(&keyword);
-                                       add_next_index_long(&keyword, T_INLINE_HTML);
-                                       add_next_index_stringl(&keyword, (char *)zendcursor, zendlimit - zendcursor);
-                                       add_next_index_long(&keyword, token_line);
-                                       add_next_index_zval(return_value, &keyword);
+                                       add_token(return_value, T_INLINE_HTML,
+                                               zendcursor, zendlimit - zendcursor, token_line);
                                }
                                break;
                        }
@@ -181,22 +187,13 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 void on_event(zend_php_scanner_event event, int token, int line, void *context)
 {
        zval *token_stream = (zval *) context;
-       zval keyword;
        HashTable *tokens_ht;
        zval *token_zv;
 
        switch (event) {
                case ON_TOKEN:
                        if (token == END) break;
-                       if (token >= 256) {
-                               array_init(&keyword);
-                               add_next_index_long(&keyword, token);
-                               add_next_index_stringl(&keyword, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng));
-                               add_next_index_long(&keyword, line);
-                               add_next_index_zval(token_stream, &keyword);
-                       } else {
-                               add_next_index_stringl(token_stream, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng));
-                       }
+                       add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
                        break;
                case ON_FEEDBACK:
                        tokens_ht = Z_ARRVAL_P(token_stream);
@@ -207,12 +204,8 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
                        break;
                case ON_STOP:
                        if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
-                               array_init(&keyword);
-                               add_next_index_long(&keyword, T_INLINE_HTML);
-                               add_next_index_stringl(&keyword,
-                                       (char *)LANG_SCNG(yy_cursor), LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor));
-                               add_next_index_long(&keyword, CG(zend_lineno));
-                               add_next_index_zval(token_stream, &keyword);
+                               add_token(token_stream, T_INLINE_HTML, LANG_SCNG(yy_cursor),
+                                       LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno));
                        }
                        break;
        }