Fixed memory leak

diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index 2e64c7f6a5d8964d6b08925dfd6536a486acad73..55cbfed97fc5d11a7aa9c0a9f9ba3a331f054fc7 100644 (file)
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -337,6 +337,8 @@ static void tokenize(zval *return_value TSRMLS_DC)
        int token_type;
        zend_bool destroy;
 
+       CG(literal_type) = UG(unicode)?IS_UNICODE:IS_STRING;
+
        array_init(return_value);
 
        ZVAL_NULL(&token);