Fix some more memory leaks.

Call error_ret() in decode_str().  It was called in some other places,
but seemed inconsistent.  It is safe to call PyTokenizer_Free() after
calling error_ret().

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 5a9bcc04e7f4be12e2c3109bb41f29e1c69ecc18..37e6c3349bac6d48917b64dc77574e7a95f81fad 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -328,6 +328,8 @@ check_bom(int get_char(struct tok_state *),
                unget_char(ch, tok);
                return 1;
        }
+       if (tok->encoding != NULL)
+               PyMem_DEL(tok->encoding);
        tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
        return 1;
   NON_BOM:
@@ -581,14 +583,14 @@ decode_str(const char *str, struct tok_state *tok)
        tok->enc = NULL;
        tok->str = str;
        if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
-               return NULL;
+               return error_ret(tok);
        str = tok->str;         /* string after BOM if any */
        assert(str);
 #ifdef Py_USING_UNICODE
        if (tok->enc != NULL) {
                utf8 = translate_into_utf8(str, tok->enc);
                if (utf8 == NULL)
-                       return NULL;
+                       return error_ret(tok);
                str = PyString_AsString(utf8);
        }
 #endif
@@ -601,7 +603,7 @@ decode_str(const char *str, struct tok_state *tok)
        }
        tok->enc = NULL;
        if (!check_coding_spec(str, s - str, tok, buf_setreadl))
-               return NULL;
+               return error_ret(tok);
 #ifdef Py_USING_UNICODE
        if (tok->enc != NULL) {
                assert(utf8 == NULL);
@@ -609,7 +611,7 @@ decode_str(const char *str, struct tok_state *tok)
                if (utf8 == NULL) {
                        PyErr_Format(PyExc_SyntaxError,
                                "unknown encoding: %s", tok->enc);
-                       return NULL;
+                       return error_ret(tok);
                }
                str = PyString_AsString(utf8);
        }
@@ -630,8 +632,11 @@ PyTokenizer_FromString(const char *str)
        if (tok == NULL)
                return NULL;
        str = (char *)decode_str(str, tok);
-       if (str == NULL)
+       if (str == NULL) {
+               PyTokenizer_Free(tok);
                return NULL;
+       }
+
        /* XXX: constify members. */
        tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
        return tok;
@@ -647,7 +652,7 @@ PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
        if (tok == NULL)
                return NULL;
        if ((tok->buf = PyMem_NEW(char, BUFSIZ)) == NULL) {
-               PyMem_DEL(tok);
+               PyTokenizer_Free(tok);
                return NULL;
        }
        tok->cur = tok->inp = tok->buf;