Make pgen compile with pydebug. Duplicate normalized names, as it may

be longer than the old string.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 0ae093e68eca2eedcfc6b1d6c05f2824b4153527..53c883f5e4c4775795303fd2664cea1883ea1304 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -128,8 +128,10 @@ tok_new(void)
        tok->read_coding_spec = 0;
        tok->issued_encoding_warning = 0;
        tok->encoding = NULL;
+#ifndef PGEN
        tok->decoding_readline = NULL;
        tok->decoding_buffer = NULL;
+#endif
        return tok;
 }
 
@@ -225,8 +227,8 @@ get_coding_spec(const char *s, int size)
                                char* r = new_string(begin, t - begin);
                                char* q = get_normal_name(r);
                                if (r != q) {
-                                       assert(strlen(r) >= strlen(q));
-                                       strcpy(r, q);
+                                       PyMem_DEL(r);
+                                       r = new_string(q, strlen(q));
                                }
                                return r;
                        }
@@ -584,8 +586,10 @@ PyTokenizer_Free(struct tok_state *tok)
 {
        if (tok->encoding != NULL)
                PyMem_DEL(tok->encoding);
+#ifndef PGEN
        Py_XDECREF(tok->decoding_readline);
        Py_XDECREF(tok->decoding_buffer);
+#endif
        if (tok->fp != NULL && tok->buf != NULL)
                PyMem_DEL(tok->buf);
        PyMem_DEL(tok);

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 9782666fd82450dfcd2f9e197f7cd9cd2c9c20cd..f3bac74acd6c9e814b321e20b6ed9bdff50b22e7 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -45,8 +45,10 @@ struct tok_state {
        int read_coding_spec;   /* whether 'coding:...' has been read  */
        int issued_encoding_warning; /* whether non-ASCII warning was issued */
        char *encoding;
+#ifndef PGEN
        PyObject *decoding_readline; /* codecs.open(...).readline */
        PyObject *decoding_buffer;
+#endif
        const char* enc;
        const char* str;
 };