SF patch #1467512, fix double free with triple quoted string in standard build.

This was the result of inconsistent use of PyMem_* and PyObject_* allocators.
By changing to use PyObject_* allocator almost everywhere, this removes
the inconsistency.

diff --git a/Parser/bitset.c b/Parser/bitset.c
index 3834e195b113b0d611da8f80f27943f798ee464c..0f3e01c02e0cb89f16f97721dd75b12c26181c1e 100644 (file)
--- a/Parser/bitset.c
+++ b/Parser/bitset.c
@@ -8,7 +8,7 @@ bitset
 newbitset(int nbits)
 {
        int nbytes = NBYTES(nbits);
-       bitset ss = PyMem_NEW(BYTE, nbytes);
+       bitset ss = PyObject_MALLOC(sizeof(BYTE) *  nbytes);
        
        if (ss == NULL)
                Py_FatalError("no mem for bitset");
@@ -22,7 +22,7 @@ newbitset(int nbits)
 void
 delbitset(bitset ss)
 {
-       PyMem_DEL(ss);
+       PyObject_FREE(ss);
 }
 
 int

diff --git a/Parser/firstsets.c b/Parser/firstsets.c
index 0f4e09d8180fb498d5a993d710b8ce3c58946e76..d6bacefec31edc30c2db9780f3765fd70fb27251 100644 (file)
--- a/Parser/firstsets.c
+++ b/Parser/firstsets.c
@@ -59,7 +59,7 @@ calcfirstset(grammar *g, dfa *d)
        nbits = g->g_ll.ll_nlabels;
        result = newbitset(nbits);
        
-       sym = PyMem_NEW(int, 1);
+       sym = PyObject_MALLOC(sizeof(int));
        if (sym == NULL)
                Py_FatalError("no mem for new sym in calcfirstset");
        nsyms = 1;
@@ -73,7 +73,7 @@ calcfirstset(grammar *g, dfa *d)
                                break;
                }
                if (j >= nsyms) { /* New label */
-                       PyMem_RESIZE(sym, int, nsyms + 1);
+                       sym = PyObject_REALLOC(sym, sizeof(int) * (nsyms + 1));
                        if (sym == NULL)
                                Py_FatalError(
                                    "no mem to resize sym in calcfirstset");
@@ -108,5 +108,5 @@ calcfirstset(grammar *g, dfa *d)
                printf(" }\n");
        }
 
-       PyMem_FREE(sym);
+       PyObject_FREE(sym);
 }

diff --git a/Parser/grammar.c b/Parser/grammar.c
index d8e38973dbfa3e07010b084739fea1e41f2661ca..880bf84b421acc5e7847e0ccba0ef0271d09ac98 100644 (file)
--- a/Parser/grammar.c
+++ b/Parser/grammar.c
@@ -20,7 +20,7 @@ newgrammar(int start)
 {
        grammar *g;
        
-       g = PyMem_NEW(grammar, 1);
+       g = PyObject_MALLOC(sizeof(grammar));
        if (g == NULL)
                Py_FatalError("no mem for new grammar");
        g->g_ndfas = 0;
@@ -37,7 +37,7 @@ adddfa(grammar *g, int type, char *name)
 {
        dfa *d;
        
-       PyMem_RESIZE(g->g_dfa, dfa, g->g_ndfas + 1);
+       g->g_dfa = PyObject_REALLOC(g->g_dfa, sizeof(dfa) * (g->g_ndfas + 1));
        if (g->g_dfa == NULL)
                Py_FatalError("no mem to resize dfa in adddfa");
        d = &g->g_dfa[g->g_ndfas++];
@@ -55,7 +55,8 @@ addstate(dfa *d)
 {
        state *s;
        
-       PyMem_RESIZE(d->d_state, state, d->d_nstates + 1);
+       d->d_state = PyObject_REALLOC(d->d_state,
+                                     sizeof(state) * (d->d_nstates + 1));
        if (d->d_state == NULL)
                Py_FatalError("no mem to resize state in addstate");
        s = &d->d_state[d->d_nstates++];
@@ -78,7 +79,7 @@ addarc(dfa *d, int from, int to, int lbl)
        assert(0 <= to && to < d->d_nstates);
        
        s = &d->d_state[from];
-       PyMem_RESIZE(s->s_arc, arc, s->s_narcs + 1);
+       s->s_arc = PyObject_REALLOC(s->s_arc, sizeof(arc) * (s->s_narcs + 1));
        if (s->s_arc == NULL)
                Py_FatalError("no mem to resize arc list in addarc");
        a = &s->s_arc[s->s_narcs++];
@@ -97,7 +98,8 @@ addlabel(labellist *ll, int type, char *str)
                        strcmp(ll->ll_label[i].lb_str, str) == 0)
                        return i;
        }
-       PyMem_RESIZE(ll->ll_label, label, ll->ll_nlabels + 1);
+       ll->ll_label = PyObject_REALLOC(ll->ll_label,
+                                       sizeof(label) * (ll->ll_nlabels + 1));
        if (ll->ll_label == NULL)
                Py_FatalError("no mem to resize labellist in addlabel");
        lb = &ll->ll_label[ll->ll_nlabels++];

diff --git a/Parser/myreadline.c b/Parser/myreadline.c
index a932a8793aa1d395bd0968fba8564624829f3da1..630997bb28bdabfd664fbb7b97c3e137a601e442 100644 (file)
--- a/Parser/myreadline.c
+++ b/Parser/myreadline.c
@@ -111,7 +111,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
        size_t n;
        char *p;
        n = 100;
-       if ((p = PyMem_MALLOC(n)) == NULL)
+       if ((p = PyObject_MALLOC(n)) == NULL)
                return NULL;
        fflush(sys_stdout);
 #ifndef RISCOS
@@ -130,7 +130,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
        case 0: /* Normal case */
                break;
        case 1: /* Interrupt */
-               PyMem_FREE(p);
+               PyObject_FREE(p);
                return NULL;
        case -1: /* EOF */
        case -2: /* Error */
@@ -141,7 +141,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
        n = strlen(p);
        while (n > 0 && p[n-1] != '\n') {
                size_t incr = n+2;
-               p = PyMem_REALLOC(p, n + incr);
+               p = PyObject_REALLOC(p, n + incr);
                if (p == NULL)
                        return NULL;
                if (incr > INT_MAX) {
@@ -151,14 +151,14 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
                        break;
                n += strlen(p+n);
        }
-       return PyMem_REALLOC(p, n+1);
+       return PyObject_REALLOC(p, n+1);
 }
 
 
 /* By initializing this function pointer, systems embedding Python can
    override the readline function.
 
-   Note: Python expects in return a buffer allocated with PyMem_Malloc. */
+   Note: Python expects in return a buffer allocated with PyObject_Malloc. */
 
 char *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, char *);
 

diff --git a/Parser/node.c b/Parser/node.c
index 7ed6c0e541f44051b8d29b3ea0dc968c18caa7b9..97f887ac9b3d6269d0353fb15f3859c0ef967ac9 100644 (file)
--- a/Parser/node.c
+++ b/Parser/node.c
@@ -62,7 +62,7 @@ fancy_roundup(int n)
  * Win98).
  *
  * In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
- * reported that, with this scheme, 89% of PyMem_RESIZE calls in
+ * reported that, with this scheme, 89% of PyObject_REALLOC calls in
  * PyNode_AddChild passed 1 for the size, and 9% passed 4.  So this usually
  * wastes very little memory, but is very effective at sidestepping
  * platform-realloc disasters on vulnernable platforms.

diff --git a/Parser/parser.c b/Parser/parser.c
index ada6be2b9c93dc829572463a64d2f249415de8d7..45b613abb68cdb1f4a970c4fe978eb5bc8ba5e80 100644 (file)
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -75,7 +75,7 @@ PyParser_New(grammar *g, int start)
        
        if (!g->g_accel)
                PyGrammar_AddAccelerators(g);
-       ps = PyMem_NEW(parser_state, 1);
+       ps = PyMem_MALLOC(sizeof(parser_state));
        if (ps == NULL)
                return NULL;
        ps->p_grammar = g;
@@ -84,7 +84,7 @@ PyParser_New(grammar *g, int start)
 #endif
        ps->p_tree = PyNode_New(start);
        if (ps->p_tree == NULL) {
-               PyMem_DEL(ps);
+               PyMem_FREE(ps);
                return NULL;
        }
        s_reset(&ps->p_stack);
@@ -98,7 +98,7 @@ PyParser_Delete(parser_state *ps)
        /* NB If you want to save the parse tree,
           you must set p_tree to NULL before calling delparser! */
        PyNode_Free(ps->p_tree);
-       PyMem_DEL(ps);
+       PyMem_FREE(ps);
 }
 
 

diff --git a/Parser/pgen.c b/Parser/pgen.c
index e643d330b4d7f1f9e38a83b9dceba255e896b9df..6aa1d19a5fab38df20732b7aee2f4e503e60bde8 100644 (file)
--- a/Parser/pgen.c
+++ b/Parser/pgen.c
@@ -49,7 +49,8 @@ addnfastate(nfa *nf)
 {
        nfastate *st;
        
-       PyMem_RESIZE(nf->nf_state, nfastate, nf->nf_nstates + 1);
+       nf->nf_state = PyObject_REALLOC(nf->nf_state, sizeof(nfastate) *
+                                                       (nf->nf_nstates + 1));
        if (nf->nf_state == NULL)
                Py_FatalError("out of mem");
        st = &nf->nf_state[nf->nf_nstates++];
@@ -65,7 +66,8 @@ addnfaarc(nfa *nf, int from, int to, int lbl)
        nfaarc *ar;
        
        st = &nf->nf_state[from];
-       PyMem_RESIZE(st->st_arc, nfaarc, st->st_narcs + 1);
+       st->st_arc = PyObject_REALLOC(st->st_arc,
+                                     sizeof(nfaarc) * (st->st_narcs + 1));
        if (st->st_arc == NULL)
                Py_FatalError("out of mem");
        ar = &st->st_arc[st->st_narcs++];
@@ -79,7 +81,7 @@ newnfa(char *name)
        nfa *nf;
        static int type = NT_OFFSET; /* All types will be disjunct */
        
-       nf = PyMem_NEW(nfa, 1);
+       nf = PyObject_MALLOC(sizeof(nfa));
        if (nf == NULL)
                Py_FatalError("no mem for new nfa");
        nf->nf_type = type++;
@@ -104,7 +106,7 @@ newnfagrammar(void)
 {
        nfagrammar *gr;
        
-       gr = PyMem_NEW(nfagrammar, 1);
+       gr = PyObject_MALLOC(sizeof(nfagrammar));
        if (gr == NULL)
                Py_FatalError("no mem for new nfa grammar");
        gr->gr_nnfas = 0;
@@ -121,7 +123,8 @@ addnfa(nfagrammar *gr, char *name)
        nfa *nf;
        
        nf = newnfa(name);
-       PyMem_RESIZE(gr->gr_nfa, nfa *, gr->gr_nnfas + 1);
+       gr->gr_nfa = PyObject_REALLOC(gr->gr_nfa,
+                                     sizeof(nfa) * (gr->gr_nnfas + 1));
        if (gr->gr_nfa == NULL)
                Py_FatalError("out of mem");
        gr->gr_nfa[gr->gr_nnfas++] = nf;
@@ -392,7 +395,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
        
        ss = newbitset(nbits);
        addclosure(ss, nf, nf->nf_start);
-       xx_state = PyMem_NEW(ss_state, 1);
+       xx_state = PyObject_MALLOC(sizeof(ss_state));
        if (xx_state == NULL)
                Py_FatalError("no mem for xx_state in makedfa");
        xx_nstates = 1;
@@ -411,6 +414,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
 
        /* For each unmarked state... */
        for (istate = 0; istate < xx_nstates; ++istate) {
+               size_t size;
                yy = &xx_state[istate];
                ss = yy->ss_ss;
                /* For all its states... */
@@ -430,8 +434,9 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
                                                goto found;
                                }
                                /* Add new arc for this state */
-                               PyMem_RESIZE(yy->ss_arc, ss_arc,
-                                            yy->ss_narcs + 1);
+                               size = sizeof(ss_arc) * (yy->ss_narcs + 1);
+                               yy->ss_arc = PyObject_REALLOC(yy->ss_arc,
+                                                             size);
                                if (yy->ss_arc == NULL)
                                        Py_FatalError("out of mem");
                                zz = &yy->ss_arc[yy->ss_narcs++];
@@ -453,7 +458,8 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
                                        goto done;
                                }
                        }
-                       PyMem_RESIZE(xx_state, ss_state, xx_nstates + 1);
+                       size = sizeof(ss_state) * (xx_nstates + 1);
+                       xx_state = PyObject_REALLOC(xx_state, size);
                        if (xx_state == NULL)
                                Py_FatalError("out of mem");
                        zz->sa_arrow = xx_nstates;

diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c
index 695e2b78802c1dbf08f8740435592c97a133bf79..6d8469f022e8d30a1af2b7d15d19c56af18fbc31 100644 (file)
--- a/Parser/pgenmain.c
+++ b/Parser/pgenmain.c
@@ -104,7 +104,7 @@ getgrammar(char *filename)
                                        putc(' ', stderr);
                        }
                        fprintf(stderr, "^\n");
-                       PyMem_DEL(err.text);
+                       PyObject_FREE(err.text);
                }
                Py_Exit(1);
        }
@@ -136,7 +136,7 @@ char *
 PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
 {
        size_t n = 1000;
-       char *p = PyMem_MALLOC(n);
+       char *p = PyObject_MALLOC(n);
        char *q;
        if (p == NULL)
                return NULL;
@@ -149,7 +149,7 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
        n = strlen(p);
        if (n > 0 && p[n-1] != '\n')
                p[n-1] = '\n';
-       return PyMem_REALLOC(p, n+1);
+       return PyObject_REALLOC(p, n+1);
 }
 
 /* No-nonsense fgets */

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 001d31a1077940d852543e0e58aac2c0171f0507..469de27b62241b484b938497eb0b76b8c6933039 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -105,7 +105,7 @@ char *_PyParser_TokenNames[] = {
 static struct tok_state *
 tok_new(void)
 {
-       struct tok_state *tok = PyMem_NEW(struct tok_state, 1);
+       struct tok_state *tok = PyMem_MALLOC(sizeof(struct tok_state));
        if (tok == NULL)
                return NULL;
        tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
@@ -721,7 +721,7 @@ tok_stdin_decode(struct tok_state *tok, char **inp)
        if (converted == NULL)
                goto error_nomem;
 
-       PyMem_FREE(*inp);
+       PyObject_FREE(*inp);
        *inp = converted;
        if (tok->encoding != NULL)
                PyObject_FREE(tok->encoding);
@@ -781,12 +781,12 @@ tok_nextc(register struct tok_state *tok)
                        if (new == NULL)
                                tok->done = E_INTR;
                        else if (*new == '\0') {
-                               PyMem_FREE(new);
+                               PyObject_FREE(new);
                                tok->done = E_EOF;
                        }
 #if !defined(PGEN) && defined(Py_USING_UNICODE)
                        else if (tok_stdin_decode(tok, &new) != 0)
-                               PyMem_FREE(new);
+                               PyObject_FREE(new);
 #endif
                        else if (tok->start != NULL) {
                                size_t start = tok->start - tok->buf;
@@ -798,7 +798,7 @@ tok_nextc(register struct tok_state *tok)
                                if (buf == NULL) {
                                        PyObject_FREE(tok->buf);
                                        tok->buf = NULL;
-                                       PyMem_FREE(new);
+                                       PyObject_FREE(new);
                                        tok->done = E_NOMEM;
                                        return EOF;
                                }
@@ -806,7 +806,7 @@ tok_nextc(register struct tok_state *tok)
                                tok->cur = tok->buf + oldlen;
                                tok->line_start = tok->cur;
                                strcpy(tok->buf + oldlen, new);
-                               PyMem_FREE(new);
+                               PyObject_FREE(new);
                                tok->inp = tok->buf + newlen;
                                tok->end = tok->inp + 1;
                                tok->start = tok->buf + start;