SF patch #578297:

Change the parser and compiler to use PyMalloc.

Only the files implementing processes that will request memory
allocations small enough for PyMalloc to be a win have been
changed, which are:-
 - Python/compile.c
 - Parser/acceler.c
 - Parser/node.c
 - Parser/parsetok.c

This augments the aggressive overallocation strategy implemented by
Tim Peters in PyNode_AddChild() [Parser/node.c], in reducing the
impact of platform malloc()/realloc()/free() corner case behaviour.
Such corner cases are known to be triggered by test_longexp and
test_import.

Jeremy Hylton, in accepting this patch, recommended this as a
bugfix candidate for 2.2.  While the changes to Python/compile.c
and Parser/node.c backport easily (and could go in), the changes
to Parser/acceler.c and Parser/parsetok.c require other not
insignificant changes as a result of the differences in the memory
APIs between 2.3 and 2.2, which I'm not in a position to work
through at the moment.  This is a pity, as the Parser/parsetok.c
changes are the most important after the Parser/node.c changes, due
to the size of the memory requests involved and their frequency.

diff --git a/Parser/acceler.c b/Parser/acceler.c
index 6ed12c3074235bb6d298cad5a1b7b536aab177b1..63919c5efca8af2fdfb669969bea36f0f9907b97 100644 (file)
--- a/Parser/acceler.c
+++ b/Parser/acceler.c
@@ -44,7 +44,7 @@ PyGrammar_RemoveAccelerators(grammar *g)
                s = d->d_state;
                for (j = 0; j < d->d_nstates; j++, s++) {
                        if (s->s_accel)
-                               PyMem_DEL(s->s_accel);
+                               PyObject_FREE(s->s_accel);
                        s->s_accel = NULL;
                }
        }
@@ -68,7 +68,7 @@ fixstate(grammar *g, state *s)
        int *accel;
        int nl = g->g_ll.ll_nlabels;
        s->s_accept = 0;
-       accel = PyMem_NEW(int, nl);
+       accel = (int *) PyObject_MALLOC(nl * sizeof(int));
        for (k = 0; k < nl; k++)
                accel[k] = -1;
        a = s->s_arc;
@@ -124,7 +124,7 @@ fixstate(grammar *g, state *s)
                k++;
        if (k < nl) {
                int i;
-               s->s_accel = PyMem_NEW(int, nl-k);
+               s->s_accel = (int *) PyObject_MALLOC((nl-k) * sizeof(int));
                if (s->s_accel == NULL) {
                        fprintf(stderr, "no mem to add parser accelerators\n");
                        exit(1);
@@ -134,5 +134,5 @@ fixstate(grammar *g, state *s)
                for (i = 0; k < nl; i++, k++)
                        s->s_accel[i] = accel[k];
        }
-       PyMem_DEL(accel);
+       PyObject_FREE(accel);
 }

diff --git a/Parser/node.c b/Parser/node.c
index 780c23080b4ee5551ca62caf056786910a340b94..75900ce8429b5df908c28ddb01eac9778676a8ab 100644 (file)
--- a/Parser/node.c
+++ b/Parser/node.c
@@ -7,7 +7,7 @@
 node *
 PyNode_New(int type)
 {
-       node *n = PyMem_NEW(node, 1);
+       node *n = (node *) PyObject_MALLOC(1 * sizeof(node));
        if (n == NULL)
                return NULL;
        n->n_type = type;
@@ -92,7 +92,8 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno)
                return E_OVERFLOW;
        if (current_capacity < required_capacity) {
                n = n1->n_child;
-               PyMem_RESIZE(n, node, required_capacity);
+               n = (node *) PyObject_REALLOC(n,
+                                             required_capacity * sizeof(node));
                if (n == NULL)
                        return E_NOMEM;
                n1->n_child = n;
@@ -116,7 +117,7 @@ PyNode_Free(node *n)
 {
        if (n != NULL) {
                freechildren(n);
-               PyMem_DEL(n);
+               PyObject_FREE(n);
        }
 }
 
@@ -127,7 +128,7 @@ freechildren(node *n)
        for (i = NCH(n); --i >= 0; )
                freechildren(CHILD(n, i));
        if (n->n_child != NULL)
-               PyMem_DEL(n->n_child);
+               PyObject_FREE(n->n_child);
        if (STR(n) != NULL)
-               PyMem_DEL(STR(n));
+               PyObject_FREE(STR(n));
 }

diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index cd3887dff9cb5c8134391e80d5f36a3dc5fca883..5758fa749ba1d4332a58420235bb13cf4b4b8148 100644 (file)
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -133,7 +133,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
                else
                        started = 1;
                len = b - a; /* XXX this may compute NULL - NULL */
-               str = PyMem_NEW(char, len + 1);
+               str = (char *) PyObject_MALLOC(len + 1);
                if (str == NULL) {
                        fprintf(stderr, "no mem for next token\n");
                        err_ret->error = E_NOMEM;
@@ -157,7 +157,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
                     PyParser_AddToken(ps, (int)type, str, tok->lineno,
                                       &(err_ret->expected))) != E_OK) {
                        if (err_ret->error != E_DONE)
-                               PyMem_DEL(str);
+                               PyObject_FREE(str);
                        break;
                }
        }
@@ -178,7 +178,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
                err_ret->offset = tok->cur - tok->buf;
                if (tok->buf != NULL) {
                        size_t len = tok->inp - tok->buf;
-                       err_ret->text = PyMem_NEW(char, len + 1);
+                       err_ret->text = (char *) PyObject_MALLOC(len + 1);
                        if (err_ret->text != NULL) {
                                if (len > 0)
                                        strncpy(err_ret->text, tok->buf, len);

diff --git a/Python/compile.c b/Python/compile.c
index 296653210a77bbd1cbeba2409cab6ba4520ae57e..3a0948e0c4ecbd6ea0034bad7a0c5fe48ccd7474 100644 (file)
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -719,7 +719,7 @@ com_free(struct compiling *c)
        Py_XDECREF(c->c_cellvars);
        Py_XDECREF(c->c_lnotab);
        if (c->c_future)
-               PyMem_Free((void *)c->c_future);
+               PyObject_FREE((void *)c->c_future);
 }
 
 static void
@@ -2020,7 +2020,7 @@ com_factor(struct compiling *c, node *n)
                        return;
                }
                if (childtype == MINUS) {
-                       char *s = PyMem_Malloc(strlen(STR(pnum)) + 2);
+                       char *s = PyObject_MALLOC(strlen(STR(pnum)) + 2);
                        if (s == NULL) {
                                com_error(c, PyExc_MemoryError, "");
                                com_addbyte(c, 255);
@@ -2028,7 +2028,7 @@ com_factor(struct compiling *c, node *n)
                        }
                        s[0] = '-';
                        strcpy(s + 1, STR(pnum));
-                       PyMem_Free(STR(pnum));
+                       PyObject_FREE(STR(pnum));
                        STR(pnum) = s;
                }
                com_atom(c, patom);
@@ -4116,7 +4116,7 @@ PyNode_CompileSymtable(node *n, char *filename)
 
        st = symtable_init();
        if (st == NULL) {
-               PyMem_Free((void *)ff);
+               PyObject_FREE((void *)ff);
                return NULL;
        }
        st->st_future = ff;
@@ -4129,7 +4129,7 @@ PyNode_CompileSymtable(node *n, char *filename)
        
        return st;
  fail:
-       PyMem_Free((void *)ff);
+       PyObject_FREE((void *)ff);
        st->st_future = NULL;
        PySymtable_Free(st);
        return NULL;
@@ -4722,7 +4722,7 @@ symtable_init()
 {
        struct symtable *st;
 
-       st = (struct symtable *)PyMem_Malloc(sizeof(struct symtable));
+       st = (struct symtable *)PyObject_MALLOC(sizeof(struct symtable));
        if (st == NULL)
                return NULL;
        st->st_pass = 1;
@@ -4749,7 +4749,7 @@ PySymtable_Free(struct symtable *st)
        Py_XDECREF(st->st_symbols);
        Py_XDECREF(st->st_stack);
        Py_XDECREF(st->st_cur);
-       PyMem_Free((void *)st);
+       PyObject_FREE((void *)st);
 }
 
 /* When the compiler exits a scope, it must should update the scope's