Implemented Martin's suggestion to clear the free lists during the garbage collection...

diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst
index f20f14c1c1c42ba05696c1743b40216e66125a2c..c104f8995ba35df5e0b10def5428d76865c341cf 100644 (file)
--- a/Doc/c-api/method.rst
+++ b/Doc/c-api/method.rst
@@ -63,3 +63,10 @@ There are some useful functions that are useful for working with method objects.
 .. cfunction:: PyObject* PyMethod_GET_SELF(PyObject *meth)
 
    Macro version of :cfunc:`PyMethod_Self` which avoids error checking.
+
+
+.. cfunction:: int PyMethod_ClearFreeList(void)
+
+   Clear the free list. Return the total number of freed items.
+
+   .. versionadded:: 2.6

diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index b25adb31edc32e91cc645014c4263287b2dce308..1fcf5fcc4003f1225a8328925cf4aacf2c1b25c8 100644 (file)
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -115,3 +115,10 @@ Tuple Objects
 
    .. versionchanged:: 2.2
       Removed unused third parameter, *last_is_sticky*.
+
+
+.. cfunction:: int PyMethod_ClearFreeList(void)
+
+   Clear the free list. Return the total number of freed items.
+
+   .. versionadded:: 2.6

diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 7c570ab78826f7f5f493eb0e3374fbf53a1796e7..5e5d1a2f1f885fcafaaf95502c329a8734832c64 100644 (file)
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -89,6 +89,13 @@ access internal read-only data of Unicode objects:
    Return a pointer to the internal buffer of the object. *o* has to be a
    :ctype:`PyUnicodeObject` (not checked).
 
+
+.. cfunction:: int PyUnicode_ClearFreeList(void)
+
+   Clear the free list. Return the total number of freed items.
+
+   .. versionadded:: 2.6
+
 Unicode provides many different character properties. The most often needed ones
 are available through these macros which are mapped to C functions depending on
 the Python configuration.

diff --git a/Include/classobject.h b/Include/classobject.h
index 8f8db7d813031a7bde3c60355fbe26359cb453aa..118dd09647a379f7a1568e57542e06d895c0b0ea 100644 (file)
--- a/Include/classobject.h
+++ b/Include/classobject.h
@@ -74,6 +74,7 @@ PyAPI_FUNC(PyObject *) _PyInstance_Lookup(PyObject *pinst, PyObject *name);
 
 PyAPI_FUNC(int) PyClass_IsSubclass(PyObject *, PyObject *);
 
+PyAPI_FUNC(int) PyMethod_ClearFreeList(void);
 
 #ifdef __cplusplus
 }

diff --git a/Include/frameobject.h b/Include/frameobject.h
index 794f6515bd5bf6139a764b20d20a7a5fac549ca5..297b66ddb2d465f2beeadf03e4f175d38da01a0d 100644 (file)
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -75,6 +75,8 @@ PyAPI_FUNC(PyObject **) PyFrame_ExtendStack(PyFrameObject *, int, int);
 PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int);
 PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *);
 
+PyAPI_FUNC(int) PyFrame_ClearFreeList(void);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/methodobject.h b/Include/methodobject.h
index 810a69c4651a84c441bdab500c2ca93ffdc9f88e..6e160b639007aba94d02433afbd0bc141ed67553 100644 (file)
--- a/Include/methodobject.h
+++ b/Include/methodobject.h
@@ -85,6 +85,8 @@ typedef struct {
     PyObject    *m_module; /* The __module__ attribute, can be anything */
 } PyCFunctionObject;
 
+PyAPI_FUNC(int) PyCFunction_ClearFreeList(void);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/tupleobject.h b/Include/tupleobject.h
index 1b51b1b742d7449a732bf21eef6bd7943a7f0caf..58479ee0ffeae3d73c6b76b3925b36137c1c95c3 100644 (file)
--- a/Include/tupleobject.h
+++ b/Include/tupleobject.h
@@ -52,6 +52,8 @@ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
 /* Macro, *only* to be used to fill in brand new tuples */
 #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
 
+PyAPI_FUNC(int) PyTuple_ClearFreeList(void);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index 8809b342b62df866bfadbcfdc701923f27dcbf43..d7ed0fa299b97a6c18e707d5b5303d2c903b911b 100644 (file)
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -209,6 +209,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
 # define _PyUnicode_Fini _PyUnicodeUCS2_Fini
 # define _PyUnicode_Init _PyUnicodeUCS2_Init
+# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
 # define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
 # define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
@@ -295,6 +296,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
 # define _PyUnicode_Fini _PyUnicodeUCS4_Fini
 # define _PyUnicode_Init _PyUnicodeUCS4_Init
+# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
 # define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
 # define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
@@ -403,6 +405,8 @@ extern const unsigned char _Py_ascii_whitespace[];
 extern "C" {
 #endif
 
+PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
+
 /* --- Unicode Type ------------------------------------------------------- */
 
 typedef struct {

diff --git a/Misc/NEWS b/Misc/NEWS
index fe4b89ff826b1ec05a827e50cd5b1641f46444aa..134ac7e71674838484707afa6430d09e260bc39f 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1?
 Core and builtins
 -----------------
 
+- Clear all free list during a gc.collect() of the highest generation in order
+  to allow pymalloc to free more arenas. Python may give back memory to the
+  OS earlier.
+
 - Issue #2045: Fix an infinite recursion triggered when printing a subclass of
   collections.defaultdict, if its default_factory is set to a bound method.
 

diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index 522cc89c50615c71c28aaf8f462b862562058c9e..4f8c85ffd9785aef380eecf5916fb0c5be1d4e3e 100644 (file)
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -19,6 +19,7 @@
 */
 
 #include "Python.h"
+#include "frameobject.h"       /* for PyFrame_ClearFreeList */
 
 /* Get an object's GC head */
 #define AS_GC(o) ((PyGC_Head *)(o)-1)
@@ -722,6 +723,21 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old)
        }
 }
 
+/* Clear all free lists
+ * All free lists are cleared during the collection of the highest generation.
+ * Allocated items in the free list may keep a pymalloc arena occupied.
+ * Clearing the free lists may give back memory to the OS earlier.
+ */
+static void
+clear_freelists(void)
+{
+       (void)PyMethod_ClearFreeList();
+       (void)PyFrame_ClearFreeList();
+       (void)PyCFunction_ClearFreeList();
+       (void)PyTuple_ClearFreeList();
+       (void)PyUnicode_ClearFreeList();
+}
+
 /* This is the main function.  Read this to understand how the
  * collection process works. */
 static Py_ssize_t
@@ -874,6 +890,12 @@ collect(int generation)
         */
        (void)handle_finalizers(&finalizers, old);
 
+       /* Clear free list only during the collection of the higest
+        * generation */
+       if (generation == NUM_GENERATIONS-1) {
+               clear_freelists();
+       }
+
        if (PyErr_Occurred()) {
                if (gc_str == NULL)
                        gc_str = PyString_FromString("garbage collection");

diff --git a/Objects/classobject.c b/Objects/classobject.c
index 759027bb93ec8777eeb0034e6ec8818223a4e61a..6c77153959b252836ac84db647a770c0b10b9be3 100644 (file)
--- a/Objects/classobject.c
+++ b/Objects/classobject.c
@@ -2626,9 +2626,11 @@ PyTypeObject PyMethod_Type = {
 
 /* Clear out the free list */
 
-void
-PyMethod_Fini(void)
+int
+PyMethod_ClearFreeList(void)
 {
+       int freelist_size = numfree;
+       
        while (free_list) {
                PyMethodObject *im = free_list;
                free_list = (PyMethodObject *)(im->im_self);
@@ -2636,4 +2638,11 @@ PyMethod_Fini(void)
                numfree--;
        }
        assert(numfree == 0);
+       return freelist_size;
+}
+
+void
+PyMethod_Fini(void)
+{
+       (void)PyMethod_ClearFreeList();
 }

diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index df9c2e01dc8f0f1504822e8db2a6854e9704c7f3..025431e56b4de2d91c1dbad5429e047948fd104b 100644 (file)
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -889,10 +889,11 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
 }
 
 /* Clear out the free list */
-
-void
-PyFrame_Fini(void)
+int
+PyFrame_ClearFreeList(void)
 {
+       int freelist_size = numfree;
+       
        while (free_list != NULL) {
                PyFrameObject *f = free_list;
                free_list = free_list->f_back;
@@ -900,6 +901,13 @@ PyFrame_Fini(void)
                --numfree;
        }
        assert(numfree == 0);
+       return freelist_size;
+}
+
+void
+PyFrame_Fini(void)
+{
+       (void)PyFrame_ClearFreeList();
        Py_XDECREF(builtin_object);
        builtin_object = NULL;
 }

diff --git a/Objects/methodobject.c b/Objects/methodobject.c
index 0d9cf1cc482c099fe848ffe588f1f1b8d225c5a2..d661c47500675573bee8bb6409a3fe6c036688fc 100644 (file)
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -353,9 +353,11 @@ Py_FindMethod(PyMethodDef *methods, PyObject *self, const char *name)
 
 /* Clear out the free list */
 
-void
-PyCFunction_Fini(void)
+int
+PyCFunction_ClearFreeList(void)
 {
+       int freelist_size = numfree;
+       
        while (free_list) {
                PyCFunctionObject *v = free_list;
                free_list = (PyCFunctionObject *)(v->m_self);
@@ -363,6 +365,13 @@ PyCFunction_Fini(void)
                numfree--;
        }
        assert(numfree == 0);
+       return freelist_size;
+}
+
+void
+PyCFunction_Fini(void)
+{
+       (void)PyCFunction_ClearFreeList();
 }
 
 /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(),

diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index dfd2a162fa3b1dd5d212f9cb7398b1c4002c943a..e9cb3efbe8e86aeb944cd29ff4bc3c82d17e5b98 100644 (file)
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -832,25 +832,38 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
        return 0;
 }
 
-void
-PyTuple_Fini(void)
+int
+PyTuple_ClearFreeList(void)
 {
+       int freelist_size = 0;
 #if PyTuple_MAXSAVESIZE > 0
        int i;
-
-       Py_XDECREF(free_list[0]);
-       free_list[0] = NULL;
-
        for (i = 1; i < PyTuple_MAXSAVESIZE; i++) {
                PyTupleObject *p, *q;
                p = free_list[i];
+               freelist_size += numfree[i];
                free_list[i] = NULL;
+               numfree[i] = 0;
                while (p) {
                        q = p;
                        p = (PyTupleObject *)(p->ob_item[0]);
                        PyObject_GC_Del(q);
                }
        }
+#endif
+       return freelist_size;
+}
+       
+void
+PyTuple_Fini(void)
+{
+#if PyTuple_MAXSAVESIZE > 0
+       /* empty tuples are used all over the place and applications may
+        * rely on the fact that an empty tuple is a singleton. */
+       Py_XDECREF(free_list[0]);
+       free_list[0] = NULL;
+
+       (void)PyTuple_ClearFreeList();
 #endif
 }
 

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 74b47964275443729d0b04af56184943b68b44a4..3e15f53fc751adc6d6d0727249d7b9809d1d14ba 100644 (file)
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -8853,10 +8853,29 @@ void _PyUnicode_Init(void)
 
 /* Finalize the Unicode implementation */
 
+int
+PyUnicode_ClearFreeList(void)
+{
+    int freelist_size = numfree;
+    PyUnicodeObject *u;
+
+    for (u = free_list; u != NULL;) {
+       PyUnicodeObject *v = u;
+       u = *(PyUnicodeObject **)u;
+       if (v->str)
+           PyMem_DEL(v->str);
+       Py_XDECREF(v->defenc);
+       PyObject_Del(v);
+       numfree--;
+    }
+    free_list = NULL;
+    assert(numfree == 0);
+    return freelist_size;
+}
+
 void
 _PyUnicode_Fini(void)
 {
-    PyUnicodeObject *u;
     int i;
 
     Py_XDECREF(unicode_empty);
@@ -8868,17 +8887,7 @@ _PyUnicode_Fini(void)
            unicode_latin1[i] = NULL;
        }
     }
-
-    for (u = free_list; u != NULL;) {
-       PyUnicodeObject *v = u;
-       u = *(PyUnicodeObject **)u;
-       if (v->str)
-           PyMem_DEL(v->str);
-       Py_XDECREF(v->defenc);
-       PyObject_Del(v);
-    }
-    free_list = NULL;
-    numfree = 0;
+    (void)PyUnicode_ClearFreeList();
 }
 
 #ifdef __cplusplus