bpo-31095: fix potential crash during GC (GH-2974) (#3196)

(cherry picked from commit a6296d34a478b4f697ea9db798146195075d496c)

diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index 003b4e505d3247db733187662ab9061154613d33..abd5da9db654544c507f7e3d11baded192aae09c 100644 (file)
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -728,8 +728,9 @@ functions.  With :c:func:`Py_VISIT`, :c:func:`Noddy_traverse` can be simplified:
    uniformity across these boring implementations.
 
 We also need to provide a method for clearing any subobjects that can
-participate in cycles.  We implement the method and reimplement the deallocator
-to use it::
+participate in cycles.
+
+::
 
    static int
    Noddy_clear(Noddy *self)
@@ -747,13 +748,6 @@ to use it::
        return 0;
    }
 
-   static void
-   Noddy_dealloc(Noddy* self)
-   {
-       Noddy_clear(self);
-       Py_TYPE(self)->tp_free((PyObject*)self);
-   }
-
 Notice the use of a temporary variable in :c:func:`Noddy_clear`. We use the
 temporary variable so that we can set each member to *NULL* before decrementing
 its reference count.  We do this because, as was discussed earlier, if the
@@ -776,6 +770,23 @@ be simplified::
        return 0;
    }
 
+Note that :c:func:`Noddy_dealloc` may call arbitrary functions through
+``__del__`` method or weakref callback. It means circular GC can be
+triggered inside the function.  Since GC assumes reference count is not zero,
+we need to untrack the object from GC by calling :c:func:`PyObject_GC_UnTrack`
+before clearing members. Here is reimplemented deallocator which uses
+:c:func:`PyObject_GC_UnTrack` and :c:func:`Noddy_clear`.
+
+::
+
+   static void
+   Noddy_dealloc(Noddy* self)
+   {
+       PyObject_GC_UnTrack(self);
+       Noddy_clear(self);
+       Py_TYPE(self)->tp_free((PyObject*)self);
+   }
+
 Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags::
 
    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */

diff --git a/Doc/includes/noddy4.c b/Doc/includes/noddy4.c
index eb9622a87d99cc97ef81dd1c4544c6c0f81385b4..08ba4c3d91a03059c1cb324633a74b950dde13d7 100644 (file)
--- a/Doc/includes/noddy4.c
+++ b/Doc/includes/noddy4.c
@@ -46,6 +46,7 @@ Noddy_clear(Noddy *self)
 static void
 Noddy_dealloc(Noddy* self)
 {
+    PyObject_GC_UnTrack(self);
     Noddy_clear(self);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }

diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-08-01-18-48-30.bpo-31095.bXWZDb.rst b/Misc/NEWS.d/next/Core and Builtins/2017-08-01-18-48-30.bpo-31095.bXWZDb.rst
new file mode 100644 (file)
index 0000000..ca1f8ba
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-08-01-18-48-30.bpo-31095.bXWZDb.rst
@@ -0,0 +1,2 @@
+Fix potential crash during GC caused by ``tp_dealloc`` which doesn't call
+``PyObject_GC_UnTrack()``.

diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
index 08fcecad9adc00f2154978ef51176b8a97407bff..25d2d4ce85c385642541d12c77911d3c06c65079 100644 (file)
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -1641,6 +1641,8 @@ dequeiter_traverse(dequeiterobject *dio, visitproc visit, void *arg)
 static void
 dequeiter_dealloc(dequeiterobject *dio)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(dio);
     Py_XDECREF(dio->deque);
     PyObject_GC_Del(dio);
 }
@@ -2021,6 +2023,8 @@ static PyMemberDef defdict_members[] = {
 static void
 defdict_dealloc(defdictobject *dd)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(dd);
     Py_CLEAR(dd->default_factory);
     PyDict_Type.tp_dealloc((PyObject *)dd);
 }

diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index fedef100cec7bfd67c62789b5e66df10a298b53b..5dba9f70a94bea4771e5988aeb02679822bd139b 100644 (file)
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -653,6 +653,7 @@ element_gc_clear(ElementObject *self)
 static void
 element_dealloc(ElementObject* self)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
     PyObject_GC_UnTrack(self);
     Py_TRASHCAN_SAFE_BEGIN(self)
 
@@ -2025,6 +2026,9 @@ static void
 elementiter_dealloc(ElementIterObject *it)
 {
     ParentLocator *p = it->parent_stack;
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(it);
+
     while (p) {
         ParentLocator *temp = p;
         Py_XDECREF(p->parent);
@@ -2034,8 +2038,6 @@ elementiter_dealloc(ElementIterObject *it)
 
     Py_XDECREF(it->sought_tag);
     Py_XDECREF(it->root_element);
-
-    PyObject_GC_UnTrack(it);
     PyObject_GC_Del(it);
 }
 

diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c
index 985bcb6985e835c64a1f2e0319663a6a21a8935b..d27e1c4a7945935fc175cde160db4170be1152c1 100644 (file)
--- a/Modules/_functoolsmodule.c
+++ b/Modules/_functoolsmodule.c
@@ -116,6 +116,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
 static void
 partial_dealloc(partialobject *pto)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
     PyObject_GC_UnTrack(pto);
     if (pto->weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject *) pto);
@@ -1039,7 +1040,11 @@ lru_cache_clear_list(lru_list_elem *link)
 static void
 lru_cache_dealloc(lru_cache_object *obj)
 {
-    lru_list_elem *list = lru_cache_unlink_list(obj);
+    lru_list_elem *list;
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(obj);
+
+    list = lru_cache_unlink_list(obj);
     Py_XDECREF(obj->maxsize_O);
     Py_XDECREF(obj->func);
     Py_XDECREF(obj->cache);

diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c
index 9e5d78b1662812c6d8262faa67f3763fb393d38a..d1c9ccb16652bb144140e4eeab5a0fdc6a6a44b0 100644 (file)
--- a/Modules/_io/bytesio.c
+++ b/Modules/_io/bytesio.c
@@ -1133,6 +1133,8 @@ bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg)
 static void
 bytesiobuf_dealloc(bytesiobuf *self)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(self);
     Py_CLEAR(self->source);
     Py_TYPE(self)->tp_free(self);
 }

diff --git a/Modules/_json.c b/Modules/_json.c
index 346bd5581b89dd2c147986494e56b281d4296904..135e2e76fb6b83526fa0db850ff7cc027c5819a7 100644 (file)
--- a/Modules/_json.c
+++ b/Modules/_json.c
@@ -655,7 +655,8 @@ py_encode_basestring(PyObject* self UNUSED, PyObject *pystr)
 static void
 scanner_dealloc(PyObject *self)
 {
-    /* Deallocate scanner object */
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(self);
     scanner_clear(self);
     Py_TYPE(self)->tp_free(self);
 }
@@ -1793,7 +1794,8 @@ bail:
 static void
 encoder_dealloc(PyObject *self)
 {
-    /* Deallocate Encoder */
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(self);
     encoder_clear(self);
     Py_TYPE(self)->tp_free(self);
 }

diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 3b2d3add2821bf3c1c8c93e07eac710e77e6f16f..86482677ae7a3e62cbc44265174f0c0d06f7c47f 100644 (file)
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -2465,6 +2465,8 @@ context_clear(PySSLContext *self)
 static void
 context_dealloc(PySSLContext *self)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(self);
     context_clear(self);
     SSL_CTX_free(self->ctx);
 #ifdef OPENSSL_NPN_NEGOTIATED

diff --git a/Modules/_struct.c b/Modules/_struct.c
index 22731295679a9dea972303c560b062284fbb4da2..5817b5edbeec3f64d6abf0b275a4bd78cad11142 100644 (file)
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -1581,6 +1581,8 @@ typedef struct {
 static void
 unpackiter_dealloc(unpackiterobject *self)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(self);
     Py_XDECREF(self->so);
     PyBuffer_Release(&self->buf);
     PyObject_GC_Del(self);

diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index ff77beeed7c243f516d5382beecceb8a74552116..a8b26d2131f977c95696b0cfed2d178a2566251d 100644 (file)
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -1636,6 +1636,8 @@ dict_dealloc(PyDictObject *mp)
     PyObject **values = mp->ma_values;
     PyDictKeysObject *keys = mp->ma_keys;
     Py_ssize_t i, n;
+
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
     PyObject_GC_UnTrack(mp);
     Py_TRASHCAN_SAFE_BEGIN(mp)
     if (values != NULL) {
@@ -2961,6 +2963,8 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype)
 static void
 dictiter_dealloc(dictiterobject *di)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    _PyObject_GC_UNTRACK(di);
     Py_XDECREF(di->di_dict);
     Py_XDECREF(di->di_result);
     PyObject_GC_Del(di);
@@ -3319,6 +3323,8 @@ dictiter_reduce(dictiterobject *di)
 static void
 dictview_dealloc(_PyDictViewObject *dv)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    _PyObject_GC_UNTRACK(dv);
     Py_XDECREF(dv->dv_dict);
     PyObject_GC_Del(dv);
 }

diff --git a/Objects/setobject.c b/Objects/setobject.c
index 3dbdb4ea5393060e1db4db9a079f24e329825cd6..1f532bc1f46e4fd39f04b7ce7dc2ee32a3b1d21d 100644 (file)
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -472,6 +472,7 @@ set_dealloc(PySetObject *so)
     setentry *entry;
     Py_ssize_t used = so->used;
 
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
     PyObject_GC_UnTrack(so);
     Py_TRASHCAN_SAFE_BEGIN(so)
     if (so->weakreflist != NULL)
@@ -736,6 +737,8 @@ typedef struct {
 static void
 setiter_dealloc(setiterobject *si)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    _PyObject_GC_UNTRACK(si);
     Py_XDECREF(si->si_set);
     PyObject_GC_Del(si);
 }

diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index ac9d2b6346ca99bf034db58383c11779556e017a..60c5a75aa2f819f502e90f118fef21b83e1e5d68 100755 (executable)
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -630,6 +630,8 @@ typedef struct {
 static void
 ast_dealloc(AST_object *self)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(self);
     Py_CLEAR(self->dict);
     Py_TYPE(self)->tp_free(self);
 }

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index ad53ba3585e83d693ebbe53724644a7b80907375..d7586f2de0eed6b161f5fb3bd3629775f8ed2b57 100644 (file)
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -486,6 +486,8 @@ typedef struct {
 static void
 ast_dealloc(AST_object *self)
 {
+    /* bpo-31095: UnTrack is needed before calling any callbacks */
+    PyObject_GC_UnTrack(self);
     Py_CLEAR(self->dict);
     Py_TYPE(self)->tp_free(self);
 }