(Issue #17807: Generators can now be finalized even when they are part of a reference cycle)
non-generator frames. See the save_exc_state and swap_exc_state
functions in ceval.c for details of their use. */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
- /* Borrowed referenced to a generator, or NULL */
- PyObject *f_gen;
PyThreadState *f_tstate;
int f_lasti; /* Last instruction if called */
/* Return the line of code the frame is currently executing. */
PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *);
-/* Generator support */
-PyAPI_FUNC(PyObject *) _PyFrame_YieldingFrom(PyFrameObject *);
-PyAPI_FUNC(PyObject *) _PyFrame_GeneratorSend(PyFrameObject *, PyObject *, int exc);
-PyAPI_FUNC(PyObject *) _PyFrame_Finalize(PyFrameObject *);
-PyAPI_FUNC(int) _PyFrame_CloseIterator(PyObject *);
-
-
#ifdef __cplusplus
}
#endif
#define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
-/* Deprecated, kept for backwards compatibility. */
PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
PyObject *_PyGen_Send(PyGenObject *, PyObject *);
-import gc
-import sys
-import unittest
-import weakref
-
-from test import support
-
-
-class FinalizationTest(unittest.TestCase):
-
- def test_frame_resurrect(self):
- # A generator frame can be resurrected by a generator's finalization.
- def gen():
- nonlocal frame
- try:
- yield
- finally:
- frame = sys._getframe()
-
- g = gen()
- wr = weakref.ref(g)
- next(g)
- del g
- support.gc_collect()
- self.assertIs(wr(), None)
- self.assertTrue(frame)
- del frame
- support.gc_collect()
-
- def test_refcycle(self):
- # A generator caught in a refcycle gets finalized anyway.
- old_garbage = gc.garbage[:]
- finalized = False
- def gen():
- nonlocal finalized
- try:
- g = yield
- yield 1
- finally:
- finalized = True
-
- g = gen()
- next(g)
- g.send(g)
- self.assertGreater(sys.getrefcount(g), 2)
- self.assertFalse(finalized)
- del g
- support.gc_collect()
- self.assertTrue(finalized)
- self.assertEqual(gc.garbage, old_garbage)
-
-
tutorial_tests = """
Let's try a simple generator:
# so this works as expected in both ways of running regrtest.
def test_main(verbose=None):
from test import support, test_generators
- support.run_unittest(__name__)
support.run_doctest(test_generators, verbose)
# This part isn't needed for regrtest, but for running the test directly.
nfrees = len(x.f_code.co_freevars)
extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
ncells + nfrees - 1
- check(x, vsize('13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+ check(x, vsize('12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
check(func, size('12P'))
- Issue #17927: Frame objects kept arguments alive if they had been
copied into a cell, even if the cell was cleared.
-- Issue #17807: Generators can now be finalized even when they are part of
- a reference cycle.
-
- Issue #1545463: At shutdown, defer finalization of codec modules so
that stderr remains usable.
static int
has_finalizer(PyObject *op)
{
- return op->ob_type->tp_del != NULL;
+ if (PyGen_CheckExact(op))
+ return PyGen_NeedsFinalizing((PyGenObject *)op);
+ else
+ return op->ob_type->tp_del != NULL;
}
/* Move the objects in unreachable with __del__ methods into `finalizers`.
return f->f_locals;
}
-/*
- * Generator support.
- */
-
-PyObject *
-_PyFrame_YieldingFrom(PyFrameObject *f)
-{
- PyObject *yf = NULL;
-
- if (f && f->f_stacktop) {
- PyObject *bytecode = f->f_code->co_code;
- unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
-
- if (code[f->f_lasti + 1] != YIELD_FROM)
- return NULL;
- yf = f->f_stacktop[-1];
- Py_INCREF(yf);
- }
- return yf;
-}
-
-PyObject *
-_PyFrame_GeneratorSend(PyFrameObject *f, PyObject *arg, int exc)
-{
- PyThreadState *tstate = PyThreadState_GET();
- PyObject *result;
- PyGenObject *gen = (PyGenObject *) f->f_gen;
-
- assert(gen == NULL || PyGen_CheckExact(gen));
- if (gen && gen->gi_running) {
- PyErr_SetString(PyExc_ValueError,
- "generator already executing");
- return NULL;
- }
- if (f->f_stacktop == NULL) {
- /* Only set exception if send() called, not throw() or next() */
- if (arg && !exc)
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
- }
-
- if (f->f_lasti == -1) {
- if (arg && arg != Py_None) {
- PyErr_SetString(PyExc_TypeError,
- "can't send non-None value to a "
- "just-started generator");
- return NULL;
- }
- } else {
- /* Push arg onto the frame's value stack */
- result = arg ? arg : Py_None;
- Py_INCREF(result);
- *(f->f_stacktop++) = result;
- }
-
- /* Generators always return to their most recent caller, not
- * necessarily their creator. */
- Py_XINCREF(tstate->frame);
- assert(f->f_back == NULL);
- f->f_back = tstate->frame;
-
- if (gen) {
- Py_INCREF(gen);
- gen->gi_running = 1;
- }
- result = PyEval_EvalFrameEx(f, exc);
- if (gen) {
- gen->gi_running = 0;
- /* In case running the frame has lost all external references
- * to gen, we must be careful not to hold on an invalid object. */
- if (Py_REFCNT(gen) == 1)
- Py_CLEAR(gen);
- else
- Py_DECREF(gen);
- }
-
- /* Don't keep the reference to f_back any longer than necessary. It
- * may keep a chain of frames alive or it could create a reference
- * cycle. */
- assert(f->f_back == tstate->frame);
- Py_CLEAR(f->f_back);
-
- /* If the generator just returned (as opposed to yielding), signal
- * that the generator is exhausted. */
- if (result && f->f_stacktop == NULL) {
- if (result == Py_None) {
- /* Delay exception instantiation if we can */
- PyErr_SetNone(PyExc_StopIteration);
- } else {
- PyObject *e = PyObject_CallFunctionObjArgs(
- PyExc_StopIteration, result, NULL);
- if (e != NULL) {
- PyErr_SetObject(PyExc_StopIteration, e);
- Py_DECREF(e);
- }
- }
- Py_CLEAR(result);
- }
-
- if (f->f_stacktop == NULL) {
- /* generator can't be rerun, so release the frame */
- /* first clean reference cycle through stored exception traceback */
- PyObject *t, *v, *tb;
- t = f->f_exc_type;
- v = f->f_exc_value;
- tb = f->f_exc_traceback;
- f->f_exc_type = NULL;
- f->f_exc_value = NULL;
- f->f_exc_traceback = NULL;
- Py_XDECREF(t);
- Py_XDECREF(v);
- Py_XDECREF(tb);
- if (gen) {
- f->f_gen = NULL;
- Py_CLEAR(gen->gi_frame);
- }
- }
-
- return result;
-}
-
-int
-_PyFrame_CloseIterator(PyObject *yf)
-{
- PyObject *retval = NULL;
- _Py_IDENTIFIER(close);
-
- if (PyGen_CheckExact(yf)) {
- PyFrameObject *f = ((PyGenObject *) yf)->gi_frame;
- assert(f != NULL);
- retval = _PyFrame_Finalize(f);
- if (retval == NULL)
- return -1;
- } else {
- PyObject *meth = _PyObject_GetAttrId(yf, &PyId_close);
- if (meth == NULL) {
- if (!PyErr_ExceptionMatches(PyExc_AttributeError))
- PyErr_WriteUnraisable(yf);
- PyErr_Clear();
- } else {
- retval = PyObject_CallFunction(meth, "");
- Py_DECREF(meth);
- if (retval == NULL)
- return -1;
- }
- }
- Py_XDECREF(retval);
- return 0;
-}
-
-PyObject *
-_PyFrame_Finalize(PyFrameObject *f)
-{
- int err = 0;
- PyObject *retval;
- PyGenObject *gen = (PyGenObject *) f->f_gen;
- PyObject *yf = _PyFrame_YieldingFrom(f);
-
- assert(gen == NULL || PyGen_CheckExact(gen));
- if (yf) {
- if (gen)
- gen->gi_running = 1;
- err = _PyFrame_CloseIterator(yf);
- if (gen)
- gen->gi_running = 0;
- Py_DECREF(yf);
- }
- if (err == 0)
- PyErr_SetNone(PyExc_GeneratorExit);
- retval = _PyFrame_GeneratorSend(f, Py_None, 1);
- if (retval) {
- Py_DECREF(retval);
- PyErr_SetString(PyExc_RuntimeError,
- "generator ignored GeneratorExit");
- return NULL;
- }
- if (PyErr_ExceptionMatches(PyExc_StopIteration)
- || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
- PyErr_Clear(); /* ignore these errors */
- Py_INCREF(Py_None);
- return Py_None;
- }
- return NULL;
-}
-
-/*
- * Line number support.
- */
-
int
PyFrame_GetLineNumber(PyFrameObject *f)
{
/* max value for numfree */
#define PyFrame_MAXFREELIST 200
-static void
-frame_clear(PyFrameObject *f);
-
static void
frame_dealloc(PyFrameObject *f)
{
+ PyObject **p, **valuestack;
PyCodeObject *co;
- Py_REFCNT(f)++;
- frame_clear(f);
- Py_REFCNT(f)--;
- if (Py_REFCNT(f) > 0) {
- /* Frame resurrected! */
- Py_ssize_t refcnt = Py_REFCNT(f);
- _Py_NewReference((PyObject *) f);
- Py_REFCNT(f) = refcnt;
- /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
- * we need to undo that. */
- _Py_DEC_REFTOTAL;
- /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
- * chain, so no more to do there.
- * If COUNT_ALLOCS, the original decref bumped tp_frees, and
- * _Py_NewReference bumped tp_allocs: both of those need to be
- * undone.
- */
-#ifdef COUNT_ALLOCS
- --(Py_TYPE(self)->tp_frees);
- --(Py_TYPE(self)->tp_allocs);
-#endif
- }
-
PyObject_GC_UnTrack(f);
Py_TRASHCAN_SAFE_BEGIN(f)
+ /* Kill all local variables */
+ valuestack = f->f_valuestack;
+ for (p = f->f_localsplus; p < valuestack; p++)
+ Py_CLEAR(*p);
+
+ /* Free stack */
+ if (f->f_stacktop != NULL) {
+ for (p = valuestack; p < f->f_stacktop; p++)
+ Py_XDECREF(*p);
+ }
Py_XDECREF(f->f_back);
Py_DECREF(f->f_builtins);
Py_DECREF(f->f_globals);
Py_CLEAR(f->f_locals);
+ Py_CLEAR(f->f_trace);
+ Py_CLEAR(f->f_exc_type);
+ Py_CLEAR(f->f_exc_value);
+ Py_CLEAR(f->f_exc_traceback);
co = f->f_code;
if (co->co_zombieframe == NULL)
{
PyObject **fastlocals, **p, **oldtop;
Py_ssize_t i, slots;
- PyObject *retval;
-
- if (f->f_back == NULL) {
- PyObject *t, *v, *tb;
- PyErr_Fetch(&t, &v, &tb);
- /* Note that this can finalize a suspended generator frame even
- * if the generator object was disposed of (i.e. if f_gen is NULL).
- */
- retval = _PyFrame_Finalize(f);
- if (retval == NULL) {
- if (PyErr_Occurred())
- PyErr_WriteUnraisable((PyObject *) f);
- }
- else
- Py_DECREF(retval);
- PyErr_Restore(t, v, tb);
- }
- /* Make sure the frame is now clearly marked as being defunct */
+ /* Before anything else, make sure that this frame is clearly marked
+ * as being defunct! Else, e.g., a generator reachable from this
+ * frame may also point to this frame, believe itself to still be
+ * active, and try cleaning up this frame again.
+ */
oldtop = f->f_stacktop;
f->f_stacktop = NULL;
f->f_lasti = -1;
f->f_lineno = code->co_firstlineno;
f->f_iblock = 0;
- f->f_gen = NULL;
_PyObject_GC_TRACK(f);
return f;
gen_dealloc(PyGenObject *gen)
{
PyObject *self = (PyObject *) gen;
- PyFrameObject *f = gen->gi_frame;
_PyObject_GC_UNTRACK(gen);
if (gen->gi_weakreflist != NULL)
PyObject_ClearWeakRefs(self);
- gen->gi_frame = NULL;
- if (f) {
- /* Close the generator by finalizing the frame */
- PyObject *retval, *t, *v, *tb;
- PyErr_Fetch(&t, &v, &tb);
- f->f_gen = NULL;
- retval = _PyFrame_Finalize(f);
- if (retval)
- Py_DECREF(retval);
- else if (PyErr_Occurred())
- PyErr_WriteUnraisable((PyObject *) gen);
- Py_DECREF(f);
- PyErr_Restore(t, v, tb);
+ _PyObject_GC_TRACK(self);
+
+ if (gen->gi_frame != NULL && gen->gi_frame->f_stacktop != NULL) {
+ /* Generator is paused, so we need to close */
+ Py_TYPE(gen)->tp_del(self);
+ if (self->ob_refcnt > 0)
+ return; /* resurrected. :( */
}
+
+ _PyObject_GC_UNTRACK(self);
+ Py_CLEAR(gen->gi_frame);
Py_CLEAR(gen->gi_code);
PyObject_GC_Del(gen);
}
+
static PyObject *
gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
{
+ PyThreadState *tstate = PyThreadState_GET();
PyFrameObject *f = gen->gi_frame;
+ PyObject *result;
- /* For compatibility, we check gi_running before f == NULL */
if (gen->gi_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
- if (f == NULL) {
- /* Only set exception if send() called, not throw() or next() */
+ if (f == NULL || f->f_stacktop == NULL) {
+ /* Only set exception if called from send() */
if (arg && !exc)
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
- return _PyFrame_GeneratorSend(f, arg, exc);
+ if (f->f_lasti == -1) {
+ if (arg && arg != Py_None) {
+ PyErr_SetString(PyExc_TypeError,
+ "can't send non-None value to a "
+ "just-started generator");
+ return NULL;
+ }
+ } else {
+ /* Push arg onto the frame's value stack */
+ result = arg ? arg : Py_None;
+ Py_INCREF(result);
+ *(f->f_stacktop++) = result;
+ }
+
+ /* Generators always return to their most recent caller, not
+ * necessarily their creator. */
+ Py_XINCREF(tstate->frame);
+ assert(f->f_back == NULL);
+ f->f_back = tstate->frame;
+
+ gen->gi_running = 1;
+ result = PyEval_EvalFrameEx(f, exc);
+ gen->gi_running = 0;
+
+ /* Don't keep the reference to f_back any longer than necessary. It
+ * may keep a chain of frames alive or it could create a reference
+ * cycle. */
+ assert(f->f_back == tstate->frame);
+ Py_CLEAR(f->f_back);
+
+ /* If the generator just returned (as opposed to yielding), signal
+ * that the generator is exhausted. */
+ if (result && f->f_stacktop == NULL) {
+ if (result == Py_None) {
+ /* Delay exception instantiation if we can */
+ PyErr_SetNone(PyExc_StopIteration);
+ } else {
+ PyObject *e = PyObject_CallFunctionObjArgs(
+ PyExc_StopIteration, result, NULL);
+ if (e != NULL) {
+ PyErr_SetObject(PyExc_StopIteration, e);
+ Py_DECREF(e);
+ }
+ }
+ Py_CLEAR(result);
+ }
+
+ if (!result || f->f_stacktop == NULL) {
+ /* generator can't be rerun, so release the frame */
+ /* first clean reference cycle through stored exception traceback */
+ PyObject *t, *v, *tb;
+ t = f->f_exc_type;
+ v = f->f_exc_value;
+ tb = f->f_exc_traceback;
+ f->f_exc_type = NULL;
+ f->f_exc_value = NULL;
+ f->f_exc_traceback = NULL;
+ Py_XDECREF(t);
+ Py_XDECREF(v);
+ Py_XDECREF(tb);
+ gen->gi_frame = NULL;
+ Py_DECREF(f);
+ }
+
+ return result;
}
PyDoc_STRVAR(send_doc,
* close a subiterator being delegated to by yield-from.
*/
+static int
+gen_close_iter(PyObject *yf)
+{
+ PyObject *retval = NULL;
+ _Py_IDENTIFIER(close);
+
+ if (PyGen_CheckExact(yf)) {
+ retval = gen_close((PyGenObject *)yf, NULL);
+ if (retval == NULL)
+ return -1;
+ } else {
+ PyObject *meth = _PyObject_GetAttrId(yf, &PyId_close);
+ if (meth == NULL) {
+ if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+ PyErr_WriteUnraisable(yf);
+ PyErr_Clear();
+ } else {
+ retval = PyObject_CallFunction(meth, "");
+ Py_DECREF(meth);
+ if (retval == NULL)
+ return -1;
+ }
+ }
+ Py_XDECREF(retval);
+ return 0;
+}
+
static PyObject *
gen_yf(PyGenObject *gen)
{
+ PyObject *yf = NULL;
PyFrameObject *f = gen->gi_frame;
- if (f)
- return _PyFrame_YieldingFrom(f);
- else
- return NULL;
+
+ if (f && f->f_stacktop) {
+ PyObject *bytecode = f->f_code->co_code;
+ unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
+
+ if (code[f->f_lasti + 1] != YIELD_FROM)
+ return NULL;
+ yf = f->f_stacktop[-1];
+ Py_INCREF(yf);
+ }
+
+ return yf;
}
static PyObject *
gen_close(PyGenObject *gen, PyObject *args)
{
- PyFrameObject *f = gen->gi_frame;
+ PyObject *retval;
+ PyObject *yf = gen_yf(gen);
+ int err = 0;
- /* For compatibility, we check gi_running before f == NULL */
- if (gen->gi_running) {
- PyErr_SetString(PyExc_ValueError,
- "generator already executing");
+ if (yf) {
+ gen->gi_running = 1;
+ err = gen_close_iter(yf);
+ gen->gi_running = 0;
+ Py_DECREF(yf);
+ }
+ if (err == 0)
+ PyErr_SetNone(PyExc_GeneratorExit);
+ retval = gen_send_ex(gen, Py_None, 1);
+ if (retval) {
+ Py_DECREF(retval);
+ PyErr_SetString(PyExc_RuntimeError,
+ "generator ignored GeneratorExit");
return NULL;
}
- if (f == NULL)
- Py_RETURN_NONE;
+ if (PyErr_ExceptionMatches(PyExc_StopIteration)
+ || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
+ PyErr_Clear(); /* ignore these errors */
+ Py_INCREF(Py_None);
+ return Py_None;
+ }
+ return NULL;
+}
- return _PyFrame_Finalize(f);
+static void
+gen_del(PyObject *self)
+{
+ PyObject *res;
+ PyObject *error_type, *error_value, *error_traceback;
+ PyGenObject *gen = (PyGenObject *)self;
+
+ if (gen->gi_frame == NULL || gen->gi_frame->f_stacktop == NULL)
+ /* Generator isn't paused, so no need to close */
+ return;
+
+ /* Temporarily resurrect the object. */
+ assert(self->ob_refcnt == 0);
+ self->ob_refcnt = 1;
+
+ /* Save the current exception, if any. */
+ PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+ res = gen_close(gen, NULL);
+
+ if (res == NULL)
+ PyErr_WriteUnraisable(self);
+ else
+ Py_DECREF(res);
+
+ /* Restore the saved exception. */
+ PyErr_Restore(error_type, error_value, error_traceback);
+
+ /* Undo the temporary resurrection; can't use DECREF here, it would
+ * cause a recursive call.
+ */
+ assert(self->ob_refcnt > 0);
+ if (--self->ob_refcnt == 0)
+ return; /* this is the normal path out */
+
+ /* close() resurrected it! Make it look like the original Py_DECREF
+ * never happened.
+ */
+ {
+ Py_ssize_t refcnt = self->ob_refcnt;
+ _Py_NewReference(self);
+ self->ob_refcnt = refcnt;
+ }
+ assert(PyType_IS_GC(Py_TYPE(self)) &&
+ _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
+
+ /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
+ * we need to undo that. */
+ _Py_DEC_REFTOTAL;
+ /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
+ * chain, so no more to do there.
+ * If COUNT_ALLOCS, the original decref bumped tp_frees, and
+ * _Py_NewReference bumped tp_allocs: both of those need to be
+ * undone.
+ */
+#ifdef COUNT_ALLOCS
+ --(Py_TYPE(self)->tp_frees);
+ --(Py_TYPE(self)->tp_allocs);
+#endif
}
+
+
PyDoc_STRVAR(throw_doc,
"throw(typ[,val[,tb]]) -> raise exception in generator,\n\
return next yielded value or raise StopIteration.");
int err;
if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) {
gen->gi_running = 1;
- err = _PyFrame_CloseIterator(yf);
+ err = gen_close_iter(yf);
gen->gi_running = 0;
Py_DECREF(yf);
if (err < 0)
0, /* tp_cache */
0, /* tp_subclasses */
0, /* tp_weaklist */
+ gen_del, /* tp_del */
};
PyObject *
return NULL;
}
gen->gi_frame = f;
- f->f_gen = (PyObject *) gen;
Py_INCREF(f->f_code);
gen->gi_code = (PyObject *)(f->f_code);
gen->gi_running = 0;
int
PyGen_NeedsFinalizing(PyGenObject *gen)
{
+ int i;
+ PyFrameObject *f = gen->gi_frame;
+
+ if (f == NULL || f->f_stacktop == NULL)
+ return 0; /* no frame or empty blockstack == no finalization */
+
+ /* Any block type besides a loop requires cleanup. */
+ for (i = 0; i < f->f_iblock; i++)
+ if (f->f_blockstack[i].b_type != SETUP_LOOP)
+ return 1;
+
+ /* No blocks except loops, it's safe to skip finalization. */
return 0;
}
projects / python / commitdiff