Issue #26494: Fixed crash on iterating exhausting iterators.

Affected classes are generic sequence iterators, iterators of bytearray,
list, tuple, set, frozenset, dict, OrderedDict and corresponding views.

diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py
index f4673d48af3206c945a41e7cbc461897313a3209..f019e8d573a1c96072f319045a1e2f19db411aaf 100644 (file)
--- a/Lib/test/seq_tests.py
+++ b/Lib/test/seq_tests.py
@@ -4,6 +4,7 @@ Tests common to tuple, list and UserList.UserList
 
 import unittest
 import sys
+from test import test_support as support
 
 # Various iterables
 # This is used for checking the constructor (here and in test_deque.py)
@@ -402,3 +403,7 @@ class CommonTest(unittest.TestCase):
         self.assertEqual(a.index(0, -4*sys.maxint, 4*sys.maxint), 2)
         self.assertRaises(ValueError, a.index, 0, 4*sys.maxint,-4*sys.maxint)
         self.assertRaises(ValueError, a.index, 2, 0, -10)
+
+    def test_free_after_iterating(self):
+        support.check_free_after_iterating(self, iter, self.type2test)
+        support.check_free_after_iterating(self, reversed, self.type2test)

diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index 02fba389ca730ae40cd4c21a8a997090b0b2ee04..82ad451db014f058485bdf88c4930bd0e7fe31d0 100644 (file)
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -518,6 +518,10 @@ class BaseBytesTest(unittest.TestCase):
         self.assertRaisesRegexp(TypeError, r'\bendswith\b', b.endswith,
                                 x, None, None, None)
 
+    def test_free_after_iterating(self):
+        test.test_support.check_free_after_iterating(self, iter, self.type2test)
+        test.test_support.check_free_after_iterating(self, reversed, self.type2test)
+
 
 class ByteArrayTest(BaseBytesTest):
     type2test = bytearray

diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
index 13ea02fe470d39eeb639694d4fd59107d68c20ad..b4efa65209d6134dae5ddcb2c1116a1833bf38ed 100644 (file)
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -669,6 +669,10 @@ class TestSubclassWithKwargs(unittest.TestCase):
         # SF bug #1486663 -- this used to erroneously raise a TypeError
         SubclassWithKwargs(newarg=1)
 
+    def test_free_after_iterating(self):
+        # For now, bypass tests that require slicing
+        self.skipTest("Exhausted deque iterator doesn't free a deque")
+
 #==============================================================================
 
 libreftest = """

diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py
index 1c63fc0e0987f84f2c3802c278e61ee781ea51e5..4474c9b91f04113ebcb1c7c51551b934c94181a7 100644 (file)
--- a/Lib/test/test_dict.py
+++ b/Lib/test/test_dict.py
@@ -681,6 +681,15 @@ class DictTest(unittest.TestCase):
         self._tracked(MyDict())
 
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, dict)
+        test_support.check_free_after_iterating(self, lambda d: d.iterkeys(), dict)
+        test_support.check_free_after_iterating(self, lambda d: d.itervalues(), dict)
+        test_support.check_free_after_iterating(self, lambda d: d.iteritems(), dict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewkeys()), dict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewvalues()), dict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewitems()), dict)
+
 from test import mapping_tests
 
 class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol):

diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py
index bd1b32d96b9cd965a3c8003a073e1aaae7a27a1a..4495fa1ad726a20a6de3d11dde4d68dfbe98973b 100644 (file)
--- a/Lib/test/test_iter.py
+++ b/Lib/test/test_iter.py
@@ -2,7 +2,8 @@
 
 import unittest
 from test.test_support import run_unittest, TESTFN, unlink, have_unicode, \
-                              check_py3k_warnings, cpython_only
+                              check_py3k_warnings, cpython_only, \
+                              check_free_after_iterating
 
 # Test result of triple loop (too big to inline)
 TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2),
@@ -921,6 +922,9 @@ class TestCase(unittest.TestCase):
         lst.extend(gen())
         self.assertEqual(len(lst), 760)
 
+    def test_free_after_iterating(self):
+        check_free_after_iterating(self, iter, SequenceClass, (0,))
+
 
 def test_main():
     run_unittest(TestCase)

diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py
index 78b3e836f4f0953f01e6ba6619222fa363fcdb5c..85e4841fbb4b2105f3c10a6e983498dca6435ba8 100644 (file)
--- a/Lib/test/test_ordered_dict.py
+++ b/Lib/test/test_ordered_dict.py
@@ -267,6 +267,15 @@ class TestOrderedDict(unittest.TestCase):
         items = [('a', 1), ('c', 3), ('b', 2)]
         self.assertEqual(list(MyOD(items).items()), items)
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: d.iterkeys(), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: d.itervalues(), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: d.iteritems(), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewkeys()), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewvalues()), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewitems()), OrderedDict)
+
 class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol):
     type2test = OrderedDict
 

diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py
index d9ea098b6d27c1d83b3eb61837e7acd8c0d8b869..6baaccf8986350b87658a477bd179bca66e9cc58 100644 (file)
--- a/Lib/test/test_set.py
+++ b/Lib/test/test_set.py
@@ -340,6 +340,9 @@ class TestJointOps(unittest.TestCase):
         gc.collect()
         self.assertTrue(ref() is None, "Cycle was not collected")
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, self.thetype)
+
 class TestSet(TestJointOps):
     thetype = set
 

diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
index 956936f87bf2c7ea5ff2e3387051b3063d95df47..85f99659e2f6ba8acbdf8894708e9f6b131ea990 100644 (file)
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -1659,3 +1659,21 @@ def strip_python_stderr(stderr):
     """
     stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip()
     return stderr
+
+
+def check_free_after_iterating(test, iter, cls, args=()):
+    class A(cls):
+        def __del__(self):
+            done[0] = True
+            try:
+                next(it)
+            except StopIteration:
+                pass
+
+    done = [False]
+    it = iter(A(*args))
+    # Issue 26494: Shouldn't crash
+    test.assertRaises(StopIteration, next, it)
+    # The sequence should be deallocated just after the end of iterating
+    gc_collect()
+    test.assertTrue(done[0])

diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 63fb8316b5f0eb994c51fc16e6a744f0f9e5aaf7..93224e17e959afbf8218efb58b29c2f018bafcb6 100644 (file)
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1857,6 +1857,11 @@ class UnicodeTest(
                     unicode_encodedecimal(u"123" + s, "xmlcharrefreplace"),
                     '123' + exp)
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, unicode)
+        test_support.check_free_after_iterating(self, reversed, unicode)
+
+
 def test_main():
     test_support.run_unittest(__name__)
 

diff --git a/Misc/NEWS b/Misc/NEWS
index 11bc97ef9da13e4bfa5e23dca170a441d1dff1aa..20b0a620958eb8bbabefffab01da5ea7a9b6f1f0 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@ What's New in Python 2.7.12?
 Core and Builtins
 -----------------
 
+- Issue #26494: Fixed crash on iterating exhausting iterators.
+  Affected classes are generic sequence iterators, iterators of bytearray,
+  list, tuple, set, frozenset, dict, OrderedDict and corresponding views.
+
 - Issue #26581: If coding cookie is specified multiple times on a line in
   Python source code file, only the first one is taken to account.
 

diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
index 1fdd9167fad7fcaf25fa977988bc58288aeb48d9..edb67a40dcd3479d8c142e7fd880050362134b85 100644 (file)
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -2982,8 +2982,8 @@ bytearrayiter_next(bytesiterobject *it)
         return item;
     }
 
-    Py_DECREF(seq);
     it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }
 

diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index fe19356de2f7b0ac4aa5a38cd471fdc3e6171a69..e3e4765d0a19c71d0686fb93537ac8d5204f2229 100644 (file)
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -2586,8 +2586,8 @@ static PyObject *dictiter_iternextkey(dictiterobject *di)
     return key;
 
 fail:
-    Py_DECREF(d);
     di->di_dict = NULL;
+    Py_DECREF(d);
     return NULL;
 }
 
@@ -2658,8 +2658,8 @@ static PyObject *dictiter_iternextvalue(dictiterobject *di)
     return value;
 
 fail:
-    Py_DECREF(d);
     di->di_dict = NULL;
+    Py_DECREF(d);
     return NULL;
 }
 
@@ -2744,8 +2744,8 @@ static PyObject *dictiter_iternextitem(dictiterobject *di)
     return result;
 
 fail:
-    Py_DECREF(d);
     di->di_dict = NULL;
+    Py_DECREF(d);
     return NULL;
 }
 

diff --git a/Objects/iterobject.c b/Objects/iterobject.c
index 9c90abe6012e7f95cb6e3063fbba188866f58568..346d2d97a99b223f181cb2abb35571ec31e201df 100644 (file)
--- a/Objects/iterobject.c
+++ b/Objects/iterobject.c
@@ -69,8 +69,8 @@ iter_iternext(PyObject *iterator)
         PyErr_ExceptionMatches(PyExc_StopIteration))
     {
         PyErr_Clear();
-        Py_DECREF(seq);
         it->it_seq = NULL;
+        Py_DECREF(seq);
     }
     return NULL;
 }

diff --git a/Objects/listobject.c b/Objects/listobject.c
index 27365b6e96bd2c879fceb79f55b7a956e8a3a5c5..c41462015169a85ba7ef83e5a24682ec56ebd6ca 100644 (file)
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -2915,8 +2915,8 @@ listiter_next(listiterobject *it)
         return item;
     }
 
-    Py_DECREF(seq);
     it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }
 
@@ -3018,9 +3018,17 @@ static PyObject *
 listreviter_next(listreviterobject *it)
 {
     PyObject *item;
-    Py_ssize_t index = it->it_index;
-    PyListObject *seq = it->it_seq;
+    Py_ssize_t index;
+    PyListObject *seq;
+
+    assert(it != NULL);
+    seq = it->it_seq;
+    if (seq == NULL) {
+        return NULL;
+    }
+    assert(PyList_Check(seq));
 
+    index = it->it_index;
     if (index>=0 && index < PyList_GET_SIZE(seq)) {
         item = PyList_GET_ITEM(seq, index);
         it->it_index--;
@@ -3028,10 +3036,8 @@ listreviter_next(listreviterobject *it)
         return item;
     }
     it->it_index = -1;
-    if (seq != NULL) {
-        it->it_seq = NULL;
-        Py_DECREF(seq);
-    }
+    it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }
 

diff --git a/Objects/setobject.c b/Objects/setobject.c
index 18e6898428b1956b4bee6f59933505bb8c7022b8..3958a6cac88683cc414fd2c91b4bdf2359b5dceb 100644 (file)
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -871,8 +871,8 @@ static PyObject *setiter_iternext(setiterobject *si)
     return key;
 
 fail:
-    Py_DECREF(so);
     si->si_set = NULL;
+    Py_DECREF(so);
     return NULL;
 }
 

diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index 5d72806959d1682383631d3a33d9ae3867124255..550719f327c4a57583e5beb8347a7fed6c0f54c8 100644 (file)
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -966,8 +966,8 @@ tupleiter_next(tupleiterobject *it)
         return item;
     }
 
-    Py_DECREF(seq);
     it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }