From: Serhiy Storchaka Date: Sun, 2 Oct 2016 06:13:14 +0000 (+0300) Subject: Issue #28322: Fixed possible crashes when unpickle itertools objects from X-Git-Tag: v3.6.0b2~73 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8f0f2056499847999fffa7af7a8872500a191203;p=python Issue #28322: Fixed possible crashes when unpickle itertools objects from incorrect pickle data. Based on patch by John Leitch. --- 8f0f2056499847999fffa7af7a8872500a191203 diff --cc Lib/test/test_itertools.py index d21d8ed32d,e054303ded..ea1f57caad --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@@ -630,56 -644,25 +643,59 @@@ class TestBasicOps(unittest.TestCase) for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # test with partial consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(2)] # consume 2 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + + # test with completely consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(7)] # consume 7 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_setstate(self): + # Verify both modes for restoring state + + # Mode 0 is efficient. It uses an incompletely consumed input + # iterator to build a cycle object and then passes in state with + # a list of previously consumed values. There is no data + # overlap between the two. + c = cycle('defg') + c.__setstate__((list('abc'), 0)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # Mode 1 is inefficient. It starts with a cycle object built + # from an iterator over the remaining elements in a partial + # cycle and then passes in state with all of the previously + # seen values (this overlaps values included in the iterator). + c = cycle('defg') + c.__setstate__((list('abcdefg'), 1)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # The first argument to setstate needs to be a tuple - with self.assertRaises(SystemError): ++ with self.assertRaises(TypeError): + cycle('defg').__setstate__([list('abcdefg'), 0]) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + c = cycle('defg') - c.__setstate__((dict.fromkeys('defg'), 0)) - take(20, c) ++ c.__setstate__((tuple('defg'), 0)) ++ take(20, c) + - # The first argument in the setstate tuple must be a list ++ # The second argument in the setstate tuple must be an int + with self.assertRaises(TypeError): + cycle('defg').__setstate__((list('abcdefg'), 'x')) + + self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, []) - self.assertRaises(TypeError, cycle('').__setstate__, 0) + self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - self.assertRaises(TypeError, cycle('').__setstate__, ((), 0)) - it = cycle('abc') - it.__setstate__((['de', 'fg'], 0)) - self.assertEqual(list(islice(it, 15)), - ['a', 'b', 'c', 'de', 'fg', - 'a', 'b', 'c', 'de', 'fg', - 'a', 'b', 'c', 'de', 'fg']) - it = cycle('abc') - it.__setstate__((['de', 'fg'], 1)) - self.assertEqual(list(islice(it, 15)), - ['a', 'b', 'c', 'de', 'fg', - 'de', 'fg', 'de', 'fg', 'de', - 'fg', 'de', 'fg', 'de', 'fg']) + def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) diff --cc Misc/NEWS index 8b102a53df,e87ed71e7a..831cb22f6b --- a/Misc/NEWS +++ b/Misc/NEWS @@@ -46,19 -85,11 +46,22 @@@ Core and Builtin Library ------- + - Issue #28322: Fixed possible crashes when unpickle itertools objects from + incorrect pickle data. Based on patch by John Leitch. + -- Issue #1703178: Fix the ability to pass the --link-objects option to the - distutils build_ext command. +- Issue #28228: imghdr now supports pathlib. + +- Issue #28226: compileall now supports pathlib. + +- Issue #28314: Fix function declaration (C flags) for the getiterator() method + of xml.etree.ElementTree.Element. + +- Issue #28148: Stop using localtime() and gmtime() in the time + module. + + Introduced platform independent _PyTime_localtime API that is + similar to POSIX localtime_r, but available on all platforms. Patch + by Ed Schouten. - Issue #28253: Fixed calendar functions for extreme months: 0001-01 and 9999-12. diff --cc Modules/itertoolsmodule.c index 62b6a0c2b5,be0f498252..6bf04cbee3 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@@ -971,13 -976,16 +981,17 @@@ cycle_setstate(cycleobject *lz, PyObjec { PyObject *saved=NULL; int firstpass; - - if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "state is not a tuple"); + return NULL; + } + if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { return NULL; + } - Py_XINCREF(saved); + Py_INCREF(saved); Py_XSETREF(lz->saved, saved); lz->firstpass = firstpass != 0; + lz->index = 0; Py_RETURN_NONE; } @@@ -3262,13 -3276,20 +3285,18 @@@ permutations_setstate(permutationsobjec PyObject *indices, *cycles, *result; Py_ssize_t n, i; + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "state is not a tuple"); + return NULL; + } if (!PyArg_ParseTuple(state, "O!O!", &PyTuple_Type, &indices, - &PyTuple_Type, &cycles)) + &PyTuple_Type, &cycles)) { return NULL; + } n = PyTuple_GET_SIZE(po->pool); - if (PyTuple_GET_SIZE(indices) != n || - PyTuple_GET_SIZE(cycles) != po->r) - { + if (PyTuple_GET_SIZE(indices) != n || PyTuple_GET_SIZE(cycles) != po->r) { PyErr_SetString(PyExc_ValueError, "invalid arguments"); return NULL; }