Issue #13555: cPickle now supports files larger than 2 GiB.

diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index c321f8e78a593f8dade95d9538fd53e90c375019..f735e955a1da9e480feb375d1a863d91162925d1 100644 (file)
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -6,7 +6,8 @@ import cStringIO
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, have_unicode, TESTFN
+from test.test_support import (TestFailed, have_unicode, TESTFN, _2G, _1M,
+                               precisionbigmemtest)
 
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
@@ -1280,3 +1281,31 @@ class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
         f.write(pickled2)
         f.seek(0)
         self.assertEqual(unpickler.load(), data2)
+
+class BigmemPickleTests(unittest.TestCase):
+
+    # Memory requirements: 1 byte per character for input strings, 1 byte
+    # for pickled data, 1 byte for unpickled strings, 1 byte for internal
+    # buffer and 1 byte of free space for resizing of internal buffer.
+
+    @precisionbigmemtest(size=_2G + 100*_1M, memuse=5)
+    def test_huge_strlist(self, size):
+        chunksize = 2**20
+        data = []
+        while size > chunksize:
+            data.append('x' * chunksize)
+            size -= chunksize
+            chunksize += 1
+        data.append('y' * size)
+
+        try:
+            for proto in protocols:
+                try:
+                    pickled = self.dumps(data, proto)
+                    res = self.loads(pickled)
+                    self.assertEqual(res, data)
+                finally:
+                    res = None
+                    pickled = None
+        finally:
+            data = None

diff --git a/Lib/test/test_cpickle.py b/Lib/test/test_cpickle.py
index c240669873a1a5734a00fcd78cf1151d249df0e2..beea3180e3dbdac02afeace18c316ea1bca9764d 100644 (file)
--- a/Lib/test/test_cpickle.py
+++ b/Lib/test/test_cpickle.py
@@ -1,7 +1,9 @@
 import cPickle, unittest
 from cStringIO import StringIO
-from test.pickletester import AbstractPickleTests, AbstractPickleModuleTests
-from test.pickletester import AbstractPicklerUnpicklerObjectTests
+from test.pickletester import (AbstractPickleTests,
+                               AbstractPickleModuleTests,
+                               AbstractPicklerUnpicklerObjectTests,
+                               BigmemPickleTests)
 from test import test_support
 
 class cPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
@@ -101,6 +103,16 @@ class cPicklePicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
     pickler_class = cPickle.Pickler
     unpickler_class = cPickle.Unpickler
 
+class cPickleBigmemPickleTests(BigmemPickleTests):
+
+    def dumps(self, arg, proto=0, fast=0):
+        # Ignore fast
+        return cPickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        # Ignore fast
+        return cPickle.loads(buf)
+
 
 class Node(object):
     pass
@@ -133,6 +145,7 @@ def test_main():
         cPickleFastPicklerTests,
         cPickleDeepRecursive,
         cPicklePicklerUnpicklerObjectTests,
+        cPickleBigmemPickleTests,
     )
 
 if __name__ == "__main__":

diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index 6c83811be821ea5bc7bd6340d32eb0eda30d6241..c3126491ad29d78f641ac9116371d115cfa12cbd 100644 (file)
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -3,10 +3,11 @@ from cStringIO import StringIO
 
 from test import test_support
 
-from test.pickletester import AbstractPickleTests
-from test.pickletester import AbstractPickleModuleTests
-from test.pickletester import AbstractPersistentPicklerTests
-from test.pickletester import AbstractPicklerUnpicklerObjectTests
+from test.pickletester import (AbstractPickleTests,
+                               AbstractPickleModuleTests,
+                               AbstractPersistentPicklerTests,
+                               AbstractPicklerUnpicklerObjectTests,
+                               BigmemPickleTests)
 
 class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
 
@@ -66,6 +67,16 @@ class PicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
     pickler_class = pickle.Pickler
     unpickler_class = pickle.Unpickler
 
+class PickleBigmemPickleTests(BigmemPickleTests):
+
+    def dumps(self, arg, proto=0, fast=0):
+        # Ignore fast
+        return pickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        # Ignore fast
+        return pickle.loads(buf)
+
 
 def test_main():
     test_support.run_unittest(
@@ -73,6 +84,7 @@ def test_main():
         PicklerTests,
         PersPicklerTests,
         PicklerUnpicklerObjectTests,
+        PickleBigmemPickleTests,
     )
     test_support.run_doctest(pickle)
 

diff --git a/Misc/NEWS b/Misc/NEWS
index a31e3a478a33099443ad70b0dea8bc8908ccd22e..b3d28fd7e5f1f48badeffb881120c752b31a8a34 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -202,6 +202,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #13555: cPickle now supports files larger than 2 GiB.
+
 - Issue #17052: unittest discovery should use self.testLoader.
 
 - Issue #4591: Uid and gid values larger than 2**31 are supported now.

diff --git a/Modules/cPickle.c b/Modules/cPickle.c
index 48b5075ec2bbbd2ac7f3072dbdf717399a71e3e8..e73bac1aceef9a4c593d309943f8c2e0455e5bae 100644 (file)
--- a/Modules/cPickle.c
+++ b/Modules/cPickle.c
@@ -139,15 +139,15 @@ static PyObject *__class___str, *__getinitargs___str, *__dict___str,
 
 typedef struct {
     PyObject_HEAD
-    int length;         /* number of initial slots in data currently used */
-    int size;           /* number of slots in data allocated */
+    Py_ssize_t length;  /* number of initial slots in data currently used */
+    Py_ssize_t size;    /* number of slots in data allocated */
     PyObject **data;
 } Pdata;
 
 static void
 Pdata_dealloc(Pdata *self)
 {
-    int i;
+    Py_ssize_t i;
     PyObject **p;
 
     for (i = self->length, p = self->data; --i >= 0; p++) {
@@ -193,9 +193,9 @@ stackUnderflow(void)
  * number of items, this is a (non-erroneous) NOP.
  */
 static int
-Pdata_clear(Pdata *self, int clearto)
+Pdata_clear(Pdata *self, Py_ssize_t clearto)
 {
-    int i;
+    Py_ssize_t i;
     PyObject **p;
 
     if (clearto < 0) return stackUnderflow();
@@ -214,18 +214,17 @@ Pdata_clear(Pdata *self, int clearto)
 static int
 Pdata_grow(Pdata *self)
 {
-    int bigger;
-    size_t nbytes;
+    Py_ssize_t bigger;
+    Py_ssize_t nbytes;
+
     PyObject **tmp;
 
-    bigger = self->size << 1;
-    if (bigger <= 0)            /* was 0, or new value overflows */
+    if (self->size > (PY_SSIZE_T_MAX >> 1))
         goto nomemory;
-    if ((int)(size_t)bigger != bigger)
-        goto nomemory;
-    nbytes = (size_t)bigger * sizeof(PyObject *);
-    if (nbytes / sizeof(PyObject *) != (size_t)bigger)
+    bigger = self->size << 1;
+    if (bigger > (PY_SSIZE_T_MAX / sizeof(PyObject *)))
         goto nomemory;
+    nbytes = bigger * sizeof(PyObject *);
     tmp = realloc(self->data, nbytes);
     if (tmp == NULL)
         goto nomemory;
@@ -280,10 +279,10 @@ Pdata_grow(Pdata *self)
 
 
 static PyObject *
-Pdata_popTuple(Pdata *self, int start)
+Pdata_popTuple(Pdata *self, Py_ssize_t start)
 {
     PyObject *r;
-    int i, j, l;
+    Py_ssize_t i, j, l;
 
     l = self->length-start;
     r = PyTuple_New(l);
@@ -297,10 +296,10 @@ Pdata_popTuple(Pdata *self, int start)
 }
 
 static PyObject *
-Pdata_popList(Pdata *self, int start)
+Pdata_popList(Pdata *self, Py_ssize_t start)
 {
     PyObject *r;
-    int i, j, l;
+    Py_ssize_t i, j, l;
 
     l=self->length-start;
     if (!( r=PyList_New(l)))  return NULL;
@@ -347,9 +346,9 @@ typedef struct Picklerobject {
     int bin;
 
     int fast; /* Fast mode doesn't save in memo, don't use if circ ref */
-    int (*write_func)(struct Picklerobject *, const char *, Py_ssize_t);
+    Py_ssize_t (*write_func)(struct Picklerobject *, const char *, Py_ssize_t);
     char *write_buf;
-    int buf_size;
+    Py_ssize_t buf_size;
     PyObject *dispatch_table;
     int fast_container; /* count nested container dumps */
     PyObject *fast_memo;
@@ -373,12 +372,12 @@ typedef struct Unpicklerobject {
     PyObject *mark;
     PyObject *pers_func;
     PyObject *last_string;
-    int *marks;
-    int num_marks;
-    int marks_size;
+    Py_ssize_t *marks;
+    Py_ssize_t num_marks;
+    Py_ssize_t marks_size;
     Py_ssize_t (*read_func)(struct Unpicklerobject *, char **, Py_ssize_t);
     Py_ssize_t (*readline_func)(struct Unpicklerobject *, char **);
-    int buf_size;
+    Py_ssize_t buf_size;
     char *buf;
     PyObject *find_class;
 } Unpicklerobject;
@@ -424,7 +423,7 @@ cPickle_ErrFormat(PyObject *ErrType, char *stringformat, char *format, ...)
     return NULL;
 }
 
-static int
+static Py_ssize_t
 write_file(Picklerobject *self, const char *s, Py_ssize_t  n)
 {
     size_t nbyteswritten;
@@ -433,11 +432,6 @@ write_file(Picklerobject *self, const char *s, Py_ssize_t  n)
         return 0;
     }
 
-    if (n > INT_MAX) {
-        /* String too large */
-        return -1;
-    }
-
     PyFile_IncUseCount((PyFileObject *)self->file);
     Py_BEGIN_ALLOW_THREADS
     nbyteswritten = fwrite(s, sizeof(char), n, self->fp);
@@ -448,40 +442,44 @@ write_file(Picklerobject *self, const char *s, Py_ssize_t  n)
         return -1;
     }
 
-    return (int)n;
+    return n;
 }
 
-static int
+static Py_ssize_t
 write_cStringIO(Picklerobject *self, const char *s, Py_ssize_t  n)
 {
+    Py_ssize_t len = n;
+
     if (s == NULL) {
         return 0;
     }
 
+    while (n > INT_MAX) {
+        if (PycStringIO->cwrite((PyObject *)self->file, s, INT_MAX) != INT_MAX) {
+            return -1;
+        }
+        n -= INT_MAX;
+    }
+
     if (PycStringIO->cwrite((PyObject *)self->file, s, n) != n) {
         return -1;
     }
 
-    return (int)n;
+    return len;
 }
 
-static int
+static Py_ssize_t
 write_none(Picklerobject *self, const char *s, Py_ssize_t  n)
 {
     if (s == NULL) return 0;
-    if (n > INT_MAX) return -1;
-    return (int)n;
+    return n;
 }
 
-static int
-write_other(Picklerobject *self, const char *s, Py_ssize_t  _n)
+static Py_ssize_t
+write_other(Picklerobject *self, const char *s, Py_ssize_t  n)
 {
     PyObject *py_str = 0, *junk = 0;
-    int n;
 
-    if (_n > INT_MAX)
-        return -1;
-    n = (int)_n;
     if (s == NULL) {
         if (!( self->buf_size ))  return 0;
         py_str = PyString_FromStringAndSize(self->write_buf,
@@ -490,7 +488,7 @@ write_other(Picklerobject *self, const char *s, Py_ssize_t  _n)
             return -1;
     }
     else {
-        if (self->buf_size && (n + self->buf_size) > WRITE_BUF_SIZE) {
+        if (self->buf_size && n > WRITE_BUF_SIZE - self->buf_size) {
             if (write_other(self, NULL, 0) < 0)
                 return -1;
         }
@@ -531,7 +529,7 @@ read_file(Unpicklerobject *self, char **s, Py_ssize_t n)
     size_t nbytesread;
 
     if (self->buf_size == 0) {
-        int size;
+        Py_ssize_t size;
 
         size = ((n < 32) ? 32 : n);
         if (!( self->buf = (char *)malloc(size))) {
@@ -575,7 +573,7 @@ read_file(Unpicklerobject *self, char **s, Py_ssize_t n)
 static Py_ssize_t
 readline_file(Unpicklerobject *self, char **s)
 {
-    int i;
+    Py_ssize_t i;
 
     if (self->buf_size == 0) {
         if (!( self->buf = (char *)malloc(40))) {
@@ -587,7 +585,7 @@ readline_file(Unpicklerobject *self, char **s)
 
     i = 0;
     while (1) {
-        int bigger;
+        Py_ssize_t bigger;
         char *newbuf;
         for (; i < (self->buf_size - 1); i++) {
             if (feof(self->fp) ||
@@ -597,13 +595,13 @@ readline_file(Unpicklerobject *self, char **s)
                 return i + 1;
             }
         }
-        bigger = self->buf_size << 1;
-        if (bigger <= 0) {              /* overflow */
+        if (self->buf_size < (PY_SSIZE_T_MAX >> 1)) {
             PyErr_NoMemory();
             return -1;
         }
+        bigger = self->buf_size << 1;
         newbuf = (char *)realloc(self->buf, bigger);
-        if (!newbuf)  {
+        if (newbuf == NULL)  {
             PyErr_NoMemory();
             return -1;
         }
@@ -616,30 +614,63 @@ readline_file(Unpicklerobject *self, char **s)
 static Py_ssize_t
 read_cStringIO(Unpicklerobject *self, char **s, Py_ssize_t  n)
 {
-    char *ptr;
+    Py_ssize_t len = n;
+    char *start, *end = NULL;
 
-    if (PycStringIO->cread((PyObject *)self->file, &ptr, n) != n) {
-        PyErr_SetNone(PyExc_EOFError);
-        return -1;
+    while (1) {
+        int k;
+        char *ptr;
+        if (n > INT_MAX)
+            k = INT_MAX;
+        else
+            k = (int)n;
+        if (PycStringIO->cread((PyObject *)self->file, &ptr, k) != k) {
+            PyErr_SetNone(PyExc_EOFError);
+            return -1;
+        }
+        if (end == NULL)
+            start = ptr;
+        else if (ptr != end) {
+            /* non-continuous area */
+            return -1;
+        }
+        if (n <= INT_MAX)
+            break;
+        end = ptr + INT_MAX;
+        n -= INT_MAX;
     }
 
-    *s = ptr;
+    *s = start;
 
-    return n;
+    return len;
 }
 
 
 static Py_ssize_t
 readline_cStringIO(Unpicklerobject *self, char **s)
 {
-    Py_ssize_t n;
-    char *ptr;
+    Py_ssize_t n = 0;
+    char *start = NULL, *end = NULL;
 
-    if ((n = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {
-        return -1;
+    while (1) {
+        int k;
+        char *ptr;
+        if ((k = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {
+            return -1;
+        }
+        n += k;
+        if (end == NULL)
+            start = ptr;
+        else if (ptr != end) {
+            /* non-continuous area */
+            return -1;
+        }
+        if (k == 0 || ptr[k - 1] == '\n')
+            break;
+        end = ptr + k;
     }
 
-    *s = ptr;
+    *s = start;
 
     return n;
 }
@@ -700,7 +731,7 @@ readline_other(Unpicklerobject *self, char **s)
  * The caller is responsible for free()'ing the return value.
  */
 static char *
-pystrndup(const char *s, int n)
+pystrndup(const char *s, Py_ssize_t n)
 {
     char *r = (char *)malloc(n+1);
     if (r == NULL)
@@ -715,7 +746,7 @@ static int
 get(Picklerobject *self, PyObject *id)
 {
     PyObject *value, *mv;
-    long c_value;
+    Py_ssize_t c_value;
     char s[30];
     size_t len;
 
@@ -735,7 +766,8 @@ get(Picklerobject *self, PyObject *id)
 
     if (!self->bin) {
         s[0] = GET;
-        PyOS_snprintf(s + 1, sizeof(s) - 1, "%ld\n", c_value);
+        PyOS_snprintf(s + 1, sizeof(s) - 1,
+                      "%" PY_FORMAT_SIZE_T "d\n", c_value);
         len = strlen(s);
     }
     else if (Pdata_Check(self->file)) {
@@ -780,8 +812,7 @@ static int
 put2(Picklerobject *self, PyObject *ob)
 {
     char c_str[30];
-    int p;
-    size_t len;
+    Py_ssize_t len, p;
     int res = -1;
     PyObject *py_ob_id = 0, *memo_len = 0, *t = 0;
 
@@ -818,7 +849,8 @@ put2(Picklerobject *self, PyObject *ob)
 
     if (!self->bin) {
         c_str[0] = PUT;
-        PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%d\n", p);
+        PyOS_snprintf(c_str + 1, sizeof(c_str) - 1,
+                      "%" PY_FORMAT_SIZE_T "d\n", p);
         len = strlen(c_str);
     }
     else if (Pdata_Check(self->file)) {
@@ -994,7 +1026,7 @@ save_int(Picklerobject *self, PyObject *args)
 {
     char c_str[32];
     long l = PyInt_AS_LONG((PyIntObject *)args);
-    int len = 0;
+    Py_ssize_t len = 0;
 
     if (!self->bin
 #if SIZEOF_LONG > 4
@@ -1201,7 +1233,7 @@ done:
 static int
 save_string(Picklerobject *self, PyObject *args, int doput)
 {
-    int size, len;
+    Py_ssize_t size, len;
     PyObject *repr=0;
 
     if ((size = PyString_Size(args)) < 0)
@@ -1448,7 +1480,7 @@ save_unicode(Picklerobject *self, PyObject *args, int doput)
 static int
 store_tuple_elements(Picklerobject *self, PyObject *t, int len)
 {
-    int i;
+    Py_ssize_t i;
     int res = -1;       /* guilty until proved innocent */
 
     assert(PyTuple_Size(t) == len);
@@ -1477,7 +1509,7 @@ static int
 save_tuple(Picklerobject *self, PyObject *args)
 {
     PyObject *py_tuple_id = NULL;
-    int len, i;
+    Py_ssize_t len, i;
     int res = -1;
 
     static char tuple = TUPLE;
@@ -1690,7 +1722,7 @@ save_list(Picklerobject *self, PyObject *args)
 {
     int res = -1;
     char s[3];
-    int len;
+    Py_ssize_t len;
     PyObject *iter;
 
     if (self->fast && !fast_save_enter(self, args))
@@ -1943,7 +1975,7 @@ save_dict(Picklerobject *self, PyObject *args)
 {
     int res = -1;
     char s[3];
-    int len;
+    Py_ssize_t len;
 
     if (self->fast && !fast_save_enter(self, args))
         goto finally;
@@ -2027,7 +2059,7 @@ save_inst(Picklerobject *self, PyObject *args)
 
     if ((getinitargs_func = PyObject_GetAttr(args, __getinitargs___str))) {
         PyObject *element = 0;
-        int i, len;
+        Py_ssize_t i, len;
 
         if (!( class_args =
                PyObject_Call(getinitargs_func, empty_tuple, NULL)))
@@ -2289,7 +2321,8 @@ static int
 save_pers(Picklerobject *self, PyObject *args, PyObject *f)
 {
     PyObject *pid = 0;
-    int size, res = -1;
+    Py_ssize_t size;
+    int res = -1;
 
     static char persid = PERSID, binpersid = BINPERSID;
 
@@ -2431,7 +2464,7 @@ save_reduce(Picklerobject *self, PyObject *args, PyObject *fn, PyObject *ob)
     if (use_newobj) {
         PyObject *cls;
         PyObject *newargtup;
-        int n, i;
+        Py_ssize_t n, i;
 
         /* Sanity checks. */
         n = PyTuple_Size(argtup);
@@ -2815,7 +2848,7 @@ Pickle_clear_memo(Picklerobject *self, PyObject *args)
 static PyObject *
 Pickle_getvalue(Picklerobject *self, PyObject *args)
 {
-    int l, i, rsize, ssize, clear=1, lm;
+    Py_ssize_t l, i, rsize, ssize, clear=1, lm;
     long ik;
     PyObject *k, *r;
     char *s, *p, *have_get;
@@ -3314,7 +3347,7 @@ find_class(PyObject *py_module_name, PyObject *py_global_name, PyObject *fc)
     return global;
 }
 
-static int
+static Py_ssize_t
 marker(Unpicklerobject *self)
 {
     if (self->num_marks < 1) {
@@ -3345,7 +3378,8 @@ load_int(Unpicklerobject *self)
 {
     PyObject *py_int = 0;
     char *endptr, *s;
-    int len, res = -1;
+    Py_ssize_t len;
+    int res = -1;
     long l;
 
     if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -3477,7 +3511,8 @@ load_long(Unpicklerobject *self)
 {
     PyObject *l = 0;
     char *end, *s;
-    int len, res = -1;
+    Py_ssize_t len;
+    int res = -1;
 
     if ((len = self->readline_func(self, &s)) < 0) return -1;
     if (len < 2) return bad_readline();
@@ -3541,7 +3576,8 @@ load_float(Unpicklerobject *self)
 {
     PyObject *py_float = 0;
     char *endptr, *s;
-    int len, res = -1;
+    Py_ssize_t len;
+    int res = -1;
     double d;
 
     if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -3597,7 +3633,8 @@ static int
 load_string(Unpicklerobject *self)
 {
     PyObject *str = 0;
-    int len, res = -1;
+    Py_ssize_t len;
+    int res = -1;
     char *s, *p;
 
     if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -3639,7 +3676,7 @@ static int
 load_binstring(Unpicklerobject *self)
 {
     PyObject *py_string = 0;
-    long l;
+    Py_ssize_t l;
     char *s;
 
     if (self->read_func(self, &s, 4) < 0) return -1;
@@ -3691,20 +3728,17 @@ static int
 load_unicode(Unpicklerobject *self)
 {
     PyObject *str = 0;
-    int len, res = -1;
+    Py_ssize_t len;
     char *s;
 
     if ((len = self->readline_func(self, &s)) < 0) return -1;
     if (len < 1) return bad_readline();
 
     if (!( str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL)))
-        goto finally;
+        return -1;
 
     PDATA_PUSH(self->stack, str, -1);
     return 0;
-
-  finally:
-    return res;
 }
 #endif
 
@@ -3714,7 +3748,7 @@ static int
 load_binunicode(Unpicklerobject *self)
 {
     PyObject *unicode;
-    long l;
+    Py_ssize_t l;
     char *s;
 
     if (self->read_func(self, &s, 4) < 0) return -1;
@@ -3745,7 +3779,7 @@ static int
 load_tuple(Unpicklerobject *self)
 {
     PyObject *tup;
-    int i;
+    Py_ssize_t i;
 
     if ((i = marker(self)) < 0) return -1;
     if (!( tup=Pdata_popTuple(self->stack, i)))  return -1;
@@ -3798,7 +3832,7 @@ static int
 load_list(Unpicklerobject *self)
 {
     PyObject *list = 0;
-    int i;
+    Py_ssize_t i;
 
     if ((i = marker(self)) < 0) return -1;
     if (!( list=Pdata_popList(self->stack, i)))  return -1;
@@ -3810,7 +3844,7 @@ static int
 load_dict(Unpicklerobject *self)
 {
     PyObject *dict, *key, *value;
-    int i, j, k;
+    Py_ssize_t i, j, k;
 
     if ((i = marker(self)) < 0) return -1;
     j=self->stack->length;
@@ -3886,7 +3920,7 @@ static int
 load_obj(Unpicklerobject *self)
 {
     PyObject *class, *tup, *obj=0;
-    int i;
+    Py_ssize_t i;
 
     if ((i = marker(self)) < 0) return -1;
     if (!( tup=Pdata_popTuple(self->stack, i+1)))  return -1;
@@ -3907,7 +3941,7 @@ static int
 load_inst(Unpicklerobject *self)
 {
     PyObject *tup, *class=0, *obj=0, *module_name, *class_name;
-    int i, len;
+    Py_ssize_t i, len;
     char *s;
 
     if ((i = marker(self)) < 0) return -1;
@@ -3993,7 +4027,7 @@ static int
 load_global(Unpicklerobject *self)
 {
     PyObject *class = 0, *module_name = 0, *class_name = 0;
-    int len;
+    Py_ssize_t len;
     char *s;
 
     if ((len = self->readline_func(self, &s)) < 0) return -1;
@@ -4024,7 +4058,7 @@ static int
 load_persid(Unpicklerobject *self)
 {
     PyObject *pid = 0;
-    int len;
+    Py_ssize_t len;
     char *s;
 
     if (self->pers_func) {
@@ -4102,7 +4136,7 @@ load_binpersid(Unpicklerobject *self)
 static int
 load_pop(Unpicklerobject *self)
 {
-    int len = self->stack->length;
+    Py_ssize_t len = self->stack->length;
 
     /* Note that we split the (pickle.py) stack into two stacks,
        an object stack and a mark stack. We have to be clever and
@@ -4127,7 +4161,7 @@ load_pop(Unpicklerobject *self)
 static int
 load_pop_mark(Unpicklerobject *self)
 {
-    int i;
+    Py_ssize_t i;
 
     if ((i = marker(self)) < 0)
         return -1;
@@ -4142,7 +4176,7 @@ static int
 load_dup(Unpicklerobject *self)
 {
     PyObject *last;
-    int len;
+    Py_ssize_t len;
 
     if ((len = self->stack->length) <= 0) return stackUnderflow();
     last=self->stack->data[len-1];
@@ -4156,7 +4190,7 @@ static int
 load_get(Unpicklerobject *self)
 {
     PyObject *py_str = 0, *value = 0;
-    int len;
+    Py_ssize_t len;
     char *s;
     int rc;
 
@@ -4214,7 +4248,7 @@ load_long_binget(Unpicklerobject *self)
     PyObject *py_key = 0, *value = 0;
     unsigned char c;
     char *s;
-    long key;
+    Py_ssize_t key;
     int rc;
 
     if (self->read_func(self, &s, 4) < 0) return -1;
@@ -4317,7 +4351,7 @@ static int
 load_put(Unpicklerobject *self)
 {
     PyObject *py_str = 0, *value = 0;
-    int len, l;
+    Py_ssize_t len, l;
     char *s;
 
     if ((l = self->readline_func(self, &s)) < 0) return -1;
@@ -4337,7 +4371,7 @@ load_binput(Unpicklerobject *self)
     PyObject *py_key = 0, *value = 0;
     unsigned char key;
     char *s;
-    int len;
+    Py_ssize_t len;
 
     if (self->read_func(self, &s, 1) < 0) return -1;
     if (!( (len=self->stack->length) > 0 ))  return stackUnderflow();
@@ -4356,10 +4390,10 @@ static int
 load_long_binput(Unpicklerobject *self)
 {
     PyObject *py_key = 0, *value = 0;
-    long key;
+    Py_ssize_t key;
     unsigned char c;
     char *s;
-    int len;
+    Py_ssize_t len;
 
     if (self->read_func(self, &s, 4) < 0) return -1;
     if (!( len=self->stack->length ))  return stackUnderflow();
@@ -4382,10 +4416,10 @@ load_long_binput(Unpicklerobject *self)
 
 
 static int
-do_append(Unpicklerobject *self, int  x)
+do_append(Unpicklerobject *self, Py_ssize_t  x)
 {
     PyObject *value = 0, *list = 0, *append_method = 0;
-    int len, i;
+    Py_ssize_t len, i;
 
     len=self->stack->length;
     if (!( len >= x && x > 0 ))  return stackUnderflow();
@@ -4451,11 +4485,11 @@ load_appends(Unpicklerobject *self)
 }
 
 
-static int
-do_setitems(Unpicklerobject *self, int  x)
+static Py_ssize_t
+do_setitems(Unpicklerobject *self, Py_ssize_t x)
 {
     PyObject *value = 0, *key = 0, *dict = 0;
-    int len, i, r=0;
+    Py_ssize_t len, i, r=0;
 
     if (!( (len=self->stack->length) >= x
            && x > 0 ))  return stackUnderflow();
@@ -4496,8 +4530,8 @@ load_build(Unpicklerobject *self)
     PyObject *state, *inst, *slotstate;
     PyObject *__setstate__;
     PyObject *d_key, *d_value;
-    Py_ssize_t i;
     int res = -1;
+    Py_ssize_t i;
 
     /* Stack is ... instance, state.  We want to leave instance at
      * the stack top, possibly mutated via instance.__setstate__(state).
@@ -4596,7 +4630,7 @@ load_build(Unpicklerobject *self)
 static int
 load_mark(Unpicklerobject *self)
 {
-    int s;
+    Py_ssize_t s;
 
     /* Note that we split the (pickle.py) stack into two stacks, an
        object stack and a mark stack. Here we push a mark onto the
@@ -4604,14 +4638,14 @@ load_mark(Unpicklerobject *self)
     */
 
     if ((self->num_marks + 1) >= self->marks_size) {
-        int *marks;
+        Py_ssize_t *marks;
         s=self->marks_size+20;
         if (s <= self->num_marks) s=self->num_marks + 1;
         if (self->marks == NULL)
-            marks=(int *)malloc(s * sizeof(int));
+            marks=(Py_ssize_t *)malloc(s * sizeof(Py_ssize_t));
         else
-            marks=(int *)realloc(self->marks,
-                                       s * sizeof(int));
+            marks=(Py_ssize_t *)realloc(self->marks,
+                                        s * sizeof(Py_ssize_t));
         if (!marks) {
             PyErr_NoMemory();
             return -1;
@@ -4981,7 +5015,7 @@ load(Unpicklerobject *self)
 static int
 noload_obj(Unpicklerobject *self)
 {
-    int i;
+    Py_ssize_t i;
 
     if ((i = marker(self)) < 0) return -1;
     return Pdata_clear(self->stack, i+1);
@@ -4991,7 +5025,7 @@ noload_obj(Unpicklerobject *self)
 static int
 noload_inst(Unpicklerobject *self)
 {
-    int i;
+    Py_ssize_t i;
     char *s;
 
     if ((i = marker(self)) < 0) return -1;
@@ -5068,7 +5102,7 @@ noload_append(Unpicklerobject *self)
 static int
 noload_appends(Unpicklerobject *self)
 {
-    int i;
+    Py_ssize_t i;
     if ((i = marker(self)) < 0) return -1;
     return Pdata_clear(self->stack, i);
 }
@@ -5082,7 +5116,7 @@ noload_setitem(Unpicklerobject *self)
 static int
 noload_setitems(Unpicklerobject *self)
 {
-    int i;
+    Py_ssize_t i;
     if ((i = marker(self)) < 0) return -1;
     return Pdata_clear(self->stack, i);
 }

diff --git a/Modules/cStringIO.c b/Modules/cStringIO.c
index 9922d03f3ac22b6d054c2655b24b0eca981c7fe9..566851f66d8533ec050e8dddfb5e70faadcfd059 100644 (file)
--- a/Modules/cStringIO.c
+++ b/Modules/cStringIO.c
@@ -210,11 +210,8 @@ IO_creadline(PyObject *self, char **output) {
     if (n < end) n++;
 
     len = n - start;
-    if (len > INT_MAX) {
-        PyErr_SetString(PyExc_OverflowError,
-                        "length too large");
-        return -1;
-    }
+    if (len > INT_MAX)
+        len = INT_MAX;
 
     *output=start;