Concatenation on a long string breaks (SF #1526585).

diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
index 26c4d0409faf787056e7f3f3ac4dfbaae37d2b22..4553838afcd6ef2f9287092e0ff7532f0476ed4d 100755 (executable)
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -66,7 +66,9 @@ reports are written to.  These parameters all have defaults (5, 4 and
 
 -M runs tests that require an exorbitant amount of memory. These tests
 typically try to ascertain containers keep working when containing more than
-2 bilion objects, and only work on 64-bit systems. The passed-in memlimit,
+2 billion objects, which only works on 64-bit systems. There are also some
+tests that try to exhaust the address space of the process, which only makes
+sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,
 which is a string in the form of '2.5Gb', determines howmuch memory the
 tests will limit themselves to (but they may go slightly over.) The number
 shouldn't be more memory than the machine has (including swap memory). You

diff --git a/Lib/test/test_bigaddrspace.py b/Lib/test/test_bigaddrspace.py
new file mode 100644 (file)
index 0000000..8c215fe
--- /dev/null
+++ b/Lib/test/test_bigaddrspace.py
@@ -0,0 +1,46 @@
+from test import test_support
+from test.test_support import bigaddrspacetest, MAX_Py_ssize_t
+
+import unittest
+import operator
+import sys
+
+
+class StrTest(unittest.TestCase):
+
+    @bigaddrspacetest
+    def test_concat(self):
+        s1 = 'x' * MAX_Py_ssize_t
+        self.assertRaises(OverflowError, operator.add, s1, '?')
+
+    @bigaddrspacetest
+    def test_optimized_concat(self):
+        x = 'x' * MAX_Py_ssize_t
+        try:
+            x = x + '?'     # this statement uses a fast path in ceval.c
+        except OverflowError:
+            pass
+        else:
+            self.fail("should have raised OverflowError")
+        try:
+            x += '?'        # this statement uses a fast path in ceval.c
+        except OverflowError:
+            pass
+        else:
+            self.fail("should have raised OverflowError")
+        self.assertEquals(len(x), MAX_Py_ssize_t)
+
+    ### the following test is pending a patch
+    #   (http://mail.python.org/pipermail/python-dev/2006-July/067774.html)
+    #@bigaddrspacetest
+    #def test_repeat(self):
+    #    self.assertRaises(OverflowError, operator.mul, 'x', MAX_Py_ssize_t + 1)
+
+
+def test_main():
+    test_support.run_unittest(StrTest)
+
+if __name__ == '__main__':
+    if len(sys.argv) > 1:
+        test_support.set_memlimit(sys.argv[1])
+    test_main()

diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
index 6532c79a997f3718c6b97230d85e1842bf295949..a9d5dabcc76af0bc502deee5b573f3c45f675614 100644 (file)
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -314,6 +314,12 @@ _1M = 1024*1024
 _1G = 1024 * _1M
 _2G = 2 * _1G
 
+# Hack to get at the maximum value an internal index can take.
+class _Dummy:
+    def __getslice__(self, i, j):
+        return j
+MAX_Py_ssize_t = _Dummy()[:]
+
 def set_memlimit(limit):
     import re
     global max_memuse
@@ -328,7 +334,9 @@ def set_memlimit(limit):
     if m is None:
         raise ValueError('Invalid memory limit %r' % (limit,))
     memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
-    if memlimit < 2.5*_1G:
+    if memlimit > MAX_Py_ssize_t:
+        memlimit = MAX_Py_ssize_t
+    if memlimit < _2G - 1:
         raise ValueError('Memory limit %r too low to be useful' % (limit,))
     max_memuse = memlimit
 
@@ -371,6 +379,17 @@ def bigmemtest(minsize, memuse, overhead=5*_1M):
         return wrapper
     return decorator
 
+def bigaddrspacetest(f):
+    """Decorator for tests that fill the address space."""
+    def wrapper(self):
+        if max_memuse < MAX_Py_ssize_t:
+            if verbose:
+                sys.stderr.write("Skipping %s because of memory "
+                                 "constraint\n" % (f.__name__,))
+        else:
+            return f(self)
+    return wrapper
+
 #=======================================================================
 # Preliminary PyUNIT integration.
 

diff --git a/Python/ceval.c b/Python/ceval.c
index ef484d9b70faeb6edabfe13dd659b5274203b8a0..a0e8b30c0526fc430ce216e1365659f7de69d9db 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4225,6 +4225,14 @@ string_concatenate(PyObject *v, PyObject *w,
 {
        /* This function implements 'variable += expr' when both arguments
           are strings. */
+       Py_ssize_t v_len = PyString_GET_SIZE(v);
+       Py_ssize_t w_len = PyString_GET_SIZE(w);
+       Py_ssize_t new_len = v_len + w_len;
+       if (new_len < 0) {
+               PyErr_SetString(PyExc_OverflowError,
+                               "strings are too large to concat");
+               return NULL;
+       }
 
        if (v->ob_refcnt == 2) {
                /* In the common case, there are 2 references to the value
@@ -4269,9 +4277,7 @@ string_concatenate(PyObject *v, PyObject *w,
                /* Now we own the last reference to 'v', so we can resize it
                 * in-place.
                 */
-               Py_ssize_t v_len = PyString_GET_SIZE(v);
-               Py_ssize_t w_len = PyString_GET_SIZE(w);
-               if (_PyString_Resize(&v, v_len + w_len) != 0) {
+               if (_PyString_Resize(&v, new_len) != 0) {
                        /* XXX if _PyString_Resize() fails, 'v' has been
                         * deallocated so it cannot be put back into 'variable'.
                         * The MemoryError is raised when there is no value in