Merged revisions 74769 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r74769 | mark.dickinson | 2009-09-13 12:56:13 +0100 (Sun, 13 Sep 2009) | 3 lines

  Fix potential signed-overflow bug in _PyLong_Format;  also fix
  a couple of whitespace issues.
........

diff --git a/Objects/longobject.c b/Objects/longobject.c
index 389b4315d32509627a5b44e61188dccc7301a0e7..91edf62b7a52aecea2e572a9a137b3a762d18ed0 100644 (file)
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -1201,7 +1201,7 @@ _PyLong_Format(PyObject *aa, int base, int addL, int newstyle)
 {
        register PyLongObject *a = (PyLongObject *)aa;
        PyStringObject *str;
-       Py_ssize_t i, j, sz;
+       Py_ssize_t i, sz;
        Py_ssize_t size_a;
        char *p;
        int bits;
@@ -1222,20 +1222,21 @@ _PyLong_Format(PyObject *aa, int base, int addL, int newstyle)
                i >>= 1;
        }
        i = 5 + (addL ? 1 : 0);
-       j = size_a*PyLong_SHIFT + bits-1;
-       sz = i + j / bits;
-       if (j / PyLong_SHIFT < size_a || sz < i) {
+       /* ensure we don't get signed overflow in sz calculation */
+       if (size_a > (PY_SSIZE_T_MAX - i) / PyLong_SHIFT) {
                PyErr_SetString(PyExc_OverflowError,
                                "long is too large to format");
                return NULL;
        }
+       sz = i + 1 + (size_a * PyLong_SHIFT - 1) / bits;
+       assert(sz >= 0);
        str = (PyStringObject *) PyString_FromStringAndSize((char *)0, sz);
        if (str == NULL)
                return NULL;
        p = PyString_AS_STRING(str) + sz;
        *p = '\0';
-        if (addL)
-                *--p = 'L';
+       if (addL)
+               *--p = 'L';
        if (a->ob_size < 0)
                sign = '-';
 
@@ -1263,7 +1264,7 @@ _PyLong_Format(PyObject *aa, int base, int addL, int newstyle)
                                accumbits -= basebits;
                                accum >>= basebits;
                        } while (i < size_a-1 ? accumbits >= basebits :
-                                               accum > 0);
+                                               accum > 0);
                }
        }
        else {
@@ -1278,7 +1279,8 @@ _PyLong_Format(PyObject *aa, int base, int addL, int newstyle)
                int power = 1;
                for (;;) {
                        unsigned long newpow = powbase * (unsigned long)base;
-                       if (newpow >> PyLong_SHIFT)  /* doesn't fit in a digit */
+                       if (newpow >> PyLong_SHIFT)
+                               /* doesn't fit in a digit */
                                break;
                        powbase = (digit)newpow;
                        ++power;
@@ -1328,7 +1330,7 @@ _PyLong_Format(PyObject *aa, int base, int addL, int newstyle)
                *--p = '0';
        }
        else if (base == 8) {
-               if (newstyle) {
+               if (newstyle) {
                        *--p = 'o';
                        *--p = '0';
                }