struct: modulo math plus warning on all endian-explicit formats for compatibility...

diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py
index 7981a5245a7b576ca129e4852a65efe499ba8e70..701f7fd1e96ed58ef8435f00d1a98e7648da68f2 100644 (file)
--- a/Lib/test/test_struct.py
+++ b/Lib/test/test_struct.py
@@ -3,6 +3,7 @@ import test.test_support
 import struct
 import array
 import unittest
+import warnings
 
 import sys
 ISBIGENDIAN = sys.byteorder == "big"
@@ -10,7 +11,14 @@ del sys
 verify((struct.pack('=i', 1)[0] == chr(0)) == ISBIGENDIAN,
        "bigendian determination appears wrong")
 
-PY_STRUCT_RANGE_CHECKING = 1
+try:
+    import _struct
+except ImportError:
+    PY_STRUCT_RANGE_CHECKING = 0
+    PY_STRUCT_WRAPPING = 1
+else:
+    PY_STRUCT_RANGE_CHECKING = getattr(_struct, '_PY_STRUCT_RANGE_CHECKING', 0)
+    PY_STRUCT_WRAPPING = getattr(_struct, '_PY_STRUCT_WRAPPING', 0)
 
 def string_reverse(s):
     chars = list(s)
@@ -35,12 +43,29 @@ def simple_err(func, *args):
 def any_err(func, *args):
     try:
         func(*args)
-    except (struct.error, OverflowError, TypeError):
+    except (struct.error, TypeError):
         pass
     else:
         raise TestFailed, "%s%s did not raise error" % (
             func.__name__, args)
 
+def deprecated_err(func, *args):
+    warnings.filterwarnings("error", r"""^struct.*""", DeprecationWarning)
+    warnings.filterwarnings("error", r""".*format requires.*""", DeprecationWarning)
+    try:
+        try:
+            func(*args)
+        except (struct.error, TypeError):
+            pass
+        except DeprecationWarning:
+            if not PY_STRUCT_WRAPPING:
+                raise TestFailed, "%s%s expected to raise struct.error" % (
+                    func.__name__, args)
+        else:
+            raise TestFailed, "%s%s did not raise error" % (
+                func.__name__, args)
+    finally:
+        warnings.resetwarnings()
 
 simple_err(struct.calcsize, 'Z')
 
@@ -272,8 +297,8 @@ class IntTester:
                 if verbose:
                     print "Skipping buggy range check for code", code
             else:
-                any_err(pack, ">" + code, x)
-                any_err(pack, "<" + code, x)
+                deprecated_err(pack, ">" + code, x)
+                deprecated_err(pack, "<" + code, x)
 
         # Much the same for unsigned.
         code = self.unsigned_code
@@ -327,8 +352,8 @@ class IntTester:
                 if verbose:
                     print "Skipping buggy range check for code", code
             else:
-                any_err(pack, ">" + code, x)
-                any_err(pack, "<" + code, x)
+                deprecated_err(pack, ">" + code, x)
+                deprecated_err(pack, "<" + code, x)
 
     def run(self):
         from random import randrange
@@ -448,13 +473,13 @@ def test_1229380():
     for endian in ('', '>', '<'):
         for cls in (int, long):
             for fmt in ('B', 'H', 'I', 'L'):
-                any_err(struct.pack, endian + fmt, cls(-1))
+                deprecated_err(struct.pack, endian + fmt, cls(-1))
 
-            any_err(struct.pack, endian + 'B', cls(300))
-            any_err(struct.pack, endian + 'H', cls(70000))
+            deprecated_err(struct.pack, endian + 'B', cls(300))
+            deprecated_err(struct.pack, endian + 'H', cls(70000))
 
-        any_err(struct.pack, endian + 'I', sys.maxint * 4L)
-        any_err(struct.pack, endian + 'L', sys.maxint * 4L)
+        deprecated_err(struct.pack, endian + 'I', sys.maxint * 4L)
+        deprecated_err(struct.pack, endian + 'L', sys.maxint * 4L)
 
 if PY_STRUCT_RANGE_CHECKING:
     test_1229380()

diff --git a/Modules/_struct.c b/Modules/_struct.c
index be641214ce68028f8a739d938c5de90330c242ee..163cd5ecc0be2d62bfa748338e4ac6ccd9e8332b 100644 (file)
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -17,6 +17,20 @@ static PyTypeObject PyStructType;
 typedef int Py_ssize_t;
 #endif
 
+/* If PY_STRUCT_WRAPPING is defined, the struct module will wrap all input
+   numbers for explicit endians such that they fit in the given type, much
+   like explicit casting in C. A warning will be raised if the number did
+   not originally fit within the range of the requested type. If it is
+   not defined, then all range errors and overflow will be struct.error
+   exceptions. */
+
+#define PY_STRUCT_WRAPPING 1
+
+#ifdef PY_STRUCT_WRAPPING
+static PyObject *pylong_ulong_mask = NULL;
+static PyObject *pyint_zero = NULL;
+#endif
+
 /* The translation function for each format character is table driven */
 typedef struct _formatdef {
        char format;
@@ -195,6 +209,75 @@ get_ulonglong(PyObject *v, unsigned PY_LONG_LONG *p)
 
 #endif
 
+#ifdef PY_STRUCT_WRAPPING
+
+/* Helper routine to get a Python integer and raise the appropriate error
+   if it isn't one */
+
+static int
+get_wrapped_long(PyObject *v, long *p)
+{
+       if (get_long(v, p) < 0) {
+               if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError)) {
+                       PyObject *wrapped;
+                       long x;
+                       PyErr_Clear();
+                       if (PyErr_Warn(PyExc_DeprecationWarning, "struct integer wrapping is deprecated") < 0)
+                               return -1;
+                       wrapped = PyNumber_And(v, pylong_ulong_mask);
+                       if (wrapped == NULL)
+                               return -1;
+                       x = (long)PyLong_AsUnsignedLong(wrapped);
+                       Py_DECREF(wrapped);
+                       if (x == -1 && PyErr_Occurred())
+                               return -1;
+                       *p = x;
+               } else {
+                       return -1;
+               }
+       }
+       return 0;
+}
+
+static int
+get_wrapped_ulong(PyObject *v, unsigned long *p)
+{
+       long x = (long)PyLong_AsUnsignedLong(v);
+       if (x == -1 && PyErr_Occurred()) {
+               PyObject *wrapped;
+               PyErr_Clear();
+               wrapped = PyNumber_And(v, pylong_ulong_mask);
+               if (wrapped == NULL)
+                       return -1;
+               if (PyErr_Warn(PyExc_DeprecationWarning, "struct integer wrapping is deprecated") < 0) {
+                       Py_DECREF(wrapped);
+                       return -1;
+               }
+               x = (long)PyLong_AsUnsignedLong(wrapped);
+               Py_DECREF(wrapped);
+               if (x == -1 && PyErr_Occurred())
+                       return -1;
+       }
+       *p = (unsigned long)x;
+       return 0;
+}
+
+#define RANGE_ERROR(x, f, flag, mask) \
+       do { \
+               if (_range_error(f, flag) < 0) \
+                       return -1; \
+               else \
+                       (x) &= (mask); \
+       } while (0)
+
+#else
+
+#define get_wrapped_long get_long
+#define get_wrapped_ulong get_ulong
+#define RANGE_ERROR(x, f, flag, mask) return _range_error(f, flag)
+
+#endif
+
 /* Floating point helpers */
 
 static PyObject *
@@ -247,6 +330,25 @@ _range_error(const formatdef *f, int is_unsigned)
                        f->format,
                        largest);
        }
+#ifdef PY_STRUCT_WRAPPING
+       {
+               PyObject *ptype, *pvalue, *ptraceback;
+               PyObject *msg;
+               int rval;
+               PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+               assert(pvalue != NULL);
+               msg = PyObject_Str(pvalue);
+               Py_XDECREF(ptype);
+               Py_XDECREF(pvalue);
+               Py_XDECREF(ptraceback);
+               if (msg == NULL)
+                       return -1;
+               rval = PyErr_Warn(PyExc_DeprecationWarning, PyString_AS_STRING(msg));
+               Py_DECREF(msg);
+               if (rval == 0)
+                       return 0;
+       }
+#endif
        return -1;
 }
 
@@ -707,15 +809,19 @@ bp_int(char *p, PyObject *v, const formatdef *f)
 {
        long x;
        Py_ssize_t i;
-       if (get_long(v, &x) < 0)
+       if (get_wrapped_long(v, &x) < 0)
                return -1;
        i = f->size;
        if (i != SIZEOF_LONG) {
                if ((i == 2) && (x < -32768 || x > 32767))
-                       return _range_error(f, 0);
+                       RANGE_ERROR(x, f, 0, 0xffffL);
 #if (SIZEOF_LONG != 4)
                else if ((i == 4) && (x < -2147483648L || x > 2147483647L))
-                       return _range_error(f, 0);
+                       RANGE_ERROR(x, f, 0, 0xffffffffL);
+#endif
+#ifdef PY_STRUCT_WRAPPING
+               else if ((i == 1) && (x < -128 || x > 127))
+                       RANGE_ERROR(x, f, 0, 0xffL);
 #endif
        }
        do {
@@ -730,14 +836,14 @@ bp_uint(char *p, PyObject *v, const formatdef *f)
 {
        unsigned long x;
        Py_ssize_t i;
-       if (get_ulong(v, &x) < 0)
+       if (get_wrapped_ulong(v, &x) < 0)
                return -1;
        i = f->size;
        if (i != SIZEOF_LONG) {
                unsigned long maxint = 1;
                maxint <<= (unsigned long)(i * 8);
                if (x >= maxint)
-                       return _range_error(f, 1);
+                       RANGE_ERROR(x, f, 1, maxint - 1);
        }
        do {
                p[--i] = (char)x;
@@ -804,8 +910,14 @@ bp_double(char *p, PyObject *v, const formatdef *f)
 
 static formatdef bigendian_table[] = {
        {'x',   1,              0,              NULL},
+#ifdef PY_STRUCT_WRAPPING
+       /* Native packers do range checking without wrapping. */
+       {'b',   1,              0,              nu_byte,        bp_int},
+       {'B',   1,              0,              nu_ubyte,       bp_uint},
+#else
        {'b',   1,              0,              nu_byte,        np_byte},
        {'B',   1,              0,              nu_ubyte,       np_ubyte},
+#endif
        {'c',   1,              0,              nu_char,        np_char},
        {'s',   1,              0,              NULL},
        {'p',   1,              0,              NULL},
@@ -915,15 +1027,19 @@ lp_int(char *p, PyObject *v, const formatdef *f)
 {
        long x;
        Py_ssize_t i;
-       if (get_long(v, &x) < 0)
+       if (get_wrapped_long(v, &x) < 0)
                return -1;
        i = f->size;
        if (i != SIZEOF_LONG) {
                if ((i == 2) && (x < -32768 || x > 32767))
-                       return _range_error(f, 0);
+                       RANGE_ERROR(x, f, 0, 0xffffL);
 #if (SIZEOF_LONG != 4)
                else if ((i == 4) && (x < -2147483648L || x > 2147483647L))
-                       return _range_error(f, 0);
+                       RANGE_ERROR(x, f, 0, 0xffffffffL);
+#endif
+#ifdef PY_STRUCT_WRAPPING
+               else if ((i == 1) && (x < -128 || x > 127))
+                       RANGE_ERROR(x, f, 0, 0xffL);
 #endif
        }
        do {
@@ -938,14 +1054,14 @@ lp_uint(char *p, PyObject *v, const formatdef *f)
 {
        unsigned long x;
        Py_ssize_t i;
-       if (get_ulong(v, &x) < 0)
+       if (get_wrapped_ulong(v, &x) < 0)
                return -1;
        i = f->size;
        if (i != SIZEOF_LONG) {
                unsigned long maxint = 1;
                maxint <<= (unsigned long)(i * 8);
                if (x >= maxint)
-                       return _range_error(f, 1);
+                       RANGE_ERROR(x, f, 1, maxint - 1);
        }
        do {
                *p++ = (char)x;
@@ -1012,8 +1128,14 @@ lp_double(char *p, PyObject *v, const formatdef *f)
 
 static formatdef lilendian_table[] = {
        {'x',   1,              0,              NULL},
+#ifdef PY_STRUCT_WRAPPING
+       /* Native packers do range checking without wrapping. */
+       {'b',   1,              0,              nu_byte,        lp_int},
+       {'B',   1,              0,              nu_ubyte,       lp_uint},
+#else
        {'b',   1,              0,              nu_byte,        np_byte},
        {'B',   1,              0,              nu_ubyte,       np_ubyte},
+#endif
        {'c',   1,              0,              nu_char,        np_char},
        {'s',   1,              0,              NULL},
        {'p',   1,              0,              NULL},
@@ -1418,8 +1540,12 @@ s_pack_internal(PyStructObject *soself, PyObject *args, int offset, char* buf)
                        *res = Py_SAFE_DOWNCAST(n, Py_ssize_t, unsigned char);
                } else {
                        v = PyTuple_GET_ITEM(args, i++);
-                       if (e->pack(res, v, e) < 0)
+                       if (e->pack(res, v, e) < 0) {
+                               if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError))
+                                       PyErr_SetString(StructError,
+                                                       "long too large to convert to int");
                                return -1;
+                       }
                }
        }
        
@@ -1614,6 +1740,26 @@ init_struct(void)
        if (PyType_Ready(&PyStructType) < 0)
                return;
 
+#ifdef PY_STRUCT_WRAPPING
+       if (pyint_zero == NULL) {
+               pyint_zero = PyInt_FromLong(0);
+               if (pyint_zero == NULL)
+                       return;
+       }
+       if (pylong_ulong_mask == NULL) {
+#if (SIZEOF_LONG == 4)
+               pylong_ulong_mask = PyLong_FromString("FFFFFFFF", NULL, 16);
+#else
+               pylong_ulong_mask = PyLong_FromString("FFFFFFFFFFFFFFFF", NULL, 16);
+#endif
+               if (pylong_ulong_mask == NULL)
+                       return;
+       }
+
+#else  
+       /* This speed trick can't be used until wrapping goes away, because
+          native endian always raises exceptions instead of wrapping. */
+       
        /* Check endian and swap in faster functions */
        {
                int one = 1;
@@ -1652,6 +1798,7 @@ init_struct(void)
                        native++;
                }
        }
+#endif
        
        /* Add some symbolic constants to the module */
        if (StructError == NULL) {
@@ -1665,4 +1812,9 @@ init_struct(void)
 
        Py_INCREF((PyObject*)&PyStructType);
        PyModule_AddObject(m, "Struct", (PyObject*)&PyStructType);
+       
+       PyModule_AddIntConstant(m, "_PY_STRUCT_RANGE_CHECKING", 1);
+#ifdef PY_STRUCT_WRAPPING
+       PyModule_AddIntConstant(m, "_PY_STRUCT_WRAPPING", 1);
+#endif
 }