This is my patch:

[ 1180995 ] binary formats for marshalling floats

Adds 2 new type codes for marshal (binary floats and binary complexes), a
new marshal version (2), updates MAGIC and fiddles the de-serializing of
code objects to be less likely to clobber the real reason for failing if
it fails.

diff --git a/Include/marshal.h b/Include/marshal.h
index fc491dda9aaa24717b5a096ba8ecd6532d7b8399..a9f73208068aa76a46d9e64b54a11d1fc687160c 100644 (file)
--- a/Include/marshal.h
+++ b/Include/marshal.h
@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-#define Py_MARSHAL_VERSION 1
+#define Py_MARSHAL_VERSION 2
 
 PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int);
 PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int);

diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
index 9901a3c2d4eac366cc4a2397950eba18954ceeaf..b66eef582f7bfb54a94277f995c313c54eeaee13 100644 (file)
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -73,20 +73,34 @@ class FloatTestCase(unittest.TestCase):
             n /= 123.4567
 
         f = 0.0
-        s = marshal.dumps(f)
+        s = marshal.dumps(f, 2)
         got = marshal.loads(s)
         self.assertEqual(f, got)
+        # and with version <= 1 (floats marshalled differently then)
+        s = marshal.dumps(f, 1)
+        got = marshal.loads(s) 
+        self.assertEqual(f, got)       
 
         n = sys.maxint * 3.7e-250
         while n < small:
             for expected in (-n, n):
                 f = float(expected)
+                
                 s = marshal.dumps(f)
                 got = marshal.loads(s)
                 self.assertEqual(f, got)
+                
+                s = marshal.dumps(f, 1)
+                got = marshal.loads(s)
+                self.assertEqual(f, got)
+                
                 marshal.dump(f, file(test_support.TESTFN, "wb"))
                 got = marshal.load(file(test_support.TESTFN, "rb"))
                 self.assertEqual(f, got)
+                
+                marshal.dump(f, file(test_support.TESTFN, "wb"), 1)
+                got = marshal.load(file(test_support.TESTFN, "rb"))
+                self.assertEqual(f, got)
             n *= 123.4567
         os.unlink(test_support.TESTFN)
 

diff --git a/Misc/NEWS b/Misc/NEWS
index 50f5a2718f722d1ee2a30a924288e9b6b4b1ad01..67ccb426370b65a2b37bc5c87ac00dbbdd5ffc8a 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@ What's New in Python 2.5 alpha 1?
 Core and builtins
 -----------------
 
+- SF patch #1180995: marshal now uses a binary format by default when
+  serializing floats.
+
 - SF patch #1181301: on platforms that appear to use IEEE 754 floats,
   the routines that promise to produce IEEE 754 binary representations
   of floats now simply copy bytes around.

diff --git a/Python/import.c b/Python/import.c
index 0362dbd47a2a29f0cb8c51a62c9d22763b91e3e4..c03d4ccf28c0c11c4e80328aa409b04c448881fa 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -50,8 +50,9 @@ extern time_t PyOS_GetLastModificationTime(char *, FILE *);
        Python 2.4a0: 62041
        Python 2.4a3: 62051
        Python 2.4b1: 62061
+       Python 2.5a0: 62071
 */
-#define MAGIC (62061 | ((long)'\r'<<16) | ((long)'\n'<<24))
+#define MAGIC (62071 | ((long)'\r'<<16) | ((long)'\n'<<24))
 
 /* Magic word as global; note that _PyImport_Init() can change the
    value of this global to accommodate for alterations of how the

diff --git a/Python/marshal.c b/Python/marshal.c
index d0fcac729699a58ce2a00283d55d24e929ed3274..2535cb5514ff48fef611dc281a8aaff86da0fedc 100644 (file)
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -15,28 +15,30 @@
  */
 #define MAX_MARSHAL_STACK_DEPTH 5000
 
-#define TYPE_NULL      '0'
-#define TYPE_NONE      'N'
-#define TYPE_FALSE     'F'
-#define TYPE_TRUE      'T'
-#define TYPE_STOPITER  'S'
-#define TYPE_ELLIPSIS   '.'
-#define TYPE_INT       'i'
-#define TYPE_INT64     'I'
-#define TYPE_FLOAT     'f'
-#define TYPE_COMPLEX   'x'
-#define TYPE_LONG      'l'
-#define TYPE_STRING    's'
-#define TYPE_INTERNED  't'
-#define TYPE_STRINGREF 'R'
-#define TYPE_TUPLE     '('
-#define TYPE_LIST      '['
-#define TYPE_DICT      '{'
-#define TYPE_CODE      'c'
-#define TYPE_UNICODE   'u'
-#define TYPE_UNKNOWN   '?'
-#define TYPE_SET       '<'
-#define TYPE_FROZENSET  '>'
+#define TYPE_NULL              '0'
+#define TYPE_NONE              'N'
+#define TYPE_FALSE             'F'
+#define TYPE_TRUE              'T'
+#define TYPE_STOPITER          'S'
+#define TYPE_ELLIPSIS          '.'
+#define TYPE_INT               'i'
+#define TYPE_INT64             'I'
+#define TYPE_FLOAT             'f'
+#define TYPE_BINARY_FLOAT      'g'
+#define TYPE_COMPLEX           'x'
+#define TYPE_BINARY_COMPLEX    'y'
+#define TYPE_LONG              'l'
+#define TYPE_STRING            's'
+#define TYPE_INTERNED          't'
+#define TYPE_STRINGREF         'R'
+#define TYPE_TUPLE             '('
+#define TYPE_LIST              '['
+#define TYPE_DICT              '{'
+#define TYPE_CODE              'c'
+#define TYPE_UNICODE           'u'
+#define TYPE_UNKNOWN           '?'
+#define TYPE_SET               '<'
+#define TYPE_FROZENSET         '>'
 
 typedef struct {
        FILE *fp;
@@ -47,6 +49,7 @@ typedef struct {
        char *ptr;
        char *end;
        PyObject *strings; /* dict on marshal, list on unmarshal */
+       int version;
 } WFILE;
 
 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
@@ -165,32 +168,62 @@ w_object(PyObject *v, WFILE *p)
                        w_short(ob->ob_digit[i], p);
        }
        else if (PyFloat_Check(v)) {
-               char buf[256]; /* Plenty to format any double */
-               PyFloat_AsReprString(buf, (PyFloatObject *)v);
-               n = strlen(buf);
-               w_byte(TYPE_FLOAT, p);
-               w_byte(n, p);
-               w_string(buf, n, p);
+               if (p->version > 1) {
+                       char buf[8];
+                       if (_PyFloat_Pack8(PyFloat_AsDouble(v), 
+                                          buf, 1) < 0) {
+                               p->error = 1;
+                               return;
+                       }
+                       w_byte(TYPE_BINARY_FLOAT, p);
+                       w_string(buf, 8, p);
+               }
+               else {
+                       char buf[256]; /* Plenty to format any double */
+                       PyFloat_AsReprString(buf, (PyFloatObject *)v);
+                       n = strlen(buf);
+                       w_byte(TYPE_FLOAT, p);
+                       w_byte(n, p);
+                       w_string(buf, n, p);
+               }
        }
 #ifndef WITHOUT_COMPLEX
        else if (PyComplex_Check(v)) {
-               char buf[256]; /* Plenty to format any double */
-               PyFloatObject *temp;
-               w_byte(TYPE_COMPLEX, p);
-               temp = (PyFloatObject*)PyFloat_FromDouble(
-                       PyComplex_RealAsDouble(v));
-               PyFloat_AsReprString(buf, temp);
-               Py_DECREF(temp);
-               n = strlen(buf);
-               w_byte(n, p);
-               w_string(buf, n, p);
-               temp = (PyFloatObject*)PyFloat_FromDouble(
-                       PyComplex_ImagAsDouble(v));
-               PyFloat_AsReprString(buf, temp);
-               Py_DECREF(temp);
-               n = strlen(buf);
-               w_byte(n, p);
-               w_string(buf, n, p);
+               if (p->version > 1) {
+                       char buf[8];
+                       if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
+                                          buf, 1) < 0) {
+                               p->error = 1;
+                               return;
+                       }
+                       w_byte(TYPE_BINARY_COMPLEX, p);
+                       w_string(buf, 8, p);
+                       if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v), 
+                                          buf, 1) < 0) {
+                               p->error = 1;
+                               return;
+                       }
+                       w_string(buf, 8, p);
+               }
+               else {
+                       char buf[256]; /* Plenty to format any double */
+                       PyFloatObject *temp;
+                       w_byte(TYPE_COMPLEX, p);
+                       temp = (PyFloatObject*)PyFloat_FromDouble(
+                               PyComplex_RealAsDouble(v));
+                       PyFloat_AsReprString(buf, temp);
+                       Py_DECREF(temp);
+                       n = strlen(buf);
+                       w_byte(n, p);
+                       w_string(buf, n, p);
+                       temp = (PyFloatObject*)PyFloat_FromDouble(
+                               PyComplex_ImagAsDouble(v));
+                       PyFloat_AsReprString(buf, temp);
+                       Py_DECREF(temp);
+                       n = strlen(buf);
+                       w_byte(n, p);
+                       w_string(buf, n, p);
+               }
        }
 #endif
        else if (PyString_Check(v)) {
@@ -335,6 +368,7 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
        wf.error = 0;
        wf.depth = 0;
        wf.strings = NULL;
+       wf.version = version;
        w_long(x, &wf);
 }
 
@@ -346,6 +380,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
        wf.error = 0;
        wf.depth = 0;
        wf.strings = (version > 0) ? PyDict_New() : NULL;
+       wf.version = version;
        w_object(x, &wf);
        Py_XDECREF(wf.strings);
 }
@@ -519,6 +554,22 @@ r_object(RFILE *p)
                        return PyFloat_FromDouble(dx);
                }
 
+       case TYPE_BINARY_FLOAT:
+               {
+                       char buf[8];
+                       double x;
+                       if (r_string(buf, 8, p) != 8) {
+                               PyErr_SetString(PyExc_EOFError,
+                                       "EOF read where object expected");
+                               return NULL;
+                       }
+                       x = _PyFloat_Unpack8(buf, 1);
+                       if (x == -1.0 && PyErr_Occurred()) {
+                               return NULL;
+                       }
+                       return PyFloat_FromDouble(x);
+               }
+
 #ifndef WITHOUT_COMPLEX
        case TYPE_COMPLEX:
                {
@@ -546,6 +597,31 @@ r_object(RFILE *p)
                        PyFPE_END_PROTECT(c)
                        return PyComplex_FromCComplex(c);
                }
+
+       case TYPE_BINARY_COMPLEX:
+               {
+                       char buf[8];
+                       Py_complex c;
+                       if (r_string(buf, 8, p) != 8) {
+                               PyErr_SetString(PyExc_EOFError,
+                                       "EOF read where object expected");
+                               return NULL;
+                       }
+                       c.real = _PyFloat_Unpack8(buf, 1);
+                       if (c.real == -1.0 && PyErr_Occurred()) {
+                               return NULL;
+                       }
+                       if (r_string(buf, 8, p) != 8) {
+                               PyErr_SetString(PyExc_EOFError,
+                                       "EOF read where object expected");
+                               return NULL;
+                       }
+                       c.imag = _PyFloat_Unpack8(buf, 1);
+                       if (c.imag == -1.0 && PyErr_Occurred()) {
+                               return NULL;
+                       }
+                       return PyComplex_FromCComplex(c);
+               }
 #endif
 
        case TYPE_INTERNED:
@@ -707,30 +783,63 @@ r_object(RFILE *p)
                        return NULL;
                }
                else {
-                       int argcount = r_long(p);
-                       int nlocals = r_long(p);
-                       int stacksize = r_long(p);
-                       int flags = r_long(p);
-                       PyObject *code = r_object(p);
-                       PyObject *consts = r_object(p);
-                       PyObject *names = r_object(p);
-                       PyObject *varnames = r_object(p);
-                       PyObject *freevars = r_object(p);
-                       PyObject *cellvars = r_object(p);
-                       PyObject *filename = r_object(p);
-                       PyObject *name = r_object(p);
-                       int firstlineno = r_long(p);
-                       PyObject *lnotab = r_object(p);
-
-                       if (!PyErr_Occurred()) {
-                               v = (PyObject *) PyCode_New(
+                       int argcount;
+                       int nlocals;
+                       int stacksize;
+                       int flags;
+                       PyObject *code = NULL;
+                       PyObject *consts = NULL;
+                       PyObject *names = NULL;
+                       PyObject *varnames = NULL;
+                       PyObject *freevars = NULL;
+                       PyObject *cellvars = NULL;
+                       PyObject *filename = NULL;
+                       PyObject *name = NULL;
+                       int firstlineno;
+                       PyObject *lnotab = NULL;
+                       
+                       v = NULL;
+
+                       argcount = r_long(p);
+                       nlocals = r_long(p);
+                       stacksize = r_long(p);
+                       flags = r_long(p);
+                       code = r_object(p);
+                       if (code == NULL)
+                               goto code_error;
+                       consts = r_object(p);
+                       if (consts == NULL)
+                               goto code_error;
+                       names = r_object(p);
+                       if (names == NULL)
+                               goto code_error;
+                       varnames = r_object(p);
+                       if (varnames == NULL)
+                               goto code_error;
+                       freevars = r_object(p);
+                       if (freevars == NULL)
+                               goto code_error;
+                       cellvars = r_object(p);
+                       if (cellvars == NULL)
+                               goto code_error;
+                       filename = r_object(p);
+                       if (filename == NULL)
+                               goto code_error;
+                       name = r_object(p);
+                       if (name == NULL)
+                               goto code_error;
+                       firstlineno = r_long(p);
+                       lnotab = r_object(p);
+                       if (lnotab == NULL)
+                               goto code_error;
+
+                       v = (PyObject *) PyCode_New(
                                        argcount, nlocals, stacksize, flags,
                                        code, consts, names, varnames,
                                        freevars, cellvars, filename, name,
                                        firstlineno, lnotab);
-                       }
-                       else
-                               v = NULL;
+
+                 code_error:
                        Py_XDECREF(code);
                        Py_XDECREF(consts);
                        Py_XDECREF(names);
@@ -882,6 +991,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
        wf.end = wf.ptr + PyString_Size(wf.str);
        wf.error = 0;
        wf.depth = 0;
+       wf.version = version;
        wf.strings = (version > 0) ? PyDict_New() : NULL;
        w_object(x, &wf);
        Py_XDECREF(wf.strings);