It's unclear whether PyMarshal_XXX() are part of the public or private API.

They're named as if public, so I did a Bad Thing by changing
PyMarshal_ReadObjectFromFile() to suck up the remainder of the file in one
gulp:  anyone who counted on that leaving the file pointer merely at the
end of the next object would be screwed.  So restored
PyMarshal_ReadObjectFromFile() to its earlier state, renamed the new greedy
code to PyMarshal_ReadLastObjectFromFile(), and changed Python internals to
call the latter instead.

diff --git a/Include/marshal.h b/Include/marshal.h
index cf11d53128bc247c4daf8afc9c713d45c2308522..722be741229b6283b1d3b4f67bb48dfb4696e28c 100644 (file)
--- a/Include/marshal.h
+++ b/Include/marshal.h
@@ -15,6 +15,7 @@ DL_IMPORT(PyObject *) PyMarshal_WriteObjectToString(PyObject *);
 DL_IMPORT(long) PyMarshal_ReadLongFromFile(FILE *);
 DL_IMPORT(int) PyMarshal_ReadShortFromFile(FILE *);
 DL_IMPORT(PyObject *) PyMarshal_ReadObjectFromFile(FILE *);
+DL_IMPORT(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *);
 DL_IMPORT(PyObject *) PyMarshal_ReadObjectFromString(char *, int);
 
 #ifdef __cplusplus

diff --git a/Python/import.c b/Python/import.c
index 8cc4449a807d22b6bd5e7a7569a2a02d67e10b5f..533fed5ae0778d21c3a79d678e6cb2754cb78e82 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -555,7 +555,7 @@ read_compiled_module(char *cpathname, FILE *fp)
 {
        PyObject *co;
 
-       co = PyMarshal_ReadObjectFromFile(fp);
+       co = PyMarshal_ReadLastObjectFromFile(fp);
        /* Ugly: rd_object() may return NULL with or without error */
        if (co == NULL || !PyCode_Check(co)) {
                if (!PyErr_Occurred())

diff --git a/Python/marshal.c b/Python/marshal.c
index 36d958abf84b7a4e991e45c450b03f467d67d77f..1b9ab9af141d4b82ce52c1d5c64f8219f3ab8eed 100644 (file)
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -110,10 +110,10 @@ w_object(PyObject *v, WFILE *p)
        PyBufferProcs *pb;
 
        p->depth++;
-       
+
        if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
                p->error = 2;
-       } 
+       }
        else if (v == NULL) {
                w_byte(TYPE_NULL, p);
        }
@@ -363,31 +363,31 @@ r_object(RFILE *p)
        PyObject *v, *v2;
        long i, n;
        int type = r_byte(p);
-       
+
        switch (type) {
-       
+
        case EOF:
                PyErr_SetString(PyExc_EOFError,
                                "EOF read where object expected");
                return NULL;
-       
+
        case TYPE_NULL:
                return NULL;
-       
+
        case TYPE_NONE:
                Py_INCREF(Py_None);
                return Py_None;
-       
+
        case TYPE_ELLIPSIS:
                Py_INCREF(Py_Ellipsis);
                return Py_Ellipsis;
-       
+
        case TYPE_INT:
                return PyInt_FromLong(r_long(p));
-       
+
        case TYPE_INT64:
                return PyInt_FromLong(r_long64(p));
-       
+
        case TYPE_LONG:
                {
                        int size;
@@ -402,7 +402,7 @@ r_object(RFILE *p)
                                ob->ob_digit[i] = r_short(p);
                        return (PyObject *)ob;
                }
-       
+
        case TYPE_FLOAT:
                {
                        char buf[256];
@@ -419,7 +419,7 @@ r_object(RFILE *p)
                        PyFPE_END_PROTECT(dx)
                        return PyFloat_FromDouble(dx);
                }
-       
+
 #ifndef WITHOUT_COMPLEX
        case TYPE_COMPLEX:
                {
@@ -448,7 +448,7 @@ r_object(RFILE *p)
                        return PyComplex_FromCComplex(c);
                }
 #endif
-       
+
        case TYPE_STRING:
                n = r_long(p);
                if (n < 0) {
@@ -465,7 +465,7 @@ r_object(RFILE *p)
                        }
                }
                return v;
-       
+
        case TYPE_UNICODE:
            {
                char *buffer;
@@ -488,7 +488,7 @@ r_object(RFILE *p)
                PyMem_DEL(buffer);
                return v;
            }
-           
+
        case TYPE_TUPLE:
                n = r_long(p);
                if (n < 0) {
@@ -508,7 +508,7 @@ r_object(RFILE *p)
                        PyTuple_SET_ITEM(v, (int)i, v2);
                }
                return v;
-       
+
        case TYPE_LIST:
                n = r_long(p);
                if (n < 0) {
@@ -528,7 +528,7 @@ r_object(RFILE *p)
                        PyList_SetItem(v, (int)i, v2);
                }
                return v;
-       
+
        case TYPE_DICT:
                v = PyDict_New();
                if (v == NULL)
@@ -545,7 +545,7 @@ r_object(RFILE *p)
                        Py_XDECREF(val);
                }
                return v;
-       
+
        case TYPE_CODE:
                {
                        int argcount = r_short(p);
@@ -562,7 +562,7 @@ r_object(RFILE *p)
                        PyObject *name = NULL;
                        int firstlineno = 0;
                        PyObject *lnotab = NULL;
-                       
+
                        code = r_object(p);
                        if (code) consts = r_object(p);
                        if (consts) names = r_object(p);
@@ -575,13 +575,13 @@ r_object(RFILE *p)
                                firstlineno = r_short(p);
                                lnotab = r_object(p);
                        }
-                       
+
                        if (!PyErr_Occurred()) {
                                v = (PyObject *) PyCode_New(
-                                       argcount, nlocals, stacksize, flags,  
+                                       argcount, nlocals, stacksize, flags,
                                        code, consts, names, varnames,
-                                       freevars, cellvars, filename, name, 
-                                       firstlineno, lnotab);  
+                                       freevars, cellvars, filename, name,
+                                       firstlineno, lnotab);
                        }
                        else
                                v = NULL;
@@ -597,13 +597,13 @@ r_object(RFILE *p)
 
                }
                return v;
-       
+
        default:
                /* Bogus data got written, which isn't ideal.
                   This will let you keep working and recover. */
                PyErr_SetString(PyExc_ValueError, "bad marshal data");
                return NULL;
-       
+
        }
 }
 
@@ -627,20 +627,21 @@ getfilesize(FILE *fp)
                return st.st_size;
 }
 #endif
- 
+
 /* If we can get the size of the file up-front, and it's reasonably small,
  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
  * than reading a byte at a time from file; speeds .pyc imports.
+ * CAUTION:  since this may read the entire remainder of the file, don't
+ * call it unless you know you're done with the file.
  */
 PyObject *
-PyMarshal_ReadObjectFromFile(FILE *fp)
+PyMarshal_ReadLastObjectFromFile(FILE *fp)
 {
 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
  * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
  */
 #define SMALL_FILE_LIMIT (1L << 14)
 #define REASONABLE_FILE_LIMIT (1L << 18)
-       RFILE rf;
 #ifdef HAVE_FSTAT
        off_t filesize;
 #endif
@@ -665,15 +666,30 @@ PyMarshal_ReadObjectFromFile(FILE *fp)
                                PyMem_FREE(pBuf);
                        return v;
                }
-               
+
        }
 #endif
-       rf.fp = fp;
-       return r_object(&rf);
+       /* We don't have fstat, or we do but the file is larger than
+        * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
+        */
+       return PyMarshal_ReadObjectFromFile(fp);
+
 #undef SMALL_FILE_LIMIT
 #undef REASONABLE_FILE_LIMIT
 }
 
+PyObject *
+PyMarshal_ReadObjectFromFile(FILE *fp)
+{
+       RFILE rf;
+       if (PyErr_Occurred()) {
+               fprintf(stderr, "XXX rd_object called with exception set\n");
+               return NULL;
+       }
+       rf.fp = fp;
+       return r_object(&rf);
+}
+
 PyObject *
 PyMarshal_ReadObjectFromString(char *str, int len)
 {
@@ -708,7 +724,7 @@ PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */
                           PyString_AS_STRING((PyStringObject *)wf.str)));
        if (wf.error) {
                Py_XDECREF(wf.str);
-               PyErr_SetString(PyExc_ValueError, 
+               PyErr_SetString(PyExc_ValueError,
                                (wf.error==1)?"unmarshallable object"
                                :"object too deeply nested to marshal");
                return NULL;
@@ -738,7 +754,7 @@ marshal_dump(PyObject *self, PyObject *args)
        wf.depth = 0;
        w_object(x, &wf);
        if (wf.error) {
-               PyErr_SetString(PyExc_ValueError, 
+               PyErr_SetString(PyExc_ValueError,
                                (wf.error==1)?"unmarshallable object"
                                :"object too deeply nested to marshal");
                return NULL;

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 4c94f9b6a90874ebc3c026e8ada2076d7bb2ab8f..5f907978abb0f5ec18d7b7391675203e62bf62b2 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -936,7 +936,7 @@ run_pyc_file(FILE *fp, char *filename, PyObject *globals, PyObject *locals)
                return NULL;
        }
        (void) PyMarshal_ReadLongFromFile(fp);
-       v = PyMarshal_ReadObjectFromFile(fp);
+       v = PyMarshal_ReadLastObjectFromFile(fp);
        fclose(fp);
        if (v == NULL || !PyCode_Check(v)) {
                Py_XDECREF(v);