Add phuang patch from Issue 708374 which adds offset parameter to mmap module.

diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index abe5b7bd07db9561dfa3827d275e3c3e64ae5664..0e982c14c17d99dbd03702b8a57c7c20ddcc38ed 100644 (file)
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -40,7 +40,7 @@ not update the underlying file.
    length.
 
 
-.. function:: mmap(fileno, length[, tagname[, access]])
+.. function:: mmap(fileno, length[, tagname[, access[, offset]]])
 
    **(Windows version)** Maps *length* bytes from the file specified by the file
    handle *fileno*, and returns a mmap object.  If *length* is larger than the
@@ -56,8 +56,12 @@ not update the underlying file.
    the mapping is created without a name.  Avoiding the use of the tag parameter
    will assist in keeping your code portable between Unix and Windows.
 
+   *offset* may be specified as a non-negative integer offset. mmap references will 
+   be relative to the offset from the beginning of the file. *offset* defaults to 0.
+   *offset* must be a multiple of the ALLOCATIONGRANULARITY.
 
-.. function:: mmap(fileno, length[, flags[, prot[, access]]])
+
+.. function:: mmap(fileno, length[, flags[, prot[, access[, offset]]]])
    :noindex:
 
    **(Unix version)** Maps *length* bytes from the file specified by the file
@@ -79,6 +83,10 @@ not update the underlying file.
    parameter.  It is an error to specify both *flags*, *prot* and *access*.  See
    the description of *access* above for information on how to use this parameter.
 
+   *offset* may be specified as a non-negative integer offset. mmap references will 
+   be relative to the offset from the beginning of the file. *offset* defaults to 0.
+   *offset* must be a multiple of the PAGESIZE or ALLOCATIONGRANULARITY.
+
 Memory-mapped file objects support the following methods:
 
 
@@ -171,3 +179,4 @@ Memory-mapped file objects support the following methods:
    created with :const:`ACCESS_READ`, then writing to it will throw a
    :exc:`TypeError` exception.
 
+

diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py
index 769eaed0b11a3be05fb0a2a03bff751137155a6a..b4f85fceef5378592cbb1bc9ce1ee5c6071e7340 100644 (file)
--- a/Lib/test/test_mmap.py
+++ b/Lib/test/test_mmap.py
@@ -340,6 +340,50 @@ class MmapTests(unittest.TestCase):
                     m[start:stop:step] = data
                     self.assertEquals(m[:], "".join(L))
 
+    def make_mmap_file (self, f, halfsize):
+        # Write 2 pages worth of data to the file
+        f.write ('\0' * halfsize)
+        f.write ('foo')
+        f.write ('\0' * (halfsize - 3))
+        f.flush ()
+        return mmap.mmap (f.fileno(), 0)
+
+    def test_offset (self):
+        f = open (TESTFN, 'w+b')
+
+        try: # unlink TESTFN no matter what
+            halfsize = mmap.ALLOCATIONGRANULARITY
+            m = self.make_mmap_file (f, halfsize)
+            m.close ()
+            f.close ()
+
+            mapsize = halfsize * 2
+            # Try invalid offset
+            f = open(TESTFN, "r+b")
+            for offset in [-2, -1, None]:
+                try:
+                    m = mmap.mmap(f.fileno(), mapsize, offset=offset)
+                    self.assertEqual(0, 1)
+                except (ValueError, TypeError, OverflowError):
+                    pass
+                else:
+                    self.assertEqual(0, 0)
+            f.close()
+
+            # Try valid offset, hopefully 8192 works on all OSes
+            f = open(TESTFN, "r+b")
+            m = mmap.mmap(f.fileno(), mapsize - halfsize, offset=halfsize)
+            self.assertEqual(m[0:3], 'foo')
+            f.close()
+            m.close()
+
+        finally:
+            f.close()
+            try:
+                os.unlink(TESTFN)
+            except OSError:
+                pass
+
 def test_main():
     run_unittest(MmapTests)
 

diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index 61144428b263ed19ae1937185e9631c832ef7b53..4474eeeaa0d51c8bcd916ff6cd182cfb28d22a33 100644 (file)
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -3,6 +3,9 @@
  /  Hacked for Unix by AMK
  /  $Id$
 
+ / Modified to support mmap with offset - to map a 'window' of a file
+ /   Author:  Yotam Medini  yotamm@mellanox.co.il
+ /
  / mmapmodule.cpp -- map a view of a file into memory
  /
  / todo: need permission flags, perhaps a 'chsize' analog
@@ -31,6 +34,16 @@ my_getpagesize(void)
        GetSystemInfo(&si);
        return si.dwPageSize;
 }
+
+static int
+my_getallocationgranularity (void)
+{
+
+       SYSTEM_INFO si;
+       GetSystemInfo(&si);
+       return si.dwAllocationGranularity;
+}
+
 #endif
 
 #ifdef UNIX
@@ -43,6 +56,8 @@ my_getpagesize(void)
 {
        return sysconf(_SC_PAGESIZE);
 }
+
+#define my_getallocationgranularity my_getpagesize
 #else
 #define my_getpagesize getpagesize
 #endif
@@ -74,7 +89,8 @@ typedef struct {
        PyObject_HEAD
        char *  data;
        size_t  size;
-       size_t  pos;
+       size_t  pos;    /* relative to offset */
+       size_t  offset; 
 
 #ifdef MS_WINDOWS
        HANDLE  map_handle;
@@ -387,18 +403,22 @@ mmap_resize_method(mmap_object *self,
 #ifdef MS_WINDOWS
        } else {
                DWORD dwErrCode = 0;
-               DWORD newSizeLow, newSizeHigh;
+               DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
                /* First, unmap the file view */
                UnmapViewOfFile(self->data);
                /* Close the mapping object */
                CloseHandle(self->map_handle);
                /* Move to the desired EOF position */
 #if SIZEOF_SIZE_T > 4
-               newSizeHigh = (DWORD)(new_size >> 32);
-               newSizeLow = (DWORD)(new_size & 0xFFFFFFFF);
+               newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
+               newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF);
+               off_hi = (DWORD)(self->offset >> 32);
+               off_lo = (DWORD)(self->offset & 0xFFFFFFFF);
 #else
                newSizeHigh = 0;
                newSizeLow = (DWORD)new_size;
+               off_hi = 0;
+               off_lo = (DWORD)self->offset;
 #endif
                SetFilePointer(self->file_handle,
                               newSizeLow, &newSizeHigh, FILE_BEGIN);
@@ -409,15 +429,15 @@ mmap_resize_method(mmap_object *self,
                        self->file_handle,
                        NULL,
                        PAGE_READWRITE,
-                       newSizeHigh,
-                       newSizeLow,
+                       0,
+                       0,
                        self->tagname);
                if (self->map_handle != NULL) {
                        self->data = (char *) MapViewOfFile(self->map_handle,
                                                            FILE_MAP_WRITE,
-                                                           0,
-                                                           0,
-                                                           0);
+                                                           off_hi,
+                                                           off_lo,
+                                                           new_size);
                        if (self->data != NULL) {
                                self->size = new_size;
                                Py_INCREF(Py_None);
@@ -962,15 +982,18 @@ static PyTypeObject mmap_object_type = {
    Returns -1 on error, with an appropriate Python exception raised. On
    success, the map size is returned. */
 static Py_ssize_t
-_GetMapSize(PyObject *o)
+_GetMapSize(PyObject *o, const char* param)
 {
+       if (o == NULL)
+               return 0;
        if (PyIndex_Check(o)) {
                Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
                if (i==-1 && PyErr_Occurred()) 
                        return -1;
                if (i < 0) {     
-                       PyErr_SetString(PyExc_OverflowError,
-                                       "memory mapped size must be positive");
+                       PyErr_Format(PyExc_OverflowError,
+                                       "memory mapped %s must be positive",
+                                        param);
                        return -1;
                }
                return i;
@@ -988,22 +1011,25 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
        struct stat st;
 #endif
        mmap_object *m_obj;
-       PyObject *map_size_obj = NULL;
-       Py_ssize_t map_size;
+       PyObject *map_size_obj = NULL, *offset_obj = NULL;
+       Py_ssize_t map_size, offset;
        int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
        int devzero = -1;
        int access = (int)ACCESS_DEFAULT;
        static char *keywords[] = {"fileno", "length",
                                          "flags", "prot",
-                                         "access", NULL};
+                                         "access", "offset", NULL};
 
-       if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii", keywords,
+       if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords,
                                         &fd, &map_size_obj, &flags, &prot,
-                                         &access))
+                                         &access, &offset_obj))
                return NULL;
-       map_size = _GetMapSize(map_size_obj);
+       map_size = _GetMapSize(map_size_obj, "size");
        if (map_size < 0)
                return NULL;
+        offset = _GetMapSize(offset_obj, "offset");
+        if (offset < 0)
+                return NULL;
 
        if ((access != (int)ACCESS_DEFAULT) &&
            ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
@@ -1038,7 +1064,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
        if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
                if (map_size == 0) {
                        map_size = st.st_size;
-               } else if ((size_t)map_size > st.st_size) {
+               } else if ((size_t)offset + (size_t)map_size > st.st_size) {
                        PyErr_SetString(PyExc_ValueError,
                                        "mmap length is greater than file size");
                        return NULL;
@@ -1050,6 +1076,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
        m_obj->data = NULL;
        m_obj->size = (size_t) map_size;
        m_obj->pos = (size_t) 0;
+        m_obj->offset = offset;
        if (fd == -1) {
                m_obj->fd = -1;
                /* Assume the caller wants to map anonymous memory.
@@ -1076,10 +1103,10 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
                        return NULL;
                }
        }
-
+       
        m_obj->data = mmap(NULL, map_size,
                           prot, flags,
-                          fd, 0);
+                          fd, offset);
 
        if (devzero != -1) {
                close(devzero);
@@ -1101,10 +1128,12 @@ static PyObject *
 new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
 {
        mmap_object *m_obj;
-       PyObject *map_size_obj = NULL;
-       Py_ssize_t map_size;
-       DWORD size_hi;  /* upper 32 bits of m_obj->size */
-       DWORD size_lo;  /* lower 32 bits of m_obj->size */
+       PyObject *map_size_obj = NULL, *offset_obj = NULL;
+       Py_ssize_t map_size, offset;
+       DWORD off_hi;   /* upper 32 bits of offset */
+       DWORD off_lo;   /* lower 32 bits of offset */
+       DWORD size_hi;  /* upper 32 bits of size */
+       DWORD size_lo;  /* lower 32 bits of size */
        char *tagname = "";
        DWORD dwErr = 0;
        int fileno;
@@ -1113,11 +1142,11 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
        DWORD flProtect, dwDesiredAccess;
        static char *keywords[] = { "fileno", "length",
                                           "tagname",
-                                          "access", NULL };
+                                          "access", "offset", NULL };
 
-       if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|zi", keywords,
+       if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords,
                                         &fileno, &map_size_obj,
-                                        &tagname, &access)) {
+                                        &tagname, &access, &offset_obj)) {
                return NULL;
        }
 
@@ -1139,9 +1168,12 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
                                    "mmap invalid access parameter.");
        }
 
-       map_size = _GetMapSize(map_size_obj);
+       map_size = _GetMapSize(map_size_obj, "size");
        if (map_size < 0)
                return NULL;
+        offset = _GetMapSize(offset_obj, "offset");
+        if (offset < 0)
+                return NULL;
 
        /* assume -1 and 0 both mean invalid filedescriptor
           to 'anonymously' map memory.
@@ -1170,6 +1202,7 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
        m_obj->file_handle = INVALID_HANDLE_VALUE;
        m_obj->map_handle = INVALID_HANDLE_VALUE;
        m_obj->tagname = NULL;
+       m_obj->offset = offset;
 
        if (fh) {
                /* It is necessary to duplicate the handle, so the
@@ -1238,12 +1271,18 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
         * right by 32, so we need different code.
         */
 #if SIZEOF_SIZE_T > 4
-       size_hi = (DWORD)(m_obj->size >> 32);
-       size_lo = (DWORD)(m_obj->size & 0xFFFFFFFF);
+       size_hi = (DWORD)((offset + m_obj->size) >> 32);
+       size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF);
+       off_hi = (DWORD)(offset >> 32);
+       off_lo = (DWORD)(offset & 0xFFFFFFFF);
 #else
        size_hi = 0;
-       size_lo = (DWORD)m_obj->size;
+       size_lo = (DWORD)(offset + m_obj->size);
+       off_hi = 0;
+       off_lo = (DWORD)offset;
 #endif
+       /* For files, it would be sufficient to pass 0 as size.
+          For anonymous maps, we have to pass the size explicitly. */
        m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
                                              NULL,
                                              flProtect,
@@ -1253,8 +1292,8 @@ new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
        if (m_obj->map_handle != NULL) {
                m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
                                                     dwDesiredAccess,
-                                                    0,
-                                                    0,
+                                                    off_hi,
+                                                    off_lo,
                                                     0);
                if (m_obj->data != NULL)
                        return (PyObject *)m_obj;
@@ -1329,6 +1368,8 @@ PyMODINIT_FUNC
 
        setint(dict, "PAGESIZE", (long)my_getpagesize());
 
+       setint(dict, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity()); 
+
        setint(dict, "ACCESS_READ", ACCESS_READ);
        setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
        setint(dict, "ACCESS_COPY", ACCESS_COPY);