http://bugs.python.org/issue6836

The debug memory api now keeps track of which external API (PyMem_* or PyObject_*) was used to allocate each block and treats any API violation as an error.  Added separate _PyMem_DebugMalloc functions for the Py_Mem API instead of having it use the _PyObject_DebugMalloc functions.

diff --git a/Include/objimpl.h b/Include/objimpl.h
index 55186b176be682814f626e47ab7ac3ee0e8fe32d..4176c6b604f4b8f324bb7a4e332a15345065c33a 100644 (file)
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -108,6 +108,13 @@ PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
 PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
 PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
 PyAPI_FUNC(void) _PyObject_DebugMallocStats(void);
+PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
+PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
+PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
+PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
+PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
 #define PyObject_MALLOC                _PyObject_DebugMalloc
 #define PyObject_Malloc                _PyObject_DebugMalloc
 #define PyObject_REALLOC       _PyObject_DebugRealloc

diff --git a/Include/pymem.h b/Include/pymem.h
index 542aceef25370db0b32f15421d23560f602dcab3..e2dfe0d3fc35dba0d1f4b10a409b5b5a59802b28 100644 (file)
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -59,9 +59,9 @@ PyAPI_FUNC(void) PyMem_Free(void *);
 /* Macros. */
 #ifdef PYMALLOC_DEBUG
 /* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC           PyObject_MALLOC
-#define PyMem_REALLOC          PyObject_REALLOC
-#define PyMem_FREE             PyObject_FREE
+#define PyMem_MALLOC           _PyMem_DebugMalloc
+#define PyMem_REALLOC          _PyMem_DebugRealloc
+#define PyMem_FREE             _PyMem_DebugFree
 
 #else  /* ! PYMALLOC_DEBUG */
 

diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index d1586c29f888bb7310f439fb4d988804a88edd79..b2c053f2f4bb3345fa09bd705701a846b7c9897d 100644 (file)
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1241,6 +1241,10 @@ PyObject_Free(void *p)
 #define DEADBYTE       0xDB    /* dead (newly freed) memory */
 #define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
 
+/* We tag each block with an API ID in order to tag API violations */
+#define _PYMALLOC_MEM_ID 'm'   /* the PyMem_Malloc() API */
+#define _PYMALLOC_OBJ_ID 'o'   /* The PyObject_Malloc() API */
+
 static size_t serialno = 0;    /* incremented on each debug {m,re}alloc */
 
 /* serialno is always incremented via calling this routine.  The point is
@@ -1331,8 +1335,49 @@ p[2*S+n+S: 2*S+n+2*S]
     instant at which this block was passed out.
 */
 
+/* debug replacements for the PyMem_* memory API */
+void *
+_PyMem_DebugMalloc(size_t nbytes)
+{
+       return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes);
+}
+void *
+_PyMem_DebugRealloc(void *p, size_t nbytes)
+{
+       return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes);
+}
+void
+_PyMem_DebugFree(void *p)
+{
+       _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p);
+}
+
+/* debug replacements for the PyObject_* memory API */
 void *
 _PyObject_DebugMalloc(size_t nbytes)
+{
+       return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes);
+}
+void *
+_PyObject_DebugRealloc(void *p, size_t nbytes)
+{
+       return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes);
+}
+void
+_PyObject_DebugFree(void *p)
+{
+       _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p);
+}
+void
+_PyObject_DebugCheckAddress(void *p)
+{
+       _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p);
+}
+
+
+/* generic debug memory api, with an "id" to identify the API in use */
+void *
+_PyObject_DebugMallocApi(char id, size_t nbytes)
 {
        uchar *p;       /* base address of malloc'ed block */
        uchar *tail;    /* p + 2*SST + nbytes == pointer to tail pad bytes */
@@ -1348,12 +1393,15 @@ _PyObject_DebugMalloc(size_t nbytes)
        if (p == NULL)
                return NULL;
 
+       /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
        write_size_t(p, nbytes);
-       memset(p + SST, FORBIDDENBYTE, SST);
+       p[SST] = (uchar)id;
+       memset(p + SST + 1 , FORBIDDENBYTE, SST-1);
 
        if (nbytes > 0)
                memset(p + 2*SST, CLEANBYTE, nbytes);
 
+       /* at tail, write pad (SST bytes) and serialno (SST bytes) */
        tail = p + 2*SST + nbytes;
        memset(tail, FORBIDDENBYTE, SST);
        write_size_t(tail + SST, serialno);
@@ -1362,27 +1410,28 @@ _PyObject_DebugMalloc(size_t nbytes)
 }
 
 /* The debug free first checks the 2*SST bytes on each end for sanity (in
-   particular, that the FORBIDDENBYTEs are still intact).
+   particular, that the FORBIDDENBYTEs with the api ID are still intact).
    Then fills the original bytes with DEADBYTE.
    Then calls the underlying free.
 */
 void
-_PyObject_DebugFree(void *p)
+_PyObject_DebugFreeApi(char api, void *p)
 {
        uchar *q = (uchar *)p - 2*SST;  /* address returned from malloc */
        size_t nbytes;
 
        if (p == NULL)
                return;
-       _PyObject_DebugCheckAddress(p);
+       _PyObject_DebugCheckAddressApi(api, p);
        nbytes = read_size_t(q);
+       nbytes += 4*SST;
        if (nbytes > 0)
                memset(q, DEADBYTE, nbytes);
        PyObject_Free(q);
 }
 
 void *
-_PyObject_DebugRealloc(void *p, size_t nbytes)
+_PyObject_DebugReallocApi(char api, void *p, size_t nbytes)
 {
        uchar *q = (uchar *)p;
        uchar *tail;
@@ -1391,9 +1440,9 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
        int i;
 
        if (p == NULL)
-               return _PyObject_DebugMalloc(nbytes);
+               return _PyObject_DebugMallocApi(api, nbytes);
 
-       _PyObject_DebugCheckAddress(p);
+       _PyObject_DebugCheckAddressApi(api, p);
        bumpserialno();
        original_nbytes = read_size_t(q - 2*SST);
        total = nbytes + 4*SST;
@@ -1403,16 +1452,20 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 
        if (nbytes < original_nbytes) {
                /* shrinking:  mark old extra memory dead */
-               memset(q + nbytes, DEADBYTE, original_nbytes - nbytes);
+               memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST);
        }
 
-       /* Resize and add decorations. */
+       /* Resize and add decorations. We may get a new pointer here, in which
+        * case we didn't get the chance to mark the old memory with DEADBYTE,
+        * but we live with that.
+        */
        q = (uchar *)PyObject_Realloc(q - 2*SST, total);
        if (q == NULL)
                return NULL;
 
        write_size_t(q, nbytes);
-       for (i = 0; i < SST; ++i)
+       assert(q[SST] == (uchar)api);
+       for (i = 1; i < SST; ++i)
                assert(q[SST + i] == FORBIDDENBYTE);
        q += 2*SST;
        tail = q + nbytes;
@@ -1431,26 +1484,38 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 /* Check the forbidden bytes on both ends of the memory allocated for p.
  * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress,
  * and call Py_FatalError to kill the program.
+ * The API id, is also checked.
  */
  void
-_PyObject_DebugCheckAddress(const void *p)
+_PyObject_DebugCheckAddressApi(char api, const void *p)
 {
        const uchar *q = (const uchar *)p;
+       char msgbuf[64];
        char *msg;
        size_t nbytes;
        const uchar *tail;
        int i;
+       char id;
 
        if (p == NULL) {
                msg = "didn't expect a NULL pointer";
                goto error;
        }
 
+       /* Check the API id */
+       id = (char)q[-SST];
+       if (id != api) {
+               msg = msgbuf;
+               snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api);
+               msgbuf[sizeof(msgbuf)-1] = 0;
+               goto error;
+       }
+
        /* Check the stuff at the start of p first:  if there's underwrite
         * corruption, the number-of-bytes field may be nuts, and checking
         * the tail could lead to a segfault then.
         */
-       for (i = SST; i >= 1; --i) {
+       for (i = SST-1; i >= 1; --i) {
                if (*(q-i) != FORBIDDENBYTE) {
                        msg = "bad leading pad byte";
                        goto error;
@@ -1482,19 +1547,24 @@ _PyObject_DebugDumpAddress(const void *p)
        size_t nbytes, serial;
        int i;
        int ok;
+       char id;
 
-       fprintf(stderr, "Debug memory block at address p=%p:\n", p);
-       if (p == NULL)
+       fprintf(stderr, "Debug memory block at address p=%p:", p);
+       if (p == NULL) {
+               fprintf(stderr, "\n");
                return;
+       }
+       id = (char)q[-SST];
+       fprintf(stderr, " API '%c'\n", id);
 
        nbytes = read_size_t(q - 2*SST);
        fprintf(stderr, "    %" PY_FORMAT_SIZE_T "u bytes originally "
                        "requested\n", nbytes);
 
        /* In case this is nuts, check the leading pad bytes first. */
-       fprintf(stderr, "    The %d pad bytes at p-%d are ", SST, SST);
+       fprintf(stderr, "    The %d pad bytes at p-%d are ", SST-1, SST-1);
        ok = 1;
-       for (i = 1; i <= SST; ++i) {
+       for (i = 1; i <= SST-1; ++i) {
                if (*(q-i) != FORBIDDENBYTE) {
                        ok = 0;
                        break;
@@ -1505,7 +1575,7 @@ _PyObject_DebugDumpAddress(const void *p)
        else {
                fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
                        FORBIDDENBYTE);
-               for (i = SST; i >= 1; --i) {
+               for (i = SST-1; i >= 1; --i) {
                        const uchar byte = *(q-i);
                        fprintf(stderr, "        at p-%d: 0x%02x", i, byte);
                        if (byte != FORBIDDENBYTE)