Ensure oom_fn callback executed on out-of-memory in calloc

(Cherry-pick commit 80f428f from 'release-7_6' branch.)

* malloc.c [REDIRECT_MALLOC] (calloc):
Call oom_fn(SIZE_MAX) (instead of returning NULL) if n*lb overflows.
* typd_mlc.c (GC_calloc_explicitly_typed): Likewise.
* typd_mlc.c (GC_calloc_explicitly_typed): If register_disappearing_link
failed due to lack of memory then call oom_fn(lb) instead of
GC_malloc(lb); update comment.

diff --git a/malloc.c b/malloc.c
index a5de562784c39df7bd4878ab3066e9f1b8a097a2..5c378de334c719ffdaa3b57cbd2ed737c51b2f61 100644 (file)
--- a/malloc.c
+++ b/malloc.c
@@ -406,7 +406,7 @@ void * calloc(size_t n, size_t lb)
 {
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial test */
         && lb && n > GC_SIZE_MAX / lb)
-      return NULL;
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
 #   if defined(GC_LINUX_THREADS) /* && !defined(USE_PROC_FOR_LIBRARIES) */
         /* libpthread allocated some memory that is only pointed to by  */
         /* mmapped thread stacks.  Make sure it is not collectible.     */

diff --git a/typd_mlc.c b/typd_mlc.c
index 3253de5c1f7e5e602d671b8899ca04694193f0c2..12bdd8acebcb4832bc6b4e8166ca4ac2270e0e48 100644 (file)
--- a/typd_mlc.c
+++ b/typd_mlc.c
@@ -693,7 +693,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
                                           &complex_descr, &leaf);
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial check */
         && lb > 0 && n > GC_SIZE_MAX / lb)
-      return NULL; /* n*lb overflow */
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
     lb *= n;
     switch(descr_type) {
         case NO_MEM: return(0);
@@ -753,9 +753,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
 #    endif
        {
            /* Couldn't register it due to lack of memory.  Punt.        */
-           /* This will probably fail too, but gives the recovery code  */
-           /* a chance.                                                 */
-            return GC_malloc(lb);
+            return (*GC_get_oom_fn())(lb);
        }
    }
    return((void *) op);