From: Ivan Maidanski <ivmai@mail.ru>
Date: Fri, 18 Nov 2016 14:16:32 +0000 (+0300)
Subject: Ensure oom_fn callback executed on out-of-memory in calloc
X-Git-Tag: v7.4.6~203
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=614cbed5a4731ece610eaa01d033f5bf6a9b1ba5;p=gc

Ensure oom_fn callback executed on out-of-memory in calloc
(Cherry-pick commit 80f428f from 'release-7_6' branch.)

* malloc.c [REDIRECT_MALLOC] (calloc):
Call oom_fn(SIZE_MAX) (instead of returning NULL) if n*lb overflows.
* typd_mlc.c (GC_calloc_explicitly_typed): Likewise.
* typd_mlc.c (GC_calloc_explicitly_typed): If register_disappearing_link
failed due to lack of memory then call oom_fn(lb) instead of
GC_malloc(lb); update comment.
---

diff --git a/malloc.c b/malloc.c
index a5de5627..5c378de3 100644
--- a/malloc.c
+++ b/malloc.c
@@ -406,7 +406,7 @@ void * calloc(size_t n, size_t lb)
 {
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial test */
         && lb && n > GC_SIZE_MAX / lb)
-      return NULL;
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
 #   if defined(GC_LINUX_THREADS) /* && !defined(USE_PROC_FOR_LIBRARIES) */
         /* libpthread allocated some memory that is only pointed to by  */
         /* mmapped thread stacks.  Make sure it is not collectible.     */
diff --git a/typd_mlc.c b/typd_mlc.c
index 3253de5c..12bdd8ac 100644
--- a/typd_mlc.c
+++ b/typd_mlc.c
@@ -693,7 +693,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
                                           &complex_descr, &leaf);
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial check */
         && lb > 0 && n > GC_SIZE_MAX / lb)
-      return NULL; /* n*lb overflow */
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
     lb *= n;
     switch(descr_type) {
         case NO_MEM: return(0);
@@ -753,9 +753,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
 #    endif
        {
            /* Couldn't register it due to lack of memory.  Punt.        */
-           /* This will probably fail too, but gives the recovery code  */
-           /* a chance.                                                 */
-            return GC_malloc(lb);
+            return (*GC_get_oom_fn())(lb);
        }
    }
    return((void *) op);