From: Ivan Maidanski <ivmai@mail.ru>
Date: Fri, 18 Nov 2016 14:16:32 +0000 (+0300)
Subject: Ensure oom_fn callback executed on out-of-memory in calloc
X-Git-Tag: v7.6.2~330
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=80f428f3980e6b819987e0293b3420ac5c1b2959;p=gc

Ensure oom_fn callback executed on out-of-memory in calloc
(fix commits e10c1eb, 4e1a6f9)

* malloc.c [REDIRECT_MALLOC && !REDIRECT_MALLOC_IN_HEADER] (calloc):
Call oom_fn(SIZE_MAX) (instead of returning NULL) if n*lb overflows.
* typd_mlc.c (GC_calloc_explicitly_typed): Likewise.
* typd_mlc.c (GC_calloc_explicitly_typed): If register_disappearing_link
failed due to lack of memory then call oom_fn(lb) instead of
GC_malloc(lb); update comment.
---

diff --git a/malloc.c b/malloc.c
index 0563cf2b..a6e0333e 100644
--- a/malloc.c
+++ b/malloc.c
@@ -435,7 +435,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_uncollectable(size_t lb)
   {
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial test */
         && lb && n > GC_SIZE_MAX / lb)
-      return NULL;
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
 #   if defined(GC_LINUX_THREADS)
       /* libpthread allocated some memory that is only pointed to by    */
       /* mmapped thread stacks.  Make sure it is not collectible.       */
diff --git a/typd_mlc.c b/typd_mlc.c
index d19e4c11..7e91fab2 100644
--- a/typd_mlc.c
+++ b/typd_mlc.c
@@ -647,7 +647,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
                                           &complex_descr, &leaf);
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial check */
         && lb > 0 && n > GC_SIZE_MAX / lb)
-      return NULL; /* n*lb overflow */
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
     lb *= n;
     switch(descr_type) {
         case NO_MEM: return(0);
@@ -690,9 +690,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
 #     endif
         {
             /* Couldn't register it due to lack of memory.  Punt.       */
-            /* This will probably fail too, but gives the recovery code */
-            /* a chance.                                                */
-            return GC_malloc(lb);
+            return (*GC_get_oom_fn())(lb);
         }
     }
     return op;