From 486d43e2be66e1da04494e2a04649eed44df5ca9 Mon Sep 17 00:00:00 2001
From: Ivan Maidanski <ivmai@mail.ru>
Date: Fri, 18 Nov 2016 17:16:32 +0300
Subject: [PATCH] Ensure oom_fn callback executed on out-of-memory in calloc
(Cherry-pick commit 80f428f from 'release-7_6' branch.)
* malloc.c [REDIRECT_MALLOC] (calloc):
Call oom_fn(SIZE_MAX) (instead of returning NULL) if n*lb overflows.
* typd_mlc.c (GC_calloc_explicitly_typed): Likewise.
* typd_mlc.c (GC_calloc_explicitly_typed): If register_disappearing_link
failed due to lack of memory then call oom_fn(lb) instead of
GC_malloc(lb); update comment.
---
malloc.c | 2 +-
typd_mlc.c | 6 ++----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/malloc.c b/malloc.c
index befd00ff..3dec405c 100644
--- a/malloc.c
+++ b/malloc.c
@@ -372,7 +372,7 @@ void * calloc(size_t n, size_t lb)
{
if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial test */
&& lb && n > GC_SIZE_MAX / lb)
- return NULL;
+ return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
# if defined(GC_LINUX_THREADS) /* && !defined(USE_PROC_FOR_LIBRARIES) */
/* libpthread allocated some memory that is only pointed to by */
/* mmapped thread stacks. Make sure it is not collectible. */
diff --git a/typd_mlc.c b/typd_mlc.c
index 6e654821..e3b26725 100644
--- a/typd_mlc.c
+++ b/typd_mlc.c
@@ -666,7 +666,7 @@ GC_API void * GC_CALL GC_calloc_explicitly_typed(size_t n, size_t lb,
&complex_descr, &leaf);
if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial check */
&& lb > 0 && n > GC_SIZE_MAX / lb)
- return NULL; /* n*lb overflow */
+ return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
lb *= n;
switch(descr_type) {
case NO_MEM: return(0);
@@ -722,9 +722,7 @@ GC_API void * GC_CALL GC_calloc_explicitly_typed(size_t n, size_t lb,
if (GC_general_register_disappearing_link((void * *)((word *)op+lw-1),
op) == GC_NO_MEMORY) {
/* Couldn't register it due to lack of memory. Punt. */
- /* This will probably fail too, but gives the recovery code */
- /* a chance. */
- return GC_malloc(lb);
+ return (*GC_get_oom_fn())(lb);
}
}
return((void *) op);
--
2.40.0