From 467b279de834a48f9d5ba731727f75949afa4000 Mon Sep 17 00:00:00 2001
From: Ivan Maidanski <ivmai@mail.ru>
Date: Tue, 25 Aug 2015 10:19:05 +0300
Subject: [PATCH] Fix out-of-memory handling in GC_toggleref_add

* finalize.c (ensure_toggleref_capacity): Change return type from void
to GC_bool; return FALSE on allocation failure (or desired capacity
value overflow).
* finalize.c (GC_toggleref_add): Change return type from void to int.
* include/gc.h (GC_toggleref_add): Likewise.
* finalize.c (GC_toggleref_add): Return GC_NO_MEMORY if
ensure_toggleref_capacity failed, GC_SUCCESS otherwise (including the
case of no callback).
---
 finalize.c   | 24 +++++++++++++++++++-----
 include/gc.h |  2 +-
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/finalize.c b/finalize.c
index bbc93894..90d5f424 100644
--- a/finalize.c
+++ b/finalize.c
@@ -372,28 +372,38 @@ void GC_toggleref_register_callback(int (*proccess_toggleref) (GC_PTR obj))
     GC_toggleref_callback = proccess_toggleref;
 }
 
-static void
+static GC_bool
 ensure_toggleref_capacity (int capacity)
 {
     if (!GC_toggleref_array) {
         GC_toggleref_array_capacity = 32;
         GC_toggleref_array = (GCToggleRef *) GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE (GC_toggleref_array_capacity * sizeof (GCToggleRef), NORMAL);
+        if (NULL == GC_toggleref_array)
+            return FALSE;
     }
-    if (GC_toggleref_array_size + capacity >= GC_toggleref_array_capacity) {
+    if ((unsigned)GC_toggleref_array_size + (unsigned)capacity
+            >= (unsigned)GC_toggleref_array_capacity) {
         GCToggleRef *tmp;
         int old_capacity = GC_toggleref_array_capacity;
-        while (GC_toggleref_array_capacity < GC_toggleref_array_size + capacity)
+        while ((unsigned)GC_toggleref_array_capacity
+                < (unsigned)GC_toggleref_array_size + (unsigned)capacity) {
             GC_toggleref_array_capacity *= 2;
+            if (GC_toggleref_array_capacity < 0) /* overflow */
+                return FALSE;
+        }
 
         tmp = (GCToggleRef *) GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE (GC_toggleref_array_capacity * sizeof (GCToggleRef), NORMAL);
+        if (NULL == tmp)
+            return FALSE;
         memcpy (tmp, GC_toggleref_array, GC_toggleref_array_size * sizeof (GCToggleRef));
 
         GC_INTERNAL_FREE(GC_toggleref_array);
         GC_toggleref_array = tmp;
     }
+    return TRUE;
 }
 
-void
+int
 GC_toggleref_add (GC_PTR object, int strong_ref)
 {
     DCL_LOCK_STATE;
@@ -402,13 +412,17 @@ GC_toggleref_add (GC_PTR object, int strong_ref)
     if (!GC_toggleref_callback)
         goto end;
 
-    ensure_toggleref_capacity (1);
+    if (!ensure_toggleref_capacity(1)) {
+        UNLOCK();
+        return GC_NO_MEMORY;
+    }
     GC_toggleref_array [GC_toggleref_array_size].strong_ref = strong_ref ? object : NULL;
     GC_toggleref_array [GC_toggleref_array_size].weak_ref = strong_ref ? (GC_hidden_pointer)NULL : GC_HIDE_POINTER (object);
     ++GC_toggleref_array_size;
 
 end:
     UNLOCK();
+    return GC_SUCCESS;
 }
 
 
diff --git a/include/gc.h b/include/gc.h
index 04cf3743..5a407cba 100644
--- a/include/gc.h
+++ b/include/gc.h
@@ -1175,7 +1175,7 @@ GC_API int GC_CALL GC_unregister_long_link(void ** /* link */);
 
 /* toggleref support */
 GC_API void GC_toggleref_register_callback (int (*proccess_toggleref) (GC_PTR obj));
-GC_API void GC_toggleref_add (GC_PTR object, int strong_ref);
+GC_API int GC_toggleref_add (GC_PTR object, int strong_ref);
 
 /* Finalizer callback support.  Invoked by the collector (with  */
 /* the allocation lock held) for each unreachable object        */
-- 
2.40.0