Fix calloc_explicitly_typed in case of lb*n overflow

(Cherry-pick commit 41a9ed4 from 'release-7_4' branch.)

* typd_mlc.c: Include limits.h (for SIZE_MAX).
* typd_mlc.c (GC_SIZE_MAX, GC_SQRT_SIZE_MAX): New macro (same as in
malloc.c).
* typd_mlc.c (GC_calloc_explicitly_typed): Return NULL if lb * n
overflows (same algorithm as in calloc defined in malloc.c); eliminate
lb *= n code duplication.

diff --git a/typd_mlc.c b/typd_mlc.c
index 982a3b2677230a3c2d53c0e402ddba9827434d18..777faa4d2ce8004a4dc1b05fa8e0d929a9b70133 100644 (file)
--- a/typd_mlc.c
+++ b/typd_mlc.c
@@ -650,6 +650,15 @@ GC_API void * GC_CALL GC_malloc_explicitly_typed_ignore_off_page(size_t lb,
    return((void *) op);
 }
 
+#include <limits.h>
+#ifdef SIZE_MAX
+# define GC_SIZE_MAX SIZE_MAX
+#else
+# define GC_SIZE_MAX (~(size_t)0)
+#endif
+
+#define GC_SQRT_SIZE_MAX ((((size_t)1) << (WORDSZ / 2)) - 1)
+
 GC_API void * GC_CALL GC_calloc_explicitly_typed(size_t n, size_t lb,
                                                  GC_descr d)
 {
@@ -662,17 +671,20 @@ GC_API void * GC_CALL GC_calloc_explicitly_typed(size_t n, size_t lb,
     struct LeafDescriptor leaf;
     DCL_LOCK_STATE;
 
-    descr_type = GC_make_array_descriptor((word)n, (word)lb, d,
-                                          &simple_descr, &complex_descr, &leaf);
+    descr_type = GC_make_array_descriptor((word)n, (word)lb, d, &simple_descr,
+                                          &complex_descr, &leaf);
+    if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial check */
+        && lb > 0 && n > GC_SIZE_MAX / lb)
+      return NULL; /* n*lb overflow */
+    lb *= n;
     switch(descr_type) {
         case NO_MEM: return(0);
-        case SIMPLE: return(GC_malloc_explicitly_typed(n*lb, simple_descr));
+        case SIMPLE:
+            return GC_malloc_explicitly_typed(lb, simple_descr);
         case LEAF:
-            lb *= n;
             lb += sizeof(struct LeafDescriptor) + TYPD_EXTRA_BYTES;
             break;
         case COMPLEX:
-            lb *= n;
             lb += TYPD_EXTRA_BYTES;
             break;
     }