From: Matthew Fernandez <matthew.fernandez@gmail.com>
Date: Sat, 30 Jul 2022 16:38:15 +0000 (-0700)
Subject: pathplan Pobsopen: use a 'size_t' when counting objects
X-Git-Tag: 5.0.1~21^2~6
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=89db2484c4c619202cb6b296d47884e4396ed938;p=graphviz

pathplan Pobsopen: use a 'size_t' when counting objects

This squashes 3 -Wsign-conversion warnings and is generally closer to what we
would like to do here. The “proper” fix is for fields like `vconfig_t.N` to
become `size_t` instead of `int`. But unfortunately they are part of the public
API, and it seems undesirable to break API for this.

Note that an assumption previously implicit in this function, that all inputs
had a non-negative polygon count, is now an explicit assertion.
---

diff --git a/lib/pathplan/cvt.c b/lib/pathplan/cvt.c
index 3e1c9ce0f..b1654ec3b 100644
--- a/lib/pathplan/cvt.c
+++ b/lib/pathplan/cvt.c
@@ -8,9 +8,11 @@
  * Contributors: Details at https://graphviz.org
  *************************************************************************/
 
+#include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <cgraph/likely.h>
+#include <limits.h>
 #include <pathplan/vis.h>
 
 typedef Ppoint_t ilcoord_t;
@@ -31,7 +33,7 @@ static void gasp_print_bezier(Ppolyline_t * route);
 vconfig_t *Pobsopen(Ppoly_t ** obs, int n_obs)
 {
     vconfig_t *rv;
-    int poly_i, pt_i, i, n;
+    int poly_i, pt_i, i;
     int start, end;
 
     rv = malloc(sizeof(vconfig_t));
@@ -40,14 +42,20 @@ vconfig_t *Pobsopen(Ppoly_t ** obs, int n_obs)
     }
 
     /* get storage */
-    n = 0;
-    for (poly_i = 0; poly_i < n_obs; poly_i++)
-	n += obs[poly_i]->pn;
+    size_t n = 0;
+    for (poly_i = 0; poly_i < n_obs; poly_i++) {
+	assert(obs[poly_i]->pn >= 0);
+	n += (size_t)obs[poly_i]->pn;
+    }
+    if (n > INT_MAX) { // will this overflow rv->N?
+	free(rv);
+	return NULL;
+    }
     rv->P = malloc(n * sizeof(Ppoint_t));
     rv->start = malloc((n_obs + 1) * sizeof(int));
     rv->next = malloc(n * sizeof(int));
     rv->prev = malloc(n * sizeof(int));
-    rv->N = n;
+    rv->N = (int)n;
     rv->Npoly = n_obs;
 
     // bail out if any above allocations failed