overflow in RectArea

in rectangle.c
changed type of area and return type of RectArea to unsigned int
added overflow check in RectArea
in split.q.h
changed type of SplitQ_t.CoverSplitArea to unsigned int

diff --git a/lib/label/rectangle.c b/lib/label/rectangle.c
index 8413f04811098c2e6a780c0645f8f7f209c803a9..94b06cc6a631cbaafd50aadfdc79836baf12d0b6 100644 (file)
--- a/lib/label/rectangle.c
+++ b/lib/label/rectangle.c
@@ -13,6 +13,7 @@
 #include "index.h"
 #include <stdio.h>
 #include <assert.h>
+#include <limits.h>
 #include "logic.h"
 #include "arith.h"
 #include "rectangle.h"
@@ -115,17 +116,30 @@ void PrintRect(Rect_t * r)
 /*-----------------------------------------------------------------------------
 | Calculate the n-dimensional area of a rectangle
 -----------------------------------------------------------------------------*/
-int RectArea(Rect_t * r)
+unsigned int RectArea(Rect_t * r)
 {
-    register int i, area;
-    assert(r);
+  register int i;
+  unsigned int area;
+  assert(r);
 
     if (Undefined(r))
        return 0;
 
+    /*
+     * XXX add overflow checks
+     */
     area = 1;
     for (i = 0; i < NUMDIMS; i++) {
-       area *= r->boundary[i + NUMDIMS] - r->boundary[i];
+#if 1  /* overflow check */
+      long long a_test = area * r->boundary[i + NUMDIMS] - r->boundary[i];
+      if( a_test > UINT_MAX) {
+       agerror("label: area too large for rtree\n");
+       return UINT_MAX;
+      }
+      area = a_test;
+#else
+      area *= r->boundary[i + NUMDIMS] - r->boundary[i];
+#endif
     }
     return area;
 }

diff --git a/lib/label/rectangle.h b/lib/label/rectangle.h
index 64fbd6fdef463125e257925e24e9d775e0ba30b0..dbda16f059c097e55f99ca6ce95fd1b68e929ce2 100644 (file)
--- a/lib/label/rectangle.h
+++ b/lib/label/rectangle.h
@@ -25,7 +25,7 @@ void InitRect(Rect_t * r);
 #ifdef RTDEBUG
 void PrintRect(Rect_t *);
 #endif
-int RectArea(Rect_t *);
+unsigned int RectArea(Rect_t *);
 int Overlap(Rect_t *, Rect_t *);
 int Contained(Rect_t *, Rect_t *);
 Rect_t CombineRect(Rect_t *, Rect_t *);

diff --git a/lib/label/split.q.c b/lib/label/split.q.c
index c382f882817b80ea2deb459a9f8647084580386d..6cc593b118bf155831eae9e7c8e26350d7b0ba0c 100644 (file)
--- a/lib/label/split.q.c
+++ b/lib/label/split.q.c
@@ -218,13 +218,16 @@ static void MethodZero(RTree_t * rtp)
 -----------------------------------------------------------------------------*/
 static void PickSeeds(RTree_t * rtp)
 {
-    register int i, j, waste, worst, seed0, seed1;
-    int area[NODECARD + 1];
+  register int i, j;
+  unsigned int waste, worst;
+  int seed0, seed1;
+  unsigned int area[NODECARD + 1];
 
     for (i = 0; i < NODECARD + 1; i++)
        area[i] = RectArea(&rtp->split.BranchBuf[i].rect);
 
-    worst = -rtp->split.CoverSplitArea - 1;
+    //worst = -rtp->split.CoverSplitArea - 1;
+    worst=0;
     for (i = 0; i < NODECARD; i++) {
        for (j = i + 1; j < NODECARD + 1; j++) {
            Rect_t rect;

diff --git a/lib/label/split.q.h b/lib/label/split.q.h
index ee341f3445ea8360831a683a791cba7dc837b814..7279bb74cc4c929035349dddba9859341f05f9a3 100644 (file)
--- a/lib/label/split.q.h
+++ b/lib/label/split.q.h
@@ -38,7 +38,7 @@ extern "C" {
 typedef struct split_q_s {
     struct Branch BranchBuf[NODECARD + 1];
     struct Rect CoverSplit;
-    int CoverSplitArea;
+    unsigned int CoverSplitArea;
     struct PartitionVars Partitions[METHODS];
 } SplitQ_t;