Avoid array overflow in ANALYZE (Closes #2985)

git-svn-id: http://svn.osgeo.org/postgis/trunk@16785 b70326c6-7e19-0410-871a-916f4a2858ee

diff --git a/postgis/gserialized_estimate.c b/postgis/gserialized_estimate.c
index 971705559450009320b83d6a9e26161350cb9b8e..a850c30efa1f87fa6a1ef85a2dff8392061b1a4f 100644 (file)
--- a/postgis/gserialized_estimate.c
+++ b/postgis/gserialized_estimate.c
@@ -775,7 +775,7 @@ nd_box_array_distribution(const ND_BOX **nd_boxes, int num_boxes, const ND_BOX *
        for ( d = 0; d < ndims; d++ )
        {
                /* Initialize counts for this dimension */
-               memset(counts, 0, sizeof(int)*NUM_BINS);
+               memset(counts, 0, sizeof(counts));
 
                smin = extent->min[d];
                smax = extent->max[d];
@@ -809,8 +809,12 @@ nd_box_array_distribution(const ND_BOX **nd_boxes, int num_boxes, const ND_BOX *
                        }
 
                        /* What bins does this range correspond to? */
-                       bmin = NUM_BINS * (minoffset) / swidth;
-                       bmax = NUM_BINS * (maxoffset) / swidth;
+                       bmin = floor(NUM_BINS * minoffset / swidth);
+                       bmax = floor(NUM_BINS * maxoffset / swidth);
+
+                       /* Should only happen when maxoffset==swidth */
+                       if (bmax >= NUM_BINS)
+                               bmax = NUM_BINS-1;
 
                        POSTGIS_DEBUGF(4, " dimension %d, feature %d: bin %d to bin %d", d, i, bmin, bmax);