infinite loop around gserialized_gist_picksplit_2d from smagen (#2449)

git-svn-id: http://svn.osgeo.org/postgis/trunk@12009 b70326c6-7e19-0410-871a-916f4a2858ee

diff --git a/postgis/gserialized_gist_2d.c b/postgis/gserialized_gist_2d.c
index ef3be4dfdc304fc9ea952f74175172b4947a86d8..87c864cc8e57724feef008347a5d84a9862e23b2 100644 (file)
--- a/postgis/gserialized_gist_2d.c
+++ b/postgis/gserialized_gist_2d.c
@@ -131,11 +131,15 @@ static void box2df_merge(BOX2DF *b_union, BOX2DF *b_new)
 
        POSTGIS_DEBUGF(5, "merging %s with %s", box2df_to_string(b_union), box2df_to_string(b_new));
        /* Adjust minimums */
-       b_union->xmin = Min(b_union->xmin, b_new->xmin);
-       b_union->ymin = Min(b_union->ymin, b_new->ymin);
+       if (b_union->xmin > b_new->xmin || isnan(b_union->xmin))
+               b_union->xmin = b_new->xmin;
+       if (b_union->ymin > b_new->ymin || isnan(b_union->ymin))
+               b_union->ymin = b_new->ymin;
        /* Adjust maximums */
-       b_union->xmax = Max(b_union->xmax, b_new->xmax);
-       b_union->ymax = Max(b_union->ymax, b_new->ymax);
+       if (b_union->xmax < b_new->xmax || isnan(b_union->xmax))
+               b_union->xmax = b_new->xmax;
+       if (b_union->ymax < b_new->ymax || isnan(b_union->ymax))
+               b_union->ymax = b_new->ymax;
 
        POSTGIS_DEBUGF(5, "merge complete %s", box2df_to_string(b_union));
        return;
@@ -1184,13 +1188,13 @@ Datum gserialized_gist_same_2d(PG_FUNCTION_ARGS)
 static void
 adjustBox(BOX2DF *b, BOX2DF *addon)
 {
-       if (b->xmax < addon->xmax)
+       if (b->xmax < addon->xmax || isnan(b->xmax))
                b->xmax = addon->xmax;
-       if (b->xmin > addon->xmin)
+       if (b->xmin > addon->xmin || isnan(b->xmin))
                b->xmin = addon->xmin;
-       if (b->ymax < addon->ymax)
+       if (b->ymax < addon->ymax || isnan(b->ymax))
                b->ymax = addon->ymax;
-       if (b->ymin > addon->ymin)
+       if (b->ymin > addon->ymin || isnan(b->ymin))
                b->ymin = addon->ymin;
 }
 
@@ -1310,14 +1314,29 @@ interval_cmp_lower(const void *i1, const void *i2)
        float           lower1 = ((const SplitInterval *) i1)->lower,
                                lower2 = ((const SplitInterval *) i2)->lower;
 
-       if (lower1 < lower2)
+       if (isnan(lower1))
+       {
+               if (isnan(lower2))
+                       return 0;
+               else
+                       return 1;
+       }
+       else if (isnan(lower2))
+       {
                return -1;
-       else if (lower1 > lower2)
-               return 1;
+       }
        else
-               return 0;
+       {
+               if (lower1 < lower2)
+                       return -1;
+               else if (lower1 > lower2)
+                       return 1;
+               else
+                       return 0;
+       }
 }
 
+
 /*
  * Interval comparison function by upper bound of the interval;
  */
@@ -1327,12 +1346,26 @@ interval_cmp_upper(const void *i1, const void *i2)
        float           upper1 = ((const SplitInterval *) i1)->upper,
                                upper2 = ((const SplitInterval *) i2)->upper;
 
-       if (upper1 < upper2)
-               return -1;
-       else if (upper1 > upper2)
+       if (isnan(upper1))
+       {
+               if (isnan(upper2))
+                       return 0;
+               else
+                       return -1;
+       }
+       else if (isnan(upper2))
+       {
                return 1;
+       }
        else
-               return 0;
+       {
+               if (upper1 < upper2)
+                       return -1;
+               else if (upper1 > upper2)
+                       return 1;
+               else
+                       return 0;
+       }
 }
 
 /*
@@ -1645,7 +1678,8 @@ Datum gserialized_gist_picksplit_2d(PG_FUNCTION_ARGS)
                        /*
                         * Find next lower bound of right group.
                         */
-                       while (i1 < nentries && rightLower == intervalsLower[i1].lower)
+                       while (i1 < nentries && (rightLower == intervalsLower[i1].lower ||
+                                       isnan(intervalsLower[i1].lower)))
                        {
                                leftUpper = Max(leftUpper, intervalsLower[i1].upper);
                                i1++;
@@ -1680,7 +1714,8 @@ Datum gserialized_gist_picksplit_2d(PG_FUNCTION_ARGS)
                        /*
                         * Find next upper bound of left group.
                         */
-                       while (i2 >= 0 && leftUpper == intervalsUpper[i2].upper)
+                       while (i2 >= 0 && (leftUpper == intervalsUpper[i2].upper ||
+                                       isnan(intervalsUpper[i2].upper)))
                        {
                                rightLower = Min(rightLower, intervalsUpper[i2].lower);
                                i2--;
@@ -1784,10 +1819,10 @@ Datum gserialized_gist_picksplit_2d(PG_FUNCTION_ARGS)
                        upper = box->ymax;
                }
 
-               if (upper <= context.leftUpper)
+               if (upper <= context.leftUpper || isnan(upper))
                {
                        /* Fits to the left group */
-                       if (lower >= context.rightLower)
+                       if (lower >= context.rightLower || isnan(lower))
                        {
                                /* Fits also to the right group, so "common entry" */
                                commonEntries[commonEntriesCount++].index = i;