Protect GIST logic that assumes penalty values can't be negative.

Apparently sane-looking penalty code might return small negative values,
for example because of roundoff error.  This will confuse places like
gistchoose().  Prevent problems by clamping negative penalty values to
zero.  (Just to be really sure, I also made it force NaNs to zero.)
Back-patch to all supported branches.

Alexander Korotkov

diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml
index 587517da1d17ce13b54f998accb6d871e2da2d3b..b0ac8e52b6a9f92e3ebee558d81ed4f157be5d7d 100644 (file)
--- a/doc/src/sgml/gist.sgml
+++ b/doc/src/sgml/gist.sgml
@@ -147,6 +147,8 @@
        Returns a value indicating the <quote>cost</quote> of inserting the new
        entry into a particular branch of the tree.  items will be inserted
        down the path of least <function>penalty</function> in the tree.
+       Values returned by <function>penalty</function> should be non-negative.
+       If a negative value is returned, it will be treated as zero.
       </para>
      </listitem>
     </varlistentry>

diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c
index d88f5ea0f81786464bd1f29a32261230250fa4aa..5bd6a9893b080b89042a72c0a2f9557026d1ea38 100644 (file)
--- a/src/backend/access/gist/gistutil.c
+++ b/src/backend/access/gist/gistutil.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include <math.h>
+
 #include "access/gist_private.h"
 #include "access/heapam.h"
 #include "access/reloptions.h"
@@ -530,16 +532,22 @@ gistpenalty(GISTSTATE *giststate, int attno,
 {
        float           penalty = 0.0;
 
-       if (giststate->penaltyFn[attno].fn_strict == FALSE || (isNullOrig == FALSE && isNullAdd == FALSE))
+       if (giststate->penaltyFn[attno].fn_strict == FALSE ||
+               (isNullOrig == FALSE && isNullAdd == FALSE))
+       {
                FunctionCall3(&giststate->penaltyFn[attno],
                                          PointerGetDatum(orig),
                                          PointerGetDatum(add),
                                          PointerGetDatum(&penalty));
+               /* disallow negative or NaN penalty */
+               if (isnan(penalty) || penalty < 0.0)
+                       penalty = 0.0;
+       }
        else if (isNullOrig && isNullAdd)
                penalty = 0.0;
        else
-               penalty = 1e10;                 /* try to prevent to mix null and non-null
-                                                                * value */
+               penalty = 1e10;                 /* try to prevent mixing null and non-null
+                                                                * values */
 
        return penalty;
 }