Protect GIST logic that assumes penalty values can't be negative.

Apparently sane-looking penalty code might return small negative values,
for example because of roundoff error.  This will confuse places like
gistchoose().  Prevent problems by clamping negative penalty values to
zero.  (Just to be really sure, I also made it force NaNs to zero.)
Back-patch to all supported branches.

Alexander Korotkov

diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml
index f6e31092aa175abd42003bd8a89dffc3028ffe06..78171cfa3fd2140e860834d4aee95ec8c46b4ddc 100644 (file)
--- a/doc/src/sgml/gist.sgml
+++ b/doc/src/sgml/gist.sgml
@@ -378,6 +378,8 @@ my_decompress(PG_FUNCTION_ARGS)
        Returns a value indicating the <quote>cost</quote> of inserting the new
        entry into a particular branch of the tree.  Items will be inserted
        down the path of least <function>penalty</function> in the tree.
+       Values returned by <function>penalty</function> should be non-negative.
+       If a negative value is returned, it will be treated as zero.
       </para>
 
       <para>

diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c
index e61b676628bc4f39b215a9c8ab3ca1bcfcee9499..1aabcc527ac54aa80a044ee29fa22f4f4201a1ad 100644 (file)
--- a/src/backend/access/gist/gistutil.c
+++ b/src/backend/access/gist/gistutil.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include <math.h>
+
 #include "access/gist_private.h"
 #include "access/reloptions.h"
 #include "storage/freespace.h"
@@ -526,16 +528,21 @@ gistpenalty(GISTSTATE *giststate, int attno,
 
        if (giststate->penaltyFn[attno].fn_strict == FALSE ||
                (isNullOrig == FALSE && isNullAdd == FALSE))
+       {
                FunctionCall3Coll(&giststate->penaltyFn[attno],
                                                  giststate->supportCollation[attno],
                                                  PointerGetDatum(orig),
                                                  PointerGetDatum(add),
                                                  PointerGetDatum(&penalty));
+               /* disallow negative or NaN penalty */
+               if (isnan(penalty) || penalty < 0.0)
+                       penalty = 0.0;
+       }
        else if (isNullOrig && isNullAdd)
                penalty = 0.0;
        else
-               penalty = 1e10;                 /* try to prevent to mix null and non-null
-                                                                * value */
+               penalty = 1e10;                 /* try to prevent mixing null and non-null
+                                                                * values */
 
        return penalty;
 }