Guard against roundoff errors in new selectivity-estimation code,

per bug report from Laurette Cisneros.

diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c
index 85b871078b14cb6794c98f628e952c7a4688c44a..edcfe91a47dc701dff53839eeaec298e1e29136d 100644 (file)
--- a/src/backend/utils/adt/selfuncs.c
+++ b/src/backend/utils/adt/selfuncs.c
@@ -15,7 +15,7 @@
  *
  *
  * IDENTIFICATION
- *       $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.102 2001/11/05 17:46:29 momjian Exp $
+ *       $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.103 2002/01/03 04:02:34 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -124,6 +124,19 @@
 #define DEFAULT_NOT_UNK_SEL            (1.0 - DEFAULT_UNK_SEL)
 #define DEFAULT_BOOL_SEL               0.5
 
+/*
+ * Clamp a computed probability estimate (which may suffer from roundoff or
+ * estimation errors) to valid range.  Argument must be a float variable.
+ */
+#define CLAMP_PROBABILITY(p) \
+       do { \
+               if (p < 0.0) \
+                       p = 0.0; \
+               else if (p > 1.0) \
+                       p = 1.0; \
+       } while (0)
+
+
 static bool convert_to_scalar(Datum value, Oid valuetypid, double *scaledvalue,
                                  Datum lobound, Datum hibound, Oid boundstypid,
                                  double *scaledlobound, double *scaledhibound);
@@ -285,6 +298,7 @@ eqsel(PG_FUNCTION_ARGS)
                                for (i = 0; i < nnumbers; i++)
                                        sumcommon += numbers[i];
                                selec = 1.0 - sumcommon - stats->stanullfrac;
+                               CLAMP_PROBABILITY(selec);
 
                                /*
                                 * and in fact it's probably a good deal less. We
@@ -356,10 +370,7 @@ eqsel(PG_FUNCTION_ARGS)
        }
 
        /* result should be in range, but make sure... */
-       if (selec < 0.0)
-               selec = 0.0;
-       else if (selec > 1.0)
-               selec = 1.0;
+       CLAMP_PROBABILITY(selec);
 
        PG_RETURN_FLOAT8((float8) selec);
 }
@@ -669,10 +680,7 @@ scalarineqsel(Query *root, Oid operator, bool isgt,
        ReleaseSysCache(statsTuple);
 
        /* result should be in range, but make sure... */
-       if (selec < 0.0)
-               selec = 0.0;
-       else if (selec > 1.0)
-               selec = 1.0;
+       CLAMP_PROBABILITY(selec);
 
        return selec;
 }
@@ -867,10 +875,7 @@ patternsel(PG_FUNCTION_ARGS, Pattern_Type ptype)
                restsel = pattern_selectivity(rest, ptype);
                selec = prefixsel * restsel;
                /* result should be in range, but make sure... */
-               if (selec < 0.0)
-                       selec = 0.0;
-               else if (selec > 1.0)
-                       selec = 1.0;
+               CLAMP_PROBABILITY(selec);
                result = selec;
        }
 
@@ -1179,10 +1184,7 @@ booltestsel(Query *root, BooleanTest *clause, int varRelid)
        }
 
        /* result should be in range, but make sure... */
-       if (selec < 0.0)
-               selec = 0.0;
-       else if (selec > 1.0)
-               selec = 1.0;
+       CLAMP_PROBABILITY(selec);
 
        return (Selectivity) selec;
 }
@@ -1285,10 +1287,7 @@ nulltestsel(Query *root, NullTest *clause, int varRelid)
        }
 
        /* result should be in range, but make sure... */
-       if (selec < 0.0)
-               selec = 0.0;
-       else if (selec > 1.0)
-               selec = 1.0;
+       CLAMP_PROBABILITY(selec);
 
        return (Selectivity) selec;
 }
@@ -1448,6 +1447,7 @@ eqjoinsel(PG_FUNCTION_ARGS)
                                        }
                                }
                        }
+                       CLAMP_PROBABILITY(matchprodfreq);
                        /* Sum up frequencies of matched and unmatched MCVs */
                        matchfreq1 = unmatchfreq1 = 0.0;
                        for (i = 0; i < nvalues1; i++)
@@ -1457,6 +1457,8 @@ eqjoinsel(PG_FUNCTION_ARGS)
                                else
                                        unmatchfreq1 += numbers1[i];
                        }
+                       CLAMP_PROBABILITY(matchfreq1);
+                       CLAMP_PROBABILITY(unmatchfreq1);
                        matchfreq2 = unmatchfreq2 = 0.0;
                        for (i = 0; i < nvalues2; i++)
                        {
@@ -1465,6 +1467,8 @@ eqjoinsel(PG_FUNCTION_ARGS)
                                else
                                        unmatchfreq2 += numbers2[i];
                        }
+                       CLAMP_PROBABILITY(matchfreq2);
+                       CLAMP_PROBABILITY(unmatchfreq2);
                        pfree(hasmatch1);
                        pfree(hasmatch2);
 
@@ -1474,6 +1478,8 @@ eqjoinsel(PG_FUNCTION_ARGS)
                         */
                        otherfreq1 = 1.0 - stats1->stanullfrac - matchfreq1 - unmatchfreq1;
                        otherfreq2 = 1.0 - stats2->stanullfrac - matchfreq2 - unmatchfreq2;
+                       CLAMP_PROBABILITY(otherfreq1);
+                       CLAMP_PROBABILITY(otherfreq2);
 
                        /*
                         * We can estimate the total selectivity from the point of
@@ -1544,6 +1550,9 @@ eqjoinsel(PG_FUNCTION_ARGS)
                if (HeapTupleIsValid(statsTuple2))
                        ReleaseSysCache(statsTuple2);
        }
+
+       CLAMP_PROBABILITY(selec);
+
        PG_RETURN_FLOAT8((float8) selec);
 }
 
@@ -2213,6 +2222,9 @@ convert_timevalue_to_scalar(Datum value, Oid typid)
  *
  * var: identifies the attribute to examine.
  * stats: pg_statistic tuple for attribute, or NULL if not available.
+ *
+ * NB: be careful to produce an integral result, since callers may compare
+ * the result to exact integer counts.
  */
 static double
 get_att_numdistinct(Query *root, Var *var, Form_pg_statistic stats)
@@ -2254,7 +2266,7 @@ get_att_numdistinct(Query *root, Var *var, Form_pg_statistic stats)
                if (stats->stadistinct > 0.0)
                        return stats->stadistinct;
                if (stats->stadistinct < 0.0)
-                       return -stats->stadistinct * ntuples;
+                       return floor((-stats->stadistinct * ntuples) + 0.5);
        }
 
        /*