From: Sandro Santilli Date: Wed, 3 Mar 2004 21:59:48 +0000 (+0000) Subject: added check to keep selectivity value in the range of validity (suggested by m.cave) X-Git-Tag: pgis_0_8_2~99 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d40c6739cf36ad6be8487096375f40405940ddaa;p=postgis added check to keep selectivity value in the range of validity (suggested by m.cave) git-svn-id: http://svn.osgeo.org/postgis/trunk@460 b70326c6-7e19-0410-871a-916f4a2858ee --- diff --git a/postgis_estimate.c b/postgis_estimate.c index b0298d3d1..d2ba189da 100644 --- a/postgis_estimate.c +++ b/postgis_estimate.c @@ -11,6 +11,9 @@ * ********************************************************************** * $Log$ + * Revision 1.15 2004/03/03 21:59:48 strk + * added check to keep selectivity value in the range of validity (suggested by m.cave) + * * Revision 1.14 2004/03/01 16:02:41 strk * histogram's boxesPerSide computed as a function of the column's statistic target * @@ -1204,10 +1207,13 @@ Datum postgis_gist_sel(PG_FUNCTION_ARGS) elog(NOTICE, " avg feat overlaps %f cells", avg_feat_cells); #endif selectivity = (float8) value / (float8) min(overlapping_cells, avg_feat_cells); - //selectivity = value/overlapping_cells; - //selectivity *= 0.1; - //selectivity = 0.9; + + /* prevent rounding overflows */ + if (selectivity > 1.0) selectivity = 1.0; + else if (selectivity < 0) selectivity = 0.0; } + + #if DEBUG_GEOMETRY_STATS elog(NOTICE, " returning computed value: %f", selectivity);