Fixed segfault caused by an empty quantile linked list being used to get the quantile...

git-svn-id: http://svn.osgeo.org/postgis/trunk@8949 b70326c6-7e19-0410-871a-916f4a2858ee

diff --git a/raster/rt_core/rt_api.c b/raster/rt_core/rt_api.c
index 10a4415f2b10f3f6149ff1ee1b749a0086e95cf6..5895577f1049907445a49a6c7e2409eb758ad62a 100644 (file)
--- a/raster/rt_core/rt_api.c
+++ b/raster/rt_core/rt_api.c
@@ -3020,7 +3020,7 @@ rt_band_get_quantiles_stream(rt_band band,
                }
                quicksort(quantiles, quantiles + quantiles_count - 1);
 
-               /* initialize rt_quantile */
+               /* initialize linked-list set */
                *qlls_count = quantiles_count * 2;
                RASTER_DEBUGF(4, "qlls_count = %d", *qlls_count);
                *qlls = rtalloc(sizeof(struct quantile_llist) * *qlls_count);
@@ -3433,6 +3433,9 @@ rt_band_get_quantiles_stream(rt_band band,
                                }
 
                        }
+                       else {
+                               RASTER_DEBUGF(5, "skipping value at (x, y) = (%d, %d)", x, y);
+                       }
 
                        z++;
                }
@@ -3461,6 +3464,11 @@ rt_band_get_quantiles_stream(rt_band band,
                RASTER_DEBUGF(5, "qll: (head, tail) = (%p, %p)", qll->head, qll->tail);
 
                rtn[k].quantile = qll->quantile;
+               rtn[k].has_value = 0;
+
+               /* check that qll->head and qll->tail have value */
+               if (qll->head == NULL || qll->tail == NULL)
+                       continue;
 
                /* AL-GEQ */
                if (qll->algeq)
@@ -3469,8 +3477,6 @@ rt_band_get_quantiles_stream(rt_band band,
                else
                        qle = qll->tail;
 
-               RASTER_DEBUGF(5, "qle: (value, count) = (%f, %d)", qle->value, qle->count);
-
                exists = 0;
                for (j = i + 1; j < *qlls_count; j++) {
                        if (FLT_EQ((*qlls)[j].quantile, qll->quantile)) {
@@ -3499,6 +3505,7 @@ rt_band_get_quantiles_stream(rt_band band,
                else {
                        rtn[k].value = qle->value;
                }
+               rtn[k].has_value = 1;
                RASTER_DEBUGF(3, "(quantile, value) = (%f, %f)\n\n", rtn[k].quantile, rtn[k].value);
 
                k++;

diff --git a/raster/rt_core/rt_api.h b/raster/rt_core/rt_api.h
index d77c1988d207c4289aa4a5f3c22cf5dcdd356bf3..9d6ab6ad987921c14bd6c1fd887522d5dce3aac9 100644 (file)
--- a/raster/rt_core/rt_api.h
+++ b/raster/rt_core/rt_api.h
@@ -1490,6 +1490,7 @@ struct rt_histogram_t {
 struct rt_quantile_t {
        double quantile;
        double value;
+       uint32_t has_value;
 };
 
 /* listed-list structures for rt_band_get_quantiles_stream */

diff --git a/raster/rt_pg/rt_pg.c b/raster/rt_pg/rt_pg.c
index 393c2da8ca5206d769790073b66d163fda2b00e8..87606d62c339c9b6eb5c2bfb870357fdbc809caf 100644 (file)
--- a/raster/rt_pg/rt_pg.c
+++ b/raster/rt_pg/rt_pg.c
@@ -5333,7 +5333,9 @@ Datum RASTER_quantileCoverage(PG_FUNCTION_ARGS)
                covquant2 = SPI_palloc(sizeof(struct rt_quantile_t) * count);
                for (i = 0; i < count; i++) {
                        covquant2[i].quantile = covquant[i].quantile;
-                       covquant2[i].value = covquant[i].value;
+                       covquant2[i].has_value = covquant[i].has_value;
+                       if (covquant2[i].has_value)
+                               covquant2[i].value = covquant[i].value;
                }
 
                if (NULL != covquant) pfree(covquant);
@@ -5392,7 +5394,10 @@ Datum RASTER_quantileCoverage(PG_FUNCTION_ARGS)
                memset(nulls, FALSE, values_length);
 
                values[0] = Float8GetDatum(covquant2[call_cntr].quantile);
-               values[1] = Float8GetDatum(covquant2[call_cntr].value);
+               if (covquant2[call_cntr].has_value)
+                       values[1] = Float8GetDatum(covquant2[call_cntr].value);
+               else
+                       nulls[1] = TRUE;
 
                /* build a tuple */
                tuple = heap_form_tuple(tupdesc, values, nulls);