From: Sandro Santilli <strk@kbt.io>
Date: Fri, 23 Jun 2017 14:47:13 +0000 (+0000)
Subject: Fix handling of EMPTY geometries in btree operator
X-Git-Tag: 2.4.0alpha~57
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=e22c44c65a31f9b725d9e66ef25b5da4cacd6883;p=postgis

Fix handling of EMPTY geometries in btree operator

Includes regress test

See #3777

git-svn-id: http://svn.osgeo.org/postgis/trunk@15447 b70326c6-7e19-0410-871a-916f4a2858ee
---

diff --git a/postgis/lwgeom_btree.c b/postgis/lwgeom_btree.c
index 328ee5c27..062d1cd83 100644
--- a/postgis/lwgeom_btree.c
+++ b/postgis/lwgeom_btree.c
@@ -53,6 +53,7 @@ Datum lwgeom_lt(PG_FUNCTION_ARGS)
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_lt called");
 
@@ -60,14 +61,19 @@ Datum lwgeom_lt(PG_FUNCTION_ARGS)
 
 	POSTGIS_DEBUG(3, "lwgeom_lt passed getSRID test");
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
 	POSTGIS_DEBUG(3, "lwgeom_lt getbox2d_p passed");
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin < box2.xmin)
@@ -102,17 +108,23 @@ Datum lwgeom_le(PG_FUNCTION_ARGS)
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_le called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin < box2.xmin)
@@ -168,9 +180,10 @@ Datum lwgeom_eq(PG_FUNCTION_ARGS)
 
 	gbox_init(&box1);
 	gbox_init(&box2);
-	
+
 	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
 	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
+
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
@@ -198,17 +211,23 @@ Datum lwgeom_ge(PG_FUNCTION_ARGS)
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_ge called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin > box2.xmin)
@@ -255,17 +274,23 @@ Datum lwgeom_gt(PG_FUNCTION_ARGS)
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_gt called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin > box2.xmin)
@@ -308,17 +333,23 @@ Datum lwgeom_cmp(PG_FUNCTION_ARGS)
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_cmp called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin < box2.xmin)
diff --git a/regress/operators.sql b/regress/operators.sql
index 1306700a0..5a2113dd1 100644
--- a/regress/operators.sql
+++ b/regress/operators.sql
@@ -140,4 +140,15 @@ WITH v(i,g) AS ( VALUES
 SELECT 'ndovm2', array_agg(i) FROM v WHERE g &&& 'POINTZ(0 0 1)'::geometry
 ORDER BY 1;
 
+-- GROUP BY on empty
+SELECT '#3777', ST_AsText(geom), count(*) FROM (
+SELECT 'POINT(0 0)'::geometry geom UNION ALL
+SELECT 'POINT(0 0)'::geometry UNION ALL
+SELECT 'POINT(0 0)'::geometry UNION ALL
+SELECT 'POINT(0 1)'::geometry UNION ALL
+SELECT 'LINESTRING(0 0,0 1)'::geometry UNION ALL
+SELECT 'GEOMETRYCOLLECTION EMPTY'::geometry UNION ALL
+SELECT 'POINT EMPTY'::geometry
+) foo
+GROUP BY geom ORDER BY 2;
 
diff --git a/regress/operators_expected b/regress/operators_expected
index c1539e6e4..b5b3761aa 100644
--- a/regress/operators_expected
+++ b/regress/operators_expected
@@ -56,3 +56,7 @@ ndov6|t
 ndov7|t
 ndovm1|{1,2,3,4,5,8}
 ndovm2|{1,2,4,6,7}
+#3777|GEOMETRYCOLLECTION EMPTY|2
+#3777|LINESTRING(0 0,0 1)|1
+#3777|POINT(0 0)|3
+#3777|POINT(0 1)|1