From 184636263c01cbe147825e44f523249d110f441f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Nicklas=20Av=C3=A9n?= Date: Mon, 26 Aug 2013 18:58:28 +0000 Subject: [PATCH] Remove homebrewed compression method. Don't add ID if not used. git-svn-id: http://svn.osgeo.org/postgis/trunk@11878 b70326c6-7e19-0410-871a-916f4a2858ee --- doc/reference_output.xml | 29 ++-- liblwgeom/liblwgeom.h.in | 9 + liblwgeom/lwout_twkb.c | 351 ++++----------------------------------- liblwgeom/lwout_twkb.h | 12 +- postgis/lwgeom_accum.c | 48 +++--- postgis/lwgeom_inout.c | 22 +-- postgis/postgis.sql.in | 12 +- 7 files changed, 96 insertions(+), 387 deletions(-) diff --git a/doc/reference_output.xml b/doc/reference_output.xml index c5dd74c97..f1d73e874 100644 --- a/doc/reference_output.xml +++ b/doc/reference_output.xml @@ -1156,21 +1156,25 @@ SELECT (ST_AsLatLonText('POINT (-302.2342342 -792.32498)')); + + bytea ST_AsTWKB + geometry g1 + integer decimaldigits + bytea ST_AsTWKB geometry g1 integer decimaldigits integer geometryID - text NDR_or_XDR Description - Returns the geometry in TWKB format. TWKB is a new compressed binary format. - + The second parameter is an integer used to define the number of coordinate decimals in the output. + The third parameter is optional. It is used to give each TWKB-geometry an unique ID TWKB is still a moving target. The format is described here , and code for building a client can be found here @@ -1183,10 +1187,10 @@ SELECT (ST_AsLatLonText('POINT (-302.2342342 -792.32498)')); Examples -SELECT ST_AsTWKB('LINESTRING(1 1,5 5)'::geometry,0,1,'NDR'); +SELECT ST_AsTWKB('LINESTRING(1 1,5 5)'::geometry,0,1); st_astwkb -------------------------------------------- - \x0142010000000200000001000000010000000404 +\x0342010202020808 @@ -1204,12 +1208,16 @@ SELECT ST_AsTWKB('LINESTRING(1 1,5 5)'::geometry,0,1,'NDR'); + + bytea ST_AsTWKB_agg + geometry g1 + integer decimaldigits + bytea ST_AsTWKB_agg geometry g1 integer decimaldigits integer geometryID - text NDR_or_XDR @@ -1219,7 +1227,8 @@ SELECT ST_AsTWKB('LINESTRING(1 1,5 5)'::geometry,0,1,'NDR'); Returns the geometry in TWKB format. TWKB is a new compressed binary format. This is the aggregate version of ST_AsTWKB. It aggregates and returns the geoemtry in TWKB-format. In the resulting TWKB-geometry each individual ID of the input geometries is stored. - + The second parameter is an integer used to define the number of coordinate decimals in the output. + The third parameter is optional. It is used to give each TWKB-geometry an unique ID TWKB is still a moving target. The format is described here , and code for building a client can be found here @@ -1232,13 +1241,13 @@ SELECT ST_AsTWKB('LINESTRING(1 1,5 5)'::geometry,0,1,'NDR'); Examples -SELECT ST_AsTWKB_agg(geom,0,id,'NDR') FROM +SELECT ST_AsTWKB_agg(geom,0,id) FROM (SELECT 'LINESTRING(1 1,5 5)'::geometry geom, 1 AS id UNION ALL -SELECT 'LINESTRING(6 5, 1 7)'::geometry AS geom, 2 AS id; +SELECT 'LINESTRING(6 5, 1 7)'::geometry AS geom, 2 AS id) foo; st_astwkb_agg ---------------------------------------------------------------------------- - \x01560200000001000000020000000100000001000000040402000000020000000100fb02 +\x035602010202020808020202000904 diff --git a/liblwgeom/liblwgeom.h.in b/liblwgeom/liblwgeom.h.in index 5d3ff05d8..ea2b7d35b 100644 --- a/liblwgeom/liblwgeom.h.in +++ b/liblwgeom/liblwgeom.h.in @@ -1783,6 +1783,15 @@ LWGEOM_UNPARSER_RESULT; #define WKT_SFSQL 0x02 #define WKT_EXTENDED 0x04 + +/* +** Variants available for TWKB, conflicting with corresponding bits for WKB and WKT +*/ +#define WKB_ID 0x01 /* This position has another meaning when building wkb! */ +#define WKB_NO_ID ~(1 << 1) +#define WKB_NO_TYPE 0x02 /* This position has another meaning when building wkb! */ + + /* ** New parsing and unparsing functions. */ diff --git a/liblwgeom/lwout_twkb.c b/liblwgeom/lwout_twkb.c index 16a529ff6..8d12d2f14 100644 --- a/liblwgeom/lwout_twkb.c +++ b/liblwgeom/lwout_twkb.c @@ -88,19 +88,6 @@ static uint8_t lwgeom_twkb_type(const LWGEOM *geom, uint8_t variant) return type_flag; } -/* -* SwapBytes? -*/ -static inline int wkb_swap_bytes(uint8_t variant) -{ - /* If requested variant matches machine arch, we don't have to swap! */ - if ( ((variant & WKB_NDR) && (getMachineEndian() == NDR)) || - ((! (variant & WKB_NDR)) && (getMachineEndian() == XDR)) ) - { - return LW_FALSE; - } - return LW_TRUE; -} /** Function for putting a Byte value into the buffer @@ -150,30 +137,6 @@ int u_getvarint_size(unsigned long val) return n; } -/** -Function for copying a variable length value into the buffer -*/ -static uint8_t* to_twkb_buf(uint8_t *iptr, uint8_t *buf, uint8_t variant, int the_size) -{ - LWDEBUGF(2, "Entered to_twkb_buf",0); - int i = 0; - - /* Machine/request arch mismatch, so flip byte order */ - if ( wkb_swap_bytes(variant)&&the_size>1 ) - { - for ( i = 0; i < the_size; i++ ) - { - buf[i] = iptr[the_size - 1 - i]; - } - } - /* If machine arch and requested arch match, don't flip byte order */ - else - { - memcpy(buf, iptr, the_size); - } - return buf + the_size; -} - /** Function for encoding a value as varInt and putting it in the buffer */ @@ -240,10 +203,10 @@ static uint8_t* empty_to_twkb_buf(const LWGEOM *geom, uint8_t *buf, uint8_t vari } uint8_t flag=0; - /* Set the endian flag */ - END_PREC_SET__ENDIANESS(flag, ((variant & WKB_NDR) ? 1 : 0)); + /* Set the id flag */ + END_PREC_SET_ID(flag, ((variant & WKB_ID) ? 1 : 0)); /* Tell what precision to use*/ - END_PREC_SET__PRECISION(flag,prec); + END_PREC_SET_PRECISION(flag,prec); /*Copy the flag to the buffer*/ buf = uint8_to_twkb_buf(flag,buf); @@ -268,9 +231,6 @@ static size_t ptarray_to_twkb_size(const POINTARRAY *pa, uint8_t variant,int pre LWDEBUGF(2, "Entered ptarray_to_twkb_size",0); switch (method) { - case 0: - return ptarray_to_twkb_size_m0(pa, variant,prec,accum_rel); - break; case 1: return ptarray_to_twkb_size_m1(pa, variant,prec,accum_rel); break; @@ -342,125 +302,6 @@ static size_t ptarray_to_twkb_size_m1(const POINTARRAY *pa, uint8_t variant,int return size; } -/** -Calculates the needed space for storing a specific pointarray as encoded with "method0" -*/ -static size_t ptarray_to_twkb_size_m0(const POINTARRAY *pa, uint8_t variant,int prec,int accum_rel[]) -{ - LWDEBUGF(2, "ptarray_to_twkb_size entered%d",0); - int dims = FLAGS_NDIMS(pa->flags); - int i, j, r, last_size,factor,test_size,r_test,j_test; - double *dbl_ptr; - size_t size = 0; - last_size=0; - int start=0; - - /*The variable factor is used to "shift" the double float coordinate to keep enough significant digits, - for demanded precision, when cast to integer*/ - factor=pow(10,prec); - - /*This is a multidimmenstional array keeping trac of the three different storage sizes. - It holds number of bytes, max-value and min-value*/ - static int size_array[3][3] = {{1,INT8_MAX,INT8_MIN},{2,INT16_MAX,INT16_MIN},{4,INT32_MAX,INT32_MIN}}; - - /* Include the npoints size if it's not a POINT type) */ - if ( ! ( variant & WKB_NO_NPOINTS ) ) - { - LWDEBUGF(2, "We add space for npoints",0); - size += u_getvarint_size(pa->npoints); - } - /*if we don't have a ref-point yet*/ - if(accum_rel[0]==INT32_MIN) - { - LWDEBUGF(2, "We don't have a ref-point yet so we give space for full coordinates",0); - /*Get a pointer to the first point of the point array*/ - dbl_ptr = (double*)getPoint_internal(pa, 0); - - - - LWDEBUGF(2, "Our geom have %d dims",dims); - /*Load the accum_rel aray with the first points dimmension*/ - for ( j = 0; j < dims; j++ ) - { - LWDEBUGF(4, "dim nr %d, refvalue is %d",j, accum_rel[j]); - r = round(factor*dbl_ptr[j]); - accum_rel[j]=r; - LWDEBUGF(4, "deltavalue = %d and resulting refvalue is %d",r,accum_rel[j]); - size += s_getvarint_size((long) r); - } - start=1; - } - - LWDEBUGF(2, "We have %d points to iterate ",pa->npoints); - for ( i = start; i < pa->npoints; i++ ) - { - dbl_ptr = (double*)getPoint_internal(pa, i); - for ( j = 0; j < dims; j++ ) - { - /*To get the relative coordinate we don't get teh distance from the last point - but instead the distance from our accumulated last point - This is important to not build up a accumulated error when rounding the coordinates*/ - r=round(factor*dbl_ptr[j]-accum_rel[j]); - - - /*last used size is too small so we have to increase*/ - if(fabs(r)>size_array[last_size][1]) - { - /*A little ugly, but we sacrify the last possible value fitting into a INT4, just to detect too big values without the need to do the substraction again with a double float instead*/ - if(fabs(r)>=size_array[2][1]) - lwerror("The relative coordinate coordinate exceeds the max_value (%d):%d",size_array[2][1],r); - /*minimum value for last used size, used to flag a change in size*/ - size += size_array[last_size][0]; - - /*Find how much space we actually need */ - while ( fabs(r)>size_array[(++last_size)][1] && last_size<3) {} - - /*register the one byte needed to tell what size we need*/ - size ++; - } - - /*We don't need that much space so let's investigate if we should decrease*/ - else if(last_size>0 && fabs(r)size_array[(test_size)][1] && test_size<=last_size) - { - LWDEBUGF(4, "testing %d bytes for value %d, dim: %d",size_array[test_size][0],r_test,j_test ); - test_size++; - } - } - if(test_sizeflags); - int i, j, r, last_size,factor,test_size,r_test,j_test; - double *dbl_ptr; - factor=pow(10,prec); - last_size=0; - int start=0; - /*This is a multidimmenstional array keeping trac of the three different storage sizes. - It holds number of bytes, max-value and min-value*/ - static int size_array[3][3] = {{1,INT8_MAX,INT8_MIN},{2,INT16_MAX,INT16_MIN},{4,INT32_MAX,INT32_MIN}}; - - /* Set the number of points (if it's not a POINT type) */ - if ( ! ( variant & WKB_NO_NPOINTS ) ) - { - buf = u_varint_to_twkb_buf(pa->npoints,buf); - LWDEBUGF(4, "Regiter npoints:%d",pa->npoints); - } - - /*if we don't have a ref-point yet*/ - if(accum_rel[0]==INT32_MIN) - { - /*Get a pointer do the first point of the point array*/ - dbl_ptr = (double*)getPoint_internal(pa, 0); - - /*the first coordinate for each dimension is copied to the buffer - and registered in accum_rel array*/ - for ( j = 0; j < dims; j++ ) - { - LWDEBUGF(4, "dim nr %d, refvalue is %d",j, accum_rel[j]); - r = round(factor*dbl_ptr[j]); - accum_rel[j]=r; - LWDEBUGF(4, "deltavalue = %d and resulting refvalue is %d",r,accum_rel[j]); - buf = s_varint_to_twkb_buf(r,buf); - } - start=1; - } - for ( i = start; i < pa->npoints; i++ ) - { - LWDEBUGF(4, "Writing point #%d", i); - dbl_ptr = (double*)getPoint_internal(pa, i); - for ( j = 0; j < dims; j++ ) - { - /*To get the relative coordinate we don't get the distance from the last point - but instead the distance from our accumulated last point - This is important to not build up a accumulated error when rounding the coordinates*/ - r=round(factor*dbl_ptr[j]-accum_rel[j]); - //accum_rel[j]+=r; - //LWDEBUGF(4, "delta value for dim %d is %d, real coordiinate is %f and accumulated coordinate is%d", j, r,dbl_ptr[j],accum_rel[j] ); - LWDEBUGF(4, "size:%d,dim: %d deltavalue: %d, coordinate: %f accumulated coordinate %d",last_size, j, r,dbl_ptr[j],accum_rel[j] ); - - - - /*last used size is too small so we have to increase*/ - if(fabs(r)>size_array[last_size][1]) - { - - LWDEBUGF(4, "increasing size from %d bytes",size_array[last_size][0]); - /*minimum value for last used size, used to flag a change in size*/ - buf = to_twkb_buf((uint8_t *) &(size_array[last_size][2]),buf, variant, size_array[last_size][0]); - - /*Find how much space we actually need */ - while ( fabs(r)>size_array[(++last_size)][1] && last_size<3) {} - LWDEBUGF(4, "to size %d bytes",size_array[last_size][0]); - - /*register needed space*/ - memcpy( buf, &(size_array[last_size][0]), 1); - buf++; - } - /*We don't need that much space so let's investigate if we should decrease - But if it is just a horizontal or vertical line, one dimmension will have short steps but another will still need bigger steps - So, to avoid size changes up and down for every point we don't decrease size if it is not possible for all dimmensions - We could here look even further forward to find out what is most optimal, but that will cost in computing instead*/ - else if (last_size>0 && fabs(r)size_array[(test_size)][1] && test_size<=last_size) - { - LWDEBUGF(4, "testing %d bytes for value %d, dim: %d",size_array[test_size][0],r_test,j_test ); - test_size++; - } - } - if(test_sizegeoms = palloc(10*sizeof(geom_id)); @@ -196,26 +193,19 @@ oldcontext = MemoryContextSwitchTo(aggcontext); /* If user specified precision, respect it */ state->precision = PG_ARGISNULL(2) ? (int) 0 : PG_GETARG_INT32(2); - - - /* If user specified endianness, respect it */ - //endianess = PG_ARGISNULL(4) ? 0 : PG_GETARG_TEXT_P(4); - - endianess = PG_GETARG_TEXT_P(4); - if ( ! strncmp(VARDATA(endianess), "xdr", 3) || - ! strncmp(VARDATA(endianess), "XDR", 3) ) + + /* If user specified method, respect it + This will probably be taken away when we can decide which compression method that is best */ + if ((PG_NARGS()>5) && (!PG_ARGISNULL(5))) { - variant = variant | WKB_XDR; + state->method = PG_GETARG_INT32(5); } else { - variant = variant | WKB_NDR; + state->method = 1; } - state->variant=variant; - - /* If user specified method, respect it - This will probably be taken away when we can decide which compression method that is best */ - state->method = PG_ARGISNULL(5) ? (int) 0 : PG_GETARG_INT32(5); + + //state->method = ((PG_NARGS()>5) && PG_ARGISNULL(5)) ? (int) 1 : PG_GETARG_INT32(5); } else @@ -237,10 +227,21 @@ oldcontext = MemoryContextSwitchTo(aggcontext); geom = (GSERIALIZED*)PG_DETOAST_DATUM(PG_GETARG_DATUM(1)); ((state->geoms)+state->n_rows)->geom = PG_ARGISNULL(1) ? (Datum) 0 : PointerGetDatum(PG_DETOAST_DATUM_COPY(PG_GETARG_DATUM(1))); - ((state->geoms)+state->n_rows)->id = PG_ARGISNULL(3) ? (int) 0 : PG_GETARG_INT32(3); - - (state->n_rows)++; -MemoryContextSwitchTo(oldcontext); + + + if ((PG_NARGS()>3) && (!PG_ARGISNULL(3))) + { + variant = variant | (WKB_ID); + ((state->geoms)+state->n_rows)->id = PG_GETARG_INT32(3); + } + else + { + variant = variant & WKB_NO_ID; + ((state->geoms)+state->n_rows)->id = 0; + } + state->variant=variant; + (state->n_rows)++; + MemoryContextSwitchTo(oldcontext); PG_RETURN_POINTER(state); } @@ -424,7 +425,6 @@ pgis_twkb_accum_finalfn(PG_FUNCTION_ARGS) } } - twkb = lwgeom_agg_to_twkb(&lwgeom_arrays, state->variant , &twkb_size,(int8_t) state->precision,state->method); diff --git a/postgis/lwgeom_inout.c b/postgis/lwgeom_inout.c index a5f463b75..6aab28bfd 100644 --- a/postgis/lwgeom_inout.c +++ b/postgis/lwgeom_inout.c @@ -412,7 +412,6 @@ Datum TWKBFromLWGEOM(PG_FUNCTION_ARGS) size_t twkb_size; uint8_t variant = 0; bytea *result; - text *type; int id,prec,method; /* If user specified precision, respect it */ @@ -428,26 +427,15 @@ Datum TWKBFromLWGEOM(PG_FUNCTION_ARGS) /* If user specified id, respect it */ if ( (PG_NARGS()>2) && (!PG_ARGISNULL(2)) ) { + variant = variant | (WKB_ID); id = PG_GETARG_INT32(2); } else - id=0; - - /* If user specified endianness, respect it */ - if ( (PG_NARGS()>3) && (!PG_ARGISNULL(3)) ) { - type = PG_GETARG_TEXT_P(3); - - if ( ! strncmp(VARDATA(type), "xdr", 3) || - ! strncmp(VARDATA(type), "XDR", 3) ) - { - variant = variant | WKB_XDR; - } - else - { - variant = variant | WKB_NDR; - } + variant = variant & WKB_NO_ID; + id=0; } + /* If user specified method, respect it This will probably be taken away when we can decide which compression method that is best */ if ( (PG_NARGS()>4) && (!PG_ARGISNULL(4)) ) @@ -455,7 +443,7 @@ Datum TWKBFromLWGEOM(PG_FUNCTION_ARGS) method = PG_GETARG_INT32(4); } else - method=0; + method=1; /* Create TWKB bin string */ lwgeom = lwgeom_from_gserialized(geom); diff --git a/postgis/postgis.sql.in b/postgis/postgis.sql.in index f5c3d1c1c..278d3c3f1 100644 --- a/postgis/postgis.sql.in +++ b/postgis/postgis.sql.in @@ -1369,13 +1369,13 @@ CREATE OR REPLACE FUNCTION ST_AsTWKB(geometry,int4,int4) LANGUAGE 'c' IMMUTABLE STRICT; -- Availability: 2.2.0 -CREATE OR REPLACE FUNCTION ST_AsTWKB(geometry,int4,int4,text) +CREATE OR REPLACE FUNCTION ST_AsTWKB(geometry,int4) RETURNS bytea AS 'MODULE_PATHNAME','TWKBFromLWGEOM' LANGUAGE 'c' IMMUTABLE STRICT; -- Availability: 2.2.0 -CREATE OR REPLACE FUNCTION ST_AsTWKB(geometry,int4,int4,text,int) +CREATE OR REPLACE FUNCTION ST_AsTWKB(geometry,int4,int4) RETURNS bytea AS 'MODULE_PATHNAME','TWKBFromLWGEOM' LANGUAGE 'c' IMMUTABLE STRICT; @@ -3353,12 +3353,12 @@ CREATE OR REPLACE FUNCTION pgis_geometry_makeline_finalfn(pgis_abs) --Availability: 2.2.0 -CREATE OR REPLACE FUNCTION pgis_twkb_accum_transfn(internal, geometry,int,int,text) +CREATE OR REPLACE FUNCTION pgis_twkb_accum_transfn(internal, geometry,int) RETURNS internal AS 'MODULE_PATHNAME' LANGUAGE c ; -CREATE OR REPLACE FUNCTION pgis_twkb_accum_transfn(internal, geometry,int,int,text,int) +CREATE OR REPLACE FUNCTION pgis_twkb_accum_transfn(internal, geometry,int,int) RETURNS internal AS 'MODULE_PATHNAME' LANGUAGE c ; @@ -3368,13 +3368,13 @@ CREATE OR REPLACE FUNCTION pgis_twkb_accum_finalfn(internal) AS 'MODULE_PATHNAME' LANGUAGE c ; -CREATE AGGREGATE st_astwkb_agg(geometry,int,int,text) ( +CREATE AGGREGATE st_astwkb_agg(geometry,int) ( SFUNC=pgis_twkb_accum_transfn, STYPE=internal, FINALFUNC=pgis_twkb_accum_finalfn ); -CREATE AGGREGATE st_astwkb_agg(geometry,int,int,text,int) ( +CREATE AGGREGATE st_astwkb_agg(geometry,int,int) ( SFUNC=pgis_twkb_accum_transfn, STYPE=internal, FINALFUNC=pgis_twkb_accum_finalfn -- 2.50.1