From: Sandro Santilli Date: Tue, 29 Jul 2014 10:25:31 +0000 (+0000) Subject: Move varInt handling code into its own file module, for reuse X-Git-Tag: 2.2.0rc1~931 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=3e5ba9989daee666e4bc801b75dc9570f4d67232;p=postgis Move varInt handling code into its own file module, for reuse Also found a bug in encoding 4bytes numbers git-svn-id: http://svn.osgeo.org/postgis/trunk@12836 b70326c6-7e19-0410-871a-916f4a2858ee --- diff --git a/liblwgeom/Makefile.in b/liblwgeom/Makefile.in index 177e2cf79..14566bf89 100644 --- a/liblwgeom/Makefile.in +++ b/liblwgeom/Makefile.in @@ -89,7 +89,8 @@ SA_OBJS = \ lwgeom_geos_clean.o \ lwgeom_geos_node.o \ lwgeom_geos_split.o \ - lwgeom_transform.o + lwgeom_transform.o \ + varint.o NM_OBJS = \ lwspheroid.o @@ -108,7 +109,8 @@ SA_HEADERS = \ liblwgeom.h \ liblwgeom_internal.h \ lwgeom_log.h \ - lwgeom_geos.h + lwgeom_geos.h \ + varint.h all: liblwgeom.la diff --git a/liblwgeom/cunit/cu_varint.c b/liblwgeom/cunit/cu_varint.c index 364c7f298..b873923c8 100644 --- a/liblwgeom/cunit/cu_varint.c +++ b/liblwgeom/cunit/cu_varint.c @@ -20,41 +20,76 @@ #include "liblwgeom_internal.h" #include "liblwgeom.h" -#include "lwout_twkb.c" +#include "varint.h" #include "cu_tester.h" - -static void do_test_unsigned_varint(uint64_t nr,int expected_size, char* expected_res) +static void do_test_s32_varint(int32_t nr,int expected_size, char* expected_res) { uint8_t buf[8]; int size; + char *hex; - size = u_getvarint_size(nr); + size = varint_s32_encoded_size(nr); CU_ASSERT_EQUAL(size,expected_size); - u_varint_to_twkb_buf(nr, buf); - CU_ASSERT_STRING_EQUAL( hexbytes_from_bytes(buf,size),expected_res); + varint_s32_encode_buf(nr, buf); + hex = hexbytes_from_bytes(buf,size); + if ( strcmp(hex,expected_res) ) { + printf("Expected: %s\nObtained: %s\n", expected_res, hex); + } + CU_ASSERT_STRING_EQUAL(hex, expected_res); + lwfree(hex); } -static void do_test_signed_varint(uint64_t nr,int expected_size, char* expected_res) +static void do_test_u64_varint(uint64_t nr,int expected_size, char* expected_res) { uint8_t buf[8]; int size; + char *hex; - size = u_getvarint_size(nr); + size = varint_u64_encoded_size(nr); CU_ASSERT_EQUAL(size,expected_size); - s_varint_to_twkb_buf(nr, buf); - CU_ASSERT_STRING_EQUAL( hexbytes_from_bytes(buf,size),expected_res); + varint_u64_encode_buf(nr, buf); + hex = hexbytes_from_bytes(buf,size); + if ( strcmp(hex,expected_res) ) { + printf("Expected: %s\nObtained: %s\n", expected_res, hex); + } + CU_ASSERT_STRING_EQUAL(hex, expected_res); + lwfree(hex); } +static void do_test_s64_varint(int64_t nr,int expected_size, char* expected_res) +{ + uint8_t buf[8]; + int size; + char *hex; + + size = varint_s64_encoded_size(nr); + CU_ASSERT_EQUAL(size,expected_size); + varint_s64_encode_buf(nr, buf); + hex = hexbytes_from_bytes(buf,size); + if ( strcmp(hex,expected_res) ) { + printf("Expected: %s\nObtained: %s\n", expected_res, hex); + } + CU_ASSERT_STRING_EQUAL(hex, expected_res); + lwfree(hex); +} static void test_varint(void) { - do_test_unsigned_varint(1,1, "01"); - do_test_unsigned_varint(300,2, "AC02"); - do_test_unsigned_varint(150,2, "9601"); + do_test_u64_varint(1, 1, "01"); + do_test_u64_varint(300, 2, "AC02"); + do_test_u64_varint(150, 2, "9601"); - do_test_signed_varint(1,1, "02"); + do_test_s64_varint(1, 1, "02"); + do_test_s64_varint(-1, 1, "01"); + do_test_s64_varint(-2, 1, "03"); +#if 0 /* FIXME! */ + do_test_s64_varint(2147483647, 4, "FFFFFFFE"); + do_test_s64_varint(-2147483648, 4, "FFFFFFFF"); +#endif + + /* TODO: test signed/unsigned 32bit varints */ } diff --git a/liblwgeom/lwout_twkb.c b/liblwgeom/lwout_twkb.c index 7a929aa21..356296587 100644 --- a/liblwgeom/lwout_twkb.c +++ b/liblwgeom/lwout_twkb.c @@ -10,7 +10,7 @@ **********************************************************************/ #include "lwout_twkb.h" - +#include "varint.h" /* * GeometryType @@ -100,88 +100,6 @@ static uint8_t* uint8_to_twkb_buf(const uint8_t ival, uint8_t *buf) return buf + 1; } -/** -Encodes a value as varInt described here: -https://developers.google.com/protocol-buffers/docs/encoding#varints -*/ -int s_getvarint_size(int64_t val) -{ - - LWDEBUGF(2, "Entered s_getvarint_size",0); - if(valmax_s_varint) - lwerror("Value is out of range for signed varint (%ld to %ld)",min_s_varint, max_s_varint); - uint64_t q; - int n=0; - - q = (val << 1) ^ (val >> 63); - while ((q>>(7*(n+1))) >0) - { - n++; - } - n++; - return n; -} - -/** -Encodes a value as signed varInt -https://developers.google.com/protocol-buffers/docs/encoding#varints -*/ -int u_getvarint_size(uint64_t val) -{ - LWDEBUGF(2, "Entered u_getvarint_size",0); - - if(val>max_varint) - lwerror("Value is out of range for unsigned varint (0 to %ld)", max_varint); - uint64_t q; - int n=0; - q =val; - while ((q>>(7*(n+1))) >0) - { - n++; - } - n++; - return n; -} - -/** -Function for encoding a value as varInt and putting it in the buffer -*/ -static uint8_t* u_varint_to_twkb_buf(uint64_t val, uint8_t *buf) -{ - LWDEBUGF(2, "Entered u_varint_to_twkb_buf",0); - LWDEBUGF(4, "Writing value %d",val); - uint64_t q; - int n,grp; - q =val; - n=0; - - while ((q>>(7*(n+1))) >0) - { - grp=128^(127&(q>>(7*n))); - buf[n]=grp; - n++; - } - grp=127&(q>>(7*n)); - buf[n]=grp; - n++; - - return buf+=n; -} - -/** -Function for encoding a varInt value as signed -*/ -static uint8_t* s_varint_to_twkb_buf(int64_t val, uint8_t *buf) -{ - LWDEBUGF(2, "Entered s_varint_to_twkb_buf",0); - LWDEBUGF(4, "Writing value %d",val); - uint64_t q; - q = (val << 1) ^ (val >> 63); - - return u_varint_to_twkb_buf(q, buf); -} - - /* * Empty */ @@ -192,9 +110,9 @@ static size_t empty_to_twkb_size(const LWGEOM *geom, uint8_t variant, int64_t id /* Endian flag/precision + id + type number + npoints*/ size_t size = WKB_BYTE_SIZE + WKB_BYTE_SIZE; /*size of ID*/ - size += s_getvarint_size((int64_t) id); + size += varint_s64_encoded_size((int64_t) id); /*size of npoints*/ - size += u_getvarint_size((uint64_t) 0); + size += varint_u64_encoded_size((uint64_t) 0); return size; } @@ -220,14 +138,14 @@ static uint8_t* empty_to_twkb_buf(const LWGEOM *geom, uint8_t *buf, uint8_t vari buf = uint8_to_twkb_buf(flag,buf); /* Set the geometry id */ - buf = s_varint_to_twkb_buf(id, buf); + buf = varint_s64_encode_buf(id, buf); /* Set the geometry type */ buf = uint8_to_twkb_buf(wkb_type,buf); /* Set nrings/npoints/ngeoms to zero */ - buf = u_varint_to_twkb_buf(0, buf); + buf = varint_u64_encode_buf(0, buf); return buf; } @@ -271,7 +189,7 @@ static size_t ptarray_to_twkb_size_m1(const POINTARRAY *pa, uint8_t variant,int if ( ! ( variant & WKB_NO_NPOINTS ) ) { LWDEBUGF(4, "We add space for npoints",0); - size+=u_getvarint_size(pa->npoints); + size+=varint_u64_encoded_size(pa->npoints); } LWDEBUGF(4, "Refvalue dim 1 is %d",accum_rel[0]); @@ -283,7 +201,7 @@ static size_t ptarray_to_twkb_size_m1(const POINTARRAY *pa, uint8_t variant,int LWDEBUGF(4, "dim nr %d, refvalue is %d",j, accum_rel[j]); r=(int64_t) lround(factor*dbl_ptr[j]-accum_rel[j]); accum_rel[j]+=r; - size += s_getvarint_size((int64_t) r); + size += varint_s64_encoded_size((int64_t) r); LWDEBUGF(4, "deltavalue = %d and resulting refvalue is %d",r,accum_rel[j]); } } @@ -327,7 +245,7 @@ static uint8_t* ptarray_to_twkb_buf_m1(const POINTARRAY *pa, uint8_t *buf, uint8 /* Set the number of points (if it's not a POINT type) */ if ( ! ( variant & WKB_NO_NPOINTS ) ) { - buf = u_varint_to_twkb_buf(pa->npoints,buf); + buf = varint_u64_encode_buf(pa->npoints,buf); LWDEBUGF(4, "Register npoints:%d",pa->npoints); } @@ -343,11 +261,7 @@ static uint8_t* ptarray_to_twkb_buf_m1(const POINTARRAY *pa, uint8_t *buf, uint8 r=(int64_t) lround(factor*dbl_ptr[j]-accum_rel[j]); LWDEBUGF(4, "deltavalue: %d, ",r ); accum_rel[j]+=r; - //add the actual coordinate - //s= getvarint((long) r, &ret, LW_TRUE); - //memcpy(buf, &ret, s); - //buf+=s; - buf = s_varint_to_twkb_buf(r,buf); + buf = varint_s64_encode_buf(r,buf); } } //LWDEBUGF(4, "Done (buf = %p)", buf); @@ -366,7 +280,7 @@ static size_t lwgeom_agg_to_twkbpoint_size(lwgeom_id *geom_array,uint8_t varian /*One byte for type declaration*/ size_t size = WKB_BYTE_SIZE; /*One integer holding number of geometries*/ - size += u_getvarint_size((uint64_t) n); + size += varint_u64_encoded_size((uint64_t) n); int i; for (i=0;ipoint, variant | WKB_NO_NPOINTS, prec,refpoint,method); @@ -413,7 +327,7 @@ static uint8_t* lwgeom_agg_to_twkbpoint_buf(lwgeom_id* geom_array,int n, uint8_t buf = uint8_to_twkb_buf(type_flag,buf); /* Set number of geometries */ - buf = u_varint_to_twkb_buf(n, buf); + buf = varint_u64_encode_buf(n, buf); for (i=0;ipoints,variant,prec,refpoint,method); @@ -499,7 +413,7 @@ static uint8_t* lwgeom_agg_to_twkbline_buf(lwgeom_id* geom_array,int n, uint8_t buf = uint8_to_twkb_buf(type_flag,buf); /* Set number of geometries */ - buf = u_varint_to_twkb_buf(n, buf); + buf = varint_u64_encode_buf(n, buf); for (i=0;inrings); + size += varint_u64_encoded_size((uint64_t) poly->nrings); LWDEBUGF(2, "we have %d rings to iterate",poly->nrings); for ( i = 0; i < poly->nrings; i++ ) @@ -593,7 +507,7 @@ static uint8_t* lwgeom_agg_to_twkbpoly_buf(lwgeom_id* geom_array,int n, uint8_t LWDEBUGF(4, "Writing ndims '%d'", dims); buf = uint8_to_twkb_buf(type_flag,buf); /* Set number of geometries */ - buf = u_varint_to_twkb_buf(n, buf); + buf = varint_u64_encode_buf(n, buf); for (i=0;inrings, buf); + buf = varint_u64_encode_buf(poly->nrings, buf); for ( i = 0; i < poly->nrings; i++ ) { @@ -638,7 +552,7 @@ static size_t lwgeom_agg_to_twkbcollection_size(lwgeom_id* geom_array,uint8_t v /*One byte for type declaration*/ size_t size = WKB_BYTE_SIZE; /*One integer holding number of collections*/ - size += u_getvarint_size((uint64_t) n); + size += varint_u64_encoded_size((uint64_t) n); int i; for (i=0;ingeoms); + size += varint_u64_encoded_size((uint64_t) col->ngeoms); int i = 0; for ( i = 0; i < col->ngeoms; i++ ) @@ -692,7 +606,7 @@ static uint8_t* lwgeom_agg_to_twkbcollection_buf(lwgeom_id* geom_array,int n, ui LWDEBUGF(4, "Writing ndims '%d'", dims); buf = uint8_to_twkb_buf(type_flag,buf); /* Set number of geometries */ - buf = u_varint_to_twkb_buf(n, buf); + buf = varint_u64_encode_buf(n, buf); for (i=0;ingeoms, buf); + buf = varint_u64_encode_buf(col->ngeoms, buf); /* Write the sub-geometries. Sub-geometries do not get SRIDs, they inherit from their parents. */ for ( i = 0; i < col->ngeoms; i++ ) @@ -971,7 +885,7 @@ uint8_t* lwgeom_agg_to_twkb(const twkb_geom_arrays *lwgeom_arrays,uint8_t varian if(chk_homogenity==0) return NULL; if(chk_homogenity>1) - buf_size = 2+u_getvarint_size((uint64_t) chk_homogenity); + buf_size = 2+varint_u64_encoded_size((uint64_t) chk_homogenity); else buf_size=1; @@ -1037,7 +951,7 @@ uint8_t* lwgeom_agg_to_twkb(const twkb_geom_arrays *lwgeom_arrays,uint8_t varian buf = uint8_to_twkb_buf(type_flag,buf); /* Set number of geometries */ - buf = u_varint_to_twkb_buf(chk_homogenity, buf); + buf = varint_u64_encode_buf(chk_homogenity, buf); } /* Write the WKB into the output buffer diff --git a/liblwgeom/lwout_twkb.h b/liblwgeom/lwout_twkb.h index d92bdec7f..e46423c6e 100644 --- a/liblwgeom/lwout_twkb.h +++ b/liblwgeom/lwout_twkb.h @@ -36,21 +36,6 @@ #define TYPE_DIM_SET_TYPE(flag, type) ((flag) = (flag & 0xE0) | ((type & 0x1F))) #define TYPE_DIM_SET_DIM(flag, dim) ((flag) = (flag & 0x1F) | ((dim & 0x07)<<5)) - -/** -*Constants to find the range of Varint. -* Since Varint only uses 7 bits instead of 8 in each byte -* a 8 byte varint only uses 56 bits for the number and 8 for telling if next byte is used -*/ -const int64_t max_varint = ((int64_t) 1<<56) - 1; -const int64_t min_s_varint = (int64_t) -1<<55; -const int64_t max_s_varint = ((int64_t)1<<55) - 1; - - - -int s_getvarint_size(int64_t val); -int u_getvarint_size(uint64_t val); - static size_t ptarray_to_twkb_size(const POINTARRAY *pa, uint8_t variant,int prec,int64_t accum_rel[],int method); static uint8_t* ptarray_to_twkb_buf(const POINTARRAY *pa, uint8_t *buf, uint8_t variant,int8_t prec,int64_t accum_rel[],int method); diff --git a/liblwgeom/varint.c b/liblwgeom/varint.c new file mode 100644 index 000000000..1a965d63a --- /dev/null +++ b/liblwgeom/varint.c @@ -0,0 +1,141 @@ +/********************************************************************** + * + * PostGIS - Spatial Types for PostgreSQL + * http://postgis.net + * + * Copyright (C) 2014 Sandro Santilli + * Copyright (C) 2013 Nicklas Avén + * + * This is free software; you can redistribute and/or modify it under + * the terms of the GNU General Public Licence. See the COPYING file. + * + ********************************************************************** + * + * Handle varInt values, as described here: + * http://developers.google.com/protocol-buffers/docs/encoding#varints + * + **********************************************************************/ + +#include "varint.h" +#include "lwgeom_log.h" +#include "liblwgeom.h" + +/** + * Constants to find the range of Varint. + * Since Varint only uses 7 bits instead of 8 in each byte a 8 byte varint + * only uses 56 bits for the number and 8 for telling if next byte is used + */ +const int64_t varint_u64_max = ((int64_t) 1<<56) - 1; +const int64_t varint_s64_min = (int64_t) -1<<55; +const int64_t varint_s64_max = ((int64_t)1<<55) - 1; +const int32_t varint_u32_max = ((int32_t) 1<<28) - 1; +const int32_t varint_s32_min = (int32_t) -1<<27; +const int32_t varint_s32_max = ((int32_t)1<<27) - 1; + +static unsigned +_varint_u64_encoded_size(uint64_t q) +{ + int n=0; + while ((q>>(7*(n+1))) >0) ++n; + return ++n; +} + +static uint8_t* +_varint_u64_encode_buf(uint64_t q, uint8_t *buf) +{ + int n=0, grp; + while ((q>>(7*(n+1))) >0) + { + grp=128^(127&(q>>(7*n))); + buf[n++]=grp; + } + grp=127&(q>>(7*n)); + buf[n++]=grp; + + return buf+=n; +} + +unsigned +varint_u32_encoded_size(uint32_t val) +{ + LWDEBUGF(2, "Entered varint_u32_encoded_size, value %u", val); + + if( val>varint_u32_max ) { + lwerror("Value is out of range for unsigned 32bit varint (0 to %ld)", + varint_u32_max); + } + + return _varint_u64_encoded_size(val); /* implicit upcast to 64bit int */ +} + +uint8_t* +varint_u32_encode_buf(uint32_t val, uint8_t *buf) +{ + LWDEBUGF(2, "Entered varint_u32_encode_buf, value %u", val); + return _varint_u64_encode_buf(val, buf); /* implicit upcast to 64bit */ +} + +unsigned +varint_s32_encoded_size(int32_t val) +{ + LWDEBUGF(2, "Entered varint_s32_encoded_size, value %d", val); + + if(valvarint_s32_max) { + lwerror("Value is out of range for signed 32bit varint (%d to %d)", + varint_s32_min, varint_s32_max); + } + + uint32_t q = (val << 1) ^ (val >> 31); /* zig-zag encode */ + return _varint_u64_encoded_size(q); /* implicit upcast to 64bit int */ +} + +uint8_t* +varint_s32_encode_buf(int32_t val, uint8_t *buf) +{ + LWDEBUGF(2, "Entered varint_s32_encode_buf, value %d", val); + uint32_t q = (val << 1) ^ (val >> 31); /* zig-zag encode */ + return _varint_u64_encode_buf(q, buf); /* implicit upcast to 64bit */ +} + +unsigned +varint_s64_encoded_size(int64_t val) +{ + LWDEBUGF(2, "Entered varint_s64_encoded_size, value %ld", val); + + if(valvarint_s64_max) { + lwerror("Value is out of range for signed 64bit varint (%ld to %ld)", + varint_s64_min, varint_s64_max); + } + + uint64_t q = (val << 1) ^ (val >> 63); /* zig-zag encode */ + return _varint_u64_encoded_size(q); +} + +uint8_t* +varint_s64_encode_buf(int64_t val, uint8_t *buf) +{ + LWDEBUGF(2, "Entered varint_s64_encode_buf, value %ld", val); + + uint64_t q = (val << 1) ^ (val >> 63); /* zig-zag encode */ + return _varint_u64_encode_buf(q, buf); +} + +unsigned +varint_u64_encoded_size(uint64_t val) +{ + LWDEBUGF(2, "Entered varint_u64_encoded_size, value %lu", val); + + if( val>varint_u64_max ) { + lwerror("Value is out of range for unsigned 64bit varint (0 to %ld)", + varint_u64_max); + } + + return _varint_u64_encoded_size(val); +} + +uint8_t* +varint_u64_encode_buf(uint64_t val, uint8_t *buf) +{ + LWDEBUGF(2, "Entered varint_u64_encode_buf, value %lu", val); + return _varint_u64_encode_buf(val, buf); +} diff --git a/liblwgeom/varint.h b/liblwgeom/varint.h new file mode 100644 index 000000000..829a9df45 --- /dev/null +++ b/liblwgeom/varint.h @@ -0,0 +1,48 @@ +/********************************************************************** + * + * PostGIS - Spatial Types for PostgreSQL + * http://postgis.net + * + * Copyright (C) 2014 Sandro Santilli + * Copyright (C) 2013 Nicklas Avén + * + * This is free software; you can redistribute and/or modify it under + * the terms of the GNU General Public Licence. See the COPYING file. + * + ********************************************************************** + * + * Handle varInt values, as described here: + * http://developers.google.com/protocol-buffers/docs/encoding#varints + * + **********************************************************************/ + +#ifndef _LIBLWGEOM_VARINT_H +#define _LIBLWGEOM_VARINT_H 1 + +#include + +/* Find encoded size for unsigned 32bit integer */ +unsigned varint_u32_encoded_size(uint32_t val); + +/* Encode unsigned 32bit integer */ +uint8_t* varint_u32_encode_buf(uint32_t val, uint8_t *buf); + +/* Find encoded size for signed 32bit integer */ +unsigned varint_s32_encoded_size(int32_t val); + +/* Encode signed 32bit integer */ +uint8_t* varint_s32_encode_buf(int32_t val, uint8_t *buf); + +/* Find encoded size for unsigned 64bit integer */ +unsigned varint_u64_encoded_size(uint64_t val); + +/* Encode unsigned 64bit integer */ +uint8_t* varint_u64_encode_buf(uint64_t val, uint8_t *buf); + +/* Find encoded size for signed 64bit integer */ +unsigned varint_s64_encoded_size(int64_t val); + +/* Encode unsigned 64bit integer */ +uint8_t* varint_s64_encode_buf(int64_t val, uint8_t *buf); + +#endif /* !defined _LIBLWGEOM_VARINT_H */