From: Sandro Santilli Date: Sun, 3 Apr 2011 19:19:30 +0000 (+0000) Subject: Ticket #855 by Loic Dachary: pgsql2shp fields conversion from predefined list X-Git-Tag: 2.0.0alpha1~1806 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=a7c93b63067deddf961f3b825f28e402efd6e873;p=postgis Ticket #855 by Loic Dachary: pgsql2shp fields conversion from predefined list git-svn-id: http://svn.osgeo.org/postgis/trunk@6996 b70326c6-7e19-0410-871a-916f4a2858ee --- diff --git a/loader/cunit/Makefile.in b/loader/cunit/Makefile.in index 0baabf56f..022612c68 100644 --- a/loader/cunit/Makefile.in +++ b/loader/cunit/Makefile.in @@ -21,12 +21,26 @@ GTK_CFLAGS = @GTK_CFLAGS@ @IGE_MAC_CFLAGS@ GTK_LIBS = @GTK_LIBS@ @IGE_MAC_LIBS@ GTK_WIN32_FLAGS = @GTK_WIN32_FLAGS@ +# PostgreSQL frontend CPPFLAGS and LDFLAGS (for compiling and linking with libpq) +PGSQL_FE_CPPFLAGS=@PGSQL_FE_CPPFLAGS@ +PGSQL_FE_LDFLAGS=@PGSQL_FE_LDFLAGS@ + +# liblwgeom +LIBLWGEOM=../../liblwgeom/liblwgeom.a + OBJS= \ cu_list.o \ + cu_pgsql2shp.o \ cu_tester.o LOADER_OBJS= \ - ../structure.o + ../structure.o \ + ../dbfopen.o \ + ../shpopen.o \ + ../getopt.o \ + ../shpcommon.o \ + ../safileio.o \ + ../pgsql2shp-core.o # If we couldn't find the cunit library then display a helpful message ifeq ($(CUNIT_LDFLAGS),) @@ -49,16 +63,16 @@ endif # Build the main unit test executable cu_tester: $(LOADER_OBJS) $(OBJS) #$(CC) $(GTK_WIN32_FLAGS) $^ -o $@ $(GTK_LIBS) $(ICONV_LDFLAGS) $(PGSQL_FE_LDFLAGS) -lm $(CUNIT_LDFLAGS) - $(CC) $^ -o $@ -lm $(CUNIT_LDFLAGS) + $(CC) $^ -o $@ $(LIBLWGEOM) $(PGSQL_FE_LDFLAGS) -lm $(CUNIT_LDFLAGS) # Command to build each of the .o files $(OBJS): %.o: %.c - $(CC) $(CFLAGS) $(CUNIT_CPPFLAGS) $(GTK_CFLAGS) -c -o $@ $< + $(CC) $(CFLAGS) $(CUNIT_CPPFLAGS) $(GTK_CFLAGS) $(PGSQL_FE_CPPFLAGS) -c -o $@ $< # Clean target clean: @rm $(OBJS) - @rm cu_tester.exe + @rm -f cu_tester.exe # Requirements message requirements_not_met_cunit: diff --git a/loader/cunit/cu_pgsql2shp.c b/loader/cunit/cu_pgsql2shp.c new file mode 100644 index 000000000..ae13f4b44 --- /dev/null +++ b/loader/cunit/cu_pgsql2shp.c @@ -0,0 +1,122 @@ +/********************************************************************** + * $Id: cu_pgsql2shp.c 5674 2010-06-03 02:04:15Z mleslie $ + * + * PostGIS - Spatial Types for PostgreSQL + * http://postgis.refractions.net + * Copyright 2010 LISAsoft Pty Ltd + * + * This is free software; you can redistribute and/or modify it under + * the terms of the GNU General Public Licence. See the COPYING file. + * + **********************************************************************/ + +#include "cu_pgsql2shp.h" +#include "cu_tester.h" +#include "../pgsql2shp-core.h" + +/* Test functions */ +void test_ShpDumperCreate(void); +void test_ShpDumperGeoMapRead(void); +void test_ShpDumperFieldnameLimit(void); + +/* +** Called from test harness to register the tests in this file. +*/ +CU_pSuite register_pgsql2shp_suite(void) +{ + CU_pSuite pSuite; + pSuite = CU_add_suite("Shapefile Loader File pgsql2shp Test", init_pgsql2shp_suite, clean_pgsql2shp_suite); + if (NULL == pSuite) + { + CU_cleanup_registry(); + return NULL; + } + + if ( + (NULL == CU_add_test(pSuite, "test_ShpDumperCreate()", test_ShpDumperCreate)) || + (NULL == CU_add_test(pSuite, "test_ShpDumperGeoMapRead()", test_ShpDumperGeoMapRead)) || + (NULL == CU_add_test(pSuite, "test_ShpDumperFieldnameLimit()", test_ShpDumperFieldnameLimit)) + ) + { + CU_cleanup_registry(); + return NULL; + } + return pSuite; +} + +/* +** The suite initialization function. +** Create any re-used objects. +*/ +int init_pgsql2shp_suite(void) +{ + return 0; +} + +/* +** The suite cleanup function. +** Frees any global objects. +*/ +int clean_pgsql2shp_suite(void) +{ + return 0; +} + +void test_ShpDumperCreate(void) +{ + SHPDUMPERCONFIG *config; + SHPDUMPERSTATE *state; + + config = (SHPDUMPERCONFIG*)calloc(1, sizeof(SHPDUMPERCONFIG)); + state = ShpDumperCreate(config); + CU_ASSERT_PTR_NOT_NULL(state); + CU_ASSERT_EQUAL(state->outtype, 's'); + free(state); + free(config); +} + +void ShpDumperGeoMapRead(char* filename, SHPDUMPERSTATE *state); + +void test_ShpDumperGeoMapRead(void) +{ + SHPDUMPERSTATE *state; + + state = malloc(sizeof(SHPDUMPERSTATE)); + ShpDumperGeoMapRead("map.txt", state); + CU_ASSERT_PTR_NOT_NULL(state->geo_map); + CU_ASSERT_EQUAL(state->geo_map_size, 3); + CU_ASSERT_STRING_EQUAL(state->geo_map[0], "0123456789toolong0"); + CU_ASSERT_STRING_EQUAL(state->geo_map[0] + strlen(state->geo_map[0]) + 1, "0123456780"); + CU_ASSERT_STRING_EQUAL(state->geo_map[1], "0123456789toolong1"); + CU_ASSERT_STRING_EQUAL(state->geo_map[1] + strlen(state->geo_map[1]) + 1, "0123456781"); + CU_ASSERT_STRING_EQUAL(state->geo_map[2], "0123456789toolong2"); + CU_ASSERT_STRING_EQUAL(state->geo_map[2] + strlen(state->geo_map[2]) + 1, "0123456782"); + free(*state->geo_map); + free(state->geo_map); + free(state); +} + +char* ShpDumperFieldnameLimit(char* ptr, SHPDUMPERSTATE *state); + +void test_ShpDumperFieldnameLimit(void) +{ + SHPDUMPERSTATE *state; + + state = malloc(sizeof(SHPDUMPERSTATE)); + ShpDumperGeoMapRead("map.txt", state); + { + char* from = "UNTOUCHED"; + char* to = ShpDumperFieldnameLimit(from, state); + CU_ASSERT_STRING_EQUAL(from, to); + free(to); + } + { + char* from = "0123456789toolong1"; + char* to = ShpDumperFieldnameLimit(from, state); + CU_ASSERT_STRING_EQUAL(to, "0123456781"); + free(to); + } + free(*state->geo_map); + free(state->geo_map); + free(state); +} diff --git a/loader/cunit/cu_pgsql2shp.h b/loader/cunit/cu_pgsql2shp.h new file mode 100644 index 000000000..c5aad4880 --- /dev/null +++ b/loader/cunit/cu_pgsql2shp.h @@ -0,0 +1,28 @@ +/********************************************************************** + * + * PostGIS - Spatial Types for PostgreSQL + * http://postgis.refractions.net + * Copyright 2010 LISAsoft Pty Ltd + * + * This is free software; you can redistribute and/or modify it under + * the terms of the GNU General Public Licence. See the COPYING file. + * + **********************************************************************/ + +#ifndef __cu_pgsql2shp_h__ +#define __cu_pgsql2shp_h__ + +#include +#include +#include +#include "CUnit/Basic.h" + +/*********************************************************************** +** for Computational Geometry Suite +*/ + +/* Admin functions */ +int init_pgsql2shp_suite(void); +int clean_pgsql2shp_suite(void); + +#endif /* __cu_pgsql2shp_h__ */ diff --git a/loader/cunit/cu_tester.c b/loader/cunit/cu_tester.c index 294f9156f..1222d8c56 100644 --- a/loader/cunit/cu_tester.c +++ b/loader/cunit/cu_tester.c @@ -33,6 +33,11 @@ int main() CU_cleanup_registry(); return CU_get_error(); } + if (NULL == register_pgsql2shp_suite()) + { + CU_cleanup_registry(); + return CU_get_error(); + } /* Run all tests using the CUnit Basic interface */ CU_basic_set_mode(CU_BRM_VERBOSE); diff --git a/loader/cunit/cu_tester.h b/loader/cunit/cu_tester.h index 0c1312758..d5c082ed6 100644 --- a/loader/cunit/cu_tester.h +++ b/loader/cunit/cu_tester.h @@ -15,6 +15,6 @@ #define __cu_tester_h__ CU_pSuite register_list_suite(void); - +CU_pSuite register_pgsql2shp_suite(void); #endif /* __cu_tester_h__ */ diff --git a/loader/cunit/map.txt b/loader/cunit/map.txt new file mode 100644 index 000000000..adfe4ea33 --- /dev/null +++ b/loader/cunit/map.txt @@ -0,0 +1,3 @@ +0123456789toolong0 0123456780 +0123456789toolong1 0123456781 +0123456789toolong2 0123456782 diff --git a/loader/pgsql2shp-cli.c b/loader/pgsql2shp-cli.c index 4faf88a96..ea7cbcb5c 100644 --- a/loader/pgsql2shp-cli.c +++ b/loader/pgsql2shp-cli.c @@ -41,6 +41,12 @@ usage() " the loader. This would not unescape attribute names\n" " and will not skip the 'gid' attribute.\n" )); printf(_(" -k Keep postgresql identifiers case.\n" )); + printf(_(" -m Remap identifiers to ten digit names.\n" + " The content of the file is lines of two symbols separated by\n" + " a single white space and no trailing or leading space:\n" + " VERYLONGSYMBOL SHORTONE\n" + " ANOTHERVERYLONGSYMBOL SHORTER\n" + " etc.\n" )); printf(_(" -? Display this help screen.\n\n" )); } @@ -63,7 +69,7 @@ main(int argc, char **argv) config = malloc(sizeof(SHPDUMPERCONFIG)); set_config_defaults(config); - while ((c = pgis_getopt(argc, argv, "bf:h:du:p:P:g:rk")) != EOF) + while ((c = pgis_getopt(argc, argv, "bf:h:du:p:P:g:rkm:")) != EOF) { switch (c) { @@ -95,6 +101,9 @@ main(int argc, char **argv) case 'g': config->geo_col_name = pgis_optarg; break; + case 'm': + config->geo_map_filename = pgis_optarg; + break; case 'k': config->keep_fieldname_case = 1; break; diff --git a/loader/pgsql2shp-core.c b/loader/pgsql2shp-core.c index fb87bc407..bfcd9e3e5 100644 --- a/loader/pgsql2shp-core.c +++ b/loader/pgsql2shp-core.c @@ -28,6 +28,7 @@ #include #include #include +#include /* Solaris9 does not provide stdint.h */ /* #include */ #include @@ -1140,8 +1141,92 @@ set_config_defaults(SHPDUMPERCONFIG *config) config->geo_col_name = NULL; config->keep_fieldname_case = 0; config->fetchsize = 100; + config->geo_map_filename = 0; } +/** + * Read the content of filename into a symbol map stored + * at state->geo_map. + * + * The content of the file is lines of two symbols separated by + * a single white space and no trailing or leading space: + * + * VERYLONGSYMBOL SHORTONE\n + * ANOTHERVERYLONGSYMBOL SHORTER\n + * + * etc. + * + * The file is read in core (one large malloc'd area), each + * space and newline is replaced by a null character. A pointer + * to the start of each line is stored in the state->geo_map + * table. + * + * It is the reponsibility of the caller to reclaim the allocated space + * as follows: + * + * free(*state->geo_map) to free the file content + * free(state->geo_map) to free the pointer list + * + * @param filename : path to a readable map file in the format + * described above. + * @param state : container of state->geo_map where the malloc'd + * symbol map will be stored. + * + * @return state->geo_map : NULL on error, symbol map pointer on + * success. + */ +void +ShpDumperGeoMapRead(char* filename, SHPDUMPERSTATE *state) +{ + struct stat stat_buf; + static char* content = 0; + { + FILE* fp = 0; + if(stat(filename, &stat_buf) < 0) + { + perror(filename); + return; + } + content = malloc(stat_buf.st_size); + fp = fopen(filename, "r"); + if(stat_buf.st_size != fread(content, 1, stat_buf.st_size, fp)) + { + free(content); + fprintf(stderr, "fread did not return the expected amount of chars"); + fclose(fp); + return; + } + fclose(fp); + } + { + int i; + state->geo_map_size = 0; + + for(i = 0; i < stat_buf.st_size; i++) + { + if(content[i] == '\n') + { + state->geo_map_size++; + } + } + state->geo_map = (char**)malloc(sizeof(char*)*state->geo_map_size); + { + char** map = state->geo_map; + *map = content; + map++; + for(i = 0; i < stat_buf.st_size; i++) + { + if(content[i] == '\n' && i + 1 < stat_buf.st_size) + { + *map = content + i + 1; + map++; + } + if(content[i] == '\n' || content[i] == ' ') + content[i] = '\0'; + } + } + } +} /* Create a new shapefile state object */ SHPDUMPERSTATE * @@ -1161,6 +1246,15 @@ ShpDumperCreate(SHPDUMPERCONFIG *config) state->table = NULL; state->geo_col_name = NULL; + if(config->geo_map_filename) + { + ShpDumperGeoMapRead(config->geo_map_filename, state); + } + else + { + state->geo_map = NULL; + state->geo_map_size = 0; + } return state; } @@ -1302,6 +1396,40 @@ ShpDumperConnectDatabase(SHPDUMPERSTATE *state) return SHPDUMPEROK; } +/** + * Map a symbol into its 10 chars equivalent according to a map. + * The map is found in state->geo_map and loaded with the -m option. + * + * @param ptr : null terminated string containing the symbol to + * be mapped + * @param state : non null state->geo_map container + * + * @return a malloc'd 10 chars symbol that is either the corresponding + * symbol found in the state->geo_map or the symbol truncated + * to its first 10 chars. The string is null terminated. + */ +char* +ShpDumperFieldnameLimit(char* ptr, SHPDUMPERSTATE *state) +{ + /* Limit dbf field name to 10-digits */ + char* dbffieldname = malloc(11); + if(state->geo_map) + { + int i; + for(i=0; igeo_map_size; i++) + { + if(!strcasecmp(state->geo_map[i], ptr)) + { + /* the replacement follows the terminating null */ + ptr = state->geo_map[i] + strlen(state->geo_map[i]) + 1; + break; + } + } + } + strncpy(dbffieldname, ptr, 10); + dbffieldname[10] = '\0'; + return dbffieldname; +} /* Open the specified table in preparation for extracting rows */ int @@ -1476,9 +1604,7 @@ ShpDumperOpenTable(SHPDUMPERSTATE *state) */ /* Limit dbf field name to 10-digits */ - dbffieldname = malloc(11); - strncpy(dbffieldname, ptr, 10); - dbffieldname[10] = '\0'; + dbffieldname = ShpDumperFieldnameLimit(ptr, state); /* * make sure the fields all have unique names, diff --git a/loader/pgsql2shp-core.h b/loader/pgsql2shp-core.h index 422372f69..5be4d6b88 100644 --- a/loader/pgsql2shp-core.h +++ b/loader/pgsql2shp-core.h @@ -95,6 +95,7 @@ typedef struct shp_dumper_config /* Number of rows to fetch in a cursor batch */ int fetchsize; + char *geo_map_filename; } SHPDUMPERCONFIG; @@ -185,6 +186,9 @@ typedef struct shp_dumper_state /* Last (error) message */ char message[SHPDUMPERMSGLEN]; + char **geo_map; + int geo_map_size; + } SHPDUMPERSTATE;