]> granicus.if.org Git - postgis/commitdiff
Ticket #855 by Loic Dachary: pgsql2shp fields conversion from predefined list
authorSandro Santilli <strk@keybit.net>
Sun, 3 Apr 2011 19:19:30 +0000 (19:19 +0000)
committerSandro Santilli <strk@keybit.net>
Sun, 3 Apr 2011 19:19:30 +0000 (19:19 +0000)
git-svn-id: http://svn.osgeo.org/postgis/trunk@6996 b70326c6-7e19-0410-871a-916f4a2858ee

loader/cunit/Makefile.in
loader/cunit/cu_pgsql2shp.c [new file with mode: 0644]
loader/cunit/cu_pgsql2shp.h [new file with mode: 0644]
loader/cunit/cu_tester.c
loader/cunit/cu_tester.h
loader/cunit/map.txt [new file with mode: 0644]
loader/pgsql2shp-cli.c
loader/pgsql2shp-core.c
loader/pgsql2shp-core.h

index 0baabf56f622caae68984f495e5ecac7668058b6..022612c681ba68cddcb9b87945df11c5fcfc9c8e 100644 (file)
@@ -21,12 +21,26 @@ GTK_CFLAGS = @GTK_CFLAGS@ @IGE_MAC_CFLAGS@
 GTK_LIBS = @GTK_LIBS@ @IGE_MAC_LIBS@
 GTK_WIN32_FLAGS = @GTK_WIN32_FLAGS@
 
+# PostgreSQL frontend CPPFLAGS and LDFLAGS (for compiling and linking with libpq)
+PGSQL_FE_CPPFLAGS=@PGSQL_FE_CPPFLAGS@
+PGSQL_FE_LDFLAGS=@PGSQL_FE_LDFLAGS@
+
+# liblwgeom
+LIBLWGEOM=../../liblwgeom/liblwgeom.a
+
 OBJS=  \
        cu_list.o \
+       cu_pgsql2shp.o \
        cu_tester.o
 
 LOADER_OBJS= \
-       ../structure.o
+       ../structure.o \
+       ../dbfopen.o \
+       ../shpopen.o \
+       ../getopt.o \
+       ../shpcommon.o \
+       ../safileio.o \
+       ../pgsql2shp-core.o
 
 # If we couldn't find the cunit library then display a helpful message
 ifeq ($(CUNIT_LDFLAGS),)
@@ -49,16 +63,16 @@ endif
 # Build the main unit test executable
 cu_tester: $(LOADER_OBJS) $(OBJS) 
        #$(CC) $(GTK_WIN32_FLAGS) $^ -o $@ $(GTK_LIBS) $(ICONV_LDFLAGS) $(PGSQL_FE_LDFLAGS) -lm $(CUNIT_LDFLAGS)
-       $(CC)  $^ -o $@ -lm $(CUNIT_LDFLAGS)
+       $(CC)  $^ -o $@ $(LIBLWGEOM) $(PGSQL_FE_LDFLAGS) -lm $(CUNIT_LDFLAGS)
 
 # Command to build each of the .o files
 $(OBJS): %.o: %.c
-       $(CC) $(CFLAGS) $(CUNIT_CPPFLAGS) $(GTK_CFLAGS) -c -o $@ $<
+       $(CC) $(CFLAGS) $(CUNIT_CPPFLAGS) $(GTK_CFLAGS) $(PGSQL_FE_CPPFLAGS) -c -o $@ $<
 
 # Clean target
 clean:
        @rm $(OBJS)
-       @rm cu_tester.exe
+       @rm -f cu_tester.exe
 
 # Requirements message
 requirements_not_met_cunit:
diff --git a/loader/cunit/cu_pgsql2shp.c b/loader/cunit/cu_pgsql2shp.c
new file mode 100644 (file)
index 0000000..ae13f4b
--- /dev/null
@@ -0,0 +1,122 @@
+/**********************************************************************
+ * $Id: cu_pgsql2shp.c 5674 2010-06-03 02:04:15Z mleslie $
+ *
+ * PostGIS - Spatial Types for PostgreSQL
+ * http://postgis.refractions.net
+ * Copyright 2010 LISAsoft Pty Ltd
+ *
+ * This is free software; you can redistribute and/or modify it under
+ * the terms of the GNU General Public Licence. See the COPYING file.
+ *
+ **********************************************************************/
+
+#include "cu_pgsql2shp.h"
+#include "cu_tester.h"
+#include "../pgsql2shp-core.h"
+
+/* Test functions */
+void test_ShpDumperCreate(void);
+void test_ShpDumperGeoMapRead(void);
+void test_ShpDumperFieldnameLimit(void);
+
+/*
+** Called from test harness to register the tests in this file.
+*/
+CU_pSuite register_pgsql2shp_suite(void)
+{
+       CU_pSuite pSuite;
+       pSuite = CU_add_suite("Shapefile Loader File pgsql2shp Test", init_pgsql2shp_suite, clean_pgsql2shp_suite);
+       if (NULL == pSuite)
+       {
+               CU_cleanup_registry();
+               return NULL;
+       }
+
+       if (
+           (NULL == CU_add_test(pSuite, "test_ShpDumperCreate()", test_ShpDumperCreate)) ||
+           (NULL == CU_add_test(pSuite, "test_ShpDumperGeoMapRead()", test_ShpDumperGeoMapRead)) ||
+           (NULL == CU_add_test(pSuite, "test_ShpDumperFieldnameLimit()", test_ShpDumperFieldnameLimit))
+       )
+       {
+               CU_cleanup_registry();
+               return NULL;
+       }
+       return pSuite;
+}
+
+/*
+** The suite initialization function.
+** Create any re-used objects.
+*/
+int init_pgsql2shp_suite(void)
+{
+       return 0;
+}
+
+/*
+** The suite cleanup function.
+** Frees any global objects.
+*/
+int clean_pgsql2shp_suite(void)
+{
+       return 0;
+}
+
+void test_ShpDumperCreate(void)
+{
+       SHPDUMPERCONFIG *config;
+       SHPDUMPERSTATE *state;
+
+       config = (SHPDUMPERCONFIG*)calloc(1, sizeof(SHPDUMPERCONFIG));
+       state = ShpDumperCreate(config);
+       CU_ASSERT_PTR_NOT_NULL(state);
+       CU_ASSERT_EQUAL(state->outtype, 's');
+       free(state);
+       free(config);
+}
+
+void ShpDumperGeoMapRead(char* filename, SHPDUMPERSTATE *state);
+
+void test_ShpDumperGeoMapRead(void)
+{
+       SHPDUMPERSTATE *state;
+
+       state = malloc(sizeof(SHPDUMPERSTATE));
+       ShpDumperGeoMapRead("map.txt", state);
+       CU_ASSERT_PTR_NOT_NULL(state->geo_map);
+       CU_ASSERT_EQUAL(state->geo_map_size, 3);
+       CU_ASSERT_STRING_EQUAL(state->geo_map[0], "0123456789toolong0");
+       CU_ASSERT_STRING_EQUAL(state->geo_map[0] + strlen(state->geo_map[0]) + 1, "0123456780");
+       CU_ASSERT_STRING_EQUAL(state->geo_map[1], "0123456789toolong1");
+       CU_ASSERT_STRING_EQUAL(state->geo_map[1] + strlen(state->geo_map[1]) + 1, "0123456781");
+       CU_ASSERT_STRING_EQUAL(state->geo_map[2], "0123456789toolong2");
+       CU_ASSERT_STRING_EQUAL(state->geo_map[2] + strlen(state->geo_map[2]) + 1, "0123456782");
+       free(*state->geo_map);
+       free(state->geo_map);
+       free(state);
+}
+
+char* ShpDumperFieldnameLimit(char* ptr, SHPDUMPERSTATE *state);
+
+void test_ShpDumperFieldnameLimit(void)
+{
+       SHPDUMPERSTATE *state;
+
+       state = malloc(sizeof(SHPDUMPERSTATE));
+       ShpDumperGeoMapRead("map.txt", state);
+       {
+               char* from = "UNTOUCHED";
+               char* to = ShpDumperFieldnameLimit(from, state);
+               CU_ASSERT_STRING_EQUAL(from, to);
+               free(to);
+       }
+       {
+               char* from = "0123456789toolong1";
+               char* to = ShpDumperFieldnameLimit(from, state);
+               CU_ASSERT_STRING_EQUAL(to, "0123456781");
+               free(to);
+       }
+       free(*state->geo_map);
+       free(state->geo_map);
+       free(state);
+}
diff --git a/loader/cunit/cu_pgsql2shp.h b/loader/cunit/cu_pgsql2shp.h
new file mode 100644 (file)
index 0000000..c5aad48
--- /dev/null
@@ -0,0 +1,28 @@
+/**********************************************************************
+ *
+ * PostGIS - Spatial Types for PostgreSQL
+ * http://postgis.refractions.net
+ * Copyright 2010 LISAsoft Pty Ltd
+ *
+ * This is free software; you can redistribute and/or modify it under
+ * the terms of the GNU General Public Licence. See the COPYING file.
+ *
+ **********************************************************************/
+
+#ifndef __cu_pgsql2shp_h__
+#define __cu_pgsql2shp_h__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "CUnit/Basic.h"
+
+/***********************************************************************
+** for Computational Geometry Suite
+*/
+
+/* Admin functions */
+int init_pgsql2shp_suite(void);
+int clean_pgsql2shp_suite(void);
+
+#endif /* __cu_pgsql2shp_h__ */
index 294f9156fe98f09ecc287bc82b833065f060096d..1222d8c56a1bd3d163f8e2135b15ed84b0e68d51 100644 (file)
@@ -33,6 +33,11 @@ int main()
                CU_cleanup_registry();
                return CU_get_error();
        }
+       if (NULL == register_pgsql2shp_suite())
+       {
+               CU_cleanup_registry();
+               return CU_get_error();
+       }
 
        /* Run all tests using the CUnit Basic interface */
        CU_basic_set_mode(CU_BRM_VERBOSE);
index 0c131275810e0de1ff0630732b8b40db177755c1..d5c082ed69bd3d2fed2ee3b97cf34269d3a72766 100644 (file)
@@ -15,6 +15,6 @@
 #define __cu_tester_h__
 
 CU_pSuite register_list_suite(void);
-
+CU_pSuite register_pgsql2shp_suite(void);
 
 #endif /* __cu_tester_h__ */
diff --git a/loader/cunit/map.txt b/loader/cunit/map.txt
new file mode 100644 (file)
index 0000000..adfe4ea
--- /dev/null
@@ -0,0 +1,3 @@
+0123456789toolong0 0123456780
+0123456789toolong1 0123456781
+0123456789toolong2 0123456782
index 4faf88a9641da8bcbd89c64884a54f9380b5d8a3..ea7cbcb5cc1f07db000a93f0785e1f2c7b64d046 100644 (file)
@@ -41,6 +41,12 @@ usage()
                 "     the loader. This would not unescape attribute names\n" 
                 "     and will not skip the 'gid' attribute.\n" ));
        printf(_("  -k Keep postgresql identifiers case.\n" ));
+       printf(_("  -m <filename> Remap identifiers to ten digit names.\n"
+                "     The content of the file is lines of two symbols separated by\n"
+                "     a single white space and no trailing or leading space:\n"
+                "     VERYLONGSYMBOL SHORTONE\n"
+                "     ANOTHERVERYLONGSYMBOL SHORTER\n"
+                "     etc.\n" ));
        printf(_("  -? Display this help screen.\n\n" ));
 }
 
@@ -63,7 +69,7 @@ main(int argc, char **argv)
        config = malloc(sizeof(SHPDUMPERCONFIG));
        set_config_defaults(config);
 
-       while ((c = pgis_getopt(argc, argv, "bf:h:du:p:P:g:rk")) != EOF)
+       while ((c = pgis_getopt(argc, argv, "bf:h:du:p:P:g:rkm:")) != EOF)
        {
                switch (c)
                {
@@ -95,6 +101,9 @@ main(int argc, char **argv)
                case 'g':
                        config->geo_col_name = pgis_optarg;
                        break;
+               case 'm':
+                       config->geo_map_filename = pgis_optarg;
+                       break;
                case 'k':
                        config->keep_fieldname_case = 1;
                        break;
index fb87bc40777ff1bc4d9160b8c8a0b1f6772ad9bb..bfcd9e3e52a6b5c4a7d169771510344638713261 100644 (file)
@@ -28,6 +28,7 @@
 #include <ctype.h>
 #include <math.h>
 #include <sys/types.h>
+#include <sys/stat.h>
 /* Solaris9 does not provide stdint.h */
 /* #include <stdint.h> */
 #include <inttypes.h>
@@ -1140,8 +1141,92 @@ set_config_defaults(SHPDUMPERCONFIG *config)
        config->geo_col_name = NULL;
        config->keep_fieldname_case = 0;
        config->fetchsize = 100;
+       config->geo_map_filename = 0;
 }
 
+/**
+ * Read the content of filename into a symbol map stored
+ * at state->geo_map.
+ *
+ * The content of the file is lines of two symbols separated by
+ * a single white space and no trailing or leading space:
+ *
+ *    VERYLONGSYMBOL SHORTONE\n
+ *    ANOTHERVERYLONGSYMBOL SHORTER\n
+ *
+ *    etc.
+ *
+ * The file is read in core (one large malloc'd area), each
+ * space and newline is replaced by a null character. A pointer
+ * to the start of each line is stored in the state->geo_map
+ * table.
+ *
+ * It is the reponsibility of the caller to reclaim the allocated space
+ * as follows:
+ *
+ * free(*state->geo_map) to free the file content
+ * free(state->geo_map) to free the pointer list
+ *
+ * @param filename : path to a readable map file in the format
+ *                   described above.
+ * @param state : container of state->geo_map where the malloc'd
+ *                symbol map will be stored.
+ *
+ * @return state->geo_map : NULL on error, symbol map pointer on
+ *                          success.
+ */
+void
+ShpDumperGeoMapRead(char* filename, SHPDUMPERSTATE *state)
+{
+       struct stat stat_buf;
+       static char* content = 0;
+       {
+               FILE* fp = 0;
+               if(stat(filename, &stat_buf) < 0)
+               {
+                       perror(filename);
+                       return;
+               }
+               content = malloc(stat_buf.st_size);
+               fp = fopen(filename, "r");
+               if(stat_buf.st_size != fread(content, 1, stat_buf.st_size, fp))
+               {
+                       free(content);
+                       fprintf(stderr, "fread did not return the expected amount of chars");
+                       fclose(fp);
+                       return;
+               }
+               fclose(fp);
+       }
+       {
+               int i;
+               state->geo_map_size = 0;
+
+               for(i = 0; i < stat_buf.st_size; i++)
+               {
+                       if(content[i] == '\n')
+                       {
+                               state->geo_map_size++;
+                       }
+               }
+               state->geo_map = (char**)malloc(sizeof(char*)*state->geo_map_size);
+               {
+                       char** map = state->geo_map;
+                       *map = content;
+                       map++;
+                       for(i = 0; i < stat_buf.st_size; i++)
+                       {
+                               if(content[i] == '\n' && i + 1 < stat_buf.st_size)
+                               {
+                                       *map = content + i + 1;
+                                       map++;
+                               }
+                               if(content[i] == '\n' || content[i] == ' ')
+                                       content[i] = '\0';
+                       }
+               }
+       }
+}
 
 /* Create a new shapefile state object */
 SHPDUMPERSTATE *
@@ -1161,6 +1246,15 @@ ShpDumperCreate(SHPDUMPERCONFIG *config)
        state->table = NULL;
        state->geo_col_name = NULL;
 
+       if(config->geo_map_filename)
+       {
+               ShpDumperGeoMapRead(config->geo_map_filename, state);
+       }
+       else
+       {
+               state->geo_map = NULL;
+               state->geo_map_size = 0;
+       }
        return state;
 }
 
@@ -1302,6 +1396,40 @@ ShpDumperConnectDatabase(SHPDUMPERSTATE *state)
        return SHPDUMPEROK;
 }
 
+/**
+ * Map a symbol into its 10 chars equivalent according to a map.
+ * The map is found in state->geo_map and loaded with the -m option.
+ *
+ * @param ptr : null terminated string containing the symbol to
+ *              be mapped
+ * @param state : non null state->geo_map container
+ *
+ * @return a malloc'd 10 chars symbol that is either the corresponding
+ *         symbol found in the state->geo_map or the symbol truncated
+ *         to its first 10 chars. The string is null terminated.
+ */
+char*
+ShpDumperFieldnameLimit(char* ptr, SHPDUMPERSTATE *state)
+{
+       /* Limit dbf field name to 10-digits */
+       char* dbffieldname = malloc(11);
+       if(state->geo_map)
+       {
+               int i;
+               for(i=0; i<state->geo_map_size; i++)
+               {
+                       if(!strcasecmp(state->geo_map[i], ptr))
+                       {
+                               /* the replacement follows the terminating null */
+                               ptr = state->geo_map[i] + strlen(state->geo_map[i]) + 1;
+                               break;
+                       }
+               }
+       }
+       strncpy(dbffieldname, ptr, 10);
+       dbffieldname[10] = '\0';
+       return dbffieldname;
+}
 
 /* Open the specified table in preparation for extracting rows */
 int
@@ -1476,9 +1604,7 @@ ShpDumperOpenTable(SHPDUMPERSTATE *state)
                 */
 
                /* Limit dbf field name to 10-digits */
-               dbffieldname = malloc(11);
-               strncpy(dbffieldname, ptr, 10);
-               dbffieldname[10] = '\0';
+               dbffieldname = ShpDumperFieldnameLimit(ptr, state);
 
                /*
                 * make sure the fields all have unique names,
index 422372f696a749460d0ea0ce5d754cd4bc0c334a..5be4d6b8898453776f3188e3e01a39d3606ceb43 100644 (file)
@@ -95,6 +95,7 @@ typedef struct shp_dumper_config
        /* Number of rows to fetch in a cursor batch */
        int fetchsize;
 
+       char *geo_map_filename;
 } SHPDUMPERCONFIG;
 
 
@@ -185,6 +186,9 @@ typedef struct shp_dumper_state
        /* Last (error) message */
        char message[SHPDUMPERMSGLEN];
 
+       char **geo_map;
+       int geo_map_size;
+
 } SHPDUMPERSTATE;