From: Mark Cave-Ayland Date: Wed, 29 Jun 2011 22:40:54 +0000 (+0000) Subject: Commit rework of #885 (pgsql2shp fields conversion from predefined list). X-Git-Tag: 2.0.0alpha1~1317 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=06b5e95f833ba70a5d62b23b2761ee5c4984638b;p=postgis Commit rework of #885 (pgsql2shp fields conversion from predefined list). This patch required extra work to ensure that it was a better fit for the new loader/dumper structure including error reporting, better use of C coding style and altering various names/code locations as appropriate. git-svn-id: http://svn.osgeo.org/postgis/trunk@7524 b70326c6-7e19-0410-871a-916f4a2858ee --- diff --git a/doc/man/pgsql2shp.1 b/doc/man/pgsql2shp.1 index 0be7f7c7e..710181e10 100644 --- a/doc/man/pgsql2shp.1 +++ b/doc/man/pgsql2shp.1 @@ -68,13 +68,13 @@ dimensions are fully encoded. Keep identifiers case (don't uppercase field names). .TP \fB\-m\fR <\fIfilename\fR> -Remap identifiers to ten digit names. The content of the file is lines -of two symbols separated by a single white space and no trailing -or leading space: +Specify a file containing a set of mappings of (long) column names to 10 +character DBF column names. The content of the file is one or more lines +of two names separated by white space and no trailing or leading space: -VERYLONGSYMBOL SHORTONE\\n +COLUMNNAME DBFFIELD1\\n .br -ANOTHERVERYLONGSYMBOL SHORTER\\n +AVERYLONGCOLUMNNAME DBFFIELD2\\n etc. .TP diff --git a/loader/cunit/cu_pgsql2shp.c b/loader/cunit/cu_pgsql2shp.c index ae13f4b44..261a8626c 100644 --- a/loader/cunit/cu_pgsql2shp.c +++ b/loader/cunit/cu_pgsql2shp.c @@ -16,8 +16,10 @@ /* Test functions */ void test_ShpDumperCreate(void); -void test_ShpDumperGeoMapRead(void); -void test_ShpDumperFieldnameLimit(void); +void test_ShpDumperDestroy(void); + +SHPDUMPERCONFIG *config; +SHPDUMPERSTATE *state; /* ** Called from test harness to register the tests in this file. @@ -34,8 +36,7 @@ CU_pSuite register_pgsql2shp_suite(void) if ( (NULL == CU_add_test(pSuite, "test_ShpDumperCreate()", test_ShpDumperCreate)) || - (NULL == CU_add_test(pSuite, "test_ShpDumperGeoMapRead()", test_ShpDumperGeoMapRead)) || - (NULL == CU_add_test(pSuite, "test_ShpDumperFieldnameLimit()", test_ShpDumperFieldnameLimit)) + (NULL == CU_add_test(pSuite, "test_ShpDumperDestroy()", test_ShpDumperDestroy)) ) { CU_cleanup_registry(); @@ -63,60 +64,14 @@ int clean_pgsql2shp_suite(void) } void test_ShpDumperCreate(void) -{ - SHPDUMPERCONFIG *config; - SHPDUMPERSTATE *state; - +{ config = (SHPDUMPERCONFIG*)calloc(1, sizeof(SHPDUMPERCONFIG)); state = ShpDumperCreate(config); CU_ASSERT_PTR_NOT_NULL(state); CU_ASSERT_EQUAL(state->outtype, 's'); - free(state); - free(config); -} - -void ShpDumperGeoMapRead(char* filename, SHPDUMPERSTATE *state); - -void test_ShpDumperGeoMapRead(void) -{ - SHPDUMPERSTATE *state; - - state = malloc(sizeof(SHPDUMPERSTATE)); - ShpDumperGeoMapRead("map.txt", state); - CU_ASSERT_PTR_NOT_NULL(state->geo_map); - CU_ASSERT_EQUAL(state->geo_map_size, 3); - CU_ASSERT_STRING_EQUAL(state->geo_map[0], "0123456789toolong0"); - CU_ASSERT_STRING_EQUAL(state->geo_map[0] + strlen(state->geo_map[0]) + 1, "0123456780"); - CU_ASSERT_STRING_EQUAL(state->geo_map[1], "0123456789toolong1"); - CU_ASSERT_STRING_EQUAL(state->geo_map[1] + strlen(state->geo_map[1]) + 1, "0123456781"); - CU_ASSERT_STRING_EQUAL(state->geo_map[2], "0123456789toolong2"); - CU_ASSERT_STRING_EQUAL(state->geo_map[2] + strlen(state->geo_map[2]) + 1, "0123456782"); - free(*state->geo_map); - free(state->geo_map); - free(state); } -char* ShpDumperFieldnameLimit(char* ptr, SHPDUMPERSTATE *state); - -void test_ShpDumperFieldnameLimit(void) +void test_ShpDumperDestroy(void) { - SHPDUMPERSTATE *state; - - state = malloc(sizeof(SHPDUMPERSTATE)); - ShpDumperGeoMapRead("map.txt", state); - { - char* from = "UNTOUCHED"; - char* to = ShpDumperFieldnameLimit(from, state); - CU_ASSERT_STRING_EQUAL(from, to); - free(to); - } - { - char* from = "0123456789toolong1"; - char* to = ShpDumperFieldnameLimit(from, state); - CU_ASSERT_STRING_EQUAL(to, "0123456781"); - free(to); - } - free(*state->geo_map); - free(state->geo_map); - free(state); + ShpDumperDestroy(state); } diff --git a/loader/pgsql2shp-cli.c b/loader/pgsql2shp-cli.c index ea7cbcb5c..36ec5b023 100644 --- a/loader/pgsql2shp-cli.c +++ b/loader/pgsql2shp-cli.c @@ -41,11 +41,11 @@ usage() " the loader. This would not unescape attribute names\n" " and will not skip the 'gid' attribute.\n" )); printf(_(" -k Keep postgresql identifiers case.\n" )); - printf(_(" -m Remap identifiers to ten digit names.\n" - " The content of the file is lines of two symbols separated by\n" - " a single white space and no trailing or leading space:\n" - " VERYLONGSYMBOL SHORTONE\n" - " ANOTHERVERYLONGSYMBOL SHORTER\n" + printf(_(" -m Specify a file containing a set of mappings of (long) column names\n" + " to 10 character DBF column names. The content of the file is one or more lines\n" + " of two names separated by white space and no trailing or leading space:\n" + " COLUMNNAME DBFFIELD1\n" + " AVERYLONGCOLUMNNAME DBFFIELD2\n" " etc.\n" )); printf(_(" -? Display this help screen.\n\n" )); } @@ -102,7 +102,7 @@ main(int argc, char **argv) config->geo_col_name = pgis_optarg; break; case 'm': - config->geo_map_filename = pgis_optarg; + config->column_map_filename = pgis_optarg; break; case 'k': config->keep_fieldname_case = 1; diff --git a/loader/pgsql2shp-core.c b/loader/pgsql2shp-core.c index bfcd9e3e5..e8995c0ea 100644 --- a/loader/pgsql2shp-core.c +++ b/loader/pgsql2shp-core.c @@ -56,19 +56,19 @@ /* Prototypes */ -int reverse_points(int num_points, double *x, double *y, double *z, double *m); -int is_clockwise(int num_points,double *x,double *y,double *z); -int is_bigendian(void); -SHPObject *create_point(SHPDUMPERSTATE *state, LWPOINT *lwpoint); -SHPObject *create_multipoint(SHPDUMPERSTATE *state, LWMPOINT *lwmultipoint); -SHPObject *create_polygon(SHPDUMPERSTATE *state, LWPOLY *lwpolygon); -SHPObject *create_multipolygon(SHPDUMPERSTATE *state, LWMPOLY *lwmultipolygon); -SHPObject *create_linestring(SHPDUMPERSTATE *state, LWLINE *lwlinestring); -SHPObject *create_multilinestring(SHPDUMPERSTATE *state, LWMLINE *lwmultilinestring); +static int reverse_points(int num_points, double *x, double *y, double *z, double *m); +static int is_clockwise(int num_points,double *x,double *y,double *z); +static int is_bigendian(void); +static SHPObject *create_point(SHPDUMPERSTATE *state, LWPOINT *lwpoint); +static SHPObject *create_multipoint(SHPDUMPERSTATE *state, LWMPOINT *lwmultipoint); +static SHPObject *create_polygon(SHPDUMPERSTATE *state, LWPOLY *lwpolygon); +static SHPObject *create_multipolygon(SHPDUMPERSTATE *state, LWMPOLY *lwmultipolygon); +static SHPObject *create_linestring(SHPDUMPERSTATE *state, LWLINE *lwlinestring); +static SHPObject *create_multilinestring(SHPDUMPERSTATE *state, LWMLINE *lwmultilinestring); static const char *nullDBFValue(char fieldType); -int getMaxFieldSize(PGconn *conn, char *schema, char *table, char *fname); -int getTableInfo(SHPDUMPERSTATE *state); -int projFileCreate(SHPDUMPERSTATE *state); +static int getMaxFieldSize(PGconn *conn, char *schema, char *table, char *fname); +static int getTableInfo(SHPDUMPERSTATE *state); +static int projFileCreate(SHPDUMPERSTATE *state); /** * @brief Make appropriate formatting of a DBF value based on type. @@ -88,7 +88,7 @@ void lwgeom_init_allocators() } -SHPObject * +static SHPObject * create_point(SHPDUMPERSTATE *state, LWPOINT *lwpoint) { SHPObject *obj; @@ -123,7 +123,7 @@ create_point(SHPDUMPERSTATE *state, LWPOINT *lwpoint) return obj; } -SHPObject * +static SHPObject * create_multipoint(SHPDUMPERSTATE *state, LWMPOINT *lwmultipoint) { SHPObject *obj; @@ -162,7 +162,7 @@ create_multipoint(SHPDUMPERSTATE *state, LWMPOINT *lwmultipoint) return obj; } -SHPObject * +static SHPObject * create_polygon(SHPDUMPERSTATE *state, LWPOLY *lwpolygon) { SHPObject *obj; @@ -250,7 +250,7 @@ create_polygon(SHPDUMPERSTATE *state, LWPOLY *lwpolygon) return obj; } -SHPObject * +static SHPObject * create_multipolygon(SHPDUMPERSTATE *state, LWMPOLY *lwmultipolygon) { SHPObject *obj; @@ -354,7 +354,7 @@ create_multipolygon(SHPDUMPERSTATE *state, LWMPOLY *lwmultipolygon) return obj; } -SHPObject * +static SHPObject * create_linestring(SHPDUMPERSTATE *state, LWLINE *lwlinestring) { SHPObject *obj; @@ -393,7 +393,7 @@ create_linestring(SHPDUMPERSTATE *state, LWLINE *lwlinestring) return obj; } -SHPObject * +static SHPObject * create_multilinestring(SHPDUMPERSTATE *state, LWMLINE *lwmultilinestring) { SHPObject *obj; @@ -454,7 +454,7 @@ create_multilinestring(SHPDUMPERSTATE *state, LWMLINE *lwmultilinestring) /*Reverse the clockwise-ness of the point list... */ -int +static int reverse_points(int num_points, double *x, double *y, double *z, double *m) { @@ -495,7 +495,7 @@ reverse_points(int num_points, double *x, double *y, double *z, double *m) } /* Return 1 if the points are in clockwise order */ -int +static int is_clockwise(int num_points, double *x, double *y, double *z) { int i; @@ -539,7 +539,7 @@ is_clockwise(int num_points, double *x, double *y, double *z) * Return the maximum octet_length from given table. * Return -1 on error. */ -int +static int getMaxFieldSize(PGconn *conn, char *schema, char *table, char *fname) { int size; @@ -586,7 +586,7 @@ getMaxFieldSize(PGconn *conn, char *schema, char *table, char *fname) return size; } -int +static int is_bigendian(void) { int test = 1; @@ -722,7 +722,8 @@ char *convert_bytes_to_hex(uchar *ewkb, size_t size) * If data is a table will use geometry_columns, if a query or view will read SRID from query output. * @warning Will give warning and not output a .prj file if SRID is -1, Unknown, mixed SRIDS or not found in spatial_ref_sys. The dbf and shp will still be output. */ -int projFileCreate(SHPDUMPERSTATE *state) +static int +projFileCreate(SHPDUMPERSTATE *state) { FILE *fp; char *pszFullname, *pszBasename; @@ -800,6 +801,7 @@ int projFileCreate(SHPDUMPERSTATE *state) { snprintf(state->message, SHPDUMPERMSGLEN, _("WARNING: Could not execute prj query: %s"), PQresultErrorMessage(res)); PQclear(res); + free(query); return SHPDUMPERWARN; } @@ -810,6 +812,7 @@ int projFileCreate(SHPDUMPERSTATE *state) { snprintf(state->message, SHPDUMPERMSGLEN, _("WARNING: Mixed set of spatial references. No prj file will be generated")); PQclear(res); + free(query); return SHPDUMPERWARN; } else @@ -818,6 +821,7 @@ int projFileCreate(SHPDUMPERSTATE *state) { snprintf(state->message, SHPDUMPERMSGLEN, _("WARNING: Cannot determine spatial reference (empty table or unknown spatial ref). No prj file will be generated.")); PQclear(res); + free(query); return SHPDUMPERWARN; } else @@ -862,7 +866,8 @@ int projFileCreate(SHPDUMPERSTATE *state) } -int getTableInfo(SHPDUMPERSTATE *state) +static int +getTableInfo(SHPDUMPERSTATE *state) { /* Get some more information from the table: @@ -1141,93 +1146,101 @@ set_config_defaults(SHPDUMPERCONFIG *config) config->geo_col_name = NULL; config->keep_fieldname_case = 0; config->fetchsize = 100; - config->geo_map_filename = 0; + config->column_map_filename = NULL; } /** * Read the content of filename into a symbol map stored - * at state->geo_map. + * at state->column_map_filename. * - * The content of the file is lines of two symbols separated by - * a single white space and no trailing or leading space: + * The content of the file is lines of two names separated by + * white space and no trailing or leading space: * - * VERYLONGSYMBOL SHORTONE\n - * ANOTHERVERYLONGSYMBOL SHORTER\n + * COLUMNNAME DBFFIELD1 + * AVERYLONGCOLUMNNAME DBFFIELD2 * * etc. * - * The file is read in core (one large malloc'd area), each - * space and newline is replaced by a null character. A pointer - * to the start of each line is stored in the state->geo_map - * table. - * * It is the reponsibility of the caller to reclaim the allocated space * as follows: * - * free(*state->geo_map) to free the file content - * free(state->geo_map) to free the pointer list + * free(state->column_map_pgfieldnames[]) to free the column names + * free(state->column_map_dbffieldnames[]) to free the dbf field names * - * @param filename : path to a readable map file in the format - * described above. - * @param state : container of state->geo_map where the malloc'd + * @param state : container of state->column_map where the malloc'd * symbol map will be stored. - * - * @return state->geo_map : NULL on error, symbol map pointer on - * success. */ -void -ShpDumperGeoMapRead(char* filename, SHPDUMPERSTATE *state) +static int +read_column_map(SHPDUMPERSTATE *state) { - struct stat stat_buf; - static char* content = 0; + FILE *fptr; + char linebuffer[1024]; + char *tmpstr, *tmpptr; + int curmapsize, fieldnamesize; + + /* Read column map file and load the column_map_dbffieldnames and column_map_pgfieldnames + arrays */ + fptr = fopen(state->config->column_map_filename, "r"); + if (!fptr) { - FILE* fp = 0; - if(stat(filename, &stat_buf) < 0) - { - perror(filename); - return; - } - content = malloc(stat_buf.st_size); - fp = fopen(filename, "r"); - if(stat_buf.st_size != fread(content, 1, stat_buf.st_size, fp)) - { - free(content); - fprintf(stderr, "fread did not return the expected amount of chars"); - fclose(fp); - return; - } - fclose(fp); + /* Return an error */ + snprintf(state->message, SHPDUMPERMSGLEN, _("ERROR: Unable to open column map file %s"), state->config->column_map_filename); + return SHPDUMPERERR; } - { - int i; - state->geo_map_size = 0; - - for(i = 0; i < stat_buf.st_size; i++) - { - if(content[i] == '\n') - { - state->geo_map_size++; - } - } - state->geo_map = (char**)malloc(sizeof(char*)*state->geo_map_size); + + /* First count how many columns we have... */ + while (fgets(linebuffer, 1024, fptr) != NULL) + state->column_map_size++; + + /* Now we know the final size, allocate the arrays and load the data */ + fseek(fptr, 0, SEEK_SET); + state->column_map_pgfieldnames = (char **)malloc(sizeof(char *) * state->column_map_size); + state->column_map_dbffieldnames = (char **)malloc(sizeof(char *) * state->column_map_size); + + /* Read in a line at a time... */ + curmapsize = 0; + while (fgets(linebuffer, 1024, fptr) != NULL) + { + /* Split into two separate strings - pgfieldname followed by dbffieldname */ + + /* First locate end of first column (pgfieldname) */ + for (tmpptr = tmpstr = linebuffer; *tmpptr != '\t' && *tmpptr != '\n' && *tmpptr != ' ' && *tmpptr != '\0'; tmpptr++); + fieldnamesize = tmpptr - tmpstr; + + /* Allocate memory and copy the string ensuring it is terminated */ + state->column_map_pgfieldnames[curmapsize] = malloc(fieldnamesize + 1); + strncpy(state->column_map_pgfieldnames[curmapsize], tmpstr, fieldnamesize); + state->column_map_pgfieldnames[curmapsize][fieldnamesize] = '\0'; + + /* Now swallow up any whitespace */ + for (tmpstr = tmpptr; *tmpptr == '\t' || *tmpptr == '\n' || *tmpptr == ' '; tmpptr++); + + /* Finally locate end of second column (dbffieldname) */ + for (tmpstr = tmpptr; *tmpptr != '\t' && *tmpptr != '\n' && *tmpptr != ' ' && *tmpptr != '\0'; tmpptr++); + fieldnamesize = tmpptr - tmpstr; + + /* Allocate memory and copy the string ensuring it is terminated */ + state->column_map_dbffieldnames[curmapsize] = malloc(fieldnamesize + 1); + strncpy(state->column_map_dbffieldnames[curmapsize], tmpstr, fieldnamesize); + state->column_map_dbffieldnames[curmapsize][fieldnamesize] = '\0'; + + /* Error out if the dbffieldname is > 10 chars */ + if (strlen(state->column_map_dbffieldnames[curmapsize]) > 10) { - char** map = state->geo_map; - *map = content; - map++; - for(i = 0; i < stat_buf.st_size; i++) - { - if(content[i] == '\n' && i + 1 < stat_buf.st_size) - { - *map = content + i + 1; - map++; - } - if(content[i] == '\n' || content[i] == ' ') - content[i] = '\0'; - } + snprintf(state->message, SHPDUMPERMSGLEN, _("ERROR: column map file specifies a DBF field name \"%s\" which is longer than 10 characters"), state->column_map_dbffieldnames[curmapsize]); + return SHPDUMPERERR; } + + curmapsize++; } + + fclose(fptr); + + /* Done; return success */ + return SHPDUMPEROK; } + /* Create a new shapefile state object */ SHPDUMPERSTATE * ShpDumperCreate(SHPDUMPERCONFIG *config) @@ -1245,16 +1258,10 @@ ShpDumperCreate(SHPDUMPERCONFIG *config) state->schema = NULL; state->table = NULL; state->geo_col_name = NULL; - - if(config->geo_map_filename) - { - ShpDumperGeoMapRead(config->geo_map_filename, state); - } - else - { - state->geo_map = NULL; - state->geo_map_size = 0; - } + state->column_map_pgfieldnames = NULL; + state->column_map_dbffieldnames = NULL; + state->column_map_size = 0; + return state; } @@ -1396,40 +1403,6 @@ ShpDumperConnectDatabase(SHPDUMPERSTATE *state) return SHPDUMPEROK; } -/** - * Map a symbol into its 10 chars equivalent according to a map. - * The map is found in state->geo_map and loaded with the -m option. - * - * @param ptr : null terminated string containing the symbol to - * be mapped - * @param state : non null state->geo_map container - * - * @return a malloc'd 10 chars symbol that is either the corresponding - * symbol found in the state->geo_map or the symbol truncated - * to its first 10 chars. The string is null terminated. - */ -char* -ShpDumperFieldnameLimit(char* ptr, SHPDUMPERSTATE *state) -{ - /* Limit dbf field name to 10-digits */ - char* dbffieldname = malloc(11); - if(state->geo_map) - { - int i; - for(i=0; igeo_map_size; i++) - { - if(!strcasecmp(state->geo_map[i], ptr)) - { - /* the replacement follows the terminating null */ - ptr = state->geo_map[i] + strlen(state->geo_map[i]) + 1; - break; - } - } - } - strncpy(dbffieldname, ptr, 10); - dbffieldname[10] = '\0'; - return dbffieldname; -} /* Open the specified table in preparation for extracting rows */ int @@ -1442,6 +1415,14 @@ ShpDumperOpenTable(SHPDUMPERSTATE *state) int gidfound = 0, i, j, ret, status; + /* Open the column map if one was specified */ + if (state->config->column_map_filename) + { + ret = read_column_map(state); + if (ret != SHPDUMPEROK) + return SHPDUMPERERR; + } + /* If a user-defined query has been specified, create and point the state to our new table */ if (state->config->usrquery) { @@ -1604,8 +1585,24 @@ ShpDumperOpenTable(SHPDUMPERSTATE *state) */ /* Limit dbf field name to 10-digits */ - dbffieldname = ShpDumperFieldnameLimit(ptr, state); + dbffieldname = malloc(11); + strncpy(dbffieldname, ptr, 10); + dbffieldname[10] = '\0'; + /* If a column map file has been passed in, use this to create the dbf field name from + the PostgreSQL column name */ + if (state->column_map_size > 0) + { + for (j = 0; j < state->column_map_size; j++) + { + if (!strcasecmp(state->column_map_pgfieldnames[j], dbffieldname)) + { + strncpy(dbffieldname, state->column_map_dbffieldnames[j], 10); + dbffieldname[10] = '\0'; + } + } + } + /* * make sure the fields all have unique names, */ @@ -2254,11 +2251,29 @@ ShpDumperDestroy(SHPDUMPERSTATE *state) free(state->dbffieldtypes); free(state->pgfieldnames); + /* Free any column map fieldnames if specified */ + if (state->column_map_size > 0) + { + for (i = 0; i < state->column_map_size; i++) + { + if (state->column_map_pgfieldnames[i]) + free(state->column_map_pgfieldnames[i]); + + if (state->column_map_dbffieldnames[i]) + free(state->column_map_dbffieldnames[i]); + } + + free(state->column_map_pgfieldnames); + free(state->column_map_dbffieldnames); + } + /* Free other names */ if (state->table) free(state->table); if (state->schema) free(state->schema); + if (state->geo_col_name) + free(state->geo_col_name); /* Free the state itself */ free(state); diff --git a/loader/pgsql2shp-core.h b/loader/pgsql2shp-core.h index 5be4d6b88..19136d046 100644 --- a/loader/pgsql2shp-core.h +++ b/loader/pgsql2shp-core.h @@ -95,7 +95,9 @@ typedef struct shp_dumper_config /* Number of rows to fetch in a cursor batch */ int fetchsize; - char *geo_map_filename; + /* Name of the column map file if specified */ + char *column_map_filename; + } SHPDUMPERCONFIG; @@ -186,8 +188,14 @@ typedef struct shp_dumper_state /* Last (error) message */ char message[SHPDUMPERMSGLEN]; - char **geo_map; - int geo_map_size; + /* Column map pgfieldnames */ + char **column_map_pgfieldnames; + + /* Column map dbffieldnames */ + char **column_map_dbffieldnames; + + /* Number of entries within column map */ + int column_map_size; } SHPDUMPERSTATE;