From: Bborie Park Date: Wed, 24 Oct 2012 16:31:39 +0000 (+0000) Subject: Added "auto" option to -t switch where raster2pgsql can compute a usable X-Git-Tag: 2.1.0beta2~472 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=0cfabd61a87fe5c349122fc769dc45073dbcd955;p=postgis Added "auto" option to -t switch where raster2pgsql can compute a usable tile size. Added warning message if generated tiles may cause memory issues. git-svn-id: http://svn.osgeo.org/postgis/trunk@10541 b70326c6-7e19-0410-871a-916f4a2858ee --- diff --git a/doc/using_raster_dataman.xml b/doc/using_raster_dataman.xml index 80c8830ff..a837e761d 100644 --- a/doc/using_raster_dataman.xml +++ b/doc/using_raster_dataman.xml @@ -151,7 +151,7 @@ -t TILE_SIZE - Cut raster into tiles to be inserted one per table row. TILE_SIZE is expressed as WIDTHxHEIGHT. + Cut raster into tiles to be inserted one per table row. TILE_SIZE is expressed as WIDTHxHEIGHT or set to the value "auto" to allow the loader to compute an appropriate tile size using the first raster and applied to all rasters. diff --git a/raster/loader/raster2pgsql.c b/raster/loader/raster2pgsql.c index af551d389..efafb0cf1 100644 --- a/raster/loader/raster2pgsql.c +++ b/raster/loader/raster2pgsql.c @@ -346,6 +346,9 @@ usage() { printf(_( " -t Cut raster into tiles to be inserted one per\n" " table row. is expressed as WIDTHxHEIGHT.\n" + " can also be \"auto\" to allow the loader to compute\n" + " an appropriate tile size using the first raster and applied to\n" + " all rasters.\n" )); printf(_( " -R Register the raster as an out-of-db (filesystem) raster. Provided\n" @@ -429,6 +432,66 @@ usage() { )); } +static void calc_tile_size( + int dimX, int dimY, + int *tileX, int *tileY +) { + int i = 0; + int j = 0; + int min = 30; + int max = 100; + + int d = 0; + double r = 0; + int _d = 0; + double _r = -1; + int _i = 0; + + /* j = 0, X */ + for (j = 0; j < 2; j++) { + _i = 0; + _d = 0; + _r = -1; + + if (j < 1 && dimX <= max) { + *tileX = dimX; + continue; + } + else if (dimY <= max) { + *tileY = dimY; + continue; + } + + for (i = max; i >= min; i--) { + if (j < 1) { + d = dimX / i; + r = (double) dimX / (double) i; + + } + else { + d = dimY / i; + r = (double) dimY / (double) i; + } + r = r - (double) d; + + if ( + FLT_EQ(_r, -1) || + (r < _r) || + FLT_EQ(r, _r) + ) { + _d = d; + _r = r; + _i = i; + } + } + + if (j < 1) + *tileX = _i; + else + *tileY = _i; + } +} + static void init_rastinfo(RASTERINFO *info) { info->srid = SRID_UNKNOWN; @@ -1420,6 +1483,7 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til int ytile = 0; double gt[6] = {0.}; const char* pszProjectionRef = NULL; + int tilesize = 0; rt_raster rast = NULL; int numbands = 0; @@ -1550,6 +1614,19 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til info->dim[0] = GDALGetRasterXSize(hdsSrc); info->dim[1] = GDALGetRasterYSize(hdsSrc); + /* tile size is "auto" */ + if ( + config->tile_size[0] == -1 && + config->tile_size[1] == -1 + ) { + calc_tile_size( + info->dim[0], info->dim[1], + &(config->tile_size[0]), &(config->tile_size[1]) + ); + + rtinfo(_("Using computed tile size: %dx%d"), config->tile_size[0], config->tile_size[1]); + } + /* decide on tile size */ if (!config->tile_size[0]) info->tile_size[0] = info->dim[0]; @@ -1566,6 +1643,9 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til if (info->tile_size[1] != info->dim[1]) ntiles[1] = (info->dim[1] + info->tile_size[1] - 1) / info->tile_size[1]; + /* estimate size of 1 tile */ + tilesize = info->tile_size[0] * info->tile_size[1]; + /* go through bands for attributes */ for (i = 0; i < info->nband_count; i++) { hbandSrc = GDALGetRasterBand(hdsSrc, info->nband[i]); @@ -1594,8 +1674,16 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til else info->nodataval[i] = 0; } + + /* update estimated size of 1 tile */ + tilesize *= rt_pixtype_size(info->bandtype[i]); } + /* roughly estimate size of one tile and all bands */ + tilesize *= 1.1; + if (tilesize > MAXTILESIZE) + rtwarn(_("The size of each output tile may exceed 1 GB. Use -t to specify a reasonable tile size")); + /* out-db raster */ if (config->outdb) { GDALClose(hdsSrc); @@ -2231,31 +2319,36 @@ main(int argc, char **argv) { } /* tile size */ else if (CSEQUAL(argv[i], "-t") && i < argc - 1) { - elements = strsplit(argv[++i], "x", &n); - if (n != 2) { - rterror(_("Could not process -t")); - rtdealloc_config(config); - exit(1); + if (CSEQUAL(argv[++i], "auto")) { + config->tile_size[0] = -1; + config->tile_size[1] = -1; } - - for (j = 0; j < n; j++) { - char *t = trim(elements[j]); - config->tile_size[j] = atoi(t); - rtdealloc(t); - rtdealloc(elements[j]); - } - rtdealloc(elements); - elements = NULL; - n = 0; - - for (j = 0; j < 2; j++) { - if (config->tile_size[j] < 1) { - rterror(_("Tile size must be greater than 0x0")); + else { + elements = strsplit(argv[i], "x", &n); + if (n != 2) { + rterror(_("Could not process -t")); rtdealloc_config(config); exit(1); } - } + for (j = 0; j < n; j++) { + char *t = trim(elements[j]); + config->tile_size[j] = atoi(t); + rtdealloc(t); + rtdealloc(elements[j]); + } + rtdealloc(elements); + elements = NULL; + n = 0; + + for (j = 0; j < 2; j++) { + if (config->tile_size[j] < 1) { + rterror(_("Tile size must be greater than 0x0")); + rtdealloc_config(config); + exit(1); + } + } + } } /* out-of-db raster */ else if (CSEQUAL(argv[i], "-R")) { diff --git a/raster/loader/raster2pgsql.h b/raster/loader/raster2pgsql.h index b4104de24..22dbbd05b 100644 --- a/raster/loader/raster2pgsql.h +++ b/raster/loader/raster2pgsql.h @@ -56,6 +56,15 @@ #define MINOVFACTOR 2 #define MAXOVFACTOR 1000 +/* + maximum tile size + based upon maximum field size as defined for PostgreSQl + http://www.postgresql.org/about/ + + 1 GB = 1024 x 1024 x 1024 +*/ +#define MAXTILESIZE 1073741824 + #define RCSID "$Id$" typedef struct raster_loader_config {