Added "auto" option to -t switch where raster2pgsql can compute a usable

tile size. Added warning message if generated tiles may cause memory
issues.

git-svn-id: http://svn.osgeo.org/postgis/trunk@10541 b70326c6-7e19-0410-871a-916f4a2858ee

diff --git a/doc/using_raster_dataman.xml b/doc/using_raster_dataman.xml
index 80c8830ffa2bc1a0f5cdc12bec1e8677b9c7f75d..a837e761d51e7f3c9d4a36e34d5cd7c37f668da7 100644 (file)
--- a/doc/using_raster_dataman.xml
+++ b/doc/using_raster_dataman.xml
@@ -151,7 +151,7 @@
                     <term>-t TILE_SIZE</term>
                     <listitem>
                        <para>
-                        Cut raster into tiles to be inserted one per table row.  <varname>TILE_SIZE</varname> is expressed as WIDTHxHEIGHT.
+                                                       Cut raster into tiles to be inserted one per table row.  <varname>TILE_SIZE</varname> is expressed as WIDTHxHEIGHT or set to the value "auto" to allow the loader to compute an appropriate tile size using the first raster and applied to all rasters.
                        </para>
                     </listitem>
                 </varlistentry>

diff --git a/raster/loader/raster2pgsql.c b/raster/loader/raster2pgsql.c
index af551d389534e8ddb090ebcb2055c69353602975..efafb0cf17b91d2b5119fdd966a9a2c85f5985b6 100644 (file)
--- a/raster/loader/raster2pgsql.c
+++ b/raster/loader/raster2pgsql.c
@@ -346,6 +346,9 @@ usage() {
        printf(_(
                "  -t <tile size> Cut raster into tiles to be inserted one per\n"
                "      table row.  <tile size> is expressed as WIDTHxHEIGHT.\n"
+               "      <tile size> can also be \"auto\" to allow the loader to compute\n"
+               "      an appropriate tile size using the first raster and applied to\n"
+               "      all rasters.\n"
        ));
        printf(_(
                "  -R  Register the raster as an out-of-db (filesystem) raster.  Provided\n"
@@ -429,6 +432,66 @@ usage() {
        ));
 }
 
+static void calc_tile_size(
+       int dimX, int dimY,
+       int *tileX, int *tileY
+) {
+       int i = 0;
+       int j = 0;
+       int min = 30;
+       int max = 100;
+
+       int d = 0;
+       double r = 0;
+       int _d = 0;
+       double _r = -1;
+       int _i = 0;
+
+       /* j = 0, X */
+       for (j = 0; j < 2; j++) {
+               _i = 0;
+               _d = 0;
+               _r = -1;
+
+               if (j < 1 && dimX <= max) {
+                       *tileX = dimX;
+                       continue;
+               }
+               else if (dimY <= max) {
+                       *tileY = dimY;
+                       continue;
+               }
+
+               for (i = max; i >= min; i--) {
+                       if (j < 1) {
+                               d = dimX / i;
+                               r = (double) dimX / (double) i;
+
+                       }
+                       else {
+                               d = dimY / i;
+                               r = (double) dimY / (double) i;
+                       }
+                       r = r - (double) d;
+
+                       if (
+                               FLT_EQ(_r, -1) ||
+                               (r < _r) ||
+                               FLT_EQ(r, _r)
+                       ) {
+                               _d = d;
+                               _r = r;
+                               _i = i;
+                       }
+               }
+
+               if (j < 1)
+                       *tileX = _i;
+               else
+                       *tileY = _i;
+       }
+}
+
 static void
 init_rastinfo(RASTERINFO *info) {
        info->srid = SRID_UNKNOWN;
@@ -1420,6 +1483,7 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til
        int ytile = 0;
        double gt[6] = {0.};
        const char* pszProjectionRef = NULL;
+       int tilesize = 0;
 
        rt_raster rast = NULL;
        int numbands = 0;
@@ -1550,6 +1614,19 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til
        info->dim[0] = GDALGetRasterXSize(hdsSrc);
        info->dim[1] = GDALGetRasterYSize(hdsSrc);
 
+       /* tile size is "auto" */
+       if (
+               config->tile_size[0] == -1 &&
+               config->tile_size[1] == -1
+       ) {
+               calc_tile_size(
+                       info->dim[0], info->dim[1],
+                       &(config->tile_size[0]), &(config->tile_size[1])
+               );
+
+               rtinfo(_("Using computed tile size: %dx%d"), config->tile_size[0], config->tile_size[1]);
+       }
+
        /* decide on tile size */
        if (!config->tile_size[0])
                info->tile_size[0] = info->dim[0];
@@ -1566,6 +1643,9 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til
        if (info->tile_size[1] != info->dim[1]) 
                ntiles[1] = (info->dim[1] + info->tile_size[1]  - 1) / info->tile_size[1];
 
+       /* estimate size of 1 tile */
+       tilesize = info->tile_size[0] * info->tile_size[1];
+
        /* go through bands for attributes */
        for (i = 0; i < info->nband_count; i++) {
                hbandSrc = GDALGetRasterBand(hdsSrc, info->nband[i]);
@@ -1594,8 +1674,16 @@ convert_raster(int idx, RTLOADERCFG *config, RASTERINFO *info, STRINGBUFFER *til
                        else
                                info->nodataval[i] = 0;
                }
+
+               /* update estimated size of 1 tile */
+               tilesize *= rt_pixtype_size(info->bandtype[i]);
        }
 
+       /* roughly estimate size of one tile and all bands */
+       tilesize *= 1.1;
+       if (tilesize > MAXTILESIZE)
+               rtwarn(_("The size of each output tile may exceed 1 GB. Use -t to specify a reasonable tile size"));
+
        /* out-db raster */
        if (config->outdb) {
                GDALClose(hdsSrc);
@@ -2231,31 +2319,36 @@ main(int argc, char **argv) {
                }
                /* tile size */
                else if (CSEQUAL(argv[i], "-t") && i < argc - 1) {
-                       elements = strsplit(argv[++i], "x", &n);
-                       if (n != 2) {
-                               rterror(_("Could not process -t"));
-                               rtdealloc_config(config);
-                               exit(1);
+                       if (CSEQUAL(argv[++i], "auto")) {
+                               config->tile_size[0] = -1;
+                               config->tile_size[1] = -1;
                        }
-
-                       for (j = 0; j < n; j++) {
-                               char *t = trim(elements[j]);
-                               config->tile_size[j] = atoi(t);
-                               rtdealloc(t);
-                               rtdealloc(elements[j]);
-                       }
-                       rtdealloc(elements);
-                       elements = NULL;
-                       n = 0;
-
-                       for (j = 0; j < 2; j++) {
-                               if (config->tile_size[j] < 1) {
-                                       rterror(_("Tile size must be greater than 0x0"));
+                       else {
+                               elements = strsplit(argv[i], "x", &n);
+                               if (n != 2) {
+                                       rterror(_("Could not process -t"));
                                        rtdealloc_config(config);
                                        exit(1);
                                }
-                       }
 
+                               for (j = 0; j < n; j++) {
+                                       char *t = trim(elements[j]);
+                                       config->tile_size[j] = atoi(t);
+                                       rtdealloc(t);
+                                       rtdealloc(elements[j]);
+                               }
+                               rtdealloc(elements);
+                               elements = NULL;
+                               n = 0;
+
+                               for (j = 0; j < 2; j++) {
+                                       if (config->tile_size[j] < 1) {
+                                               rterror(_("Tile size must be greater than 0x0"));
+                                               rtdealloc_config(config);
+                                               exit(1);
+                                       }
+                               }
+                       }
                }
                /* out-of-db raster */
                else if (CSEQUAL(argv[i], "-R")) {

diff --git a/raster/loader/raster2pgsql.h b/raster/loader/raster2pgsql.h
index b4104de24b68d507aff8fd6919f310a9b771ce7f..22dbbd05b9e684d16e82a40face8d2b0e7f6effa 100644 (file)
--- a/raster/loader/raster2pgsql.h
+++ b/raster/loader/raster2pgsql.h
@@ -56,6 +56,15 @@
 #define MINOVFACTOR 2
 #define MAXOVFACTOR 1000
 
+/*
+       maximum tile size
+       based upon maximum field size as defined for PostgreSQl
+       http://www.postgresql.org/about/
+
+       1 GB = 1024 x 1024 x 1024
+*/
+#define MAXTILESIZE 1073741824
+
 #define RCSID "$Id$"
 
 typedef struct raster_loader_config {