pg_upgrade: dramatically reduce memory consumption

Report from Jeff Janes

diff --git a/contrib/pg_upgrade/check.c b/contrib/pg_upgrade/check.c
index f04707f9c6f510349ad3375d47ebf18a8cf04ee8..acf8660044d42ae6685353167b27e9873bd25eee 100644 (file)
--- a/contrib/pg_upgrade/check.c
+++ b/contrib/pg_upgrade/check.c
@@ -707,20 +707,18 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name)
                        fprintf(script, "\n");
                        /* remove PG_VERSION? */
                        if (GET_MAJOR_VERSION(old_cluster.major_version) <= 804)
-                               fprintf(script, RM_CMD " %s%s%cPG_VERSION\n",
+                               fprintf(script, RM_CMD " %s%cPG_VERSION\n",
                                                fix_path_separator(os_info.old_tablespaces[tblnum]),
-                                               fix_path_separator(old_cluster.tablespace_suffix),
                                                PATH_SEPARATOR);
 
                        for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
-                       {
-                               fprintf(script, RMDIR_CMD " %s%s%c%d\n",
+                               fprintf(script, RMDIR_CMD " %s%c%d\n",
                                                fix_path_separator(os_info.old_tablespaces[tblnum]),
-                                               fix_path_separator(old_cluster.tablespace_suffix),
                                                PATH_SEPARATOR, old_cluster.dbarr.dbs[dbnum].db_oid);
-                       }
                }
                else
+               {
+                       char    *suffix_path = pg_strdup(old_cluster.tablespace_suffix);
 
                        /*
                         * Simply delete the tablespace directory, which might be ".old"
@@ -728,7 +726,9 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name)
                         */
                        fprintf(script, RMDIR_CMD " %s%s\n",
                                        fix_path_separator(os_info.old_tablespaces[tblnum]),
-                                       fix_path_separator(old_cluster.tablespace_suffix));
+                                       fix_path_separator(suffix_path));
+                       pfree(suffix_path);
+               }
        }
 
        fclose(script);

diff --git a/contrib/pg_upgrade/info.c b/contrib/pg_upgrade/info.c
index 3d0dd1a1015152e375f3f930e1a45e83bc1228ab..fd083de821f1c4bdef611a1fc3fd9f197a65cae6 100644 (file)
--- a/contrib/pg_upgrade/info.c
+++ b/contrib/pg_upgrade/info.c
@@ -108,20 +108,18 @@ create_rel_filename_map(const char *old_data, const char *new_data,
                 * relation belongs to the default tablespace, hence relfiles should
                 * exist in the data directories.
                 */
-               strlcpy(map->old_tablespace, old_data, sizeof(map->old_tablespace));
-               strlcpy(map->new_tablespace, new_data, sizeof(map->new_tablespace));
-               strlcpy(map->old_tablespace_suffix, "/base", sizeof(map->old_tablespace_suffix));
-               strlcpy(map->new_tablespace_suffix, "/base", sizeof(map->new_tablespace_suffix));
+               map->old_tablespace = old_data;
+               map->new_tablespace = new_data;
+               map->old_tablespace_suffix = "/base";
+               map->new_tablespace_suffix = "/base";
        }
        else
        {
                /* relation belongs to a tablespace, so use the tablespace location */
-               strlcpy(map->old_tablespace, old_rel->tablespace, sizeof(map->old_tablespace));
-               strlcpy(map->new_tablespace, new_rel->tablespace, sizeof(map->new_tablespace));
-               strlcpy(map->old_tablespace_suffix, old_cluster.tablespace_suffix,
-                               sizeof(map->old_tablespace_suffix));
-               strlcpy(map->new_tablespace_suffix, new_cluster.tablespace_suffix,
-                               sizeof(map->new_tablespace_suffix));
+               map->old_tablespace = old_rel->tablespace;
+               map->new_tablespace = new_rel->tablespace;
+               map->old_tablespace_suffix = old_cluster.tablespace_suffix;
+               map->new_tablespace_suffix = new_cluster.tablespace_suffix;
        }
 
        map->old_db_oid = old_db->db_oid;
@@ -231,7 +229,7 @@ get_db_infos(ClusterInfo *cluster)
        {
                dbinfos[tupnum].db_oid = atooid(PQgetvalue(res, tupnum, i_oid));
                dbinfos[tupnum].db_name = pg_strdup(PQgetvalue(res, tupnum, i_datname));
-               snprintf(dbinfos[tupnum].db_tblspace, sizeof(dbinfos[tupnum].db_tblspace), "%s",
+               snprintf(dbinfos[tupnum].db_tablespace, sizeof(dbinfos[tupnum].db_tablespace), "%s",
                                 PQgetvalue(res, tupnum, i_spclocation));
        }
        PQclear(res);
@@ -264,6 +262,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
        int                     num_rels = 0;
        char       *nspname = NULL;
        char       *relname = NULL;
+       char       *tablespace = NULL;
        int                     i_spclocation,
                                i_nspname,
                                i_relname,
@@ -271,6 +270,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
                                i_relfilenode,
                                i_reltablespace;
        char            query[QUERY_ALLOC];
+       char       *last_namespace = NULL, *last_tablespace = NULL;
 
        /*
         * pg_largeobject contains user data that does not appear in pg_dumpall
@@ -366,26 +366,53 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
        for (relnum = 0; relnum < ntups; relnum++)
        {
                RelInfo    *curr = &relinfos[num_rels++];
-               const char *tblspace;
 
                curr->reloid = atooid(PQgetvalue(res, relnum, i_oid));
 
                nspname = PQgetvalue(res, relnum, i_nspname);
-               curr->nspname = pg_strdup(nspname);
+               curr->nsp_alloc = false;
+
+               /*
+                * Many of the namespace and tablespace strings are identical,
+                * so we try to reuse the allocated string pointers where possible
+                * to reduce memory consumption.
+                */
+               /* Can we reuse the previous string allocation? */
+               if (last_namespace && strcmp(nspname, last_namespace) == 0)
+                       curr->nspname = last_namespace;
+               else
+               {
+                       last_namespace = curr->nspname = pg_strdup(nspname);
+                       curr->nsp_alloc = true;
+               }
 
                relname = PQgetvalue(res, relnum, i_relname);
                curr->relname = pg_strdup(relname);
 
                curr->relfilenode = atooid(PQgetvalue(res, relnum, i_relfilenode));
+               curr->tblsp_alloc = false;
 
+               /* Is the tablespace oid non-zero? */
                if (atooid(PQgetvalue(res, relnum, i_reltablespace)) != 0)
-                       /* Might be "", meaning the cluster default location. */
-                       tblspace = PQgetvalue(res, relnum, i_spclocation);
+               {
+                       /*
+                        * The tablespace location might be "", meaning the cluster
+                        * default location, i.e. pg_default or pg_global.
+                        */
+                       tablespace = PQgetvalue(res, relnum, i_spclocation);
+
+                       /* Can we reuse the previous string allocation? */
+                       if (last_tablespace && strcmp(tablespace, last_tablespace) == 0)
+                               curr->tablespace = last_tablespace;
+                       else
+                       {
+                               last_tablespace = curr->tablespace = pg_strdup(tablespace);
+                               curr->tblsp_alloc = true;
+                       }
+               }
                else
-                       /* A zero reltablespace indicates the database tablespace. */
-                       tblspace = dbinfo->db_tblspace;
-
-               strlcpy(curr->tablespace, tblspace, sizeof(curr->tablespace));
+                       /* A zero reltablespace oid indicates the database tablespace. */
+                       curr->tablespace = dbinfo->db_tablespace;
        }
        PQclear(res);
 
@@ -419,8 +446,11 @@ free_rel_infos(RelInfoArr *rel_arr)
 
        for (relnum = 0; relnum < rel_arr->nrels; relnum++)
        {
-               pg_free(rel_arr->rels[relnum].nspname);
+               if (rel_arr->rels[relnum].nsp_alloc)
+                       pg_free(rel_arr->rels[relnum].nspname);
                pg_free(rel_arr->rels[relnum].relname);
+               if (rel_arr->rels[relnum].tblsp_alloc)
+                       pg_free(rel_arr->rels[relnum].tablespace);
        }
        pg_free(rel_arr->rels);
        rel_arr->nrels = 0;

diff --git a/contrib/pg_upgrade/pg_upgrade.h b/contrib/pg_upgrade/pg_upgrade.h
index de487bcb21eadd90acdbb3ea6bbc427fc6431198..389ec936d04858d1dfdb62be9e741e0d3bffcdbf 100644 (file)
--- a/contrib/pg_upgrade/pg_upgrade.h
+++ b/contrib/pg_upgrade/pg_upgrade.h
@@ -127,7 +127,9 @@ typedef struct
        Oid                     reloid;                 /* relation oid */
        Oid                     relfilenode;    /* relation relfile node */
        /* relation tablespace path, or "" for the cluster default */
-       char            tablespace[MAXPGPATH];
+       char       *tablespace;
+       bool            nsp_alloc;
+       bool            tblsp_alloc;
 } RelInfo;
 
 typedef struct
@@ -141,10 +143,10 @@ typedef struct
  */
 typedef struct
 {
-       char            old_tablespace[MAXPGPATH];
-       char            new_tablespace[MAXPGPATH];
-       char            old_tablespace_suffix[MAXPGPATH];
-       char            new_tablespace_suffix[MAXPGPATH];
+       const char              *old_tablespace;
+       const char              *new_tablespace;
+       const char              *old_tablespace_suffix;
+       const char              *new_tablespace_suffix;
        Oid                     old_db_oid;
        Oid                     new_db_oid;
 
@@ -166,7 +168,7 @@ typedef struct
 {
        Oid                     db_oid;                 /* oid of the database */
        char       *db_name;            /* database name */
-       char            db_tblspace[MAXPGPATH]; /* database default tablespace path */
+       char            db_tablespace[MAXPGPATH]; /* database default tablespace path */
        RelInfoArr      rel_arr;                /* array of all user relinfos */
 } DbInfo;
 
@@ -256,7 +258,7 @@ typedef struct
        Oid                     pg_database_oid;        /* OID of pg_database relation */
        Oid                     install_role_oid;               /* OID of connected role */
        Oid                     role_count;             /* number of roles defined in the cluster */
-       char       *tablespace_suffix;          /* directory specification */
+       const char *tablespace_suffix;          /* directory specification */
 } ClusterInfo;