From 2a2e40aaaad6d9d8a2c6a61817542a2d8c369935 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 12 Feb 2014 16:35:24 -0500 Subject: [PATCH] pg_upgrade: dramatically reduce memory consumption Report from Jeff Janes --- contrib/pg_upgrade/check.c | 14 +++---- contrib/pg_upgrade/info.c | 70 +++++++++++++++++++++++---------- contrib/pg_upgrade/pg_upgrade.h | 16 ++++---- 3 files changed, 66 insertions(+), 34 deletions(-) diff --git a/contrib/pg_upgrade/check.c b/contrib/pg_upgrade/check.c index f04707f9c6..acf8660044 100644 --- a/contrib/pg_upgrade/check.c +++ b/contrib/pg_upgrade/check.c @@ -707,20 +707,18 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name) fprintf(script, "\n"); /* remove PG_VERSION? */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 804) - fprintf(script, RM_CMD " %s%s%cPG_VERSION\n", + fprintf(script, RM_CMD " %s%cPG_VERSION\n", fix_path_separator(os_info.old_tablespaces[tblnum]), - fix_path_separator(old_cluster.tablespace_suffix), PATH_SEPARATOR); for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++) - { - fprintf(script, RMDIR_CMD " %s%s%c%d\n", + fprintf(script, RMDIR_CMD " %s%c%d\n", fix_path_separator(os_info.old_tablespaces[tblnum]), - fix_path_separator(old_cluster.tablespace_suffix), PATH_SEPARATOR, old_cluster.dbarr.dbs[dbnum].db_oid); - } } else + { + char *suffix_path = pg_strdup(old_cluster.tablespace_suffix); /* * Simply delete the tablespace directory, which might be ".old" @@ -728,7 +726,9 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name) */ fprintf(script, RMDIR_CMD " %s%s\n", fix_path_separator(os_info.old_tablespaces[tblnum]), - fix_path_separator(old_cluster.tablespace_suffix)); + fix_path_separator(suffix_path)); + pfree(suffix_path); + } } fclose(script); diff --git a/contrib/pg_upgrade/info.c b/contrib/pg_upgrade/info.c index 3d0dd1a101..fd083de821 100644 --- a/contrib/pg_upgrade/info.c +++ b/contrib/pg_upgrade/info.c @@ -108,20 +108,18 @@ create_rel_filename_map(const char *old_data, const char *new_data, * relation belongs to the default tablespace, hence relfiles should * exist in the data directories. */ - strlcpy(map->old_tablespace, old_data, sizeof(map->old_tablespace)); - strlcpy(map->new_tablespace, new_data, sizeof(map->new_tablespace)); - strlcpy(map->old_tablespace_suffix, "/base", sizeof(map->old_tablespace_suffix)); - strlcpy(map->new_tablespace_suffix, "/base", sizeof(map->new_tablespace_suffix)); + map->old_tablespace = old_data; + map->new_tablespace = new_data; + map->old_tablespace_suffix = "/base"; + map->new_tablespace_suffix = "/base"; } else { /* relation belongs to a tablespace, so use the tablespace location */ - strlcpy(map->old_tablespace, old_rel->tablespace, sizeof(map->old_tablespace)); - strlcpy(map->new_tablespace, new_rel->tablespace, sizeof(map->new_tablespace)); - strlcpy(map->old_tablespace_suffix, old_cluster.tablespace_suffix, - sizeof(map->old_tablespace_suffix)); - strlcpy(map->new_tablespace_suffix, new_cluster.tablespace_suffix, - sizeof(map->new_tablespace_suffix)); + map->old_tablespace = old_rel->tablespace; + map->new_tablespace = new_rel->tablespace; + map->old_tablespace_suffix = old_cluster.tablespace_suffix; + map->new_tablespace_suffix = new_cluster.tablespace_suffix; } map->old_db_oid = old_db->db_oid; @@ -231,7 +229,7 @@ get_db_infos(ClusterInfo *cluster) { dbinfos[tupnum].db_oid = atooid(PQgetvalue(res, tupnum, i_oid)); dbinfos[tupnum].db_name = pg_strdup(PQgetvalue(res, tupnum, i_datname)); - snprintf(dbinfos[tupnum].db_tblspace, sizeof(dbinfos[tupnum].db_tblspace), "%s", + snprintf(dbinfos[tupnum].db_tablespace, sizeof(dbinfos[tupnum].db_tablespace), "%s", PQgetvalue(res, tupnum, i_spclocation)); } PQclear(res); @@ -264,6 +262,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) int num_rels = 0; char *nspname = NULL; char *relname = NULL; + char *tablespace = NULL; int i_spclocation, i_nspname, i_relname, @@ -271,6 +270,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) i_relfilenode, i_reltablespace; char query[QUERY_ALLOC]; + char *last_namespace = NULL, *last_tablespace = NULL; /* * pg_largeobject contains user data that does not appear in pg_dumpall @@ -366,26 +366,53 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) for (relnum = 0; relnum < ntups; relnum++) { RelInfo *curr = &relinfos[num_rels++]; - const char *tblspace; curr->reloid = atooid(PQgetvalue(res, relnum, i_oid)); nspname = PQgetvalue(res, relnum, i_nspname); - curr->nspname = pg_strdup(nspname); + curr->nsp_alloc = false; + + /* + * Many of the namespace and tablespace strings are identical, + * so we try to reuse the allocated string pointers where possible + * to reduce memory consumption. + */ + /* Can we reuse the previous string allocation? */ + if (last_namespace && strcmp(nspname, last_namespace) == 0) + curr->nspname = last_namespace; + else + { + last_namespace = curr->nspname = pg_strdup(nspname); + curr->nsp_alloc = true; + } relname = PQgetvalue(res, relnum, i_relname); curr->relname = pg_strdup(relname); curr->relfilenode = atooid(PQgetvalue(res, relnum, i_relfilenode)); + curr->tblsp_alloc = false; + /* Is the tablespace oid non-zero? */ if (atooid(PQgetvalue(res, relnum, i_reltablespace)) != 0) - /* Might be "", meaning the cluster default location. */ - tblspace = PQgetvalue(res, relnum, i_spclocation); + { + /* + * The tablespace location might be "", meaning the cluster + * default location, i.e. pg_default or pg_global. + */ + tablespace = PQgetvalue(res, relnum, i_spclocation); + + /* Can we reuse the previous string allocation? */ + if (last_tablespace && strcmp(tablespace, last_tablespace) == 0) + curr->tablespace = last_tablespace; + else + { + last_tablespace = curr->tablespace = pg_strdup(tablespace); + curr->tblsp_alloc = true; + } + } else - /* A zero reltablespace indicates the database tablespace. */ - tblspace = dbinfo->db_tblspace; - - strlcpy(curr->tablespace, tblspace, sizeof(curr->tablespace)); + /* A zero reltablespace oid indicates the database tablespace. */ + curr->tablespace = dbinfo->db_tablespace; } PQclear(res); @@ -419,8 +446,11 @@ free_rel_infos(RelInfoArr *rel_arr) for (relnum = 0; relnum < rel_arr->nrels; relnum++) { - pg_free(rel_arr->rels[relnum].nspname); + if (rel_arr->rels[relnum].nsp_alloc) + pg_free(rel_arr->rels[relnum].nspname); pg_free(rel_arr->rels[relnum].relname); + if (rel_arr->rels[relnum].tblsp_alloc) + pg_free(rel_arr->rels[relnum].tablespace); } pg_free(rel_arr->rels); rel_arr->nrels = 0; diff --git a/contrib/pg_upgrade/pg_upgrade.h b/contrib/pg_upgrade/pg_upgrade.h index de487bcb21..389ec936d0 100644 --- a/contrib/pg_upgrade/pg_upgrade.h +++ b/contrib/pg_upgrade/pg_upgrade.h @@ -127,7 +127,9 @@ typedef struct Oid reloid; /* relation oid */ Oid relfilenode; /* relation relfile node */ /* relation tablespace path, or "" for the cluster default */ - char tablespace[MAXPGPATH]; + char *tablespace; + bool nsp_alloc; + bool tblsp_alloc; } RelInfo; typedef struct @@ -141,10 +143,10 @@ typedef struct */ typedef struct { - char old_tablespace[MAXPGPATH]; - char new_tablespace[MAXPGPATH]; - char old_tablespace_suffix[MAXPGPATH]; - char new_tablespace_suffix[MAXPGPATH]; + const char *old_tablespace; + const char *new_tablespace; + const char *old_tablespace_suffix; + const char *new_tablespace_suffix; Oid old_db_oid; Oid new_db_oid; @@ -166,7 +168,7 @@ typedef struct { Oid db_oid; /* oid of the database */ char *db_name; /* database name */ - char db_tblspace[MAXPGPATH]; /* database default tablespace path */ + char db_tablespace[MAXPGPATH]; /* database default tablespace path */ RelInfoArr rel_arr; /* array of all user relinfos */ } DbInfo; @@ -256,7 +258,7 @@ typedef struct Oid pg_database_oid; /* OID of pg_database relation */ Oid install_role_oid; /* OID of connected role */ Oid role_count; /* number of roles defined in the cluster */ - char *tablespace_suffix; /* directory specification */ + const char *tablespace_suffix; /* directory specification */ } ClusterInfo; -- 2.40.0