X-Git-Url: https://granicus.if.org/sourcecode?a=blobdiff_plain;f=contrib%2Fpg_upgrade%2Fpg_upgrade.c;h=b32d81efe711535699d66f7a621e5a07de313b50;hb=fbb1d7d73f8e23a3a61e702629c53cef48cb0918;hp=7297efd124035f56e15f5dfdc9b779b7a9470f6b;hpb=476291bef0985e242685fcd37866fec7ab1b97c7;p=postgresql diff --git a/contrib/pg_upgrade/pg_upgrade.c b/contrib/pg_upgrade/pg_upgrade.c index 7297efd124..b32d81efe7 100644 --- a/contrib/pg_upgrade/pg_upgrade.c +++ b/contrib/pg_upgrade/pg_upgrade.c @@ -3,7 +3,7 @@ * * main source file * - * Copyright (c) 2010-2012, PostgreSQL Global Development Group + * Copyright (c) 2010-2014, PostgreSQL Global Development Group * contrib/pg_upgrade/pg_upgrade.c */ @@ -15,13 +15,12 @@ * oids are the same between old and new clusters. This is important * because toast oids are stored as toast pointers in user tables. * - * FYI, while pg_class.oid and pg_class.relfilenode are initially the same - * in a cluster, but they can diverge due to CLUSTER, REINDEX, or VACUUM - * FULL. The new cluster will have matching pg_class.oid and - * pg_class.relfilenode values and be based on the old oid value. This can - * cause the old and new pg_class.relfilenode values to differ. In summary, - * old and new pg_class.oid and new pg_class.relfilenode will have the - * same value, and old pg_class.relfilenode might differ. + * While pg_class.oid and pg_class.relfilenode are initially the same + * in a cluster, they can diverge due to CLUSTER, REINDEX, or VACUUM + * FULL. In the new cluster, pg_class.oid and pg_class.relfilenode will + * be the same and will match the old pg_class.oid value. Because of + * this, old/new pg_class.relfilenode values will not match if CLUSTER, + * REINDEX, or VACUUM FULL have been performed in the old cluster. * * We control all assignments of pg_type.oid because these oids are stored * in user composite type values. @@ -29,13 +28,13 @@ * We control all assignments of pg_enum.oid because these oids are stored * in user tables as enum values. * - * We control all assignments of pg_auth.oid because these oids are stored + * We control all assignments of pg_authid.oid because these oids are stored * in pg_largeobject_metadata. */ -#include "postgres.h" +#include "postgres_fe.h" #include "pg_upgrade.h" @@ -48,20 +47,19 @@ static void prepare_new_databases(void); static void create_new_objects(void); static void copy_clog_xlog_xid(void); static void set_frozenxids(void); -static void setup(char *argv0, bool live_check); +static void setup(char *argv0, bool *live_check); static void cleanup(void); ClusterInfo old_cluster, new_cluster; OSInfo os_info; -char *output_files[] = { +char *output_files[] = { SERVER_LOG_FILE, #ifdef WIN32 - /* file is unique on Win32 */ - SERVER_LOG_FILE2, + /* unique file for pg_ctl start */ + SERVER_START_LOG_FILE, #endif - RESTORE_LOG_FILE, UTILITY_LOG_FILE, INTERNAL_LOG_FILE, NULL @@ -81,18 +79,22 @@ main(int argc, char **argv) adjust_data_dir(&old_cluster); adjust_data_dir(&new_cluster); - output_check_banner(&live_check); + setup(argv[0], &live_check); - setup(argv[0], live_check); + output_check_banner(live_check); check_cluster_versions(); + + get_sock_dir(&old_cluster, live_check); + get_sock_dir(&new_cluster, false); + check_cluster_compatibility(live_check); - check_old_cluster(live_check, &sequence_script_file_name); + check_and_dump_old_cluster(live_check, &sequence_script_file_name); /* -- NEW -- */ - start_postmaster(&new_cluster); + start_postmaster(&new_cluster, true); check_new_cluster(); report_clusters_compatible(); @@ -113,7 +115,7 @@ main(int argc, char **argv) /* New now using xids of the old system */ /* -- NEW -- */ - start_postmaster(&new_cluster); + start_postmaster(&new_cluster, true); prepare_new_databases(); @@ -122,17 +124,16 @@ main(int argc, char **argv) stop_postmaster(false); /* - * Most failures happen in create_new_objects(), which has - * completed at this point. We do this here because it is just - * before linking, which will link the old and new cluster data - * files, preventing the old cluster from being safely started - * once the new cluster is started. + * Most failures happen in create_new_objects(), which has completed at + * this point. We do this here because it is just before linking, which + * will link the old and new cluster data files, preventing the old + * cluster from being safely started once the new cluster is started. */ if (user_opts.transfer_mode == TRANSFER_MODE_LINK) disable_old_cluster(); - transfer_all_new_dbs(&old_cluster.dbarr, &new_cluster.dbarr, - old_cluster.pgdata, new_cluster.pgdata); + transfer_all_new_tablespaces(&old_cluster.dbarr, &new_cluster.dbarr, + old_cluster.pgdata, new_cluster.pgdata); /* * Assuming OIDs are only used in system tables, there is no need to @@ -141,11 +142,16 @@ main(int argc, char **argv) * because there is no need to have the schema load use new oids. */ prep_status("Setting next OID for new cluster"); - exec_prog(true, true, UTILITY_LOG_FILE, - SYSTEMQUOTE "\"%s/pg_resetxlog\" -o %u \"%s\" >> \"%s\" 2>&1" - SYSTEMQUOTE, + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/pg_resetxlog\" -o %u \"%s\"", new_cluster.bindir, old_cluster.controldata.chkpnt_nxtoid, - new_cluster.pgdata, UTILITY_LOG_FILE); + new_cluster.pgdata); + check_ok(); + + prep_status("Sync data directory to disk"); + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/initdb\" --sync-only \"%s\"", new_cluster.bindir, + new_cluster.pgdata); check_ok(); create_script_for_cluster_analyze(&analyze_script_file_name); @@ -170,7 +176,7 @@ main(int argc, char **argv) static void -setup(char *argv0, bool live_check) +setup(char *argv0, bool *live_check) { char exec_path[MAXPGPATH]; /* full path to my executable */ @@ -182,19 +188,42 @@ setup(char *argv0, bool live_check) verify_directories(); - /* no postmasters should be running */ - if (!live_check && is_server_running(old_cluster.pgdata)) - pg_log(PG_FATAL, "There seems to be a postmaster servicing the old cluster.\n" - "Please shutdown that postmaster and try again.\n"); + /* no postmasters should be running, except for a live check */ + if (pid_lock_file_exists(old_cluster.pgdata)) + { + /* + * If we have a postmaster.pid file, try to start the server. If it + * starts, the pid file was stale, so stop the server. If it doesn't + * start, assume the server is running. If the pid file is left over + * from a server crash, this also allows any committed transactions + * stored in the WAL to be replayed so they are not lost, because WAL + * files are not transfered from old to new servers. + */ + if (start_postmaster(&old_cluster, false)) + stop_postmaster(false); + else + { + if (!user_opts.check) + pg_fatal("There seems to be a postmaster servicing the old cluster.\n" + "Please shutdown that postmaster and try again.\n"); + else + *live_check = true; + } + } /* same goes for the new postmaster */ - if (is_server_running(new_cluster.pgdata)) - pg_log(PG_FATAL, "There seems to be a postmaster servicing the new cluster.\n" - "Please shutdown that postmaster and try again.\n"); + if (pid_lock_file_exists(new_cluster.pgdata)) + { + if (start_postmaster(&new_cluster, false)) + stop_postmaster(false); + else + pg_fatal("There seems to be a postmaster servicing the new cluster.\n" + "Please shutdown that postmaster and try again.\n"); + } /* get path to pg_upgrade executable */ if (find_my_exec(argv0, exec_path) < 0) - pg_log(PG_FATAL, "Could not get path name to pg_upgrade: %s\n", getErrorText(errno)); + pg_fatal("Could not get path name to pg_upgrade: %s\n", getErrorText(errno)); /* Trim off program name and keep just path */ *last_dir_separator(exec_path) = '\0'; @@ -212,11 +241,10 @@ prepare_new_cluster(void) * --analyze so autovacuum doesn't update statistics later */ prep_status("Analyzing all rows in the new cluster"); - exec_prog(true, true, UTILITY_LOG_FILE, - SYSTEMQUOTE "\"%s/vacuumdb\" --port %d --username \"%s\" " - "--all --analyze %s >> \"%s\" 2>&1" SYSTEMQUOTE, - new_cluster.bindir, new_cluster.port, os_info.user, - log_opts.verbose ? "--verbose" : "", UTILITY_LOG_FILE); + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/vacuumdb\" %s --all --analyze %s", + new_cluster.bindir, cluster_conn_opts(&new_cluster), + log_opts.verbose ? "--verbose" : ""); check_ok(); /* @@ -226,11 +254,10 @@ prepare_new_cluster(void) * later. */ prep_status("Freezing all rows on the new cluster"); - exec_prog(true, true, UTILITY_LOG_FILE, - SYSTEMQUOTE "\"%s/vacuumdb\" --port %d --username \"%s\" " - "--all --freeze %s >> \"%s\" 2>&1" SYSTEMQUOTE, - new_cluster.bindir, new_cluster.port, os_info.user, - log_opts.verbose ? "--verbose" : "", UTILITY_LOG_FILE); + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/vacuumdb\" %s --all --freeze %s", + new_cluster.bindir, cluster_conn_opts(&new_cluster), + log_opts.verbose ? "--verbose" : ""); check_ok(); get_pg_database_relfilenode(&new_cluster); @@ -248,7 +275,7 @@ prepare_new_databases(void) set_frozenxids(); - prep_status("Creating databases in the new cluster"); + prep_status("Restoring global objects in the new cluster"); /* * Install support functions in the global-object restore database to @@ -264,14 +291,10 @@ prepare_new_databases(void) * support functions in template1 but pg_dumpall creates database using * the template0 template. */ - exec_prog(true, true, RESTORE_LOG_FILE, - SYSTEMQUOTE "\"%s/psql\" --echo-queries " - "--set ON_ERROR_STOP=on " - /* --no-psqlrc prevents AUTOCOMMIT=off */ - "--no-psqlrc --port %d --username \"%s\" " - "-f \"%s\" --dbname template1 >> \"%s\" 2>&1" SYSTEMQUOTE, - new_cluster.bindir, new_cluster.port, os_info.user, - GLOBALS_DUMP_FILE, RESTORE_LOG_FILE); + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/psql\" " EXEC_PSQL_ARGS " %s -f \"%s\"", + new_cluster.bindir, cluster_conn_opts(&new_cluster), + GLOBALS_DUMP_FILE); check_ok(); /* we load this to get a current list of databases */ @@ -286,6 +309,11 @@ create_new_objects(void) prep_status("Adding support functions to new cluster"); + /* + * Technically, we only need to install these support functions in new + * databases that also exist in the old cluster, but for completeness we + * process all new databases. + */ for (dbnum = 0; dbnum < new_cluster.dbarr.ndbs; dbnum++) { DbInfo *new_db = &new_cluster.dbarr.dbs[dbnum]; @@ -296,14 +324,36 @@ create_new_objects(void) } check_ok(); - prep_status("Restoring database schema to new cluster"); - exec_prog(true, true, RESTORE_LOG_FILE, - SYSTEMQUOTE "\"%s/psql\" --echo-queries " - "--set ON_ERROR_STOP=on " - "--no-psqlrc --port %d --username \"%s\" " - "-f \"%s\" --dbname template1 >> \"%s\" 2>&1" SYSTEMQUOTE, - new_cluster.bindir, new_cluster.port, os_info.user, - DB_DUMP_FILE, RESTORE_LOG_FILE); + prep_status("Restoring database schemas in the new cluster\n"); + + for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++) + { + char sql_file_name[MAXPGPATH], + log_file_name[MAXPGPATH]; + DbInfo *old_db = &old_cluster.dbarr.dbs[dbnum]; + + pg_log(PG_STATUS, "%s", old_db->db_name); + snprintf(sql_file_name, sizeof(sql_file_name), DB_DUMP_FILE_MASK, old_db->db_oid); + snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid); + + /* + * pg_dump only produces its output at the end, so there is little + * parallelism if using the pipe. + */ + parallel_exec_prog(log_file_name, + NULL, + "\"%s/pg_restore\" %s --exit-on-error --verbose --dbname \"%s\" \"%s\"", + new_cluster.bindir, + cluster_conn_opts(&new_cluster), + old_db->db_name, + sql_file_name); + } + + /* reap all children */ + while (reap_child(true) == true) + ; + + end_progress_output(); check_ok(); /* regenerate now that we have objects in the databases */ @@ -312,55 +362,126 @@ create_new_objects(void) uninstall_support_functions_from_new_cluster(); } - +/* + * Delete the given subdirectory contents from the new cluster + */ static void -copy_clog_xlog_xid(void) +remove_new_subdir(char *subdir, bool rmtopdir) { - char old_clog_path[MAXPGPATH]; - char new_clog_path[MAXPGPATH]; + char new_path[MAXPGPATH]; - /* copy old commit logs to new data dir */ - prep_status("Deleting new commit clogs"); + prep_status("Deleting files from new %s", subdir); + + snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, subdir); + if (!rmtree(new_path, rmtopdir)) + pg_fatal("could not delete directory \"%s\"\n", new_path); - snprintf(old_clog_path, sizeof(old_clog_path), "%s/pg_clog", old_cluster.pgdata); - snprintf(new_clog_path, sizeof(new_clog_path), "%s/pg_clog", new_cluster.pgdata); - if (!rmtree(new_clog_path, true)) - pg_log(PG_FATAL, "could not delete directory \"%s\"\n", new_clog_path); check_ok(); +} - prep_status("Copying old commit clogs to new server"); - exec_prog(true, false, UTILITY_LOG_FILE, +/* + * Copy the files from the old cluster into it + */ +static void +copy_subdir_files(char *subdir) +{ + char old_path[MAXPGPATH]; + char new_path[MAXPGPATH]; + + remove_new_subdir(subdir, true); + + snprintf(old_path, sizeof(old_path), "%s/%s", old_cluster.pgdata, subdir); + snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, subdir); + + prep_status("Copying old %s to new server", subdir); + + exec_prog(UTILITY_LOG_FILE, NULL, true, #ifndef WIN32 - SYSTEMQUOTE "%s \"%s\" \"%s\" >> \"%s\" 2>&1" SYSTEMQUOTE, - "cp -Rf", + "cp -Rf \"%s\" \"%s\"", #else /* flags: everything, no confirm, quiet, overwrite read-only */ - SYSTEMQUOTE "%s \"%s\" \"%s\\\" >> \"%s\" 2>&1" SYSTEMQUOTE, - "xcopy /e /y /q /r", + "xcopy /e /y /q /r \"%s\" \"%s\\\"", #endif - old_clog_path, new_clog_path, UTILITY_LOG_FILE); + old_path, new_path); + check_ok(); +} + +static void +copy_clog_xlog_xid(void) +{ + /* copy old commit logs to new data dir */ + copy_subdir_files("pg_clog"); /* set the next transaction id of the new cluster */ prep_status("Setting next transaction ID for new cluster"); - exec_prog(true, true, UTILITY_LOG_FILE, - SYSTEMQUOTE - "\"%s/pg_resetxlog\" -f -x %u \"%s\" >> \"%s\" 2>&1" - SYSTEMQUOTE, new_cluster.bindir, - old_cluster.controldata.chkpnt_nxtxid, - new_cluster.pgdata, UTILITY_LOG_FILE); + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/pg_resetxlog\" -f -x %u \"%s\"", + new_cluster.bindir, old_cluster.controldata.chkpnt_nxtxid, + new_cluster.pgdata); check_ok(); + /* + * If the old server is before the MULTIXACT_FORMATCHANGE_CAT_VER change + * (see pg_upgrade.h) and the new server is after, then we don't copy + * pg_multixact files, but we need to reset pg_control so that the new + * server doesn't attempt to read multis older than the cutoff value. + */ + if (old_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER && + new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER) + { + copy_subdir_files("pg_multixact/offsets"); + copy_subdir_files("pg_multixact/members"); + + prep_status("Setting next multixact ID and offset for new cluster"); + + /* + * we preserve all files and contents, so we must preserve both "next" + * counters here and the oldest multi present on system. + */ + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/pg_resetxlog\" -O %u -m %u,%u \"%s\"", + new_cluster.bindir, + old_cluster.controldata.chkpnt_nxtmxoff, + old_cluster.controldata.chkpnt_nxtmulti, + old_cluster.controldata.chkpnt_oldstMulti, + new_cluster.pgdata); + check_ok(); + } + else if (new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER) + { + /* + * Remove files created by initdb that no longer match the + * new multi-xid value. + */ + remove_new_subdir("pg_multixact/offsets", false); + remove_new_subdir("pg_multixact/members", false); + + prep_status("Setting oldest multixact ID on new cluster"); + + /* + * We don't preserve files in this case, but it's important that the + * oldest multi is set to the latest value used by the old system, so + * that multixact.c returns the empty set for multis that might be + * present on disk. We set next multi to the value following that; it + * might end up wrapped around (i.e. 0) if the old cluster had + * next=MaxMultiXactId, but multixact.c can cope with that just fine. + */ + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/pg_resetxlog\" -m %u,%u \"%s\"", + new_cluster.bindir, + old_cluster.controldata.chkpnt_nxtmulti + 1, + old_cluster.controldata.chkpnt_nxtmulti, + new_cluster.pgdata); + check_ok(); + } + /* now reset the wal archives in the new cluster */ prep_status("Resetting WAL archives"); - exec_prog(true, true, UTILITY_LOG_FILE, - SYSTEMQUOTE - "\"%s/pg_resetxlog\" -l %u,%u,%u \"%s\" >> \"%s\" 2>&1" - SYSTEMQUOTE, new_cluster.bindir, - old_cluster.controldata.chkpnt_tli, - old_cluster.controldata.logid, - old_cluster.controldata.nxtlogseg, - new_cluster.pgdata, UTILITY_LOG_FILE); + exec_prog(UTILITY_LOG_FILE, NULL, true, + "\"%s/pg_resetxlog\" -l %s \"%s\"", new_cluster.bindir, + old_cluster.controldata.nextxlogfile, + new_cluster.pgdata); check_ok(); } @@ -370,7 +491,7 @@ copy_clog_xlog_xid(void) * * We have frozen all xids, so set relfrozenxid and datfrozenxid * to be the old cluster's xid counter, which we just set in the new - * cluster. User-table frozenxid values will be set by pg_dumpall + * cluster. User-table frozenxid values will be set by pg_dump * --binary-upgrade, but objects not set by the pg_dump must have * proper frozen counters. */ @@ -419,9 +540,8 @@ set_frozenxids(void) */ if (strcmp(datallowconn, "f") == 0) PQclear(executeQueryOrDie(conn_template1, - "UPDATE pg_catalog.pg_database " - "SET datallowconn = true " - "WHERE datname = '%s'", datname)); + "ALTER DATABASE %s ALLOW_CONNECTIONS = true", + quote_identifier(datname))); conn = connectToServer(&new_cluster, datname); @@ -429,17 +549,16 @@ set_frozenxids(void) PQclear(executeQueryOrDie(conn, "UPDATE pg_catalog.pg_class " "SET relfrozenxid = '%u' " - /* only heap and TOAST are vacuumed */ - "WHERE relkind IN ('r', 't')", + /* only heap, materialized view, and TOAST are vacuumed */ + "WHERE relkind IN ('r', 'm', 't')", old_cluster.controldata.chkpnt_nxtxid)); PQfinish(conn); /* Reset datallowconn flag */ if (strcmp(datallowconn, "f") == 0) PQclear(executeQueryOrDie(conn_template1, - "UPDATE pg_catalog.pg_database " - "SET datallowconn = false " - "WHERE datname = '%s'", datname)); + "ALTER DATABASE %s ALLOW_CONNECTIONS = false", + quote_identifier(datname))); } PQclear(dbres); @@ -453,20 +572,32 @@ set_frozenxids(void) static void cleanup(void) { - fclose(log_opts.internal); /* Remove dump and log files? */ if (!log_opts.retain) { - char **filename; + int dbnum; + char **filename; for (filename = output_files; *filename != NULL; filename++) unlink(*filename); - /* remove SQL files */ - unlink(ALL_DUMP_FILE); + /* remove dump files */ unlink(GLOBALS_DUMP_FILE); - unlink(DB_DUMP_FILE); + + if (old_cluster.dbarr.dbs) + for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++) + { + char sql_file_name[MAXPGPATH], + log_file_name[MAXPGPATH]; + DbInfo *old_db = &old_cluster.dbarr.dbs[dbnum]; + + snprintf(sql_file_name, sizeof(sql_file_name), DB_DUMP_FILE_MASK, old_db->db_oid); + unlink(sql_file_name); + + snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid); + unlink(log_file_name); + } } }