granicus.if.org Git - postgresql/blob - contrib/pg_upgrade/pg_upgrade.c

   1 /*
   2  *      pg_upgrade.c
   3  *
   4  *      main source file
   5  *
   6  *      Copyright (c) 2010-2013, PostgreSQL Global Development Group
   7  *      contrib/pg_upgrade/pg_upgrade.c
   8  */
   9
  10 /*
  11  *      To simplify the upgrade process, we force certain system values to be
  12  *      identical between old and new clusters:
  13  *
  14  *      We control all assignments of pg_class.oid (and relfilenode) so toast
  15  *      oids are the same between old and new clusters.  This is important
  16  *      because toast oids are stored as toast pointers in user tables.
  17  *
  18  *      FYI, while pg_class.oid and pg_class.relfilenode are initially the same
  19  *      in a cluster, but they can diverge due to CLUSTER, REINDEX, or VACUUM
  20  *      FULL.  The new cluster will have matching pg_class.oid and
  21  *      pg_class.relfilenode values and be based on the old oid value.  This can
  22  *      cause the old and new pg_class.relfilenode values to differ.  In summary,
  23  *      old and new pg_class.oid and new pg_class.relfilenode will have the
  24  *      same value, and old pg_class.relfilenode might differ.
  25  *
  26  *      We control all assignments of pg_type.oid because these oids are stored
  27  *      in user composite type values.
  28  *
  29  *      We control all assignments of pg_enum.oid because these oids are stored
  30  *      in user tables as enum values.
  31  *
  32  *      We control all assignments of pg_authid.oid because these oids are stored
  33  *      in pg_largeobject_metadata.
  34  */
  35
  36
  37
  38 #include "postgres.h"
  39
  40 #include "pg_upgrade.h"
  41
  42 #ifdef HAVE_LANGINFO_H
  43 #include <langinfo.h>
  44 #endif
  45
  46 static void prepare_new_cluster(void);
  47 static void prepare_new_databases(void);
  48 static void create_new_objects(void);
  49 static void copy_clog_xlog_xid(void);
  50 static void set_frozenxids(void);
  51 static void setup(char *argv0, bool live_check);
  52 static void cleanup(void);
  53
  54 ClusterInfo old_cluster,
  55                         new_cluster;
  56 OSInfo          os_info;
  57
  58 char       *output_files[] = {
  59         SERVER_LOG_FILE,
  60 #ifdef WIN32
  61         /* unique file for pg_ctl start */
  62         SERVER_START_LOG_FILE,
  63 #endif
  64         UTILITY_LOG_FILE,
  65         INTERNAL_LOG_FILE,
  66         NULL
  67 };
  68
  69
  70 int
  71 main(int argc, char **argv)
  72 {
  73         char       *sequence_script_file_name = NULL;
  74         char       *analyze_script_file_name = NULL;
  75         char       *deletion_script_file_name = NULL;
  76         bool            live_check = false;
  77
  78         parseCommandLine(argc, argv);
  79
  80         adjust_data_dir(&old_cluster);
  81         adjust_data_dir(&new_cluster);
  82
  83         output_check_banner(&live_check);
  84
  85         setup(argv[0], live_check);
  86
  87         check_cluster_versions();
  88
  89         get_sock_dir(&old_cluster, live_check);
  90         get_sock_dir(&new_cluster, false);
  91
  92         check_cluster_compatibility(live_check);
  93
  94         check_and_dump_old_cluster(live_check, &sequence_script_file_name);
  95
  96
  97         /* -- NEW -- */
  98         start_postmaster(&new_cluster);
  99
 100         check_new_cluster();
 101         report_clusters_compatible();
 102
 103         pg_log(PG_REPORT, "\nPerforming Upgrade\n");
 104         pg_log(PG_REPORT, "------------------\n");
 105
 106         prepare_new_cluster();
 107
 108         stop_postmaster(false);
 109
 110         /*
 111          * Destructive Changes to New Cluster
 112          */
 113
 114         copy_clog_xlog_xid();
 115
 116         /* New now using xids of the old system */
 117
 118         /* -- NEW -- */
 119         start_postmaster(&new_cluster);
 120
 121         prepare_new_databases();
 122
 123         create_new_objects();
 124
 125         stop_postmaster(false);
 126
 127         /*
 128          * Most failures happen in create_new_objects(), which has completed at
 129          * this point.  We do this here because it is just before linking, which
 130          * will link the old and new cluster data files, preventing the old
 131          * cluster from being safely started once the new cluster is started.
 132          */
 133         if (user_opts.transfer_mode == TRANSFER_MODE_LINK)
 134                 disable_old_cluster();
 135
 136         transfer_all_new_dbs(&old_cluster.dbarr, &new_cluster.dbarr,
 137                                                  old_cluster.pgdata, new_cluster.pgdata);
 138
 139         /*
 140          * Assuming OIDs are only used in system tables, there is no need to
 141          * restore the OID counter because we have not transferred any OIDs from
 142          * the old system, but we do it anyway just in case.  We do it late here
 143          * because there is no need to have the schema load use new oids.
 144          */
 145         prep_status("Setting next OID for new cluster");
 146         exec_prog(UTILITY_LOG_FILE, NULL, true,
 147                           "\"%s/pg_resetxlog\" -o %u \"%s\"",
 148                           new_cluster.bindir, old_cluster.controldata.chkpnt_nxtoid,
 149                           new_cluster.pgdata);
 150         check_ok();
 151
 152         prep_status("Sync data directory to disk");
 153         exec_prog(UTILITY_LOG_FILE, NULL, true,
 154                           "\"%s/initdb\" --sync-only \"%s\"", new_cluster.bindir,
 155                           new_cluster.pgdata);
 156         check_ok();
 157
 158         create_script_for_cluster_analyze(&analyze_script_file_name);
 159         create_script_for_old_cluster_deletion(&deletion_script_file_name);
 160
 161         issue_warnings(sequence_script_file_name);
 162
 163         pg_log(PG_REPORT, "\nUpgrade Complete\n");
 164         pg_log(PG_REPORT, "----------------\n");
 165
 166         output_completion_banner(analyze_script_file_name,
 167                                                          deletion_script_file_name);
 168
 169         pg_free(analyze_script_file_name);
 170         pg_free(deletion_script_file_name);
 171         pg_free(sequence_script_file_name);
 172
 173         cleanup();
 174
 175         return 0;
 176 }
 177
 178
 179 static void
 180 setup(char *argv0, bool live_check)
 181 {
 182         char            exec_path[MAXPGPATH];   /* full path to my executable */
 183
 184         /*
 185          * make sure the user has a clean environment, otherwise, we may confuse
 186          * libpq when we connect to one (or both) of the servers.
 187          */
 188         check_pghost_envvar();
 189
 190         verify_directories();
 191
 192         /* no postmasters should be running */
 193         if (!live_check && is_server_running(old_cluster.pgdata))
 194                 pg_log(PG_FATAL, "There seems to be a postmaster servicing the old cluster.\n"
 195                            "Please shutdown that postmaster and try again.\n");
 196
 197         /* same goes for the new postmaster */
 198         if (is_server_running(new_cluster.pgdata))
 199                 pg_log(PG_FATAL, "There seems to be a postmaster servicing the new cluster.\n"
 200                            "Please shutdown that postmaster and try again.\n");
 201
 202         /* get path to pg_upgrade executable */
 203         if (find_my_exec(argv0, exec_path) < 0)
 204                 pg_log(PG_FATAL, "Could not get path name to pg_upgrade: %s\n", getErrorText(errno));
 205
 206         /* Trim off program name and keep just path */
 207         *last_dir_separator(exec_path) = '\0';
 208         canonicalize_path(exec_path);
 209         os_info.exec_path = pg_strdup(exec_path);
 210 }
 211
 212
 213 static void
 214 prepare_new_cluster(void)
 215 {
 216         /*
 217          * It would make more sense to freeze after loading the schema, but that
 218          * would cause us to lose the frozenids restored by the load. We use
 219          * --analyze so autovacuum doesn't update statistics later
 220          */
 221         prep_status("Analyzing all rows in the new cluster");
 222         exec_prog(UTILITY_LOG_FILE, NULL, true,
 223                           "\"%s/vacuumdb\" %s --all --analyze %s",
 224                           new_cluster.bindir, cluster_conn_opts(&new_cluster),
 225                           log_opts.verbose ? "--verbose" : "");
 226         check_ok();
 227
 228         /*
 229          * We do freeze after analyze so pg_statistic is also frozen. template0 is
 230          * not frozen here, but data rows were frozen by initdb, and we set its
 231          * datfrozenxid and relfrozenxids later to match the new xid counter
 232          * later.
 233          */
 234         prep_status("Freezing all rows on the new cluster");
 235         exec_prog(UTILITY_LOG_FILE, NULL, true,
 236                           "\"%s/vacuumdb\" %s --all --freeze %s",
 237                           new_cluster.bindir, cluster_conn_opts(&new_cluster),
 238                           log_opts.verbose ? "--verbose" : "");
 239         check_ok();
 240
 241         get_pg_database_relfilenode(&new_cluster);
 242 }
 243
 244
 245 static void
 246 prepare_new_databases(void)
 247 {
 248         /*
 249          * We set autovacuum_freeze_max_age to its maximum value so autovacuum
 250          * does not launch here and delete clog files, before the frozen xids are
 251          * set.
 252          */
 253
 254         set_frozenxids();
 255
 256         prep_status("Restoring global objects in the new cluster");
 257
 258         /*
 259          * Install support functions in the global-object restore database to
 260          * preserve pg_authid.oid.      pg_dumpall uses 'template0' as its template
 261          * database so objects we add into 'template1' are not propogated.      They
 262          * are removed on pg_upgrade exit.
 263          */
 264         install_support_functions_in_new_db("template1");
 265
 266         /*
 267          * We have to create the databases first so we can install support
 268          * functions in all the other databases.  Ideally we could create the
 269          * support functions in template1 but pg_dumpall creates database using
 270          * the template0 template.
 271          */
 272         exec_prog(UTILITY_LOG_FILE, NULL, true,
 273                           "\"%s/psql\" " EXEC_PSQL_ARGS " %s -f \"%s\"",
 274                           new_cluster.bindir, cluster_conn_opts(&new_cluster),
 275                           GLOBALS_DUMP_FILE);
 276         check_ok();
 277
 278         /* we load this to get a current list of databases */
 279         get_db_and_rel_infos(&new_cluster);
 280 }
 281
 282
 283 static void
 284 create_new_objects(void)
 285 {
 286         int                     dbnum;
 287
 288         prep_status("Adding support functions to new cluster");
 289
 290         /*
 291          *      Technically, we only need to install these support functions in new
 292          *      databases that also exist in the old cluster, but for completeness
 293          *      we process all new databases.
 294          */
 295         for (dbnum = 0; dbnum < new_cluster.dbarr.ndbs; dbnum++)
 296         {
 297                 DbInfo     *new_db = &new_cluster.dbarr.dbs[dbnum];
 298
 299                 /* skip db we already installed */
 300                 if (strcmp(new_db->db_name, "template1") != 0)
 301                         install_support_functions_in_new_db(new_db->db_name);
 302         }
 303         check_ok();
 304
 305         prep_status("Restoring database schemas in the new cluster\n");
 306
 307         for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
 308         {
 309                 char sql_file_name[MAXPGPATH], log_file_name[MAXPGPATH];
 310                 DbInfo     *old_db = &old_cluster.dbarr.dbs[dbnum];
 311
 312                 pg_log(PG_STATUS, "%s", old_db->db_name);
 313                 snprintf(sql_file_name, sizeof(sql_file_name), DB_DUMP_FILE_MASK, old_db->db_oid);
 314                 snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
 315
 316                 /*
 317                  *      Using pg_restore --single-transaction is faster than other
 318                  *      methods, like --jobs.  pg_dump only produces its output at the
 319                  *      end, so there is little parallelism using the pipe.
 320                  */
 321                 parallel_exec_prog(log_file_name, NULL,
 322                                   "\"%s/pg_restore\" %s --exit-on-error --single-transaction --verbose --dbname \"%s\" \"%s\"",
 323                                   new_cluster.bindir, cluster_conn_opts(&new_cluster),
 324                                   old_db->db_name, sql_file_name);
 325         }
 326
 327         /* reap all children */
 328         while (reap_child(true) == true)
 329                 ;
 330
 331         end_progress_output();
 332         check_ok();
 333
 334         /* regenerate now that we have objects in the databases */
 335         get_db_and_rel_infos(&new_cluster);
 336
 337         uninstall_support_functions_from_new_cluster();
 338 }
 339
 340 /*
 341  * Delete the given subdirectory contents from the new cluster, and copy the
 342  * files from the old cluster into it.
 343  */
 344 static void
 345 copy_subdir_files(char *subdir)
 346 {
 347         char            old_path[MAXPGPATH];
 348         char            new_path[MAXPGPATH];
 349
 350         prep_status("Deleting files from new %s", subdir);
 351
 352         snprintf(old_path, sizeof(old_path), "%s/%s", old_cluster.pgdata, subdir);
 353         snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, subdir);
 354         if (!rmtree(new_path, true))
 355                 pg_log(PG_FATAL, "could not delete directory \"%s\"\n", new_path);
 356         check_ok();
 357
 358         prep_status("Copying old %s to new server", subdir);
 359
 360         exec_prog(UTILITY_LOG_FILE, NULL, true,
 361 #ifndef WIN32
 362                           "cp -Rf \"%s\" \"%s\"",
 363 #else
 364         /* flags: everything, no confirm, quiet, overwrite read-only */
 365                           "xcopy /e /y /q /r \"%s\" \"%s\\\"",
 366 #endif
 367                           old_path, new_path);
 368
 369         check_ok();
 370 }
 371
 372 static void
 373 copy_clog_xlog_xid(void)
 374 {
 375         /* copy old commit logs to new data dir */
 376         copy_subdir_files("pg_clog");
 377
 378         /* set the next transaction id of the new cluster */
 379         prep_status("Setting next transaction ID for new cluster");
 380         exec_prog(UTILITY_LOG_FILE, NULL, true,
 381                           "\"%s/pg_resetxlog\" -f -x %u \"%s\"",
 382                           new_cluster.bindir, old_cluster.controldata.chkpnt_nxtxid,
 383                           new_cluster.pgdata);
 384         check_ok();
 385
 386         /* now reset the wal archives in the new cluster */
 387         prep_status("Resetting WAL archives");
 388         exec_prog(UTILITY_LOG_FILE, NULL, true,
 389                           "\"%s/pg_resetxlog\" -l %s \"%s\"", new_cluster.bindir,
 390                           old_cluster.controldata.nextxlogfile,
 391                           new_cluster.pgdata);
 392         check_ok();
 393 }
 394
 395
 396 /*
 397  *      set_frozenxids()
 398  *
 399  *      We have frozen all xids, so set relfrozenxid and datfrozenxid
 400  *      to be the old cluster's xid counter, which we just set in the new
 401  *      cluster.  User-table frozenxid values will be set by pg_dumpall
 402  *      --binary-upgrade, but objects not set by the pg_dump must have
 403  *      proper frozen counters.
 404  */
 405 static
 406 void
 407 set_frozenxids(void)
 408 {
 409         int                     dbnum;
 410         PGconn     *conn,
 411                            *conn_template1;
 412         PGresult   *dbres;
 413         int                     ntups;
 414         int                     i_datname;
 415         int                     i_datallowconn;
 416
 417         prep_status("Setting frozenxid counters in new cluster");
 418
 419         conn_template1 = connectToServer(&new_cluster, "template1");
 420
 421         /* set pg_database.datfrozenxid */
 422         PQclear(executeQueryOrDie(conn_template1,
 423                                                           "UPDATE pg_catalog.pg_database "
 424                                                           "SET  datfrozenxid = '%u'",
 425                                                           old_cluster.controldata.chkpnt_nxtxid));
 426
 427         /* get database names */
 428         dbres = executeQueryOrDie(conn_template1,
 429                                                           "SELECT       datname, datallowconn "
 430                                                           "FROM pg_catalog.pg_database");
 431
 432         i_datname = PQfnumber(dbres, "datname");
 433         i_datallowconn = PQfnumber(dbres, "datallowconn");
 434
 435         ntups = PQntuples(dbres);
 436         for (dbnum = 0; dbnum < ntups; dbnum++)
 437         {
 438                 char       *datname = PQgetvalue(dbres, dbnum, i_datname);
 439                 char       *datallowconn = PQgetvalue(dbres, dbnum, i_datallowconn);
 440
 441                 /*
 442                  * We must update databases where datallowconn = false, e.g.
 443                  * template0, because autovacuum increments their datfrozenxids and
 444                  * relfrozenxids even if autovacuum is turned off, and even though all
 445                  * the data rows are already frozen  To enable this, we temporarily
 446                  * change datallowconn.
 447                  */
 448                 if (strcmp(datallowconn, "f") == 0)
 449                         PQclear(executeQueryOrDie(conn_template1,
 450                                                                           "UPDATE pg_catalog.pg_database "
 451                                                                           "SET  datallowconn = true "
 452                                                                           "WHERE datname = '%s'", datname));
 453
 454                 conn = connectToServer(&new_cluster, datname);
 455
 456                 /* set pg_class.relfrozenxid */
 457                 PQclear(executeQueryOrDie(conn,
 458                                                                   "UPDATE       pg_catalog.pg_class "
 459                                                                   "SET  relfrozenxid = '%u' "
 460                 /* only heap and TOAST are vacuumed */
 461                                                                   "WHERE        relkind IN ('r', 't')",
 462                                                                   old_cluster.controldata.chkpnt_nxtxid));
 463                 PQfinish(conn);
 464
 465                 /* Reset datallowconn flag */
 466                 if (strcmp(datallowconn, "f") == 0)
 467                         PQclear(executeQueryOrDie(conn_template1,
 468                                                                           "UPDATE pg_catalog.pg_database "
 469                                                                           "SET  datallowconn = false "
 470                                                                           "WHERE datname = '%s'", datname));
 471         }
 472
 473         PQclear(dbres);
 474
 475         PQfinish(conn_template1);
 476
 477         check_ok();
 478 }
 479
 480
 481 static void
 482 cleanup(void)
 483 {
 484
 485         fclose(log_opts.internal);
 486
 487         /* Remove dump and log files? */
 488         if (!log_opts.retain)
 489         {
 490                 int                     dbnum;
 491                 char      **filename;
 492
 493                 for (filename = output_files; *filename != NULL; filename++)
 494                         unlink(*filename);
 495
 496                 /* remove dump files */
 497                 unlink(GLOBALS_DUMP_FILE);
 498
 499                 if (old_cluster.dbarr.dbs)
 500                         for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
 501                         {
 502                                 char sql_file_name[MAXPGPATH], log_file_name[MAXPGPATH];
 503                                 DbInfo     *old_db = &old_cluster.dbarr.dbs[dbnum];
 504
 505                                 snprintf(sql_file_name, sizeof(sql_file_name), DB_DUMP_FILE_MASK, old_db->db_oid);
 506                                 unlink(sql_file_name);
 507
 508                                 snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_FILE_MASK, old_db->db_oid);
 509                                 unlink(log_file_name);
 510                         }
 511         }
 512 }