granicus.if.org Git - postgresql/blob - contrib/pg_upgrade/pg_upgrade.c

   1 /*
   2  *      pg_upgrade.c
   3  *
   4  *      main source file
   5  *
   6  *      Copyright (c) 2010-2014, PostgreSQL Global Development Group
   7  *      contrib/pg_upgrade/pg_upgrade.c
   8  */
   9
  10 /*
  11  *      To simplify the upgrade process, we force certain system values to be
  12  *      identical between old and new clusters:
  13  *
  14  *      We control all assignments of pg_class.oid (and relfilenode) so toast
  15  *      oids are the same between old and new clusters.  This is important
  16  *      because toast oids are stored as toast pointers in user tables.
  17  *
  18  *      FYI, while pg_class.oid and pg_class.relfilenode are initially the same
  19  *      in a cluster, but they can diverge due to CLUSTER, REINDEX, or VACUUM
  20  *      FULL.  The new cluster will have matching pg_class.oid and
  21  *      pg_class.relfilenode values and be based on the old oid value.  This can
  22  *      cause the old and new pg_class.relfilenode values to differ.  In summary,
  23  *      old and new pg_class.oid and new pg_class.relfilenode will have the
  24  *      same value, and old pg_class.relfilenode might differ.
  25  *
  26  *      We control all assignments of pg_type.oid because these oids are stored
  27  *      in user composite type values.
  28  *
  29  *      We control all assignments of pg_enum.oid because these oids are stored
  30  *      in user tables as enum values.
  31  *
  32  *      We control all assignments of pg_authid.oid because these oids are stored
  33  *      in pg_largeobject_metadata.
  34  */
  35
  36
  37
  38 #include "postgres_fe.h"
  39
  40 #include "pg_upgrade.h"
  41
  42 #ifdef HAVE_LANGINFO_H
  43 #include <langinfo.h>
  44 #endif
  45
  46 static void prepare_new_cluster(void);
  47 static void prepare_new_databases(void);
  48 static void create_new_objects(void);
  49 static void copy_clog_xlog_xid(void);
  50 static void set_frozenxids(void);
  51 static void setup(char *argv0, bool *live_check);
  52 static void cleanup(void);
  53
  54 ClusterInfo old_cluster,
  55                         new_cluster;
  56 OSInfo          os_info;
  57
  58 char       *output_files[] = {
  59         SERVER_LOG_FILE,
  60 #ifdef WIN32
  61         /* unique file for pg_ctl start */
  62         SERVER_START_LOG_FILE,
  63 #endif
  64         UTILITY_LOG_FILE,
  65         INTERNAL_LOG_FILE,
  66         NULL
  67 };
  68
  69
  70 int
  71 main(int argc, char **argv)
  72 {
  73         char       *sequence_script_file_name = NULL;
  74         char       *analyze_script_file_name = NULL;
  75         char       *deletion_script_file_name = NULL;
  76         bool            live_check = false;
  77
  78         parseCommandLine(argc, argv);
  79
  80         adjust_data_dir(&old_cluster);
  81         adjust_data_dir(&new_cluster);
  82
  83         setup(argv[0], &live_check);
  84
  85         output_check_banner(live_check);
  86
  87         check_cluster_versions();
  88
  89         get_sock_dir(&old_cluster, live_check);
  90         get_sock_dir(&new_cluster, false);
  91
  92         check_cluster_compatibility(live_check);
  93
  94         check_and_dump_old_cluster(live_check, &sequence_script_file_name);
  95
  96
  97         /* -- NEW -- */
  98         start_postmaster(&new_cluster, true);
  99
 100         check_new_cluster();
 101         report_clusters_compatible();
 102
 103         pg_log(PG_REPORT, "\nPerforming Upgrade\n");
 104         pg_log(PG_REPORT, "------------------\n");
 105
 106         prepare_new_cluster();
 107
 108         stop_postmaster(false);
 109
 110         /*
 111          * Destructive Changes to New Cluster
 112          */
 113
 114         copy_clog_xlog_xid();
 115
 116         /* New now using xids of the old system */
 117
 118         /* -- NEW -- */
 119         start_postmaster(&new_cluster, true);
 120
 121         prepare_new_databases();
 122
 123         create_new_objects();
 124
 125         stop_postmaster(false);
 126
 127         /*
 128          * Most failures happen in create_new_objects(), which has completed at
 129          * this point.  We do this here because it is just before linking, which
 130          * will link the old and new cluster data files, preventing the old
 131          * cluster from being safely started once the new cluster is started.
 132          */
 133         if (user_opts.transfer_mode == TRANSFER_MODE_LINK)
 134                 disable_old_cluster();
 135
 136         transfer_all_new_tablespaces(&old_cluster.dbarr, &new_cluster.dbarr,
 137                                                                  old_cluster.pgdata, new_cluster.pgdata);
 138
 139         /*
 140          * Assuming OIDs are only used in system tables, there is no need to
 141          * restore the OID counter because we have not transferred any OIDs from
 142          * the old system, but we do it anyway just in case.  We do it late here
 143          * because there is no need to have the schema load use new oids.
 144          */
 145         prep_status("Setting next OID for new cluster");
 146         exec_prog(UTILITY_LOG_FILE, NULL, true,
 147                           "\"%s/pg_resetxlog\" -o %u \"%s\"",
 148                           new_cluster.bindir, old_cluster.controldata.chkpnt_nxtoid,
 149                           new_cluster.pgdata);
 150         check_ok();
 151
 152         prep_status("Sync data directory to disk");
 153         exec_prog(UTILITY_LOG_FILE, NULL, true,
 154                           "\"%s/initdb\" --sync-only \"%s\"", new_cluster.bindir,
 155                           new_cluster.pgdata);
 156         check_ok();
 157
 158         create_script_for_cluster_analyze(&analyze_script_file_name);
 159         create_script_for_old_cluster_deletion(&deletion_script_file_name);
 160
 161         issue_warnings(sequence_script_file_name);
 162
 163         pg_log(PG_REPORT, "\nUpgrade Complete\n");
 164         pg_log(PG_REPORT, "----------------\n");
 165
 166         output_completion_banner(analyze_script_file_name,
 167                                                          deletion_script_file_name);
 168
 169         pg_free(analyze_script_file_name);
 170         pg_free(deletion_script_file_name);
 171         pg_free(sequence_script_file_name);
 172
 173         cleanup();
 174
 175         return 0;
 176 }
 177
 178
 179 static void
 180 setup(char *argv0, bool *live_check)
 181 {
 182         char            exec_path[MAXPGPATH];   /* full path to my executable */
 183
 184         /*
 185          * make sure the user has a clean environment, otherwise, we may confuse
 186          * libpq when we connect to one (or both) of the servers.
 187          */
 188         check_pghost_envvar();
 189
 190         verify_directories();
 191
 192         /* no postmasters should be running, except for a live check */
 193         if (pid_lock_file_exists(old_cluster.pgdata))
 194         {
 195                 /*
 196                  * If we have a postmaster.pid file, try to start the server.  If it
 197                  * starts, the pid file was stale, so stop the server.  If it doesn't
 198                  * start, assume the server is running.  If the pid file is left over
 199                  * from a server crash, this also allows any committed transactions
 200                  * stored in the WAL to be replayed so they are not lost, because WAL
 201                  * files are not transfered from old to new servers.
 202                  */
 203                 if (start_postmaster(&old_cluster, false))
 204                         stop_postmaster(false);
 205                 else
 206                 {
 207                         if (!user_opts.check)
 208                                 pg_fatal("There seems to be a postmaster servicing the old cluster.\n"
 209                                            "Please shutdown that postmaster and try again.\n");
 210                         else
 211                                 *live_check = true;
 212                 }
 213         }
 214
 215         /* same goes for the new postmaster */
 216         if (pid_lock_file_exists(new_cluster.pgdata))
 217         {
 218                 if (start_postmaster(&new_cluster, false))
 219                         stop_postmaster(false);
 220                 else
 221                         pg_fatal("There seems to be a postmaster servicing the new cluster.\n"
 222                                    "Please shutdown that postmaster and try again.\n");
 223         }
 224
 225         /* get path to pg_upgrade executable */
 226         if (find_my_exec(argv0, exec_path) < 0)
 227                 pg_fatal("Could not get path name to pg_upgrade: %s\n", getErrorText(errno));
 228
 229         /* Trim off program name and keep just path */
 230         *last_dir_separator(exec_path) = '\0';
 231         canonicalize_path(exec_path);
 232         os_info.exec_path = pg_strdup(exec_path);
 233 }
 234
 235
 236 static void
 237 prepare_new_cluster(void)
 238 {
 239         /*
 240          * It would make more sense to freeze after loading the schema, but that
 241          * would cause us to lose the frozenids restored by the load. We use
 242          * --analyze so autovacuum doesn't update statistics later
 243          */
 244         prep_status("Analyzing all rows in the new cluster");
 245         exec_prog(UTILITY_LOG_FILE, NULL, true,
 246                           "\"%s/vacuumdb\" %s --all --analyze %s",
 247                           new_cluster.bindir, cluster_conn_opts(&new_cluster),
 248                           log_opts.verbose ? "--verbose" : "");
 249         check_ok();
 250
 251         /*
 252          * We do freeze after analyze so pg_statistic is also frozen. template0 is
 253          * not frozen here, but data rows were frozen by initdb, and we set its
 254          * datfrozenxid and relfrozenxids later to match the new xid counter
 255          * later.
 256          */
 257         prep_status("Freezing all rows on the new cluster");
 258         exec_prog(UTILITY_LOG_FILE, NULL, true,
 259                           "\"%s/vacuumdb\" %s --all --freeze %s",
 260                           new_cluster.bindir, cluster_conn_opts(&new_cluster),
 261                           log_opts.verbose ? "--verbose" : "");
 262         check_ok();
 263
 264         get_pg_database_relfilenode(&new_cluster);
 265 }
 266
 267
 268 static void
 269 prepare_new_databases(void)
 270 {
 271         /*
 272          * We set autovacuum_freeze_max_age to its maximum value so autovacuum
 273          * does not launch here and delete clog files, before the frozen xids are
 274          * set.
 275          */
 276
 277         set_frozenxids();
 278
 279         prep_status("Restoring global objects in the new cluster");
 280
 281         /*
 282          * Install support functions in the global-object restore database to
 283          * preserve pg_authid.oid.      pg_dumpall uses 'template0' as its template
 284          * database so objects we add into 'template1' are not propogated.      They
 285          * are removed on pg_upgrade exit.
 286          */
 287         install_support_functions_in_new_db("template1");
 288
 289         /*
 290          * We have to create the databases first so we can install support
 291          * functions in all the other databases.  Ideally we could create the
 292          * support functions in template1 but pg_dumpall creates database using
 293          * the template0 template.
 294          */
 295         exec_prog(UTILITY_LOG_FILE, NULL, true,
 296                           "\"%s/psql\" " EXEC_PSQL_ARGS " %s -f \"%s\"",
 297                           new_cluster.bindir, cluster_conn_opts(&new_cluster),
 298                           GLOBALS_DUMP_FILE);
 299         check_ok();
 300
 301         /* we load this to get a current list of databases */
 302         get_db_and_rel_infos(&new_cluster);
 303 }
 304
 305
 306 static void
 307 create_new_objects(void)
 308 {
 309         int                     dbnum;
 310
 311         prep_status("Adding support functions to new cluster");
 312
 313         /*
 314          * Technically, we only need to install these support functions in new
 315          * databases that also exist in the old cluster, but for completeness we
 316          * process all new databases.
 317          */
 318         for (dbnum = 0; dbnum < new_cluster.dbarr.ndbs; dbnum++)
 319         {
 320                 DbInfo     *new_db = &new_cluster.dbarr.dbs[dbnum];
 321
 322                 /* skip db we already installed */
 323                 if (strcmp(new_db->db_name, "template1") != 0)
 324                         install_support_functions_in_new_db(new_db->db_name);
 325         }
 326         check_ok();
 327
 328         prep_status("Restoring database schemas in the new cluster\n");
 329
 330         for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
 331         {
 332                 char            sql_file_name[MAXPGPATH],
 333                                         log_file_name[MAXPGPATH];
 334                 DbInfo     *old_db = &old_cluster.dbarr.dbs[dbnum];
 335
 336                 pg_log(PG_STATUS, "%s", old_db->db_name);
 337                 snprintf(sql_file_name, sizeof(sql_file_name), DB_DUMP_FILE_MASK, old_db->db_oid);
 338                 snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
 339
 340                 /*
 341                  * pg_dump only produces its output at the end, so there is little
 342                  * parallelism if using the pipe.
 343                  */
 344                 parallel_exec_prog(log_file_name,
 345                                                    NULL,
 346                                                    "\"%s/pg_restore\" %s --exit-on-error --verbose --dbname \"%s\" \"%s\"",
 347                                                    new_cluster.bindir,
 348                                                    cluster_conn_opts(&new_cluster),
 349                                                    old_db->db_name,
 350                                                    sql_file_name);
 351         }
 352
 353         /* reap all children */
 354         while (reap_child(true) == true)
 355                 ;
 356
 357         end_progress_output();
 358         check_ok();
 359
 360         /* regenerate now that we have objects in the databases */
 361         get_db_and_rel_infos(&new_cluster);
 362
 363         uninstall_support_functions_from_new_cluster();
 364 }
 365
 366 /*
 367  * Delete the given subdirectory contents from the new cluster, and copy the
 368  * files from the old cluster into it.
 369  */
 370 static void
 371 copy_subdir_files(char *subdir)
 372 {
 373         char            old_path[MAXPGPATH];
 374         char            new_path[MAXPGPATH];
 375
 376         prep_status("Deleting files from new %s", subdir);
 377
 378         snprintf(old_path, sizeof(old_path), "%s/%s", old_cluster.pgdata, subdir);
 379         snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, subdir);
 380         if (!rmtree(new_path, true))
 381                 pg_fatal("could not delete directory \"%s\"\n", new_path);
 382         check_ok();
 383
 384         prep_status("Copying old %s to new server", subdir);
 385
 386         exec_prog(UTILITY_LOG_FILE, NULL, true,
 387 #ifndef WIN32
 388                           "cp -Rf \"%s\" \"%s\"",
 389 #else
 390         /* flags: everything, no confirm, quiet, overwrite read-only */
 391                           "xcopy /e /y /q /r \"%s\" \"%s\\\"",
 392 #endif
 393                           old_path, new_path);
 394
 395         check_ok();
 396 }
 397
 398 static void
 399 copy_clog_xlog_xid(void)
 400 {
 401         /* copy old commit logs to new data dir */
 402         copy_subdir_files("pg_clog");
 403
 404         /* set the next transaction id of the new cluster */
 405         prep_status("Setting next transaction ID for new cluster");
 406         exec_prog(UTILITY_LOG_FILE, NULL, true,
 407                           "\"%s/pg_resetxlog\" -f -x %u \"%s\"",
 408                           new_cluster.bindir, old_cluster.controldata.chkpnt_nxtxid,
 409                           new_cluster.pgdata);
 410         check_ok();
 411
 412         /*
 413          * If the old server is before the MULTIXACT_FORMATCHANGE_CAT_VER change
 414          * (see pg_upgrade.h) and the new server is after, then we don't copy
 415          * pg_multixact files, but we need to reset pg_control so that the new
 416          * server doesn't attempt to read multis older than the cutoff value.
 417          */
 418         if (old_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER &&
 419                 new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
 420         {
 421                 copy_subdir_files("pg_multixact/offsets");
 422                 copy_subdir_files("pg_multixact/members");
 423                 prep_status("Setting next multixact ID and offset for new cluster");
 424
 425                 /*
 426                  * we preserve all files and contents, so we must preserve both "next"
 427                  * counters here and the oldest multi present on system.
 428                  */
 429                 exec_prog(UTILITY_LOG_FILE, NULL, true,
 430                                   "\"%s/pg_resetxlog\" -O %u -m %u,%u \"%s\"",
 431                                   new_cluster.bindir,
 432                                   old_cluster.controldata.chkpnt_nxtmxoff,
 433                                   old_cluster.controldata.chkpnt_nxtmulti,
 434                                   old_cluster.controldata.chkpnt_oldstMulti,
 435                                   new_cluster.pgdata);
 436                 check_ok();
 437         }
 438         else if (new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
 439         {
 440                 prep_status("Setting oldest multixact ID on new cluster");
 441
 442                 /*
 443                  * We don't preserve files in this case, but it's important that the
 444                  * oldest multi is set to the latest value used by the old system, so
 445                  * that multixact.c returns the empty set for multis that might be
 446                  * present on disk.  We set next multi to the value following that; it
 447                  * might end up wrapped around (i.e. 0) if the old cluster had
 448                  * next=MaxMultiXactId, but multixact.c can cope with that just fine.
 449                  */
 450                 exec_prog(UTILITY_LOG_FILE, NULL, true,
 451                                   "\"%s/pg_resetxlog\" -m %u,%u \"%s\"",
 452                                   new_cluster.bindir,
 453                                   old_cluster.controldata.chkpnt_nxtmulti + 1,
 454                                   old_cluster.controldata.chkpnt_nxtmulti,
 455                                   new_cluster.pgdata);
 456                 check_ok();
 457         }
 458
 459         /* now reset the wal archives in the new cluster */
 460         prep_status("Resetting WAL archives");
 461         exec_prog(UTILITY_LOG_FILE, NULL, true,
 462                           "\"%s/pg_resetxlog\" -l %s \"%s\"", new_cluster.bindir,
 463                           old_cluster.controldata.nextxlogfile,
 464                           new_cluster.pgdata);
 465         check_ok();
 466 }
 467
 468
 469 /*
 470  *      set_frozenxids()
 471  *
 472  *      We have frozen all xids, so set relfrozenxid and datfrozenxid
 473  *      to be the old cluster's xid counter, which we just set in the new
 474  *      cluster.  User-table frozenxid values will be set by pg_dumpall
 475  *      --binary-upgrade, but objects not set by the pg_dump must have
 476  *      proper frozen counters.
 477  */
 478 static
 479 void
 480 set_frozenxids(void)
 481 {
 482         int                     dbnum;
 483         PGconn     *conn,
 484                            *conn_template1;
 485         PGresult   *dbres;
 486         int                     ntups;
 487         int                     i_datname;
 488         int                     i_datallowconn;
 489
 490         prep_status("Setting frozenxid counters in new cluster");
 491
 492         conn_template1 = connectToServer(&new_cluster, "template1");
 493
 494         /* set pg_database.datfrozenxid */
 495         PQclear(executeQueryOrDie(conn_template1,
 496                                                           "UPDATE pg_catalog.pg_database "
 497                                                           "SET  datfrozenxid = '%u'",
 498                                                           old_cluster.controldata.chkpnt_nxtxid));
 499
 500         /* get database names */
 501         dbres = executeQueryOrDie(conn_template1,
 502                                                           "SELECT       datname, datallowconn "
 503                                                           "FROM pg_catalog.pg_database");
 504
 505         i_datname = PQfnumber(dbres, "datname");
 506         i_datallowconn = PQfnumber(dbres, "datallowconn");
 507
 508         ntups = PQntuples(dbres);
 509         for (dbnum = 0; dbnum < ntups; dbnum++)
 510         {
 511                 char       *datname = PQgetvalue(dbres, dbnum, i_datname);
 512                 char       *datallowconn = PQgetvalue(dbres, dbnum, i_datallowconn);
 513
 514                 /*
 515                  * We must update databases where datallowconn = false, e.g.
 516                  * template0, because autovacuum increments their datfrozenxids and
 517                  * relfrozenxids even if autovacuum is turned off, and even though all
 518                  * the data rows are already frozen  To enable this, we temporarily
 519                  * change datallowconn.
 520                  */
 521                 if (strcmp(datallowconn, "f") == 0)
 522                         PQclear(executeQueryOrDie(conn_template1,
 523                                                                           "UPDATE pg_catalog.pg_database "
 524                                                                           "SET  datallowconn = true "
 525                                                                           "WHERE datname = '%s'", datname));
 526
 527                 conn = connectToServer(&new_cluster, datname);
 528
 529                 /* set pg_class.relfrozenxid */
 530                 PQclear(executeQueryOrDie(conn,
 531                                                                   "UPDATE       pg_catalog.pg_class "
 532                                                                   "SET  relfrozenxid = '%u' "
 533                 /* only heap, materialized view, and TOAST are vacuumed */
 534                                                                   "WHERE        relkind IN ('r', 'm', 't')",
 535                                                                   old_cluster.controldata.chkpnt_nxtxid));
 536                 PQfinish(conn);
 537
 538                 /* Reset datallowconn flag */
 539                 if (strcmp(datallowconn, "f") == 0)
 540                         PQclear(executeQueryOrDie(conn_template1,
 541                                                                           "UPDATE pg_catalog.pg_database "
 542                                                                           "SET  datallowconn = false "
 543                                                                           "WHERE datname = '%s'", datname));
 544         }
 545
 546         PQclear(dbres);
 547
 548         PQfinish(conn_template1);
 549
 550         check_ok();
 551 }
 552
 553
 554 static void
 555 cleanup(void)
 556 {
 557         fclose(log_opts.internal);
 558
 559         /* Remove dump and log files? */
 560         if (!log_opts.retain)
 561         {
 562                 int                     dbnum;
 563                 char      **filename;
 564
 565                 for (filename = output_files; *filename != NULL; filename++)
 566                         unlink(*filename);
 567
 568                 /* remove dump files */
 569                 unlink(GLOBALS_DUMP_FILE);
 570
 571                 if (old_cluster.dbarr.dbs)
 572                         for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
 573                         {
 574                                 char            sql_file_name[MAXPGPATH],
 575                                                         log_file_name[MAXPGPATH];
 576                                 DbInfo     *old_db = &old_cluster.dbarr.dbs[dbnum];
 577
 578                                 snprintf(sql_file_name, sizeof(sql_file_name), DB_DUMP_FILE_MASK, old_db->db_oid);
 579                                 unlink(sql_file_name);
 580
 581                                 snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
 582                                 unlink(log_file_name);
 583                         }
 584         }
 585 }