granicus.if.org Git - postgresql/blob - src/bin/pg_basebackup/pg_basebackup.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * pg_basebackup.c - receive a base backup using streaming replication protocol
   4  *
   5  * Author: Magnus Hagander <magnus@hagander.net>
   6  *
   7  * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
   8  *
   9  * IDENTIFICATION
  10  *                src/bin/pg_basebackup/pg_basebackup.c
  11  *-------------------------------------------------------------------------
  12  */
  13
  14 #include "postgres_fe.h"
  15 #include "libpq-fe.h"
  16 #include "pqexpbuffer.h"
  17 #include "pgtar.h"
  18
  19 #include <unistd.h>
  20 #include <dirent.h>
  21 #include <sys/stat.h>
  22 #include <sys/types.h>
  23 #include <sys/wait.h>
  24 #include <time.h>
  25
  26 #ifdef HAVE_LIBZ
  27 #include <zlib.h>
  28 #endif
  29
  30 #include "getopt_long.h"
  31
  32 #include "receivelog.h"
  33 #include "streamutil.h"
  34
  35
  36 /* Global options */
  37 char       *basedir = NULL;
  38 char            format = 'p';           /* p(lain)/t(ar) */
  39 char       *label = "pg_basebackup base backup";
  40 bool            showprogress = false;
  41 int                     verbose = 0;
  42 int                     compresslevel = 0;
  43 bool            includewal = false;
  44 bool            streamwal = false;
  45 bool            fastcheckpoint = false;
  46 bool            writerecoveryconf = false;
  47 int                     standby_message_timeout = 10 * 1000;            /* 10 sec = default */
  48
  49 /* Progress counters */
  50 static uint64 totalsize;
  51 static uint64 totaldone;
  52 static int      tablespacecount;
  53
  54 /* Pipe to communicate with background wal receiver process */
  55 #ifndef WIN32
  56 static int      bgpipe[2] = {-1, -1};
  57 #endif
  58
  59 /* Handle to child process */
  60 static pid_t bgchild = -1;
  61
  62 /* End position for xlog streaming, empty string if unknown yet */
  63 static XLogRecPtr xlogendptr;
  64
  65 #ifndef WIN32
  66 static int      has_xlogendptr = 0;
  67 #else
  68 static volatile LONG has_xlogendptr = 0;
  69 #endif
  70
  71 /* Contents of recovery.conf to be generated */
  72 static PQExpBuffer recoveryconfcontents = NULL;
  73
  74 /* Function headers */
  75 static void usage(void);
  76 static void verify_dir_is_empty_or_create(char *dirname);
  77 static void progress_report(int tablespacenum, const char *filename);
  78
  79 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
  80 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
  81 static void GenerateRecoveryConf(PGconn *conn);
  82 static void WriteRecoveryConf(void);
  83 static void BaseBackup(void);
  84
  85 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
  86                                          bool segment_finished);
  87
  88 #ifdef HAVE_LIBZ
  89 static const char *
  90 get_gz_error(gzFile gzf)
  91 {
  92         int                     errnum;
  93         const char *errmsg;
  94
  95         errmsg = gzerror(gzf, &errnum);
  96         if (errnum == Z_ERRNO)
  97                 return strerror(errno);
  98         else
  99                 return errmsg;
 100 }
 101 #endif
 102
 103 static void
 104 usage(void)
 105 {
 106         printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
 107                    progname);
 108         printf(_("Usage:\n"));
 109         printf(_("  %s [OPTION]...\n"), progname);
 110         printf(_("\nOptions controlling the output:\n"));
 111         printf(_("  -D, --pgdata=DIRECTORY receive base backup into directory\n"));
 112         printf(_("  -F, --format=p|t       output format (plain (default), tar)\n"));
 113         printf(_("  -R, --write-recovery-conf\n"
 114                          "                         write recovery.conf after backup\n"));
 115         printf(_("  -x, --xlog             include required WAL files in backup (fetch mode)\n"));
 116         printf(_("  -X, --xlog-method=fetch|stream\n"
 117                          "                         include required WAL files with specified method\n"));
 118         printf(_("  -z, --gzip             compress tar output\n"));
 119         printf(_("  -Z, --compress=0-9     compress tar output with given compression level\n"));
 120         printf(_("\nGeneral options:\n"));
 121         printf(_("  -c, --checkpoint=fast|spread\n"
 122                          "                         set fast or spread checkpointing\n"));
 123         printf(_("  -l, --label=LABEL      set backup label\n"));
 124         printf(_("  -P, --progress         show progress information\n"));
 125         printf(_("  -v, --verbose          output verbose messages\n"));
 126         printf(_("  -V, --version          output version information, then exit\n"));
 127         printf(_("  -?, --help             show this help, then exit\n"));
 128         printf(_("\nConnection options:\n"));
 129         printf(_("  -d, --dbname=CONNSTR   connection string\n"));
 130         printf(_("  -h, --host=HOSTNAME    database server host or socket directory\n"));
 131         printf(_("  -p, --port=PORT        database server port number\n"));
 132         printf(_("  -s, --status-interval=INTERVAL\n"
 133                          "                         time between status packets sent to server (in seconds)\n"));
 134         printf(_("  -U, --username=NAME    connect as specified database user\n"));
 135         printf(_("  -w, --no-password      never prompt for password\n"));
 136         printf(_("  -W, --password         force password prompt (should happen automatically)\n"));
 137         printf(_("\nReport bugs to <pgsql-bugs@postgresql.org>.\n"));
 138 }
 139
 140
 141 /*
 142  * Called in the background process every time data is received.
 143  * On Unix, we check to see if there is any data on our pipe
 144  * (which would mean we have a stop position), and if it is, check if
 145  * it is time to stop.
 146  * On Windows, we are in a single process, so we can just check if it's
 147  * time to stop.
 148  */
 149 static bool
 150 reached_end_position(XLogRecPtr segendpos, uint32 timeline,
 151                                          bool segment_finished)
 152 {
 153         if (!has_xlogendptr)
 154         {
 155 #ifndef WIN32
 156                 fd_set          fds;
 157                 struct timeval tv;
 158                 int                     r;
 159
 160                 /*
 161                  * Don't have the end pointer yet - check our pipe to see if it has
 162                  * been sent yet.
 163                  */
 164                 FD_ZERO(&fds);
 165                 FD_SET(bgpipe[0], &fds);
 166
 167                 MemSet(&tv, 0, sizeof(tv));
 168
 169                 r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
 170                 if (r == 1)
 171                 {
 172                         char            xlogend[64];
 173                         uint32          hi,
 174                                                 lo;
 175
 176                         MemSet(xlogend, 0, sizeof(xlogend));
 177                         r = read(bgpipe[0], xlogend, sizeof(xlogend)-1);
 178                         if (r < 0)
 179                         {
 180                                 fprintf(stderr, _("%s: could not read from ready pipe: %s\n"),
 181                                                 progname, strerror(errno));
 182                                 exit(1);
 183                         }
 184
 185                         if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
 186                         {
 187                                 fprintf(stderr,
 188                                   _("%s: could not parse transaction log location \"%s\"\n"),
 189                                                 progname, xlogend);
 190                                 exit(1);
 191                         }
 192                         xlogendptr = ((uint64) hi) << 32 | lo;
 193                         has_xlogendptr = 1;
 194
 195                         /*
 196                          * Fall through to check if we've reached the point further
 197                          * already.
 198                          */
 199                 }
 200                 else
 201                 {
 202                         /*
 203                          * No data received on the pipe means we don't know the end
 204                          * position yet - so just say it's not time to stop yet.
 205                          */
 206                         return false;
 207                 }
 208 #else
 209
 210                 /*
 211                  * On win32, has_xlogendptr is set by the main thread, so if it's not
 212                  * set here, we just go back and wait until it shows up.
 213                  */
 214                 return false;
 215 #endif
 216         }
 217
 218         /*
 219          * At this point we have an end pointer, so compare it to the current
 220          * position to figure out if it's time to stop.
 221          */
 222         if (segendpos >= xlogendptr)
 223                 return true;
 224
 225         /*
 226          * Have end pointer, but haven't reached it yet - so tell the caller to
 227          * keep streaming.
 228          */
 229         return false;
 230 }
 231
 232 typedef struct
 233 {
 234         PGconn     *bgconn;
 235         XLogRecPtr      startptr;
 236         char            xlogdir[MAXPGPATH];
 237         char       *sysidentifier;
 238         int                     timeline;
 239 } logstreamer_param;
 240
 241 static int
 242 LogStreamerMain(logstreamer_param *param)
 243 {
 244         if (!ReceiveXlogStream(param->bgconn, param->startptr, param->timeline,
 245                                                    param->sysidentifier, param->xlogdir,
 246                                                    reached_end_position, standby_message_timeout,
 247                                                    NULL))
 248
 249                 /*
 250                  * Any errors will already have been reported in the function process,
 251                  * but we need to tell the parent that we didn't shutdown in a nice
 252                  * way.
 253                  */
 254                 return 1;
 255
 256         PQfinish(param->bgconn);
 257         return 0;
 258 }
 259
 260 /*
 261  * Initiate background process for receiving xlog during the backup.
 262  * The background stream will use its own database connection so we can
 263  * stream the logfile in parallel with the backups.
 264  */
 265 static void
 266 StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
 267 {
 268         logstreamer_param *param;
 269         uint32          hi,
 270                                 lo;
 271
 272         param = pg_malloc0(sizeof(logstreamer_param));
 273         param->timeline = timeline;
 274         param->sysidentifier = sysidentifier;
 275
 276         /* Convert the starting position */
 277         if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
 278         {
 279                 fprintf(stderr,
 280                                 _("%s: could not parse transaction log location \"%s\"\n"),
 281                                 progname, startpos);
 282                 disconnect_and_exit(1);
 283         }
 284         param->startptr = ((uint64) hi) << 32 | lo;
 285         /* Round off to even segment position */
 286         param->startptr -= param->startptr % XLOG_SEG_SIZE;
 287
 288 #ifndef WIN32
 289         /* Create our background pipe */
 290         if (pipe(bgpipe) < 0)
 291         {
 292                 fprintf(stderr,
 293                                 _("%s: could not create pipe for background process: %s\n"),
 294                                 progname, strerror(errno));
 295                 disconnect_and_exit(1);
 296         }
 297 #endif
 298
 299         /* Get a second connection */
 300         param->bgconn = GetConnection();
 301         if (!param->bgconn)
 302                 /* Error message already written in GetConnection() */
 303                 exit(1);
 304
 305         /*
 306          * Always in plain format, so we can write to basedir/pg_xlog. But the
 307          * directory entry in the tar file may arrive later, so make sure it's
 308          * created before we start.
 309          */
 310         snprintf(param->xlogdir, sizeof(param->xlogdir), "%s/pg_xlog", basedir);
 311         verify_dir_is_empty_or_create(param->xlogdir);
 312
 313         /*
 314          * Start a child process and tell it to start streaming. On Unix, this is
 315          * a fork(). On Windows, we create a thread.
 316          */
 317 #ifndef WIN32
 318         bgchild = fork();
 319         if (bgchild == 0)
 320         {
 321                 /* in child process */
 322                 exit(LogStreamerMain(param));
 323         }
 324         else if (bgchild < 0)
 325         {
 326                 fprintf(stderr, _("%s: could not create background process: %s\n"),
 327                                 progname, strerror(errno));
 328                 disconnect_and_exit(1);
 329         }
 330
 331         /*
 332          * Else we are in the parent process and all is well.
 333          */
 334 #else                                                   /* WIN32 */
 335         bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
 336         if (bgchild == 0)
 337         {
 338                 fprintf(stderr, _("%s: could not create background thread: %s\n"),
 339                                 progname, strerror(errno));
 340                 disconnect_and_exit(1);
 341         }
 342 #endif
 343 }
 344
 345 /*
 346  * Verify that the given directory exists and is empty. If it does not
 347  * exist, it is created. If it exists but is not empty, an error will
 348  * be give and the process ended.
 349  */
 350 static void
 351 verify_dir_is_empty_or_create(char *dirname)
 352 {
 353         switch (pg_check_dir(dirname))
 354         {
 355                 case 0:
 356
 357                         /*
 358                          * Does not exist, so create
 359                          */
 360                         if (pg_mkdir_p(dirname, S_IRWXU) == -1)
 361                         {
 362                                 fprintf(stderr,
 363                                                 _("%s: could not create directory \"%s\": %s\n"),
 364                                                 progname, dirname, strerror(errno));
 365                                 disconnect_and_exit(1);
 366                         }
 367                         return;
 368                 case 1:
 369
 370                         /*
 371                          * Exists, empty
 372                          */
 373                         return;
 374                 case 2:
 375                 case 3:
 376                 case 4:
 377
 378                         /*
 379                          * Exists, not empty
 380                          */
 381                         fprintf(stderr,
 382                                         _("%s: directory \"%s\" exists but is not empty\n"),
 383                                         progname, dirname);
 384                         disconnect_and_exit(1);
 385                 case -1:
 386
 387                         /*
 388                          * Access problem
 389                          */
 390                         fprintf(stderr, _("%s: could not access directory \"%s\": %s\n"),
 391                                         progname, dirname, strerror(errno));
 392                         disconnect_and_exit(1);
 393         }
 394 }
 395
 396
 397 /*
 398  * Print a progress report based on the global variables. If verbose output
 399  * is enabled, also print the current file name.
 400  */
 401 static void
 402 progress_report(int tablespacenum, const char *filename)
 403 {
 404         int                     percent = (int) ((totaldone / 1024) * 100 / totalsize);
 405         char            totaldone_str[32];
 406         char            totalsize_str[32];
 407
 408         /*
 409          * Avoid overflowing past 100% or the full size. This may make the total
 410          * size number change as we approach the end of the backup (the estimate
 411          * will always be wrong if WAL is included), but that's better than having
 412          * the done column be bigger than the total.
 413          */
 414         if (percent > 100)
 415                 percent = 100;
 416         if (totaldone / 1024 > totalsize)
 417                 totalsize = totaldone / 1024;
 418
 419         /*
 420          * Separate step to keep platform-dependent format code out of
 421          * translatable strings.  And we only test for INT64_FORMAT availability
 422          * in snprintf, not fprintf.
 423          */
 424         snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
 425                          totaldone / 1024);
 426         snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize);
 427
 428 #define VERBOSE_FILENAME_LENGTH 35
 429         if (verbose)
 430         {
 431                 if (!filename)
 432
 433                         /*
 434                          * No filename given, so clear the status line (used for last
 435                          * call)
 436                          */
 437                         fprintf(stderr,
 438                                         ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
 439                                                          "%*s/%s kB (100%%), %d/%d tablespaces %*s",
 440                                                          tablespacecount),
 441                                         (int) strlen(totalsize_str),
 442                                         totaldone_str, totalsize_str,
 443                                         tablespacenum, tablespacecount,
 444                                         VERBOSE_FILENAME_LENGTH + 5, "");
 445                 else
 446                 {
 447                         bool            truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
 448
 449                         fprintf(stderr,
 450                                         ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
 451                                                          "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
 452                                                          tablespacecount),
 453                                         (int) strlen(totalsize_str),
 454                                         totaldone_str, totalsize_str, percent,
 455                                         tablespacenum, tablespacecount,
 456                         /* Prefix with "..." if we do leading truncation */
 457                                         truncate ? "..." : "",
 458                         truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
 459                         truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
 460                         /* Truncate filename at beginning if it's too long */
 461                                         truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
 462                 }
 463         }
 464         else
 465                 fprintf(stderr,
 466                                 ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
 467                                                  "%*s/%s kB (%d%%), %d/%d tablespaces",
 468                                                  tablespacecount),
 469                                 (int) strlen(totalsize_str),
 470                                 totaldone_str, totalsize_str, percent,
 471                                 tablespacenum, tablespacecount);
 472
 473         fprintf(stderr, "\r");
 474 }
 475
 476
 477 /*
 478  * Write a piece of tar data
 479  */
 480 static void
 481 writeTarData(
 482 #ifdef HAVE_LIBZ
 483                          gzFile ztarfile,
 484 #endif
 485                          FILE *tarfile, char *buf, int r, char *current_file)
 486 {
 487 #ifdef HAVE_LIBZ
 488         if (ztarfile != NULL)
 489         {
 490                 if (gzwrite(ztarfile, buf, r) != r)
 491                 {
 492                         fprintf(stderr,
 493                                         _("%s: could not write to compressed file \"%s\": %s\n"),
 494                                         progname, current_file, get_gz_error(ztarfile));
 495                         disconnect_and_exit(1);
 496                 }
 497         }
 498         else
 499 #endif
 500         {
 501                 if (fwrite(buf, r, 1, tarfile) != 1)
 502                 {
 503                         fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"),
 504                                         progname, current_file, strerror(errno));
 505                         disconnect_and_exit(1);
 506                 }
 507         }
 508 }
 509
 510 #ifdef HAVE_LIBZ
 511 #define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename)
 512 #else
 513 #define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename)
 514 #endif
 515
 516 /*
 517  * Receive a tar format file from the connection to the server, and write
 518  * the data from this file directly into a tar file. If compression is
 519  * enabled, the data will be compressed while written to the file.
 520  *
 521  * The file will be named base.tar[.gz] if it's for the main data directory
 522  * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
 523  *
 524  * No attempt to inspect or validate the contents of the file is done.
 525  */
 526 static void
 527 ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
 528 {
 529         char            filename[MAXPGPATH];
 530         char       *copybuf = NULL;
 531         FILE       *tarfile = NULL;
 532         char            tarhdr[512];
 533         bool            basetablespace = PQgetisnull(res, rownum, 0);
 534         bool            in_tarhdr = true;
 535         bool            skip_file = false;
 536         size_t          tarhdrsz = 0;
 537         size_t          filesz = 0;
 538
 539 #ifdef HAVE_LIBZ
 540         gzFile          ztarfile = NULL;
 541 #endif
 542
 543         if (basetablespace)
 544         {
 545                 /*
 546                  * Base tablespaces
 547                  */
 548                 if (strcmp(basedir, "-") == 0)
 549                 {
 550 #ifdef HAVE_LIBZ
 551                         if (compresslevel != 0)
 552                         {
 553                                 ztarfile = gzdopen(dup(fileno(stdout)), "wb");
 554                                 if (gzsetparams(ztarfile, compresslevel,
 555                                                                 Z_DEFAULT_STRATEGY) != Z_OK)
 556                                 {
 557                                         fprintf(stderr,
 558                                                         _("%s: could not set compression level %d: %s\n"),
 559                                                         progname, compresslevel, get_gz_error(ztarfile));
 560                                         disconnect_and_exit(1);
 561                                 }
 562                         }
 563                         else
 564 #endif
 565                                 tarfile = stdout;
 566                 }
 567                 else
 568                 {
 569 #ifdef HAVE_LIBZ
 570                         if (compresslevel != 0)
 571                         {
 572                                 snprintf(filename, sizeof(filename), "%s/base.tar.gz", basedir);
 573                                 ztarfile = gzopen(filename, "wb");
 574                                 if (gzsetparams(ztarfile, compresslevel,
 575                                                                 Z_DEFAULT_STRATEGY) != Z_OK)
 576                                 {
 577                                         fprintf(stderr,
 578                                                         _("%s: could not set compression level %d: %s\n"),
 579                                                         progname, compresslevel, get_gz_error(ztarfile));
 580                                         disconnect_and_exit(1);
 581                                 }
 582                         }
 583                         else
 584 #endif
 585                         {
 586                                 snprintf(filename, sizeof(filename), "%s/base.tar", basedir);
 587                                 tarfile = fopen(filename, "wb");
 588                         }
 589                 }
 590         }
 591         else
 592         {
 593                 /*
 594                  * Specific tablespace
 595                  */
 596 #ifdef HAVE_LIBZ
 597                 if (compresslevel != 0)
 598                 {
 599                         snprintf(filename, sizeof(filename), "%s/%s.tar.gz", basedir,
 600                                          PQgetvalue(res, rownum, 0));
 601                         ztarfile = gzopen(filename, "wb");
 602                         if (gzsetparams(ztarfile, compresslevel,
 603                                                         Z_DEFAULT_STRATEGY) != Z_OK)
 604                         {
 605                                 fprintf(stderr,
 606                                                 _("%s: could not set compression level %d: %s\n"),
 607                                                 progname, compresslevel, get_gz_error(ztarfile));
 608                                 disconnect_and_exit(1);
 609                         }
 610                 }
 611                 else
 612 #endif
 613                 {
 614                         snprintf(filename, sizeof(filename), "%s/%s.tar", basedir,
 615                                          PQgetvalue(res, rownum, 0));
 616                         tarfile = fopen(filename, "wb");
 617                 }
 618         }
 619
 620 #ifdef HAVE_LIBZ
 621         if (compresslevel != 0)
 622         {
 623                 if (!ztarfile)
 624                 {
 625                         /* Compression is in use */
 626                         fprintf(stderr,
 627                                         _("%s: could not create compressed file \"%s\": %s\n"),
 628                                         progname, filename, get_gz_error(ztarfile));
 629                         disconnect_and_exit(1);
 630                 }
 631         }
 632         else
 633 #endif
 634         {
 635                 /* Either no zlib support, or zlib support but compresslevel = 0 */
 636                 if (!tarfile)
 637                 {
 638                         fprintf(stderr, _("%s: could not create file \"%s\": %s\n"),
 639                                         progname, filename, strerror(errno));
 640                         disconnect_and_exit(1);
 641                 }
 642         }
 643
 644         /*
 645          * Get the COPY data stream
 646          */
 647         res = PQgetResult(conn);
 648         if (PQresultStatus(res) != PGRES_COPY_OUT)
 649         {
 650                 fprintf(stderr, _("%s: could not get COPY data stream: %s"),
 651                                 progname, PQerrorMessage(conn));
 652                 disconnect_and_exit(1);
 653         }
 654
 655         while (1)
 656         {
 657                 int                     r;
 658
 659                 if (copybuf != NULL)
 660                 {
 661                         PQfreemem(copybuf);
 662                         copybuf = NULL;
 663                 }
 664
 665                 r = PQgetCopyData(conn, &copybuf, 0);
 666                 if (r == -1)
 667                 {
 668                         /*
 669                          * End of chunk. If requested, and this is the base tablespace,
 670                          * write recovery.conf into the tarfile. When done, close the file
 671                          * (but not stdout).
 672                          *
 673                          * Also, write two completely empty blocks at the end of the tar
 674                          * file, as required by some tar programs.
 675                          */
 676                         char            zerobuf[1024];
 677
 678                         MemSet(zerobuf, 0, sizeof(zerobuf));
 679
 680                         if (basetablespace && writerecoveryconf)
 681                         {
 682                                 char            header[512];
 683                                 int                     padding;
 684
 685                                 tarCreateHeader(header, "recovery.conf", NULL,
 686                                                                 recoveryconfcontents->len,
 687                                                                 0600, 04000, 02000,
 688                                                                 time(NULL));
 689
 690                                 padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
 691
 692                                 WRITE_TAR_DATA(header, sizeof(header));
 693                                 WRITE_TAR_DATA(recoveryconfcontents->data, recoveryconfcontents->len);
 694                                 if (padding)
 695                                         WRITE_TAR_DATA(zerobuf, padding);
 696                         }
 697
 698                         /* 2 * 512 bytes empty data at end of file */
 699                         WRITE_TAR_DATA(zerobuf, sizeof(zerobuf));
 700
 701 #ifdef HAVE_LIBZ
 702                         if (ztarfile != NULL)
 703                         {
 704                                 if (gzclose(ztarfile) != 0)
 705                                 {
 706                                         fprintf(stderr,
 707                                            _("%s: could not close compressed file \"%s\": %s\n"),
 708                                                         progname, filename, get_gz_error(ztarfile));
 709                                         disconnect_and_exit(1);
 710                                 }
 711                         }
 712                         else
 713 #endif
 714                         {
 715                                 if (strcmp(basedir, "-") != 0)
 716                                 {
 717                                         if (fclose(tarfile) != 0)
 718                                         {
 719                                                 fprintf(stderr,
 720                                                                 _("%s: could not close file \"%s\": %s\n"),
 721                                                                 progname, filename, strerror(errno));
 722                                                 disconnect_and_exit(1);
 723                                         }
 724                                 }
 725                         }
 726
 727                         break;
 728                 }
 729                 else if (r == -2)
 730                 {
 731                         fprintf(stderr, _("%s: could not read COPY data: %s"),
 732                                         progname, PQerrorMessage(conn));
 733                         disconnect_and_exit(1);
 734                 }
 735
 736                 if (!writerecoveryconf || !basetablespace)
 737                 {
 738                         /*
 739                          * When not writing recovery.conf, or when not working on the base
 740                          * tablespace, we never have to look for an existing recovery.conf
 741                          * file in the stream.
 742                          */
 743                         WRITE_TAR_DATA(copybuf, r);
 744                 }
 745                 else
 746                 {
 747                         /*
 748                          * Look for a recovery.conf in the existing tar stream. If it's
 749                          * there, we must skip it so we can later overwrite it with our
 750                          * own version of the file.
 751                          *
 752                          * To do this, we have to process the individual files inside the
 753                          * TAR stream. The stream consists of a header and zero or more
 754                          * chunks, all 512 bytes long. The stream from the server is
 755                          * broken up into smaller pieces, so we have to track the size of
 756                          * the files to find the next header structure.
 757                          */
 758                         int                     rr = r;
 759                         int                     pos = 0;
 760
 761                         while (rr > 0)
 762                         {
 763                                 if (in_tarhdr)
 764                                 {
 765                                         /*
 766                                          * We're currently reading a header structure inside the
 767                                          * TAR stream, i.e. the file metadata.
 768                                          */
 769                                         if (tarhdrsz < 512)
 770                                         {
 771                                                 /*
 772                                                  * Copy the header structure into tarhdr in case the
 773                                                  * header is not aligned to 512 bytes or it's not
 774                                                  * returned in whole by the last PQgetCopyData call.
 775                                                  */
 776                                                 int                     hdrleft;
 777                                                 int                     bytes2copy;
 778
 779                                                 hdrleft = 512 - tarhdrsz;
 780                                                 bytes2copy = (rr > hdrleft ? hdrleft : rr);
 781
 782                                                 memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy);
 783
 784                                                 rr -= bytes2copy;
 785                                                 pos += bytes2copy;
 786                                                 tarhdrsz += bytes2copy;
 787                                         }
 788                                         else
 789                                         {
 790                                                 /*
 791                                                  * We have the complete header structure in tarhdr,
 792                                                  * look at the file metadata: - the subsequent file
 793                                                  * contents have to be skipped if the filename is
 794                                                  * recovery.conf - find out the size of the file
 795                                                  * padded to the next multiple of 512
 796                                                  */
 797                                                 int                     padding;
 798
 799                                                 skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
 800
 801                                                 sscanf(&tarhdr[124], "%11o", (unsigned int *) &filesz);
 802
 803                                                 padding = ((filesz + 511) & ~511) - filesz;
 804                                                 filesz += padding;
 805
 806                                                 /* Next part is the file, not the header */
 807                                                 in_tarhdr = false;
 808
 809                                                 /*
 810                                                  * If we're not skipping the file, write the tar
 811                                                  * header unmodified.
 812                                                  */
 813                                                 if (!skip_file)
 814                                                         WRITE_TAR_DATA(tarhdr, 512);
 815                                         }
 816                                 }
 817                                 else
 818                                 {
 819                                         /*
 820                                          * We're processing a file's contents.
 821                                          */
 822                                         if (filesz > 0)
 823                                         {
 824                                                 /*
 825                                                  * We still have data to read (and possibly write).
 826                                                  */
 827                                                 int                     bytes2write;
 828
 829                                                 bytes2write = (filesz > rr ? rr : filesz);
 830
 831                                                 if (!skip_file)
 832                                                         WRITE_TAR_DATA(copybuf + pos, bytes2write);
 833
 834                                                 rr -= bytes2write;
 835                                                 pos += bytes2write;
 836                                                 filesz -= bytes2write;
 837                                         }
 838                                         else
 839                                         {
 840                                                 /*
 841                                                  * No more data in the current file, the next piece of
 842                                                  * data (if any) will be a new file header structure.
 843                                                  */
 844                                                 in_tarhdr = true;
 845                                                 skip_file = false;
 846                                                 tarhdrsz = 0;
 847                                                 filesz = 0;
 848                                         }
 849                                 }
 850                         }
 851                 }
 852                 totaldone += r;
 853                 if (showprogress)
 854                         progress_report(rownum, filename);
 855         }                                                       /* while (1) */
 856
 857         if (copybuf != NULL)
 858                 PQfreemem(copybuf);
 859 }
 860
 861 /*
 862  * Receive a tar format stream from the connection to the server, and unpack
 863  * the contents of it into a directory. Only files, directories and
 864  * symlinks are supported, no other kinds of special files.
 865  *
 866  * If the data is for the main data directory, it will be restored in the
 867  * specified directory. If it's for another tablespace, it will be restored
 868  * in the original directory, since relocation of tablespaces is not
 869  * supported.
 870  */
 871 static void
 872 ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
 873 {
 874         char            current_path[MAXPGPATH];
 875         char            filename[MAXPGPATH];
 876         int                     current_len_left;
 877         int                     current_padding = 0;
 878         bool            basetablespace = PQgetisnull(res, rownum, 0);
 879         char       *copybuf = NULL;
 880         FILE       *file = NULL;
 881
 882         if (basetablespace)
 883                 strcpy(current_path, basedir);
 884         else
 885                 strcpy(current_path, PQgetvalue(res, rownum, 1));
 886
 887         /*
 888          * Get the COPY data
 889          */
 890         res = PQgetResult(conn);
 891         if (PQresultStatus(res) != PGRES_COPY_OUT)
 892         {
 893                 fprintf(stderr, _("%s: could not get COPY data stream: %s"),
 894                                 progname, PQerrorMessage(conn));
 895                 disconnect_and_exit(1);
 896         }
 897
 898         while (1)
 899         {
 900                 int                     r;
 901
 902                 if (copybuf != NULL)
 903                 {
 904                         PQfreemem(copybuf);
 905                         copybuf = NULL;
 906                 }
 907
 908                 r = PQgetCopyData(conn, &copybuf, 0);
 909
 910                 if (r == -1)
 911                 {
 912                         /*
 913                          * End of chunk
 914                          */
 915                         if (file)
 916                                 fclose(file);
 917
 918                         break;
 919                 }
 920                 else if (r == -2)
 921                 {
 922                         fprintf(stderr, _("%s: could not read COPY data: %s"),
 923                                         progname, PQerrorMessage(conn));
 924                         disconnect_and_exit(1);
 925                 }
 926
 927                 if (file == NULL)
 928                 {
 929                         int                     filemode;
 930
 931                         /*
 932                          * No current file, so this must be the header for a new file
 933                          */
 934                         if (r != 512)
 935                         {
 936                                 fprintf(stderr, _("%s: invalid tar block header size: %d\n"),
 937                                                 progname, r);
 938                                 disconnect_and_exit(1);
 939                         }
 940                         totaldone += 512;
 941
 942                         if (sscanf(copybuf + 124, "%11o", &current_len_left) != 1)
 943                         {
 944                                 fprintf(stderr, _("%s: could not parse file size\n"),
 945                                                 progname);
 946                                 disconnect_and_exit(1);
 947                         }
 948
 949                         /* Set permissions on the file */
 950                         if (sscanf(&copybuf[100], "%07o ", &filemode) != 1)
 951                         {
 952                                 fprintf(stderr, _("%s: could not parse file mode\n"),
 953                                                 progname);
 954                                 disconnect_and_exit(1);
 955                         }
 956
 957                         /*
 958                          * All files are padded up to 512 bytes
 959                          */
 960                         current_padding =
 961                                 ((current_len_left + 511) & ~511) - current_len_left;
 962
 963                         /*
 964                          * First part of header is zero terminated filename
 965                          */
 966                         snprintf(filename, sizeof(filename), "%s/%s", current_path,
 967                                          copybuf);
 968                         if (filename[strlen(filename) - 1] == '/')
 969                         {
 970                                 /*
 971                                  * Ends in a slash means directory or symlink to directory
 972                                  */
 973                                 if (copybuf[156] == '5')
 974                                 {
 975                                         /*
 976                                          * Directory
 977                                          */
 978                                         filename[strlen(filename) - 1] = '\0';          /* Remove trailing slash */
 979                                         if (mkdir(filename, S_IRWXU) != 0)
 980                                         {
 981                                                 /*
 982                                                  * When streaming WAL, pg_xlog will have been created
 983                                                  * by the wal receiver process, so just ignore failure
 984                                                  * on that.
 985                                                  */
 986                                                 if (!streamwal || strcmp(filename + strlen(filename) - 8, "/pg_xlog") != 0)
 987                                                 {
 988                                                         fprintf(stderr,
 989                                                         _("%s: could not create directory \"%s\": %s\n"),
 990                                                                         progname, filename, strerror(errno));
 991                                                         disconnect_and_exit(1);
 992                                                 }
 993                                         }
 994 #ifndef WIN32
 995                                         if (chmod(filename, (mode_t) filemode))
 996                                                 fprintf(stderr,
 997                                                                 _("%s: could not set permissions on directory \"%s\": %s\n"),
 998                                                                 progname, filename, strerror(errno));
 999 #endif
1000                                 }
1001                                 else if (copybuf[156] == '2')
1002                                 {
1003                                         /*
1004                                          * Symbolic link
1005                                          */
1006                                         filename[strlen(filename) - 1] = '\0';          /* Remove trailing slash */
1007                                         if (symlink(&copybuf[157], filename) != 0)
1008                                         {
1009                                                 fprintf(stderr,
1010                                                                 _("%s: could not create symbolic link from \"%s\" to \"%s\": %s\n"),
1011                                                  progname, filename, &copybuf[157], strerror(errno));
1012                                                 disconnect_and_exit(1);
1013                                         }
1014                                 }
1015                                 else
1016                                 {
1017                                         fprintf(stderr,
1018                                                         _("%s: unrecognized link indicator \"%c\"\n"),
1019                                                         progname, copybuf[156]);
1020                                         disconnect_and_exit(1);
1021                                 }
1022                                 continue;               /* directory or link handled */
1023                         }
1024
1025                         /*
1026                          * regular file
1027                          */
1028                         file = fopen(filename, "wb");
1029                         if (!file)
1030                         {
1031                                 fprintf(stderr, _("%s: could not create file \"%s\": %s\n"),
1032                                                 progname, filename, strerror(errno));
1033                                 disconnect_and_exit(1);
1034                         }
1035
1036 #ifndef WIN32
1037                         if (chmod(filename, (mode_t) filemode))
1038                                 fprintf(stderr, _("%s: could not set permissions on file \"%s\": %s\n"),
1039                                                 progname, filename, strerror(errno));
1040 #endif
1041
1042                         if (current_len_left == 0)
1043                         {
1044                                 /*
1045                                  * Done with this file, next one will be a new tar header
1046                                  */
1047                                 fclose(file);
1048                                 file = NULL;
1049                                 continue;
1050                         }
1051                 }                                               /* new file */
1052                 else
1053                 {
1054                         /*
1055                          * Continuing blocks in existing file
1056                          */
1057                         if (current_len_left == 0 && r == current_padding)
1058                         {
1059                                 /*
1060                                  * Received the padding block for this file, ignore it and
1061                                  * close the file, then move on to the next tar header.
1062                                  */
1063                                 fclose(file);
1064                                 file = NULL;
1065                                 totaldone += r;
1066                                 continue;
1067                         }
1068
1069                         if (fwrite(copybuf, r, 1, file) != 1)
1070                         {
1071                                 fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"),
1072                                                 progname, filename, strerror(errno));
1073                                 disconnect_and_exit(1);
1074                         }
1075                         totaldone += r;
1076                         if (showprogress)
1077                                 progress_report(rownum, filename);
1078
1079                         current_len_left -= r;
1080                         if (current_len_left == 0 && current_padding == 0)
1081                         {
1082                                 /*
1083                                  * Received the last block, and there is no padding to be
1084                                  * expected. Close the file and move on to the next tar
1085                                  * header.
1086                                  */
1087                                 fclose(file);
1088                                 file = NULL;
1089                                 continue;
1090                         }
1091                 }                                               /* continuing data in existing file */
1092         }                                                       /* loop over all data blocks */
1093
1094         if (file != NULL)
1095         {
1096                 fprintf(stderr,
1097                                 _("%s: COPY stream ended before last file was finished\n"),
1098                                 progname);
1099                 disconnect_and_exit(1);
1100         }
1101
1102         if (copybuf != NULL)
1103                 PQfreemem(copybuf);
1104
1105         if (basetablespace && writerecoveryconf)
1106                 WriteRecoveryConf();
1107 }
1108
1109 /*
1110  * Escape a parameter value so that it can be used as part of a libpq
1111  * connection string, e.g. in:
1112  *
1113  * application_name=<value>
1114  *
1115  * The returned string is malloc'd. Return NULL on out-of-memory.
1116  */
1117 static char *
1118 escapeConnectionParameter(const char *src)
1119 {
1120         bool            need_quotes = false;
1121         bool            need_escaping = false;
1122         const char *p;
1123         char       *dstbuf;
1124         char       *dst;
1125
1126         /*
1127          * First check if quoting is needed. Any quote (') or backslash (\)
1128          * characters need to be escaped. Parameters are separated by whitespace,
1129          * so any string containing whitespace characters need to be quoted. An
1130          * empty string is represented by ''.
1131          */
1132         if (strchr(src, '\'') != NULL || strchr(src, '\\') != NULL)
1133                 need_escaping = true;
1134
1135         for (p = src; *p; p++)
1136         {
1137                 if (isspace((unsigned char) *p))
1138                 {
1139                         need_quotes = true;
1140                         break;
1141                 }
1142         }
1143
1144         if (*src == '\0')
1145                 return pg_strdup("''");
1146
1147         if (!need_quotes && !need_escaping)
1148                 return pg_strdup(src);  /* no quoting or escaping needed */
1149
1150         /*
1151          * Allocate a buffer large enough for the worst case that all the source
1152          * characters need to be escaped, plus quotes.
1153          */
1154         dstbuf = pg_malloc(strlen(src) * 2 + 2 + 1);
1155
1156         dst = dstbuf;
1157         if (need_quotes)
1158                 *(dst++) = '\'';
1159         for (; *src; src++)
1160         {
1161                 if (*src == '\'' || *src == '\\')
1162                         *(dst++) = '\\';
1163                 *(dst++) = *src;
1164         }
1165         if (need_quotes)
1166                 *(dst++) = '\'';
1167         *dst = '\0';
1168
1169         return dstbuf;
1170 }
1171
1172 /*
1173  * Escape a string so that it can be used as a value in a key-value pair
1174  * a configuration file.
1175  */
1176 static char *
1177 escape_quotes(const char *src)
1178 {
1179         char       *result = escape_single_quotes_ascii(src);
1180
1181         if (!result)
1182         {
1183                 fprintf(stderr, _("%s: out of memory\n"), progname);
1184                 exit(1);
1185         }
1186         return result;
1187 }
1188
1189 /*
1190  * Create a recovery.conf file in memory using a PQExpBuffer
1191  */
1192 static void
1193 GenerateRecoveryConf(PGconn *conn)
1194 {
1195         PQconninfoOption *connOptions;
1196         PQconninfoOption *option;
1197         PQExpBufferData conninfo_buf;
1198         char       *escaped;
1199
1200         recoveryconfcontents = createPQExpBuffer();
1201         if (!recoveryconfcontents)
1202         {
1203                 fprintf(stderr, _("%s: out of memory\n"), progname);
1204                 disconnect_and_exit(1);
1205         }
1206
1207         connOptions = PQconninfo(conn);
1208         if (connOptions == NULL)
1209         {
1210                 fprintf(stderr, _("%s: out of memory\n"), progname);
1211                 disconnect_and_exit(1);
1212         }
1213
1214         appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
1215
1216         initPQExpBuffer(&conninfo_buf);
1217         for (option = connOptions; option && option->keyword; option++)
1218         {
1219                 /*
1220                  * Do not emit this setting if: - the setting is "replication",
1221                  * "dbname" or "fallback_application_name", since these would be
1222                  * overridden by the libpqwalreceiver module anyway. - not set or
1223                  * empty.
1224                  */
1225                 if (strcmp(option->keyword, "replication") == 0 ||
1226                         strcmp(option->keyword, "dbname") == 0 ||
1227                         strcmp(option->keyword, "fallback_application_name") == 0 ||
1228                         (option->val == NULL) ||
1229                         (option->val != NULL && option->val[0] == '\0'))
1230                         continue;
1231
1232                 /* Separate key-value pairs with spaces */
1233                 if (conninfo_buf.len != 0)
1234                         appendPQExpBufferStr(&conninfo_buf, " ");
1235
1236                 /*
1237                  * Write "keyword=value" pieces, the value string is escaped and/or
1238                  * quoted if necessary.
1239                  */
1240                 escaped = escapeConnectionParameter(option->val);
1241                 appendPQExpBuffer(&conninfo_buf, "%s=%s", option->keyword, escaped);
1242                 free(escaped);
1243         }
1244
1245         /*
1246          * Escape the connection string, so that it can be put in the config file.
1247          * Note that this is different from the escaping of individual connection
1248          * options above!
1249          */
1250         escaped = escape_quotes(conninfo_buf.data);
1251         appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
1252         free(escaped);
1253
1254         if (PQExpBufferBroken(recoveryconfcontents) ||
1255                 PQExpBufferDataBroken(conninfo_buf))
1256         {
1257                 fprintf(stderr, _("%s: out of memory\n"), progname);
1258                 disconnect_and_exit(1);
1259         }
1260
1261         termPQExpBuffer(&conninfo_buf);
1262
1263         PQconninfoFree(connOptions);
1264 }
1265
1266
1267 /*
1268  * Write a recovery.conf file into the directory specified in basedir,
1269  * with the contents already collected in memory.
1270  */
1271 static void
1272 WriteRecoveryConf(void)
1273 {
1274         char            filename[MAXPGPATH];
1275         FILE       *cf;
1276
1277         sprintf(filename, "%s/recovery.conf", basedir);
1278
1279         cf = fopen(filename, "w");
1280         if (cf == NULL)
1281         {
1282                 fprintf(stderr, _("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno));
1283                 disconnect_and_exit(1);
1284         }
1285
1286         if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
1287         {
1288                 fprintf(stderr,
1289                                 _("%s: could not write to file \"%s\": %s\n"),
1290                                 progname, filename, strerror(errno));
1291                 disconnect_and_exit(1);
1292         }
1293
1294         fclose(cf);
1295 }
1296
1297
1298 static void
1299 BaseBackup(void)
1300 {
1301         PGresult   *res;
1302         char       *sysidentifier;
1303         uint32          latesttli;
1304         uint32          starttli;
1305         char            current_path[MAXPGPATH];
1306         char            escaped_label[MAXPGPATH];
1307         int                     i;
1308         char            xlogstart[64];
1309         char            xlogend[64];
1310         int                     minServerMajor,
1311                                 maxServerMajor;
1312         int                     serverMajor;
1313
1314         /*
1315          * Connect in replication mode to the server
1316          */
1317         conn = GetConnection();
1318         if (!conn)
1319                 /* Error message already written in GetConnection() */
1320                 exit(1);
1321
1322         /*
1323          * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1324          * can't work with servers older than 9.1.
1325          */
1326         minServerMajor = 901;
1327         maxServerMajor = PG_VERSION_NUM / 100;
1328         serverMajor = PQserverVersion(conn) / 100;
1329         if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1330         {
1331                 const char *serverver = PQparameterStatus(conn, "server_version");
1332
1333                 fprintf(stderr, _("%s: incompatible server version %s\n"),
1334                                 progname, serverver ? serverver : "'unknown'");
1335                 disconnect_and_exit(1);
1336         }
1337
1338         /*
1339          * If WAL streaming was requested, also check that the server is new
1340          * enough for that.
1341          */
1342         if (streamwal && !CheckServerVersionForStreaming(conn))
1343         {
1344                 /* Error message already written in CheckServerVersionForStreaming() */
1345                 disconnect_and_exit(1);
1346         }
1347
1348         /*
1349          * Build contents of recovery.conf if requested
1350          */
1351         if (writerecoveryconf)
1352                 GenerateRecoveryConf(conn);
1353
1354         /*
1355          * Run IDENTIFY_SYSTEM so we can get the timeline
1356          */
1357         res = PQexec(conn, "IDENTIFY_SYSTEM");
1358         if (PQresultStatus(res) != PGRES_TUPLES_OK)
1359         {
1360                 fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
1361                                 progname, "IDENTIFY_SYSTEM", PQerrorMessage(conn));
1362                 disconnect_and_exit(1);
1363         }
1364         if (PQntuples(res) != 1 || PQnfields(res) != 3)
1365         {
1366                 fprintf(stderr,
1367                                 _("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d fields\n"),
1368                                 progname, PQntuples(res), PQnfields(res), 1, 3);
1369                 disconnect_and_exit(1);
1370         }
1371         sysidentifier = pg_strdup(PQgetvalue(res, 0, 0));
1372         latesttli = atoi(PQgetvalue(res, 0, 1));
1373         PQclear(res);
1374
1375         /*
1376          * Start the actual backup
1377          */
1378         PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1379         snprintf(current_path, sizeof(current_path),
1380                          "BASE_BACKUP LABEL '%s' %s %s %s %s",
1381                          escaped_label,
1382                          showprogress ? "PROGRESS" : "",
1383                          includewal && !streamwal ? "WAL" : "",
1384                          fastcheckpoint ? "FAST" : "",
1385                          includewal ? "NOWAIT" : "");
1386
1387         if (PQsendQuery(conn, current_path) == 0)
1388         {
1389                 fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
1390                                 progname, "BASE_BACKUP", PQerrorMessage(conn));
1391                 disconnect_and_exit(1);
1392         }
1393
1394         /*
1395          * Get the starting xlog position
1396          */
1397         res = PQgetResult(conn);
1398         if (PQresultStatus(res) != PGRES_TUPLES_OK)
1399         {
1400                 fprintf(stderr, _("%s: could not initiate base backup: %s"),
1401                                 progname, PQerrorMessage(conn));
1402                 disconnect_and_exit(1);
1403         }
1404         if (PQntuples(res) != 1)
1405         {
1406                 fprintf(stderr,
1407                                 _("%s: server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields\n"),
1408                                 progname, PQntuples(res), PQnfields(res), 1, 2);
1409                 disconnect_and_exit(1);
1410         }
1411
1412         strcpy(xlogstart, PQgetvalue(res, 0, 0));
1413
1414         /*
1415          * 9.3 and later sends the TLI of the starting point. With older servers,
1416          * assume it's the same as the latest timeline reported by
1417          * IDENTIFY_SYSTEM.
1418          */
1419         if (PQnfields(res) >= 2)
1420                 starttli = atoi(PQgetvalue(res, 0, 1));
1421         else
1422                 starttli = latesttli;
1423         PQclear(res);
1424         MemSet(xlogend, 0, sizeof(xlogend));
1425
1426         if (verbose && includewal)
1427                 fprintf(stderr, _("transaction log start point: %s on timeline %u\n"),
1428                                 xlogstart, starttli);
1429
1430         /*
1431          * Get the header
1432          */
1433         res = PQgetResult(conn);
1434         if (PQresultStatus(res) != PGRES_TUPLES_OK)
1435         {
1436                 fprintf(stderr, _("%s: could not get backup header: %s"),
1437                                 progname, PQerrorMessage(conn));
1438                 disconnect_and_exit(1);
1439         }
1440         if (PQntuples(res) < 1)
1441         {
1442                 fprintf(stderr, _("%s: no data returned from server\n"), progname);
1443                 disconnect_and_exit(1);
1444         }
1445
1446         /*
1447          * Sum up the total size, for progress reporting
1448          */
1449         totalsize = totaldone = 0;
1450         tablespacecount = PQntuples(res);
1451         for (i = 0; i < PQntuples(res); i++)
1452         {
1453                 if (showprogress)
1454                         totalsize += atol(PQgetvalue(res, i, 2));
1455
1456                 /*
1457                  * Verify tablespace directories are empty. Don't bother with the
1458                  * first once since it can be relocated, and it will be checked before
1459                  * we do anything anyway.
1460                  */
1461                 if (format == 'p' && !PQgetisnull(res, i, 1))
1462                         verify_dir_is_empty_or_create(PQgetvalue(res, i, 1));
1463         }
1464
1465         /*
1466          * When writing to stdout, require a single tablespace
1467          */
1468         if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1469         {
1470                 fprintf(stderr,
1471                                 _("%s: can only write single tablespace to stdout, database has %d\n"),
1472                                 progname, PQntuples(res));
1473                 disconnect_and_exit(1);
1474         }
1475
1476         /*
1477          * If we're streaming WAL, start the streaming session before we start
1478          * receiving the actual data chunks.
1479          */
1480         if (streamwal)
1481         {
1482                 if (verbose)
1483                         fprintf(stderr, _("%s: starting background WAL receiver\n"),
1484                                         progname);
1485                 StartLogStreamer(xlogstart, starttli, sysidentifier);
1486         }
1487
1488         /*
1489          * Start receiving chunks
1490          */
1491         for (i = 0; i < PQntuples(res); i++)
1492         {
1493                 if (format == 't')
1494                         ReceiveTarFile(conn, res, i);
1495                 else
1496                         ReceiveAndUnpackTarFile(conn, res, i);
1497         }                                                       /* Loop over all tablespaces */
1498
1499         if (showprogress)
1500         {
1501                 progress_report(PQntuples(res), NULL);
1502                 fprintf(stderr, "\n");  /* Need to move to next line */
1503         }
1504         PQclear(res);
1505
1506         /*
1507          * Get the stop position
1508          */
1509         res = PQgetResult(conn);
1510         if (PQresultStatus(res) != PGRES_TUPLES_OK)
1511         {
1512                 fprintf(stderr,
1513                  _("%s: could not get transaction log end position from server: %s"),
1514                                 progname, PQerrorMessage(conn));
1515                 disconnect_and_exit(1);
1516         }
1517         if (PQntuples(res) != 1)
1518         {
1519                 fprintf(stderr,
1520                          _("%s: no transaction log end position returned from server\n"),
1521                                 progname);
1522                 disconnect_and_exit(1);
1523         }
1524         strcpy(xlogend, PQgetvalue(res, 0, 0));
1525         if (verbose && includewal)
1526                 fprintf(stderr, "transaction log end point: %s\n", xlogend);
1527         PQclear(res);
1528
1529         res = PQgetResult(conn);
1530         if (PQresultStatus(res) != PGRES_COMMAND_OK)
1531         {
1532                 fprintf(stderr, _("%s: final receive failed: %s"),
1533                                 progname, PQerrorMessage(conn));
1534                 disconnect_and_exit(1);
1535         }
1536
1537         if (bgchild > 0)
1538         {
1539 #ifndef WIN32
1540                 int                     status;
1541                 int                     r;
1542 #else
1543                 DWORD           status;
1544                 uint32          hi,
1545                                         lo;
1546 #endif
1547
1548                 if (verbose)
1549                         fprintf(stderr,
1550                                         _("%s: waiting for background process to finish streaming ...\n"), progname);
1551
1552 #ifndef WIN32
1553                 if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
1554                 {
1555                         fprintf(stderr,
1556                                         _("%s: could not send command to background pipe: %s\n"),
1557                                         progname, strerror(errno));
1558                         disconnect_and_exit(1);
1559                 }
1560
1561                 /* Just wait for the background process to exit */
1562                 r = waitpid(bgchild, &status, 0);
1563                 if (r == -1)
1564                 {
1565                         fprintf(stderr, _("%s: could not wait for child process: %s\n"),
1566                                         progname, strerror(errno));
1567                         disconnect_and_exit(1);
1568                 }
1569                 if (r != bgchild)
1570                 {
1571                         fprintf(stderr, _("%s: child %d died, expected %d\n"),
1572                                         progname, r, (int) bgchild);
1573                         disconnect_and_exit(1);
1574                 }
1575                 if (!WIFEXITED(status))
1576                 {
1577                         fprintf(stderr, _("%s: child process did not exit normally\n"),
1578                                         progname);
1579                         disconnect_and_exit(1);
1580                 }
1581                 if (WEXITSTATUS(status) != 0)
1582                 {
1583                         fprintf(stderr, _("%s: child process exited with error %d\n"),
1584                                         progname, WEXITSTATUS(status));
1585                         disconnect_and_exit(1);
1586                 }
1587                 /* Exited normally, we're happy! */
1588 #else                                                   /* WIN32 */
1589
1590                 /*
1591                  * On Windows, since we are in the same process, we can just store the
1592                  * value directly in the variable, and then set the flag that says
1593                  * it's there.
1594                  */
1595                 if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
1596                 {
1597                         fprintf(stderr,
1598                                   _("%s: could not parse transaction log location \"%s\"\n"),
1599                                         progname, xlogend);
1600                         disconnect_and_exit(1);
1601                 }
1602                 xlogendptr = ((uint64) hi) << 32 | lo;
1603                 InterlockedIncrement(&has_xlogendptr);
1604
1605                 /* First wait for the thread to exit */
1606                 if (WaitForSingleObjectEx((HANDLE) bgchild, INFINITE, FALSE) !=
1607                         WAIT_OBJECT_0)
1608                 {
1609                         _dosmaperr(GetLastError());
1610                         fprintf(stderr, _("%s: could not wait for child thread: %s\n"),
1611                                         progname, strerror(errno));
1612                         disconnect_and_exit(1);
1613                 }
1614                 if (GetExitCodeThread((HANDLE) bgchild, &status) == 0)
1615                 {
1616                         _dosmaperr(GetLastError());
1617                         fprintf(stderr, _("%s: could not get child thread exit status: %s\n"),
1618                                         progname, strerror(errno));
1619                         disconnect_and_exit(1);
1620                 }
1621                 if (status != 0)
1622                 {
1623                         fprintf(stderr, _("%s: child thread exited with error %u\n"),
1624                                         progname, (unsigned int) status);
1625                         disconnect_and_exit(1);
1626                 }
1627                 /* Exited normally, we're happy */
1628 #endif
1629         }
1630
1631         /* Free the recovery.conf contents */
1632         destroyPQExpBuffer(recoveryconfcontents);
1633
1634         /*
1635          * End of copy data. Final result is already checked inside the loop.
1636          */
1637         PQclear(res);
1638         PQfinish(conn);
1639
1640         if (verbose)
1641                 fprintf(stderr, "%s: base backup completed\n", progname);
1642 }
1643
1644
1645 int
1646 main(int argc, char **argv)
1647 {
1648         static struct option long_options[] = {
1649                 {"help", no_argument, NULL, '?'},
1650                 {"version", no_argument, NULL, 'V'},
1651                 {"pgdata", required_argument, NULL, 'D'},
1652                 {"format", required_argument, NULL, 'F'},
1653                 {"checkpoint", required_argument, NULL, 'c'},
1654                 {"write-recovery-conf", no_argument, NULL, 'R'},
1655                 {"xlog", no_argument, NULL, 'x'},
1656                 {"xlog-method", required_argument, NULL, 'X'},
1657                 {"gzip", no_argument, NULL, 'z'},
1658                 {"compress", required_argument, NULL, 'Z'},
1659                 {"label", required_argument, NULL, 'l'},
1660                 {"dbname", required_argument, NULL, 'd'},
1661                 {"host", required_argument, NULL, 'h'},
1662                 {"port", required_argument, NULL, 'p'},
1663                 {"username", required_argument, NULL, 'U'},
1664                 {"no-password", no_argument, NULL, 'w'},
1665                 {"password", no_argument, NULL, 'W'},
1666                 {"status-interval", required_argument, NULL, 's'},
1667                 {"verbose", no_argument, NULL, 'v'},
1668                 {"progress", no_argument, NULL, 'P'},
1669                 {NULL, 0, NULL, 0}
1670         };
1671         int                     c;
1672
1673         int                     option_index;
1674
1675         progname = get_progname(argv[0]);
1676         set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
1677
1678         if (argc > 1)
1679         {
1680                 if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
1681                 {
1682                         usage();
1683                         exit(0);
1684                 }
1685                 else if (strcmp(argv[1], "-V") == 0
1686                                  || strcmp(argv[1], "--version") == 0)
1687                 {
1688                         puts("pg_basebackup (PostgreSQL) " PG_VERSION);
1689                         exit(0);
1690                 }
1691         }
1692
1693         while ((c = getopt_long(argc, argv, "D:F:RxX:l:zZ:d:c:h:p:U:s:wWvP",
1694                                                         long_options, &option_index)) != -1)
1695         {
1696                 switch (c)
1697                 {
1698                         case 'D':
1699                                 basedir = pg_strdup(optarg);
1700                                 break;
1701                         case 'F':
1702                                 if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
1703                                         format = 'p';
1704                                 else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
1705                                         format = 't';
1706                                 else
1707                                 {
1708                                         fprintf(stderr,
1709                                                         _("%s: invalid output format \"%s\", must be \"plain\" or \"tar\"\n"),
1710                                                         progname, optarg);
1711                                         exit(1);
1712                                 }
1713                                 break;
1714                         case 'R':
1715                                 writerecoveryconf = true;
1716                                 break;
1717                         case 'x':
1718                                 if (includewal)
1719                                 {
1720                                         fprintf(stderr,
1721                                          _("%s: cannot specify both --xlog and --xlog-method\n"),
1722                                                         progname);
1723                                         exit(1);
1724                                 }
1725
1726                                 includewal = true;
1727                                 streamwal = false;
1728                                 break;
1729                         case 'X':
1730                                 if (includewal)
1731                                 {
1732                                         fprintf(stderr,
1733                                          _("%s: cannot specify both --xlog and --xlog-method\n"),
1734                                                         progname);
1735                                         exit(1);
1736                                 }
1737
1738                                 includewal = true;
1739                                 if (strcmp(optarg, "f") == 0 ||
1740                                         strcmp(optarg, "fetch") == 0)
1741                                         streamwal = false;
1742                                 else if (strcmp(optarg, "s") == 0 ||
1743                                                  strcmp(optarg, "stream") == 0)
1744                                         streamwal = true;
1745                                 else
1746                                 {
1747                                         fprintf(stderr,
1748                                                         _("%s: invalid xlog-method option \"%s\", must be \"fetch\" or \"stream\"\n"),
1749                                                         progname, optarg);
1750                                         exit(1);
1751                                 }
1752                                 break;
1753                         case 'l':
1754                                 label = pg_strdup(optarg);
1755                                 break;
1756                         case 'z':
1757 #ifdef HAVE_LIBZ
1758                                 compresslevel = Z_DEFAULT_COMPRESSION;
1759 #else
1760                                 compresslevel = 1;              /* will be rejected below */
1761 #endif
1762                                 break;
1763                         case 'Z':
1764                                 compresslevel = atoi(optarg);
1765                                 if (compresslevel <= 0 || compresslevel > 9)
1766                                 {
1767                                         fprintf(stderr, _("%s: invalid compression level \"%s\"\n"),
1768                                                         progname, optarg);
1769                                         exit(1);
1770                                 }
1771                                 break;
1772                         case 'c':
1773                                 if (pg_strcasecmp(optarg, "fast") == 0)
1774                                         fastcheckpoint = true;
1775                                 else if (pg_strcasecmp(optarg, "spread") == 0)
1776                                         fastcheckpoint = false;
1777                                 else
1778                                 {
1779                                         fprintf(stderr, _("%s: invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"\n"),
1780                                                         progname, optarg);
1781                                         exit(1);
1782                                 }
1783                                 break;
1784                         case 'd':
1785                                 connection_string = pg_strdup(optarg);
1786                                 break;
1787                         case 'h':
1788                                 dbhost = pg_strdup(optarg);
1789                                 break;
1790                         case 'p':
1791                                 dbport = pg_strdup(optarg);
1792                                 break;
1793                         case 'U':
1794                                 dbuser = pg_strdup(optarg);
1795                                 break;
1796                         case 'w':
1797                                 dbgetpassword = -1;
1798                                 break;
1799                         case 'W':
1800                                 dbgetpassword = 1;
1801                                 break;
1802                         case 's':
1803                                 standby_message_timeout = atoi(optarg) * 1000;
1804                                 if (standby_message_timeout < 0)
1805                                 {
1806                                         fprintf(stderr, _("%s: invalid status interval \"%s\"\n"),
1807                                                         progname, optarg);
1808                                         exit(1);
1809                                 }
1810                                 break;
1811                         case 'v':
1812                                 verbose++;
1813                                 break;
1814                         case 'P':
1815                                 showprogress = true;
1816                                 break;
1817                         default:
1818
1819                                 /*
1820                                  * getopt_long already emitted a complaint
1821                                  */
1822                                 fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
1823                                                 progname);
1824                                 exit(1);
1825                 }
1826         }
1827
1828         /*
1829          * Any non-option arguments?
1830          */
1831         if (optind < argc)
1832         {
1833                 fprintf(stderr,
1834                                 _("%s: too many command-line arguments (first is \"%s\")\n"),
1835                                 progname, argv[optind]);
1836                 fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
1837                                 progname);
1838                 exit(1);
1839         }
1840
1841         /*
1842          * Required arguments
1843          */
1844         if (basedir == NULL)
1845         {
1846                 fprintf(stderr, _("%s: no target directory specified\n"), progname);
1847                 fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
1848                                 progname);
1849                 exit(1);
1850         }
1851
1852         /*
1853          * Mutually exclusive arguments
1854          */
1855         if (format == 'p' && compresslevel != 0)
1856         {
1857                 fprintf(stderr,
1858                                 _("%s: only tar mode backups can be compressed\n"),
1859                                 progname);
1860                 fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
1861                                 progname);
1862                 exit(1);
1863         }
1864
1865         if (format != 'p' && streamwal)
1866         {
1867                 fprintf(stderr,
1868                                 _("%s: WAL streaming can only be used in plain mode\n"),
1869                                 progname);
1870                 fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
1871                                 progname);
1872                 exit(1);
1873         }
1874
1875 #ifndef HAVE_LIBZ
1876         if (compresslevel != 0)
1877         {
1878                 fprintf(stderr,
1879                                 _("%s: this build does not support compression\n"),
1880                                 progname);
1881                 exit(1);
1882         }
1883 #endif
1884
1885         /*
1886          * Verify that the target directory exists, or create it. For plaintext
1887          * backups, always require the directory. For tar backups, require it
1888          * unless we are writing to stdout.
1889          */
1890         if (format == 'p' || strcmp(basedir, "-") != 0)
1891                 verify_dir_is_empty_or_create(basedir);
1892
1893         BaseBackup();
1894
1895         return 0;
1896 }