pg_basebackup: Add --nosync option

This is useful for testing, similar to initdb's --nosync.

From: Michael Paquier <michael.paquier@gmail.com>

diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index fe557ed002f2eb1edf37f00852ea079715d68bbe..55e913f70de511baf6ab277b48df80e23dd7a199 100644 (file)
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -438,6 +438,21 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-N</option></term>
+      <term><option>--nosync</option></term>
+      <listitem>
+       <para>
+        By default, <command>pg_basebackup</command> will wait for all files
+        to be written safely to disk.  This option causes
+        <command>pg_basebackup</command> to return without waiting, which is
+        faster, but means that a subsequent operating system crash can leave
+        the base backup corrupt.  Generally, this option is useful for testing
+        but should not be used when creating a production installation.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-v</option></term>
       <term><option>--verbose</option></term>

diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index cd7d095103df953f1d1e68ed63be42d83ee84d47..0f5d9d6a87f57358d053994a22700a0b76f4cdab 100644 (file)
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -69,6 +69,7 @@ static bool includewal = false;
 static bool streamwal = false;
 static bool fastcheckpoint = false;
 static bool writerecoveryconf = false;
+static bool do_sync = true;
 static int     standby_message_timeout = 10 * 1000;            /* 10 sec = default */
 static pg_time_t last_progress_report = 0;
 static int32 maxrate = 0;              /* no limit by default */
@@ -329,6 +330,7 @@ usage(void)
                         "                         set fast or spread checkpointing\n"));
        printf(_("  -l, --label=LABEL      set backup label\n"));
        printf(_("  -n, --noclean          do not clean up after errors\n"));
+       printf(_("  -N, --nosync           do not wait for changes to be written safely to disk\n"));
        printf(_("  -P, --progress         show progress information\n"));
        printf(_("  -v, --verbose          output verbose messages\n"));
        printf(_("  -V, --version          output version information, then exit\n"));
@@ -460,6 +462,7 @@ LogStreamerMain(logstreamer_param *param)
        stream.stream_stop = reached_end_position;
        stream.standby_message_timeout = standby_message_timeout;
        stream.synchronous = false;
+       stream.do_sync = do_sync;
        stream.mark_done = true;
        stream.basedir = param->xlogdir;
        stream.partial_suffix = NULL;
@@ -1199,7 +1202,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
                PQfreemem(copybuf);
 
        /* sync the resulting tar file, errors are not considered fatal */
-       if (strcmp(basedir, "-") != 0)
+       if (do_sync && strcmp(basedir, "-") != 0)
                (void) fsync_fname(filename, false, progname);
 }
 
@@ -1967,14 +1970,17 @@ BaseBackup(void)
         * all the data of the base directory is synced, taking into account
         * all the tablespaces. Errors are not considered fatal.
         */
-       if (format == 't')
+       if (do_sync)
        {
-               if (strcmp(basedir, "-") != 0)
-                       (void) fsync_fname(basedir, true, progname);
-       }
-       else
-       {
-               (void) fsync_pgdata(basedir, progname);
+               if (format == 't')
+               {
+                       if (strcmp(basedir, "-") != 0)
+                               (void) fsync_fname(basedir, true, progname);
+               }
+               else
+               {
+                       (void) fsync_pgdata(basedir, progname);
+               }
        }
 
        if (verbose)
@@ -2001,6 +2007,7 @@ main(int argc, char **argv)
                {"compress", required_argument, NULL, 'Z'},
                {"label", required_argument, NULL, 'l'},
                {"noclean", no_argument, NULL, 'n'},
+               {"nosync", no_argument, NULL, 'N'},
                {"dbname", required_argument, NULL, 'd'},
                {"host", required_argument, NULL, 'h'},
                {"port", required_argument, NULL, 'p'},
@@ -2037,7 +2044,7 @@ main(int argc, char **argv)
 
        atexit(cleanup_directories_atexit);
 
-       while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP",
+       while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nNzZ:d:c:h:p:U:s:S:wWvP",
                                                        long_options, &option_index)) != -1)
        {
                switch (c)
@@ -2115,6 +2122,9 @@ main(int argc, char **argv)
                        case 'n':
                                noclean = true;
                                break;
+                       case 'N':
+                               do_sync = false;
+                               break;
                        case 'z':
 #ifdef HAVE_LIBZ
                                compresslevel = Z_DEFAULT_COMPRESSION;

diff --git a/src/bin/pg_basebackup/pg_receivexlog.c b/src/bin/pg_basebackup/pg_receivexlog.c
index 7f7ee9dc9baeb6e0986e3ee06f5146bb195b04d1..a58a251a59f82086d22e577965245ddf3218abb1 100644 (file)
--- a/src/bin/pg_basebackup/pg_receivexlog.c
+++ b/src/bin/pg_basebackup/pg_receivexlog.c
@@ -336,6 +336,7 @@ StreamLog(void)
        stream.stream_stop = stop_streaming;
        stream.standby_message_timeout = standby_message_timeout;
        stream.synchronous = synchronous;
+       stream.do_sync = true;
        stream.mark_done = false;
        stream.basedir = basedir;
        stream.partial_suffix = ".partial";

diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c
index 6b78a60f2780bf9a4cbafa80e5a64215dd57cdfc..8f29d19114469e89e4400c09c1cb208e57cc6756 100644 (file)
--- a/src/bin/pg_basebackup/receivelog.c
+++ b/src/bin/pg_basebackup/receivelog.c
@@ -41,8 +41,8 @@ static PGresult *HandleCopyStream(PGconn *conn, StreamCtl *stream,
                                 XLogRecPtr *stoppos);
 static int     CopyStreamPoll(PGconn *conn, long timeout_ms);
 static int     CopyStreamReceive(PGconn *conn, long timeout, char **buffer);
-static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
-                                       XLogRecPtr blockpos, int64 *last_status);
+static bool ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf,
+                                       int len, XLogRecPtr blockpos, int64 *last_status);
 static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
                                   XLogRecPtr *blockpos);
 static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf,
@@ -56,7 +56,7 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos,
                                                 uint32 *timeline);
 
 static bool
-mark_file_as_archived(const char *basedir, const char *fname)
+mark_file_as_archived(const char *basedir, const char *fname, bool do_sync)
 {
        int                     fd;
        static char tmppath[MAXPGPATH];
@@ -74,10 +74,10 @@ mark_file_as_archived(const char *basedir, const char *fname)
 
        close(fd);
 
-       if (fsync_fname(tmppath, false, progname) != 0)
+       if (do_sync && fsync_fname(tmppath, false, progname) != 0)
                return false;
 
-       if (fsync_parent_path(tmppath, progname) != 0)
+       if (do_sync && fsync_parent_path(tmppath, progname) != 0)
                return false;
 
        return true;
@@ -134,9 +134,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
                 * fsync, in case of a previous crash between padding and fsyncing the
                 * file.
                 */
-               if (fsync_fname(fn, false, progname) != 0)
+               if (stream->do_sync && fsync_fname(fn, false, progname) != 0)
                        return false;
-               if (fsync_parent_path(fn, progname) != 0)
+               if (stream->do_sync && fsync_parent_path(fn, progname) != 0)
                        return false;
 
                return true;
@@ -173,9 +173,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint)
         * using synchronous mode, where the file is modified and fsynced
         * in-place, without a directory fsync.
         */
-       if (fsync_fname(fn, false, progname) != 0)
+       if (stream->do_sync && fsync_fname(fn, false, progname) != 0)
                return false;
-       if (fsync_parent_path(fn, progname) != 0)
+       if (stream->do_sync && fsync_parent_path(fn, progname) != 0)
                return false;
 
        if (lseek(f, SEEK_SET, 0) != 0)
@@ -212,7 +212,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
                return false;
        }
 
-       if (fsync(walfile) != 0)
+       if (stream->do_sync && fsync(walfile) != 0)
        {
                fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
                                progname, current_walfile_name, strerror(errno));
@@ -258,7 +258,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
        if (currpos == XLOG_SEG_SIZE && stream->mark_done)
        {
                /* writes error message if failed */
-               if (!mark_file_as_archived(stream->basedir, current_walfile_name))
+               if (!mark_file_as_archived(stream->basedir, current_walfile_name,
+                                                                  stream->do_sync))
                        return false;
        }
 
@@ -378,7 +379,8 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content)
        if (stream->mark_done)
        {
                /* writes error message if failed */
-               if (!mark_file_as_archived(stream->basedir, histfname))
+               if (!mark_file_as_archived(stream->basedir, histfname,
+                                                                  stream->do_sync))
                        return false;
        }
 
@@ -836,7 +838,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream,
                 */
                if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1)
                {
-                       if (fsync(walfile) != 0)
+                       if (stream->do_sync && fsync(walfile) != 0)
                        {
                                fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
                                                progname, current_walfile_name, strerror(errno));
@@ -890,7 +892,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream,
                        /* Check the message type. */
                        if (copybuf[0] == 'k')
                        {
-                               if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos,
+                               if (!ProcessKeepaliveMsg(conn, stream, copybuf, r, blockpos,
                                                                                 &last_status))
                                        goto error;
                        }
@@ -1043,7 +1045,7 @@ CopyStreamReceive(PGconn *conn, long timeout, char **buffer)
  * Process the keepalive message.
  */
 static bool
-ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
+ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
                                        XLogRecPtr blockpos, int64 *last_status)
 {
        int                     pos;
@@ -1079,7 +1081,7 @@ ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len,
                         * data has been successfully replicated or not, at the normal
                         * shutdown of the server.
                         */
-                       if (fsync(walfile) != 0)
+                       if (stream->do_sync && fsync(walfile) != 0)
                        {
                                fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"),
                                                progname, current_walfile_name, strerror(errno));

diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h
index 554ff8b5b28610df1385f3d2342b57f9c485a91d..7a3bbc50800d1f540ed93b311c36b15b7293e92c 100644 (file)
--- a/src/bin/pg_basebackup/receivelog.h
+++ b/src/bin/pg_basebackup/receivelog.h
@@ -34,8 +34,10 @@ typedef struct StreamCtl
                                                                 * timeline */
        int                     standby_message_timeout;                /* Send status messages this
                                                                                                 * often */
-       bool            synchronous;    /* Flush data on write */
+       bool            synchronous;    /* Flush immediately WAL data on write */
        bool            mark_done;              /* Mark segment as done in generated archive */
+       bool            do_sync;                /* Flush to disk to ensure consistent state
+                                                                * of data */
 
        stream_stop_callback stream_stop;       /* Stop streaming when returns true */