From 6ed2d8584cc680a2d6898480de74a57cd96176b5 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: [PATCH] pg_basebackup: Add --nosync option This is useful for testing, similar to initdb's --nosync. From: Michael Paquier --- doc/src/sgml/ref/pg_basebackup.sgml | 15 ++++++++++++ src/bin/pg_basebackup/pg_basebackup.c | 28 ++++++++++++++------- src/bin/pg_basebackup/pg_receivexlog.c | 1 + src/bin/pg_basebackup/receivelog.c | 34 ++++++++++++++------------ src/bin/pg_basebackup/receivelog.h | 4 ++- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index fe557ed002..55e913f70d 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -438,6 +438,21 @@ PostgreSQL documentation + + + + + + By default, pg_basebackup will wait for all files + to be written safely to disk. This option causes + pg_basebackup to return without waiting, which is + faster, but means that a subsequent operating system crash can leave + the base backup corrupt. Generally, this option is useful for testing + but should not be used when creating a production installation. + + + + diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index cd7d095103..0f5d9d6a87 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -69,6 +69,7 @@ static bool includewal = false; static bool streamwal = false; static bool fastcheckpoint = false; static bool writerecoveryconf = false; +static bool do_sync = true; static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ @@ -329,6 +330,7 @@ usage(void) " set fast or spread checkpointing\n")); printf(_(" -l, --label=LABEL set backup label\n")); printf(_(" -n, --noclean do not clean up after errors\n")); + printf(_(" -N, --nosync do not wait for changes to be written safely to disk\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -460,6 +462,7 @@ LogStreamerMain(logstreamer_param *param) stream.stream_stop = reached_end_position; stream.standby_message_timeout = standby_message_timeout; stream.synchronous = false; + stream.do_sync = do_sync; stream.mark_done = true; stream.basedir = param->xlogdir; stream.partial_suffix = NULL; @@ -1199,7 +1202,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) PQfreemem(copybuf); /* sync the resulting tar file, errors are not considered fatal */ - if (strcmp(basedir, "-") != 0) + if (do_sync && strcmp(basedir, "-") != 0) (void) fsync_fname(filename, false, progname); } @@ -1967,14 +1970,17 @@ BaseBackup(void) * all the data of the base directory is synced, taking into account * all the tablespaces. Errors are not considered fatal. */ - if (format == 't') + if (do_sync) { - if (strcmp(basedir, "-") != 0) - (void) fsync_fname(basedir, true, progname); - } - else - { - (void) fsync_pgdata(basedir, progname); + if (format == 't') + { + if (strcmp(basedir, "-") != 0) + (void) fsync_fname(basedir, true, progname); + } + else + { + (void) fsync_pgdata(basedir, progname); + } } if (verbose) @@ -2001,6 +2007,7 @@ main(int argc, char **argv) {"compress", required_argument, NULL, 'Z'}, {"label", required_argument, NULL, 'l'}, {"noclean", no_argument, NULL, 'n'}, + {"nosync", no_argument, NULL, 'N'}, {"dbname", required_argument, NULL, 'd'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, @@ -2037,7 +2044,7 @@ main(int argc, char **argv) atexit(cleanup_directories_atexit); - while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP", + while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nNzZ:d:c:h:p:U:s:S:wWvP", long_options, &option_index)) != -1) { switch (c) @@ -2115,6 +2122,9 @@ main(int argc, char **argv) case 'n': noclean = true; break; + case 'N': + do_sync = false; + break; case 'z': #ifdef HAVE_LIBZ compresslevel = Z_DEFAULT_COMPRESSION; diff --git a/src/bin/pg_basebackup/pg_receivexlog.c b/src/bin/pg_basebackup/pg_receivexlog.c index 7f7ee9dc9b..a58a251a59 100644 --- a/src/bin/pg_basebackup/pg_receivexlog.c +++ b/src/bin/pg_basebackup/pg_receivexlog.c @@ -336,6 +336,7 @@ StreamLog(void) stream.stream_stop = stop_streaming; stream.standby_message_timeout = standby_message_timeout; stream.synchronous = synchronous; + stream.do_sync = true; stream.mark_done = false; stream.basedir = basedir; stream.partial_suffix = ".partial"; diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 6b78a60f27..8f29d19114 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -41,8 +41,8 @@ static PGresult *HandleCopyStream(PGconn *conn, StreamCtl *stream, XLogRecPtr *stoppos); static int CopyStreamPoll(PGconn *conn, long timeout_ms); static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer); -static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, - XLogRecPtr blockpos, int64 *last_status); +static bool ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, + int len, XLogRecPtr blockpos, int64 *last_status); static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, XLogRecPtr *blockpos); static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf, @@ -56,7 +56,7 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, uint32 *timeline); static bool -mark_file_as_archived(const char *basedir, const char *fname) +mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) { int fd; static char tmppath[MAXPGPATH]; @@ -74,10 +74,10 @@ mark_file_as_archived(const char *basedir, const char *fname) close(fd); - if (fsync_fname(tmppath, false, progname) != 0) + if (do_sync && fsync_fname(tmppath, false, progname) != 0) return false; - if (fsync_parent_path(tmppath, progname) != 0) + if (do_sync && fsync_parent_path(tmppath, progname) != 0) return false; return true; @@ -134,9 +134,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) * fsync, in case of a previous crash between padding and fsyncing the * file. */ - if (fsync_fname(fn, false, progname) != 0) + if (stream->do_sync && fsync_fname(fn, false, progname) != 0) return false; - if (fsync_parent_path(fn, progname) != 0) + if (stream->do_sync && fsync_parent_path(fn, progname) != 0) return false; return true; @@ -173,9 +173,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) * using synchronous mode, where the file is modified and fsynced * in-place, without a directory fsync. */ - if (fsync_fname(fn, false, progname) != 0) + if (stream->do_sync && fsync_fname(fn, false, progname) != 0) return false; - if (fsync_parent_path(fn, progname) != 0) + if (stream->do_sync && fsync_parent_path(fn, progname) != 0) return false; if (lseek(f, SEEK_SET, 0) != 0) @@ -212,7 +212,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) return false; } - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); @@ -258,7 +258,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) if (currpos == XLOG_SEG_SIZE && stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, current_walfile_name)) + if (!mark_file_as_archived(stream->basedir, current_walfile_name, + stream->do_sync)) return false; } @@ -378,7 +379,8 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) if (stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, histfname)) + if (!mark_file_as_archived(stream->basedir, histfname, + stream->do_sync)) return false; } @@ -836,7 +838,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, */ if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1) { - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); @@ -890,7 +892,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, /* Check the message type. */ if (copybuf[0] == 'k') { - if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos, + if (!ProcessKeepaliveMsg(conn, stream, copybuf, r, blockpos, &last_status)) goto error; } @@ -1043,7 +1045,7 @@ CopyStreamReceive(PGconn *conn, long timeout, char **buffer) * Process the keepalive message. */ static bool -ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, +ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, XLogRecPtr blockpos, int64 *last_status) { int pos; @@ -1079,7 +1081,7 @@ ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, * data has been successfully replicated or not, at the normal * shutdown of the server. */ - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h index 554ff8b5b2..7a3bbc5080 100644 --- a/src/bin/pg_basebackup/receivelog.h +++ b/src/bin/pg_basebackup/receivelog.h @@ -34,8 +34,10 @@ typedef struct StreamCtl * timeline */ int standby_message_timeout; /* Send status messages this * often */ - bool synchronous; /* Flush data on write */ + bool synchronous; /* Flush immediately WAL data on write */ bool mark_done; /* Mark segment as done in generated archive */ + bool do_sync; /* Flush to disk to ensure consistent state + * of data */ stream_stop_callback stream_stop; /* Stop streaming when returns true */ -- 2.40.0