From ffd37740ee6fcd434416ec0c5461f7040e0a11de Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 15 May 2015 18:55:24 +0300 Subject: [PATCH] Add archive_mode='always' option. In 'always' mode, the standby independently archives all files it receives from the primary. Original patch by Fujii Masao, docs and review by me. --- doc/src/sgml/config.sgml | 13 ++++++- doc/src/sgml/high-availability.sgml | 39 +++++++++++++++++++ src/backend/access/transam/xlog.c | 22 ++++++++++- src/backend/access/transam/xlogarchive.c | 5 ++- src/backend/postmaster/postmaster.c | 37 +++++++++++++----- src/backend/replication/walreceiver.c | 10 ++++- src/backend/utils/misc/guc.c | 21 +++++----- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog.h | 13 ++++++- 9 files changed, 133 insertions(+), 29 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0d8624af71..5549b7dce6 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2521,7 +2521,7 @@ include_dir 'conf.d' - archive_mode (boolean) + archive_mode (enum) archive_mode configuration parameter @@ -2530,7 +2530,16 @@ include_dir 'conf.d' When archive_mode is enabled, completed WAL segments are sent to archive storage by setting - . + . In addition to off, + to disable, there are two modes: on, and + always. During normal operation, there is no + difference between the two modes, but when set to always + the WAL archiver is enabled also during archive recovery or standby + mode. In always mode, all files restored from the archive + or streamed with streaming replication will be archived (again). See + for details. + + archive_mode and archive_command are separate variables so that archive_command can be changed without leaving archiving mode. diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index a17f55545c..46518262cd 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1220,6 +1220,45 @@ primary_slot_name = 'node_a_slot' + + + Continuous archiving in standby + + + continuous archiving + in standby + + + + When continuous WAL archiving is used in a standby, there are two + different scenarios: the WAL archive can be shared between the primary + and the standby, or the standby can have its own WAL archive. When + the standby has its own WAL archive, set archive_mode + to always, and the standby will call the archive + command for every WAL segment it receives, whether it's by restoring + from the archive or by streaming replication. The shared archive can + be handled similarly, but the archive_command must test if the file + being archived exists already, and if the existing file has identical + contents. This requires more care in the archive_command, as it must + be careful to not overwrite an existing file with different contents, + but return success if the exactly same file is archived twice. And + all that must be done free of race conditions, if two servers attempt + to archive the same file at the same time. + + + + If archive_mode is set to on, the + archiver is not enabled during recovery or standby mode. If the standby + server is promoted, it will start archiving after the promotion, but + will not archive any WAL it did not generate itself. To get a complete + series of WAL files in the archive, you must ensure that all WAL is + archived, before it reaches the standby. This is inherently true with + file-based log shipping, as the standby can only restore files that + are found in the archive, but not if streaming replication is enabled. + When a server is not in recovery mode, there is no difference between + on and always modes. + + diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 5f0551a3cb..0485bb5201 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -86,7 +86,7 @@ int min_wal_size = 5; /* 80 MB */ int wal_keep_segments = 0; int XLOGbuffers = -1; int XLogArchiveTimeout = 0; -bool XLogArchiveMode = false; +int XLogArchiveMode = ARCHIVE_MODE_OFF; char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; @@ -140,6 +140,24 @@ const struct config_enum_entry sync_method_options[] = { {NULL, 0, false} }; + +/* + * Although only "on", "off", and "always" are documented, + * we accept all the likely variants of "on" and "off". + */ +const struct config_enum_entry archive_mode_options[] = { + {"always", ARCHIVE_MODE_ALWAYS, false}, + {"on", ARCHIVE_MODE_ON, false}, + {"off", ARCHIVE_MODE_OFF, false}, + {"true", ARCHIVE_MODE_ON, true}, + {"false", ARCHIVE_MODE_OFF, true}, + {"yes", ARCHIVE_MODE_ON, true}, + {"no", ARCHIVE_MODE_OFF, true}, + {"1", ARCHIVE_MODE_ON, true}, + {"0", ARCHIVE_MODE_OFF, true}, + {NULL, 0, false} +}; + /* * Statistics for current checkpoint are collected in this global struct. * Because only the checkpointer or a stand-alone backend can perform @@ -767,7 +785,7 @@ static MemoryContext walDebugCxt = NULL; #endif static void readRecoveryCommandFile(void); -static void exitArchiveRecovery(TimeLineID endTLI, XLogSegNo endLogSegNo); +static void exitArchiveRecovery(TimeLineID endTLI, XLogRecPtr endOfLog); static bool recoveryStopsBefore(XLogReaderState *record); static bool recoveryStopsAfter(XLogReaderState *record); static void recoveryPausesHere(void); diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index f435f65e98..4c69b738bc 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -480,7 +480,10 @@ KeepFileRestoredFromArchive(char *path, char *xlogfname) * Create .done file forcibly to prevent the restored segment from being * archived again later. */ - XLogArchiveForceDone(xlogfname); + if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS) + XLogArchiveForceDone(xlogfname); + else + XLogArchiveNotify(xlogfname); /* * If the existing file was replaced, since walsenders might have it open, diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index a9f20ac1b4..36440cbdcc 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -828,9 +828,9 @@ PostmasterMain(int argc, char *argv[]) write_stderr("%s: max_wal_senders must be less than max_connections\n", progname); ExitPostmaster(1); } - if (XLogArchiveMode && wal_level == WAL_LEVEL_MINIMAL) + if (XLogArchiveMode > ARCHIVE_MODE_OFF && wal_level == WAL_LEVEL_MINIMAL) ereport(ERROR, - (errmsg("WAL archival (archive_mode=on) requires wal_level \"archive\", \"hot_standby\", or \"logical\""))); + (errmsg("WAL archival cannot be enabled when wal_level is \"minimal\""))); if (max_wal_senders > 0 && wal_level == WAL_LEVEL_MINIMAL) ereport(ERROR, (errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"archive\", \"hot_standby\", or \"logical\""))); @@ -1645,13 +1645,21 @@ ServerLoop(void) start_autovac_launcher = false; /* signal processed */ } - /* If we have lost the archiver, try to start a new one */ - if (XLogArchivingActive() && PgArchPID == 0 && pmState == PM_RUN) - PgArchPID = pgarch_start(); - - /* If we have lost the stats collector, try to start a new one */ - if (PgStatPID == 0 && pmState == PM_RUN) - PgStatPID = pgstat_start(); + /* + * If we have lost the archiver, try to start a new one. + * + * If WAL archiving is enabled always, we try to start a new archiver + * even during recovery. + */ + if (PgArchPID == 0 && wal_level >= WAL_LEVEL_ARCHIVE) + { + if ((pmState == PM_RUN && XLogArchiveMode > ARCHIVE_MODE_OFF) || + ((pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY) && + XLogArchiveMode == ARCHIVE_MODE_ALWAYS)) + { + PgArchPID = pgarch_start(); + } + } /* If we need to signal the autovacuum launcher, do so now */ if (avlauncher_needs_signal) @@ -4807,6 +4815,17 @@ sigusr1_handler(SIGNAL_ARGS) Assert(BgWriterPID == 0); BgWriterPID = StartBackgroundWriter(); + /* + * Start the archiver if we're responsible for (re-)archiving received + * files. + */ + Assert(PgArchPID == 0); + if (wal_level >= WAL_LEVEL_ARCHIVE && + XLogArchiveMode == ARCHIVE_MODE_ALWAYS) + { + PgArchPID = pgarch_start(); + } + pmState = PM_RECOVERY; } if (CheckPostmasterSignal(PMSIGNAL_BEGIN_HOT_STANDBY) && diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 9c7710f0db..41e57f2439 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -540,7 +540,10 @@ WalReceiverMain(void) * being archived later. */ XLogFileName(xlogfname, recvFileTLI, recvSegNo); - XLogArchiveForceDone(xlogfname); + if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS) + XLogArchiveForceDone(xlogfname); + else + XLogArchiveNotify(xlogfname); } recvFile = -1; @@ -897,7 +900,10 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr) * from being archived later. */ XLogFileName(xlogfname, recvFileTLI, recvSegNo); - XLogArchiveForceDone(xlogfname); + if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS) + XLogArchiveForceDone(xlogfname); + else + XLogArchiveNotify(xlogfname); } recvFile = -1; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 222228ae5a..3038d7c9dd 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -396,6 +396,7 @@ static const struct config_enum_entry row_security_options[] = { * Options for enum values stored in other modules */ extern const struct config_enum_entry wal_level_options[]; +extern const struct config_enum_entry archive_mode_options[]; extern const struct config_enum_entry sync_method_options[]; extern const struct config_enum_entry dynamic_shared_memory_options[]; @@ -1529,16 +1530,6 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, - { - {"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING, - gettext_noop("Allows archiving of WAL files using archive_command."), - NULL - }, - &XLogArchiveMode, - false, - NULL, NULL, NULL - }, - { {"hot_standby", PGC_POSTMASTER, REPLICATION_STANDBY, gettext_noop("Allows connections and queries during recovery."), @@ -3551,6 +3542,16 @@ static struct config_enum ConfigureNamesEnum[] = NULL, assign_synchronous_commit, NULL }, + { + {"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING, + gettext_noop("Allows archiving of WAL files using archive_command."), + NULL + }, + &XLogArchiveMode, + ARCHIVE_MODE_OFF, archive_mode_options, + NULL, NULL, NULL + }, + { {"trace_recovery_messages", PGC_SIGHUP, DEVELOPER_OPTIONS, gettext_noop("Enables logging of recovery-related debugging information."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 110983f176..06dfc067b0 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -206,7 +206,7 @@ # - Archiving - -#archive_mode = off # allows archiving to be done +#archive_mode = off # enables archiving; off, on, or always # (change requires restart) #archive_command = '' # command to use to archive a logfile segment # placeholders: %p = path of file to archive diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 961e050622..9567379f49 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -98,7 +98,6 @@ extern int wal_keep_segments; extern int XLOGbuffers; extern int XLogArchiveTimeout; extern int wal_retrieve_retry_interval; -extern bool XLogArchiveMode; extern char *XLogArchiveCommand; extern bool EnableHotStandby; extern bool fullPageWrites; @@ -108,6 +107,15 @@ extern bool log_checkpoints; extern int CheckPointSegments; +/* Archive modes */ +typedef enum ArchiveMode +{ + ARCHIVE_MODE_OFF = 0, /* disabled */ + ARCHIVE_MODE_ON, /* enabled while server is running normally */ + ARCHIVE_MODE_ALWAYS /* enabled always (even during recovery) */ +} ArchiveMode; +extern int XLogArchiveMode; + /* WAL levels */ typedef enum WalLevel { @@ -118,7 +126,8 @@ typedef enum WalLevel } WalLevel; extern int wal_level; -#define XLogArchivingActive() (XLogArchiveMode && wal_level >= WAL_LEVEL_ARCHIVE) +#define XLogArchivingActive() \ + (XLogArchiveMode > ARCHIVE_MODE_OFF && wal_level >= WAL_LEVEL_ARCHIVE) #define XLogArchiveCommandSet() (XLogArchiveCommand[0] != '\0') /* -- 2.40.0