Track the timeline associated with minRecoveryPoint, for more sanity checks.

This allows recovery to notice certain incorrect recovery scenarios.
If a server has recovered to point X on timeline 5, and you restart
recovery, it better be on timeline 5 when it reaches point X again, not on
some timeline with a higher ID. This can happen e.g if you a standby server
is shut down, a new timeline appears in the WAL archive, and the standby
server is restarted. It will try to follow the new timeline, which is wrong
because some WAL on the old timeline was already replayed before shutdown.

Requires an initdb (or at least pg_resetxlog), because this adds a field to
the control file.

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 754dfe7bdc66ba08a1fccc590971ab675c4b9d8a..b3356fd43491bed3b4b732cc37d265abeb10eac7 100644 (file)
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -447,6 +447,7 @@ typedef struct XLogCtlData
 
        /* end+1 of the last record replayed (or being replayed) */
        XLogRecPtr      replayEndRecPtr;
+       TimeLineID      replayEndTLI;
        /* end+1 of the last record replayed */
        XLogRecPtr      recoveryLastRecPtr;
        /* timestamp of last COMMIT/ABORT record replayed (or being replayed) */
@@ -580,6 +581,7 @@ static TimeLineID lastSegmentTLI = 0;
 
 static XLogRecPtr minRecoveryPoint;            /* local copy of
                                                                                 * ControlFile->minRecoveryPoint */
+static TimeLineID minRecoveryPointTLI;
 static bool updateMinRecoveryPoint = true;
 
 /*
@@ -1778,6 +1780,7 @@ UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force)
 
        /* update local copy */
        minRecoveryPoint = ControlFile->minRecoveryPoint;
+       minRecoveryPointTLI = ControlFile->minRecoveryPointTLI;
 
        /*
         * An invalid minRecoveryPoint means that we need to recover all the WAL,
@@ -1791,6 +1794,7 @@ UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force)
                /* use volatile pointer to prevent code rearrangement */
                volatile XLogCtlData *xlogctl = XLogCtl;
                XLogRecPtr      newMinRecoveryPoint;
+               TimeLineID      newMinRecoveryPointTLI;
 
                /*
                 * To avoid having to update the control file too often, we update it
@@ -1807,6 +1811,7 @@ UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force)
                 */
                SpinLockAcquire(&xlogctl->info_lck);
                newMinRecoveryPoint = xlogctl->replayEndRecPtr;
+               newMinRecoveryPointTLI = xlogctl->replayEndTLI;
                SpinLockRelease(&xlogctl->info_lck);
 
                if (!force && XLByteLT(newMinRecoveryPoint, lsn))
@@ -1820,13 +1825,16 @@ UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force)
                if (XLByteLT(ControlFile->minRecoveryPoint, newMinRecoveryPoint))
                {
                        ControlFile->minRecoveryPoint = newMinRecoveryPoint;
+                       ControlFile->minRecoveryPointTLI = newMinRecoveryPointTLI;
                        UpdateControlFile();
                        minRecoveryPoint = newMinRecoveryPoint;
+                       minRecoveryPointTLI = newMinRecoveryPointTLI;
 
                        ereport(DEBUG2,
-                                       (errmsg("updated min recovery point to %X/%X",
+                                       (errmsg("updated min recovery point to %X/%X on timeline %u",
                                                        (uint32) (minRecoveryPoint >> 32),
-                                                       (uint32) minRecoveryPoint)));
+                                                       (uint32) minRecoveryPoint,
+                                                       newMinRecoveryPointTLI)));
                }
        }
        LWLockRelease(ControlFileLock);
@@ -2132,6 +2140,7 @@ XLogNeedsFlush(XLogRecPtr record)
                if (!LWLockConditionalAcquire(ControlFileLock, LW_SHARED))
                        return true;
                minRecoveryPoint = ControlFile->minRecoveryPoint;
+               minRecoveryPointTLI = ControlFile->minRecoveryPointTLI;
                LWLockRelease(ControlFileLock);
 
                /*
@@ -5305,6 +5314,19 @@ StartupXLOG(void)
                                                recoveryTargetTLI,
                                                ControlFile->checkPointCopy.ThisTimeLineID)));
 
+       /*
+        * The min recovery point should be part of the requested timeline's
+        * history, too.
+        */
+       if (!XLogRecPtrIsInvalid(ControlFile->minRecoveryPoint) &&
+               !list_member_int(expectedTLIs, ControlFile->minRecoveryPointTLI))
+               ereport(FATAL,
+                               (errmsg("requested timeline %u does not contain minimum recovery point %X/%X on timeline %u",
+                                               recoveryTargetTLI,
+                                               (uint32) (ControlFile->minRecoveryPoint >> 32),
+                                               (uint32) ControlFile->minRecoveryPoint,
+                                               ControlFile->minRecoveryPointTLI)));
+
        /*
         * Save the selected recovery target timeline ID and
         * archive_cleanup_command in shared memory so that other processes can
@@ -5523,7 +5545,10 @@ StartupXLOG(void)
                {
                        /* initialize minRecoveryPoint if not set yet */
                        if (XLByteLT(ControlFile->minRecoveryPoint, checkPoint.redo))
+                       {
                                ControlFile->minRecoveryPoint = checkPoint.redo;
+                               ControlFile->minRecoveryPointTLI = checkPoint.ThisTimeLineID;
+                       }
                }
 
                /*
@@ -5556,6 +5581,7 @@ StartupXLOG(void)
 
                /* initialize our local copy of minRecoveryPoint */
                minRecoveryPoint = ControlFile->minRecoveryPoint;
+               minRecoveryPointTLI = ControlFile->minRecoveryPointTLI;
 
                /*
                 * Reset pgstat data, because it may be invalid after recovery.
@@ -5681,6 +5707,7 @@ StartupXLOG(void)
                 */
                SpinLockAcquire(&xlogctl->info_lck);
                xlogctl->replayEndRecPtr = ReadRecPtr;
+               xlogctl->replayEndTLI = ThisTimeLineID;
                xlogctl->recoveryLastRecPtr = EndRecPtr;
                xlogctl->recoveryLastXTime = 0;
                xlogctl->currentChunkStartTime = 0;
@@ -7202,6 +7229,7 @@ CreateCheckPoint(int flags)
        ControlFile->time = (pg_time_t) time(NULL);
        /* crash recovery should always recover to the end of WAL */
        MemSet(&ControlFile->minRecoveryPoint, 0, sizeof(XLogRecPtr));
+       ControlFile->minRecoveryPointTLI = 0;
        UpdateControlFile();
        LWLockRelease(ControlFileLock);
 
@@ -7878,16 +7906,42 @@ xlog_redo(XLogRecPtr lsn, XLogRecord *record)
                }
 
                /*
-                * TLI may change in a shutdown checkpoint, but it shouldn't decrease
+                * TLI may change in a shutdown checkpoint.
                 */
                if (checkPoint.ThisTimeLineID != ThisTimeLineID)
                {
+                       /*
+                        * The new timeline better be in the list of timelines we expect
+                        * to see, according to the timeline history. It should also not
+                        * decrease.
+                        */
                        if (checkPoint.ThisTimeLineID < ThisTimeLineID ||
                                !list_member_int(expectedTLIs,
                                                                 (int) checkPoint.ThisTimeLineID))
                                ereport(PANIC,
                                                (errmsg("unexpected timeline ID %u (after %u) in checkpoint record",
                                                                checkPoint.ThisTimeLineID, ThisTimeLineID)));
+
+                       /*
+                        * If we have not yet reached min recovery point, and we're about
+                        * to switch to a timeline greater than the timeline of the min
+                        * recovery point: trouble. After switching to the new timeline,
+                        * we could not possibly visit the min recovery point on the
+                        * correct timeline anymore. This can happen if there is a newer
+                        * timeline in the archive that branched before the timeline the
+                        * min recovery point is on, and you attempt to do PITR to the
+                        * new timeline.
+                        */
+                       if (!XLogRecPtrIsInvalid(minRecoveryPoint) &&
+                               XLByteLT(lsn, minRecoveryPoint) &&
+                               checkPoint.ThisTimeLineID > minRecoveryPointTLI)
+                               ereport(PANIC,
+                                               (errmsg("unexpected timeline ID %u in checkpoint record, before reaching minimum recovery point %X/%X on timeline %u",
+                                                               checkPoint.ThisTimeLineID,
+                                                               (uint32) (minRecoveryPoint >> 32),
+                                                               (uint32) minRecoveryPoint,
+                                                               minRecoveryPointTLI)));
+
                        /* Following WAL records should be run with new TLI */
                        ThisTimeLineID = checkPoint.ThisTimeLineID;
                }
@@ -7972,7 +8026,10 @@ xlog_redo(XLogRecPtr lsn, XLogRecord *record)
                        LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
 
                        if (XLByteLT(ControlFile->minRecoveryPoint, lsn))
+                       {
                                ControlFile->minRecoveryPoint = lsn;
+                               ControlFile->minRecoveryPointTLI = ThisTimeLineID;
+                       }
                        MemSet(&ControlFile->backupStartPoint, 0, sizeof(XLogRecPtr));
                        ControlFile->backupEndRequired = false;
                        UpdateControlFile();
@@ -8002,9 +8059,11 @@ xlog_redo(XLogRecPtr lsn, XLogRecord *record)
                 * decreasing max_* settings.
                 */
                minRecoveryPoint = ControlFile->minRecoveryPoint;
+               minRecoveryPointTLI = ControlFile->minRecoveryPointTLI;
                if (minRecoveryPoint != 0 && XLByteLT(minRecoveryPoint, lsn))
                {
                        ControlFile->minRecoveryPoint = lsn;
+                       ControlFile->minRecoveryPointTLI = ThisTimeLineID;
                }
 
                UpdateControlFile();

diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 129c4d0dda3588e5444ccd9baddc14bdbe35f82e..0fe68bb9e1357d9c9849d39977883da5626a8e37 100644 (file)
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -234,9 +234,11 @@ main(int argc, char *argv[])
                   ControlFile.checkPointCopy.oldestActiveXid);
        printf(_("Time of latest checkpoint:            %s\n"),
                   ckpttime_str);
-       printf(_("Minimum recovery ending location:     %X/%X\n"),
+       printf(_("Min recovery ending location:         %X/%X\n"),
                   (uint32) (ControlFile.minRecoveryPoint >> 32),
                   (uint32) ControlFile.minRecoveryPoint);
+       printf(_("Min recovery ending loc's timeline:   %u\n"),
+                  ControlFile.minRecoveryPointTLI);
        printf(_("Backup start location:                %X/%X\n"),
                   (uint32) (ControlFile.backupStartPoint >> 32),
                   (uint32) ControlFile.backupStartPoint);

diff --git a/src/bin/pg_resetxlog/pg_resetxlog.c b/src/bin/pg_resetxlog/pg_resetxlog.c
index 3d9a6698b390434342c195662c56c5953603c770..35be03d9db5fee74c35e6ee886d18f6b9ecd28f7 100644 (file)
--- a/src/bin/pg_resetxlog/pg_resetxlog.c
+++ b/src/bin/pg_resetxlog/pg_resetxlog.c
@@ -610,6 +610,7 @@ RewriteControlFile(void)
        ControlFile.checkPoint = ControlFile.checkPointCopy.redo;
        ControlFile.prevCheckPoint = 0;
        ControlFile.minRecoveryPoint = 0;
+       ControlFile.minRecoveryPointTLI = 0;
        ControlFile.backupStartPoint = 0;
        ControlFile.backupEndPoint = 0;
        ControlFile.backupEndRequired = false;

diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index c9ee80531c3b122392f5a5d210ed8e4fdce055ee..1408be9c3a1c4f762795a148c29d721cffeedc84 100644 (file)
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -21,7 +21,7 @@
 
 
 /* Version identifier for this pg_control format */
-#define PG_CONTROL_VERSION     931
+#define PG_CONTROL_VERSION     932
 
 /*
  * Body of CheckPoint XLOG records.  This is declared here because we keep
@@ -153,6 +153,7 @@ typedef struct ControlFileData
         * pg_start_backup() call, not accompanied by pg_stop_backup().
         */
        XLogRecPtr      minRecoveryPoint;
+       TimeLineID      minRecoveryPointTLI;
        XLogRecPtr      backupStartPoint;
        XLogRecPtr      backupEndPoint;
        bool            backupEndRequired;