static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir);
static void parse_basebackup_options(List *options, basebackup_options *opt);
static void SendXlogRecPtrResult(XLogRecPtr ptr);
+static int compareWalFileNames(const void *a, const void *b);
/*
* Size of each block sent into the tar stream for larger files.
- *
- * XLogSegSize *MUST* be evenly dividable by this
*/
#define TAR_SEND_SIZE 32768
* We've left the last tar file "open", so we can now append the
* required WAL files to it.
*/
+ char pathbuf[MAXPGPATH];
uint32 logid,
logseg;
+ uint32 startlogid,
+ startlogseg;
uint32 endlogid,
endlogseg;
struct stat statbuf;
+ List *historyFileList = NIL;
+ List *walFileList = NIL;
+ char **walFiles;
+ int nWalFiles;
+ char firstoff[MAXFNAMELEN];
+ char lastoff[MAXFNAMELEN];
+ DIR *dir;
+ struct dirent *de;
+ int i;
+ ListCell *lc;
+ TimeLineID tli;
- MemSet(&statbuf, 0, sizeof(statbuf));
- statbuf.st_mode = S_IRUSR | S_IWUSR;
-#ifndef WIN32
- statbuf.st_uid = geteuid();
- statbuf.st_gid = getegid();
-#endif
- statbuf.st_size = XLogSegSize;
- statbuf.st_mtime = time(NULL);
-
- XLByteToSeg(startptr, logid, logseg);
+ /*
+ * I'd rather not worry about timelines here, so scan pg_xlog and
+ * include all WAL files in the range between 'startptr' and 'endptr',
+ * regardless of the timeline the file is stamped with. If there are
+ * some spurious WAL files belonging to timelines that don't belong
+ * in this server's history, they will be included too. Normally there
+ * shouldn't be such files, but if there are, there's little harm in
+ * including them.
+ */
+ XLByteToSeg(startptr, startlogid, startlogseg);
+ XLogFileName(firstoff, ThisTimeLineID, startlogid, startlogseg);
XLByteToPrevSeg(endptr, endlogid, endlogseg);
+ XLogFileName(lastoff, ThisTimeLineID, endlogid, endlogseg);
- while (true)
+ dir = AllocateDir("pg_xlog");
+ if (!dir)
+ ereport(ERROR,
+ (errmsg("could not open directory \"%s\": %m", "pg_xlog")));
+ while ((de = ReadDir(dir, "pg_xlog")) != NULL)
{
- /* Send another xlog segment */
- char fn[MAXPGPATH];
- int i;
+ /* Does it look like a WAL segment, and is it in the range? */
+ if (strlen(de->d_name) == 24 &&
+ strspn(de->d_name, "0123456789ABCDEF") == 24 &&
+ strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
+ strcmp(de->d_name + 8, lastoff + 8) <= 0)
+ {
+ walFileList = lappend(walFileList, pstrdup(de->d_name));
+ }
+ /* Does it look like a timeline history file? */
+ else if (strlen(de->d_name) == 8 + strlen(".history") &&
+ strspn(de->d_name, "0123456789ABCDEF") == 8 &&
+ strcmp(de->d_name + 8, ".history") == 0)
+ {
+ historyFileList = lappend(historyFileList, pstrdup(de->d_name));
+ }
+ }
+ FreeDir(dir);
- XLogFilePath(fn, ThisTimeLineID, logid, logseg);
- _tarWriteHeader(fn, NULL, &statbuf);
+ /*
+ * Before we go any further, check that none of the WAL segments we
+ * need were removed.
+ */
+ CheckXLogRemoved(startlogid, startlogseg, ThisTimeLineID);
+
+ /*
+ * Put the WAL filenames into an array, and sort. We send the files
+ * in order from oldest to newest, to reduce the chance that a file
+ * is recycled before we get a chance to send it over.
+ */
+ nWalFiles = list_length(walFileList);
+ walFiles = palloc(nWalFiles * sizeof(char *));
+ i = 0;
+ foreach(lc, walFileList)
+ {
+ walFiles[i++] = lfirst(lc);
+ }
+ qsort(walFiles, nWalFiles, sizeof(char *), compareWalFileNames);
- /* Send the actual WAL file contents, block-by-block */
- for (i = 0; i < XLogSegSize / TAR_SEND_SIZE; i++)
+ /*
+ * Sanity check: the first and last segment should cover startptr and
+ * endptr, with no gaps in between.
+ */
+ XLogFromFileName(walFiles[0], &tli, &logid, &logseg);
+ if (logid != startlogid || logseg != startlogseg)
+ {
+ char startfname[MAXFNAMELEN];
+ XLogFileName(startfname, ThisTimeLineID, startlogid, startlogseg);
+ ereport(ERROR,
+ (errmsg("could not find WAL file %s", startfname)));
+ }
+ for (i = 0; i < nWalFiles; i++)
+ {
+ int currlogid = logid,
+ currlogseg = logseg;
+ int nextlogid = logid,
+ nextlogseg = logseg;
+ NextLogSeg(nextlogid, nextlogseg);
+
+ XLogFromFileName(walFiles[i], &tli, &logid, &logseg);
+ if (!((nextlogid == logid && nextlogseg == logseg) ||
+ (currlogid == logid && currlogseg == logseg)))
{
- char buf[TAR_SEND_SIZE];
- XLogRecPtr ptr;
+ char nextfname[MAXFNAMELEN];
+ XLogFileName(nextfname, ThisTimeLineID, nextlogid, nextlogseg);
+ ereport(ERROR,
+ (errmsg("could not find WAL file %s", nextfname)));
+ }
+ }
+ if (logid != endlogid || logseg != endlogseg)
+ {
+ char endfname[MAXFNAMELEN];
+ XLogFileName(endfname, ThisTimeLineID, endlogid, endlogseg);
+ ereport(ERROR,
+ (errmsg("could not find WAL file %s", endfname)));
+ }
+
+ /* Ok, we have everything we need. Send the WAL files. */
+ for (i = 0; i < nWalFiles; i++)
+ {
+ FILE *fp;
+ char buf[TAR_SEND_SIZE];
+ size_t cnt;
+ pgoff_t len = 0;
- ptr.xlogid = logid;
- ptr.xrecoff = logseg * XLogSegSize + TAR_SEND_SIZE * i;
+ snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
+ XLogFromFileName(walFiles[i], &tli, &logid, &logseg);
+ fp = AllocateFile(pathbuf, "rb");
+ if (fp == NULL)
+ {
/*
- * Some old compilers, e.g. gcc 2.95.3/x86, think that passing
- * a struct in the same function as a longjump might clobber a
- * variable. bjm 2011-02-04
- * http://lists.apple.com/archives/xcode-users/2003/Dec//msg000
- * 51.html
+ * Most likely reason for this is that the file was already
+ * removed by a checkpoint, so check for that to get a better
+ * error message.
*/
- XLogRead(buf, ptr, TAR_SEND_SIZE);
- if (pq_putmessage('d', buf, TAR_SEND_SIZE))
+ CheckXLogRemoved(logid, logseg, tli);
+
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\": %m", pathbuf)));
+ }
+
+ if (fstat(fileno(fp), &statbuf) != 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ pathbuf)));
+ if (statbuf.st_size != XLogSegSize)
+ {
+ CheckXLogRemoved(logid, logseg, tli);
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
+ }
+
+ _tarWriteHeader(pathbuf, NULL, &statbuf);
+
+ while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0)
+ {
+ CheckXLogRemoved(logid, logseg, tli);
+ /* Send the chunk as a CopyData message */
+ if (pq_putmessage('d', buf, cnt))
ereport(ERROR,
(errmsg("base backup could not send data, aborting backup")));
+
+ len += cnt;
+ if (len == XLogSegSize)
+ break;
}
- /*
- * Files are always fixed size, and always end on a 512 byte
- * boundary, so padding is never necessary.
- */
+ if (len != XLogSegSize)
+ {
+ CheckXLogRemoved(logid, logseg, tli);
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
+ }
+ /* XLogSegSize is a multiple of 512, so no need for padding */
+ FreeFile(fp);
+ }
+
+ /*
+ * Send timeline history files too. Only the latest timeline history
+ * file is required for recovery, and even that only if there happens
+ * to be a timeline switch in the first WAL segment that contains the
+ * checkpoint record, or if we're taking a base backup from a standby
+ * server and the target timeline changes while the backup is taken.
+ * But they are small and highly useful for debugging purposes, so
+ * better include them all, always.
+ */
+ foreach(lc, historyFileList)
+ {
+ char *fname = lfirst(lc);
+ snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
- /* Advance to the next WAL file */
- NextLogSeg(logid, logseg);
+ if (lstat(pathbuf, &statbuf) != 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m", pathbuf)));
- /* Have we reached our stop position yet? */
- if (logid > endlogid ||
- (logid == endlogid && logseg > endlogseg))
- break;
+ sendFile(pathbuf, pathbuf, &statbuf, false);
}
/* Send CopyDone message for the last tar file */
SendXlogRecPtrResult(endptr);
}
+/*
+ * qsort comparison function, to compare log/seg portion of WAL segment
+ * filenames, ignoring the timeline portion.
+ */
+static int
+compareWalFileNames(const void *a, const void *b)
+{
+ char *fna = *((char **) a);
+ char *fnb = *((char **) b);
+
+ return strcmp(fna + 8, fnb + 8);
+}
+
/*
* Parse the base backup options passed down by the parser
*/
projects / postgresql / commitdiff