with_blocksize
with_segsize
with_wal_blocksize
-with_wal_segsize
with_CC
enable_depend
enable_cassert
--with-segsize=SEGSIZE set table segment size in GB [1]
--with-wal-blocksize=BLOCKSIZE
set WAL block size in kB [8]
- --with-wal-segsize=SEGSIZE
- set WAL segment size in MB [16]
--with-CC=CMD set compiler (deprecated)
--with-icu build with ICU support
--with-tcl build Tcl modules (PL/Tcl)
_ACEOF
-#
-# WAL segment size
-#
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for WAL segment size" >&5
-$as_echo_n "checking for WAL segment size... " >&6; }
-
-
-
-# Check whether --with-wal-segsize was given.
-if test "${with_wal_segsize+set}" = set; then :
- withval=$with_wal_segsize;
- case $withval in
- yes)
- as_fn_error $? "argument required for --with-wal-segsize option" "$LINENO" 5
- ;;
- no)
- as_fn_error $? "argument required for --with-wal-segsize option" "$LINENO" 5
- ;;
- *)
- wal_segsize=$withval
- ;;
- esac
-
-else
- wal_segsize=16
-fi
-
-
-case ${wal_segsize} in
- 1) ;;
- 2) ;;
- 4) ;;
- 8) ;;
- 16) ;;
- 32) ;;
- 64) ;;
- 128) ;;
- 256) ;;
- 512) ;;
- 1024) ;;
- *) as_fn_error $? "Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024." "$LINENO" 5
-esac
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: ${wal_segsize}MB" >&5
-$as_echo "${wal_segsize}MB" >&6; }
-
-
-cat >>confdefs.h <<_ACEOF
-#define XLOG_SEG_SIZE (${wal_segsize} * 1024 * 1024)
-_ACEOF
-
-
#
# C compiler
#
Changing XLOG_BLCKSZ requires an initdb.
])
-#
-# WAL segment size
-#
-AC_MSG_CHECKING([for WAL segment size])
-PGAC_ARG_REQ(with, wal-segsize, [SEGSIZE], [set WAL segment size in MB [16]],
- [wal_segsize=$withval],
- [wal_segsize=16])
-case ${wal_segsize} in
- 1) ;;
- 2) ;;
- 4) ;;
- 8) ;;
- 16) ;;
- 32) ;;
- 64) ;;
- 128) ;;
- 256) ;;
- 512) ;;
- 1024) ;;
- *) AC_MSG_ERROR([Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024.])
-esac
-AC_MSG_RESULT([${wal_segsize}MB])
-
-AC_DEFINE_UNQUOTED([XLOG_SEG_SIZE], [(${wal_segsize} * 1024 * 1024)], [
- XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2
- and larger than XLOG_BLCKSZ (preferably, a great deal larger than
- XLOG_BLCKSZ).
-
- Changing XLOG_SEG_SIZE requires an initdb.
-])
-
#
# C compiler
#
const char *progname;
+int WalSegSz = -1;
+
/* Options and defaults */
int sleeptime = 5; /* amount of time to sleep between file checks */
int waittime = -1; /* how long we have been waiting, -1 no wait
struct stat stat_buf;
+static bool SetWALFileNameForCleanup(void);
+static bool SetWALSegSize(void);
+
+
/* =====================================================================
*
* Customizable section
{
if (stat(WALFilePath, &stat_buf) == 0)
{
+ /*
+ * If we've not seen any WAL segments, we don't know the WAL segment
+ * size, which we need. If it looks like a WAL segment, determine size
+ * of segments for the cluster.
+ */
+ if (WalSegSz == -1 && IsXLogFileName(nextWALFileName))
+ {
+ if (SetWALSegSize())
+ {
+ /*
+ * Successfully determined WAL segment size. Can compute
+ * cleanup cutoff now.
+ */
+ need_cleanup = SetWALFileNameForCleanup();
+ if (debug)
+ {
+ fprintf(stderr,
+ _("WAL segment size: %d \n"), WalSegSz);
+ fprintf(stderr, "Keep archive history: ");
+
+ if (need_cleanup)
+ fprintf(stderr, "%s and later\n",
+ exclusiveCleanupFileName);
+ else
+ fprintf(stderr, "no cleanup required\n");
+ }
+ }
+ }
+
/*
* If it's a backup file, return immediately. If it's a regular file
* return only if it's the right size already.
nextWALFileType = XLOG_BACKUP_LABEL;
return true;
}
- else if (stat_buf.st_size == XLOG_SEG_SIZE)
+ else if (WalSegSz > 0 && stat_buf.st_size == WalSegSz)
{
#ifdef WIN32
/*
* If still too small, wait until it is the correct size
*/
- if (stat_buf.st_size > XLOG_SEG_SIZE)
+ if (WalSegSz > 0 && stat_buf.st_size > WalSegSz)
{
if (debug)
{
return false;
}
-#define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE )
-
static void
CustomizableCleanupPriorWALFiles(void)
{
uint32 log_diff = 0,
seg_diff = 0;
bool cleanup = false;
+ int max_segments_per_logfile = (0xFFFFFFFF / WalSegSz);
if (restartWALFileName)
{
sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
if (tli > 0 && seg > 0)
{
- log_diff = keepfiles / MaxSegmentsPerLogFile;
- seg_diff = keepfiles % MaxSegmentsPerLogFile;
+ log_diff = keepfiles / max_segments_per_logfile;
+ seg_diff = keepfiles % max_segments_per_logfile;
if (seg_diff > seg)
{
log_diff++;
- seg = MaxSegmentsPerLogFile - (seg_diff - seg);
+ seg = max_segments_per_logfile - (seg_diff - seg);
}
else
seg -= seg_diff;
return cleanup;
}
+/*
+ * Try to set the wal segment size from the WAL file specified by WALFilePath.
+ *
+ * Return true if size could be determined, false otherwise.
+ */
+static bool
+SetWALSegSize(void)
+{
+ bool ret_val = false;
+ int fd;
+ char *buf = (char *) malloc(XLOG_BLCKSZ);
+
+ Assert(WalSegSz == -1);
+
+ if ((fd = open(WALFilePath, O_RDWR, 0)) < 0)
+ {
+ fprintf(stderr, "%s: couldn't open WAL file \"%s\"\n",
+ progname, WALFilePath);
+ return false;
+ }
+ if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ)
+ {
+ XLogLongPageHeader longhdr = (XLogLongPageHeader) buf;
+
+ WalSegSz = longhdr->xlp_seg_size;
+
+ if (IsValidWalSegSize(WalSegSz))
+ {
+ /* successfully retrieved WAL segment size */
+ ret_val = true;
+ }
+ else
+ fprintf(stderr,
+ "%s: WAL segment size must be a power of two between 1MB and 1GB, but the WAL file header specifies %d bytes\n",
+ progname, WalSegSz);
+ close(fd);
+ }
+ else
+ {
+ /*
+ * Don't complain loudly, this is to be expected for segments being
+ * created.
+ */
+ if (errno != 0)
+ {
+ if (debug)
+ fprintf(stderr, "could not read file \"%s\": %s",
+ WALFilePath, strerror(errno));
+ }
+ else
+ {
+ if (debug)
+ fprintf(stderr, "not enough data in file \"%s\"", WALFilePath);
+ }
+ }
+
+ fflush(stderr);
+ return ret_val;
+}
+
/*
* CheckForExternalTrigger()
*
CustomizableInitialize();
- need_cleanup = SetWALFileNameForCleanup();
-
if (debug)
{
fprintf(stderr, "Trigger file: %s\n", triggerPath ? triggerPath : "<not set>");
fprintf(stderr, "Max wait interval: %d %s\n",
maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
fprintf(stderr, "Command for restore: %s\n", restoreCommand);
- fprintf(stderr, "Keep archive history: ");
- if (need_cleanup)
- fprintf(stderr, "%s and later\n", exclusiveCleanupFileName);
- else
- fprintf(stderr, "no cleanup required\n");
fflush(stderr);
}
produces an indefinitely long sequence of WAL records. The system
physically divides this sequence into WAL <firstterm>segment
files</>, which are normally 16MB apiece (although the segment size
- can be altered when building <productname>PostgreSQL</>). The segment
+ can be altered during <application>initdb</>). The segment
files are given numeric names that reflect their position in the
abstract WAL sequence. When not using WAL archiving, the system
normally creates just a few segment files and then
</listitem>
</varlistentry>
- <varlistentry>
- <term><option>--with-wal-segsize=<replaceable>SEGSIZE</replaceable></option></term>
- <listitem>
- <para>
- Set the <firstterm>WAL segment size</>, in megabytes. This is
- the size of each individual file in the WAL log. It may be useful
- to adjust this size to control the granularity of WAL log shipping.
- The default size is 16 megabytes.
- The value must be a power of 2 between 1 and 1024 (megabytes).
- Note that changing this value requires an initdb.
- </para>
- </listitem>
- </varlistentry>
-
<varlistentry>
<term><option>--with-wal-blocksize=<replaceable>BLOCKSIZE</replaceable></option></term>
<listitem>
</varlistentry>
<varlistentry>
+ <term><option>--wal-segsize=<replaceable>SEGSIZE</replaceable></option></term>
+ <listitem>
+ <para>
+ Set the <firstterm>WAL segment size</>, in megabytes. This is
+ the size of each individual file in the WAL log. It may be useful
+ to adjust this size to control the granularity of WAL log shipping.
+ This option can only be set during initialization, and cannot be
+ changed later.
+ The default size is 16 megabytes.
+ The value must be a power of 2 between 1 and 1024 (megabytes).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-X <replaceable class="parameter">directory</replaceable></option></term>
<term><option>--waldir=<replaceable class="parameter">directory</replaceable></option></term>
<listitem>
<acronym>WAL</acronym> logs are stored in the directory
<filename>pg_wal</filename> under the data directory, as a set of
segment files, normally each 16 MB in size (but the size can be changed
- by altering the <option>--with-wal-segsize</> configure option when
- building the server). Each segment is divided into pages, normally
- 8 kB each (this size can be changed via the <option>--with-wal-blocksize</>
- configure option). The log record headers are described in
- <filename>access/xlogrecord.h</filename>; the record content is dependent
- on the type of event that is being logged. Segment files are given
- ever-increasing numbers as names, starting at
+ by altering the <option>--wal-segsize</> initdb option). Each segment is
+ divided into pages, normally 8 kB each (this size can be changed via the
+ <option>--with-wal-blocksize</> configure option). The log record headers
+ are described in <filename>access/xlogrecord.h</filename>; the record
+ content is dependent on the type of event that is being logged. Segment
+ files are given ever-increasing numbers as names, starting at
<filename>000000010000000000000000</filename>. The numbers do not wrap,
but it will take a very, very long time to exhaust the
available stock of numbers.
XLogReaderState *xlogreader;
char *errormsg;
- xlogreader = XLogReaderAllocate(&read_local_xlog_page, NULL);
+ xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page,
+ NULL);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
bool XLOG_DEBUG = false;
#endif
+int wal_segment_size = DEFAULT_XLOG_SEG_SIZE;
+
/*
* Number of WAL insertion locks to use. A higher value allows more insertions
* to happen concurrently, but adds some CPU overhead to flushing the WAL,
(((recptr) / XLOG_BLCKSZ) % (XLogCtl->XLogCacheBlck + 1))
/*
- * These are the number of bytes in a WAL page and segment usable for WAL data.
+ * These are the number of bytes in a WAL page usable for WAL data.
*/
#define UsableBytesInPage (XLOG_BLCKSZ - SizeOfXLogShortPHD)
-#define UsableBytesInSegment ((XLOG_SEG_SIZE / XLOG_BLCKSZ) * UsableBytesInPage - (SizeOfXLogLongPHD - SizeOfXLogShortPHD))
/* Convert min_wal_size_mb and max wal_size_mb to equivalent segment count */
-#define ConvertToXSegs(x) \
- (x / (XLOG_SEG_SIZE / (1024 * 1024)))
+#define ConvertToXSegs(x, segsize) \
+ (x / ((segsize) / (1024 * 1024)))
+
+/* The number of bytes in a WAL segment usable for WAL data. */
+static int UsableBytesInSegment;
/*
* Private, possibly out-of-date copy of shared LogwrtResult.
EndPos = StartPos + SizeOfXLogRecord;
if (StartPos / XLOG_BLCKSZ != EndPos / XLOG_BLCKSZ)
{
- if (EndPos % XLOG_SEG_SIZE == EndPos % XLOG_BLCKSZ)
+ uint64 offset = XLogSegmentOffset(EndPos, wal_segment_size);
+
+ if (offset == EndPos % XLOG_BLCKSZ)
EndPos += SizeOfXLogLongPHD;
else
EndPos += SizeOfXLogShortPHD;
appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
if (!debug_reader)
- debug_reader = XLogReaderAllocate(NULL, NULL);
+ debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
if (!debug_reader)
{
startbytepos = Insert->CurrBytePos;
ptr = XLogBytePosToEndRecPtr(startbytepos);
- if (ptr % XLOG_SEG_SIZE == 0)
+ if (XLogSegmentOffset(ptr, wal_segment_size) == 0)
{
SpinLockRelease(&Insert->insertpos_lck);
*EndPos = *StartPos = ptr;
*StartPos = XLogBytePosToRecPtr(startbytepos);
*EndPos = XLogBytePosToEndRecPtr(endbytepos);
- segleft = XLOG_SEG_SIZE - ((*EndPos) % XLOG_SEG_SIZE);
- if (segleft != XLOG_SEG_SIZE)
+ segleft = wal_segment_size - XLogSegmentOffset(*EndPos, wal_segment_size);
+ if (segleft != wal_segment_size)
{
/* consume the rest of the segment */
*EndPos += segleft;
*PrevPtr = XLogBytePosToRecPtr(prevbytepos);
- Assert((*EndPos) % XLOG_SEG_SIZE == 0);
+ Assert(XLogSegmentOffset(*EndPos, wal_segment_size) == 0);
Assert(XLogRecPtrToBytePos(*EndPos) == endbytepos);
Assert(XLogRecPtrToBytePos(*StartPos) == startbytepos);
Assert(XLogRecPtrToBytePos(*PrevPtr) == prevbytepos);
pagehdr->xlp_info |= XLP_FIRST_IS_CONTRECORD;
/* skip over the page header */
- if (CurrPos % XLogSegSize == 0)
+ if (XLogSegmentOffset(CurrPos, wal_segment_size) == 0)
{
CurrPos += SizeOfXLogLongPHD;
currpos += SizeOfXLogLongPHD;
* allocated and zeroed in the WAL buffers so that when the caller (or
* someone else) does XLogWrite(), it can really write out all the zeros.
*/
- if (isLogSwitch && CurrPos % XLOG_SEG_SIZE != 0)
+ if (isLogSwitch && XLogSegmentOffset(CurrPos, wal_segment_size) != 0)
{
/* An xlog-switch record doesn't contain any data besides the header */
Assert(write_len == SizeOfXLogRecord);
/*
* We do this one page at a time, to make sure we don't deadlock
- * against ourselves if wal_buffers < XLOG_SEG_SIZE.
+ * against ourselves if wal_buffers < wal_segment_size.
*/
- Assert(EndPos % XLogSegSize == 0);
+ Assert(XLogSegmentOffset(EndPos, wal_segment_size) == 0);
/* Use up all the remaining space on the first page */
CurrPos += freespace;
* the page header.
*/
if (ptr % XLOG_BLCKSZ == SizeOfXLogShortPHD &&
- ptr % XLOG_SEG_SIZE > XLOG_BLCKSZ)
+ XLogSegmentOffset(ptr, wal_segment_size) > XLOG_BLCKSZ)
initializedUpto = ptr - SizeOfXLogShortPHD;
else if (ptr % XLOG_BLCKSZ == SizeOfXLogLongPHD &&
- ptr % XLOG_SEG_SIZE < XLOG_BLCKSZ)
+ XLogSegmentOffset(ptr, wal_segment_size) < XLOG_BLCKSZ)
initializedUpto = ptr - SizeOfXLogLongPHD;
else
initializedUpto = ptr;
seg_offset += fullpages * XLOG_BLCKSZ + bytesleft + SizeOfXLogShortPHD;
}
- XLogSegNoOffsetToRecPtr(fullsegs, seg_offset, result);
+ XLogSegNoOffsetToRecPtr(fullsegs, seg_offset, result, wal_segment_size);
return result;
}
seg_offset += fullpages * XLOG_BLCKSZ + bytesleft + SizeOfXLogShortPHD;
}
- XLogSegNoOffsetToRecPtr(fullsegs, seg_offset, result);
+ XLogSegNoOffsetToRecPtr(fullsegs, seg_offset, result, wal_segment_size);
return result;
}
uint32 offset;
uint64 result;
- XLByteToSeg(ptr, fullsegs);
+ XLByteToSeg(ptr, fullsegs, wal_segment_size);
- fullpages = (ptr % XLOG_SEG_SIZE) / XLOG_BLCKSZ;
+ fullpages = (XLogSegmentOffset(ptr, wal_segment_size)) / XLOG_BLCKSZ;
offset = ptr % XLOG_BLCKSZ;
if (fullpages == 0)
/*
* If first page of an XLOG segment file, make it a long header.
*/
- if ((NewPage->xlp_pageaddr % XLogSegSize) == 0)
+ if ((XLogSegmentOffset(NewPage->xlp_pageaddr, wal_segment_size)) == 0)
{
XLogLongPageHeader NewLongPage = (XLogLongPageHeader) NewPage;
NewLongPage->xlp_sysid = ControlFile->system_identifier;
- NewLongPage->xlp_seg_size = XLogSegSize;
+ NewLongPage->xlp_seg_size = wal_segment_size;
NewLongPage->xlp_xlog_blcksz = XLOG_BLCKSZ;
NewPage->xlp_info |= XLP_LONG_HEADER;
}
* number of segments consumed between checkpoints.
*-------
*/
- target = (double) ConvertToXSegs(max_wal_size_mb) / (2.0 + CheckPointCompletionTarget);
+ target = (double) ConvertToXSegs(max_wal_size_mb, wal_segment_size) /
+ (2.0 + CheckPointCompletionTarget);
/* round down */
CheckPointSegments = (int) target;
* correspond to. Always recycle enough segments to meet the minimum, and
* remove enough segments to stay below the maximum.
*/
- minSegNo = PriorRedoPtr / XLOG_SEG_SIZE + ConvertToXSegs(min_wal_size_mb) - 1;
- maxSegNo = PriorRedoPtr / XLOG_SEG_SIZE + ConvertToXSegs(max_wal_size_mb) - 1;
+ minSegNo = PriorRedoPtr / wal_segment_size +
+ ConvertToXSegs(min_wal_size_mb, wal_segment_size) - 1;
+ maxSegNo = PriorRedoPtr / wal_segment_size +
+ ConvertToXSegs(max_wal_size_mb, wal_segment_size) - 1;
/*
* Between those limits, recycle enough segments to get us through to the
/* add 10% for good measure. */
distance *= 1.10;
- recycleSegNo = (XLogSegNo) ceil(((double) PriorRedoPtr + distance) / XLOG_SEG_SIZE);
+ recycleSegNo = (XLogSegNo) ceil(((double) PriorRedoPtr + distance) /
+ wal_segment_size);
if (recycleSegNo < minSegNo)
recycleSegNo = minSegNo;
{
XLogSegNo old_segno;
- XLByteToSeg(RedoRecPtr, old_segno);
+ XLByteToSeg(RedoRecPtr, old_segno, wal_segment_size);
if (new_segno >= old_segno + (uint64) (CheckPointSegments - 1))
return true;
LogwrtResult.Write = EndPtr;
ispartialpage = WriteRqst.Write < LogwrtResult.Write;
- if (!XLByteInPrevSeg(LogwrtResult.Write, openLogSegNo))
+ if (!XLByteInPrevSeg(LogwrtResult.Write, openLogSegNo,
+ wal_segment_size))
{
/*
* Switch to new logfile segment. We cannot have any pending
Assert(npages == 0);
if (openLogFile >= 0)
XLogFileClose();
- XLByteToPrevSeg(LogwrtResult.Write, openLogSegNo);
+ XLByteToPrevSeg(LogwrtResult.Write, openLogSegNo,
+ wal_segment_size);
/* create/use new log file */
use_existent = true;
/* Make sure we have the current logfile open */
if (openLogFile < 0)
{
- XLByteToPrevSeg(LogwrtResult.Write, openLogSegNo);
+ XLByteToPrevSeg(LogwrtResult.Write, openLogSegNo,
+ wal_segment_size);
openLogFile = XLogFileOpen(openLogSegNo);
openLogOff = 0;
}
{
/* first of group */
startidx = curridx;
- startoffset = (LogwrtResult.Write - XLOG_BLCKSZ) % XLogSegSize;
+ startoffset = XLogSegmentOffset(LogwrtResult.Write - XLOG_BLCKSZ,
+ wal_segment_size);
}
npages++;
last_iteration = WriteRqst.Write <= LogwrtResult.Write;
finishing_seg = !ispartialpage &&
- (startoffset + npages * XLOG_BLCKSZ) >= XLogSegSize;
+ (startoffset + npages * XLOG_BLCKSZ) >= wal_segment_size;
if (last_iteration ||
curridx == XLogCtl->XLogCacheBlck ||
sync_method != SYNC_METHOD_OPEN_DSYNC)
{
if (openLogFile >= 0 &&
- !XLByteInPrevSeg(LogwrtResult.Write, openLogSegNo))
+ !XLByteInPrevSeg(LogwrtResult.Write, openLogSegNo,
+ wal_segment_size))
XLogFileClose();
if (openLogFile < 0)
{
- XLByteToPrevSeg(LogwrtResult.Write, openLogSegNo);
+ XLByteToPrevSeg(LogwrtResult.Write, openLogSegNo,
+ wal_segment_size);
openLogFile = XLogFileOpen(openLogSegNo);
openLogOff = 0;
}
{
if (openLogFile >= 0)
{
- if (!XLByteInPrevSeg(LogwrtResult.Write, openLogSegNo))
+ if (!XLByteInPrevSeg(LogwrtResult.Write, openLogSegNo,
+ wal_segment_size))
{
XLogFileClose();
}
int fd;
int nbytes;
- XLogFilePath(path, ThisTimeLineID, logsegno);
+ XLogFilePath(path, ThisTimeLineID, logsegno, wal_segment_size);
/*
* Try to use existent file (checkpoint maker may have created it already)
*/
zbuffer = (char *) MAXALIGN(zbuffer_raw);
memset(zbuffer, 0, XLOG_BLCKSZ);
- for (nbytes = 0; nbytes < XLogSegSize; nbytes += XLOG_BLCKSZ)
+ for (nbytes = 0; nbytes < wal_segment_size; nbytes += XLOG_BLCKSZ)
{
errno = 0;
pgstat_report_wait_start(WAIT_EVENT_WAL_INIT_WRITE);
/*
* Open the source file
*/
- XLogFilePath(path, srcTLI, srcsegno);
+ XLogFilePath(path, srcTLI, srcsegno, wal_segment_size);
srcfd = OpenTransientFile(path, O_RDONLY | PG_BINARY, 0);
if (srcfd < 0)
ereport(ERROR,
/*
* Do the data copying.
*/
- for (nbytes = 0; nbytes < XLogSegSize; nbytes += sizeof(buffer))
+ for (nbytes = 0; nbytes < wal_segment_size; nbytes += sizeof(buffer))
{
int nread;
char path[MAXPGPATH];
struct stat stat_buf;
- XLogFilePath(path, ThisTimeLineID, *segno);
+ XLogFilePath(path, ThisTimeLineID, *segno, wal_segment_size);
/*
* We want to be sure that only one process does this at a time.
return false;
}
(*segno)++;
- XLogFilePath(path, ThisTimeLineID, *segno);
+ XLogFilePath(path, ThisTimeLineID, *segno, wal_segment_size);
}
}
char path[MAXPGPATH];
int fd;
- XLogFilePath(path, ThisTimeLineID, segno);
+ XLogFilePath(path, ThisTimeLineID, segno, wal_segment_size);
fd = BasicOpenFile(path, O_RDWR | PG_BINARY | get_sync_bit(sync_method),
S_IRUSR | S_IWUSR);
char path[MAXPGPATH];
int fd;
- XLogFileName(xlogfname, tli, segno);
+ XLogFileName(xlogfname, tli, segno, wal_segment_size);
switch (source)
{
restoredFromArchive = RestoreArchivedFile(path, xlogfname,
"RECOVERYXLOG",
- XLogSegSize,
+ wal_segment_size,
InRedo);
if (!restoredFromArchive)
return -1;
case XLOG_FROM_PG_WAL:
case XLOG_FROM_STREAM:
- XLogFilePath(path, tli, segno);
+ XLogFilePath(path, tli, segno, wal_segment_size);
restoredFromArchive = false;
break;
}
/* Couldn't find it. For simplicity, complain about front timeline */
- XLogFilePath(path, recoveryTargetTLI, segno);
+ XLogFilePath(path, recoveryTargetTLI, segno, wal_segment_size);
errno = ENOENT;
ereport(emode,
(errcode_for_file_access(),
XLogSegNo _logSegNo;
int lf;
bool use_existent;
+ uint64 offset;
- XLByteToPrevSeg(endptr, _logSegNo);
- if ((endptr - 1) % XLogSegSize >= (uint32) (0.75 * XLogSegSize))
+ XLByteToPrevSeg(endptr, _logSegNo, wal_segment_size);
+ offset = XLogSegmentOffset(endptr - 1, wal_segment_size);
+ if (offset >= (uint32) (0.75 * wal_segment_size))
{
_logSegNo++;
use_existent = true;
{
char filename[MAXFNAMELEN];
- XLogFileName(filename, tli, segno);
+ XLogFileName(filename, tli, segno, wal_segment_size);
ereport(ERROR,
(errcode_for_file_access(),
errmsg("requested WAL segment %s has already been removed",
uint32 tli;
XLogSegNo segno;
- XLogFromFileName(filename, &tli, &segno);
+ XLogFromFileName(filename, &tli, &segno, wal_segment_size);
SpinLockAcquire(&XLogCtl->info_lck);
if (segno > XLogCtl->lastRemovedSegNo)
* doesn't matter, we ignore that in the comparison. (During recovery,
* ThisTimeLineID isn't set, so we can't use that.)
*/
- XLogFileName(lastoff, 0, segno);
+ XLogFileName(lastoff, 0, segno, wal_segment_size);
elog(DEBUG2, "attempting to remove WAL segments older than log file %s",
lastoff);
char switchseg[MAXFNAMELEN];
XLogSegNo endLogSegNo;
- XLByteToPrevSeg(switchpoint, endLogSegNo);
+ XLByteToPrevSeg(switchpoint, endLogSegNo, wal_segment_size);
xldir = AllocateDir(XLOGDIR);
if (xldir == NULL)
/*
* Construct a filename of the last segment to be kept.
*/
- XLogFileName(switchseg, newTLI, endLogSegNo);
+ XLogFileName(switchseg, newTLI, endLogSegNo, wal_segment_size);
elog(DEBUG2, "attempting to remove WAL segments newer than log file %s",
switchseg);
/*
* Initialize info about where to try to recycle to.
*/
- XLByteToSeg(endptr, endlogSegNo);
+ XLByteToSeg(endptr, endlogSegNo, wal_segment_size);
if (PriorRedoPtr == InvalidXLogRecPtr)
recycleSegNo = endlogSegNo + 10;
else
XLogSegNo segno;
int32 offset;
- XLByteToSeg(xlogreader->latestPagePtr, segno);
- offset = xlogreader->latestPagePtr % XLogSegSize;
- XLogFileName(fname, xlogreader->readPageTLI, segno);
+ XLByteToSeg(xlogreader->latestPagePtr, segno, wal_segment_size);
+ offset = XLogSegmentOffset(xlogreader->latestPagePtr,
+ wal_segment_size);
+ XLogFileName(fname, xlogreader->readPageTLI, segno,
+ wal_segment_size);
ereport(emode_for_corrupt_record(emode,
RecPtr ? RecPtr : EndRecPtr),
(errmsg("unexpected timeline ID %u in log segment %s, offset %u",
ControlFile->blcksz = BLCKSZ;
ControlFile->relseg_size = RELSEG_SIZE;
ControlFile->xlog_blcksz = XLOG_BLCKSZ;
- ControlFile->xlog_seg_size = XLOG_SEG_SIZE;
+ ControlFile->xlog_seg_size = wal_segment_size;
ControlFile->nameDataLen = NAMEDATALEN;
ControlFile->indexMaxKeys = INDEX_MAX_KEYS;
{
pg_crc32c crc;
int fd;
+ static char wal_segsz_str[20];
/*
* Read data...
" but the server was compiled with XLOG_BLCKSZ %d.",
ControlFile->xlog_blcksz, XLOG_BLCKSZ),
errhint("It looks like you need to recompile or initdb.")));
- if (ControlFile->xlog_seg_size != XLOG_SEG_SIZE)
- ereport(FATAL,
- (errmsg("database files are incompatible with server"),
- errdetail("The database cluster was initialized with XLOG_SEG_SIZE %d,"
- " but the server was compiled with XLOG_SEG_SIZE %d.",
- ControlFile->xlog_seg_size, XLOG_SEG_SIZE),
- errhint("It looks like you need to recompile or initdb.")));
if (ControlFile->nameDataLen != NAMEDATALEN)
ereport(FATAL,
(errmsg("database files are incompatible with server"),
errhint("It looks like you need to recompile or initdb.")));
#endif
+ wal_segment_size = ControlFile->xlog_seg_size;
+
+ if (!IsValidWalSegSize(wal_segment_size))
+ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("WAL segment size must be a power of two between 1MB and 1GB, but the control file specifies %d bytes",
+ wal_segment_size)));
+
+ snprintf(wal_segsz_str, sizeof(wal_segsz_str), "%d", wal_segment_size);
+ SetConfigOption("wal_segment_size", wal_segsz_str, PGC_INTERNAL,
+ PGC_S_OVERRIDE);
+
+ /* check and update variables dependent on wal_segment_size */
+ if (ConvertToXSegs(min_wal_size_mb, wal_segment_size) < 2)
+ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"min_wal_size\" must be at least twice \"wal_segment_size\".")));
+
+ if (ConvertToXSegs(max_wal_size_mb, wal_segment_size) < 2)
+ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"max_wal_size\" must be at least twice \"wal_segment_size\".")));
+
+ UsableBytesInSegment =
+ (wal_segment_size / XLOG_BLCKSZ * UsableBytesInPage) -
+ (SizeOfXLogLongPHD - SizeOfXLogShortPHD);
+
+ CalculateCheckpointSegments();
+
/* Make the initdb settings visible as GUC variables, too */
SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no",
PGC_INTERNAL, PGC_S_OVERRIDE);
int xbuffers;
xbuffers = NBuffers / 32;
- if (xbuffers > XLOG_SEG_SIZE / XLOG_BLCKSZ)
- xbuffers = XLOG_SEG_SIZE / XLOG_BLCKSZ;
+ if (xbuffers > (wal_segment_size / XLOG_BLCKSZ))
+ xbuffers = (wal_segment_size / XLOG_BLCKSZ);
if (xbuffers < 8)
xbuffers = 8;
return xbuffers;
* segment with logid=0 logseg=1. The very first WAL segment, 0/0, is not
* used, so that we can use 0/0 to mean "before any valid WAL segment".
*/
- checkPoint.redo = XLogSegSize + SizeOfXLogLongPHD;
+ checkPoint.redo = wal_segment_size + SizeOfXLogLongPHD;
checkPoint.ThisTimeLineID = ThisTimeLineID;
checkPoint.PrevTimeLineID = ThisTimeLineID;
checkPoint.fullPageWrites = fullPageWrites;
page->xlp_magic = XLOG_PAGE_MAGIC;
page->xlp_info = XLP_LONG_HEADER;
page->xlp_tli = ThisTimeLineID;
- page->xlp_pageaddr = XLogSegSize;
+ page->xlp_pageaddr = wal_segment_size;
longpage = (XLogLongPageHeader) page;
longpage->xlp_sysid = sysidentifier;
- longpage->xlp_seg_size = XLogSegSize;
+ longpage->xlp_seg_size = wal_segment_size;
longpage->xlp_xlog_blcksz = XLOG_BLCKSZ;
/* Insert the initial checkpoint record */
* they are the same, but if the switch happens exactly at a segment
* boundary, startLogSegNo will be endLogSegNo + 1.
*/
- XLByteToPrevSeg(endOfLog, endLogSegNo);
- XLByteToSeg(endOfLog, startLogSegNo);
+ XLByteToPrevSeg(endOfLog, endLogSegNo, wal_segment_size);
+ XLByteToSeg(endOfLog, startLogSegNo, wal_segment_size);
/*
* Initialize the starting WAL segment for the new timeline. If the switch
* avoid emplacing a bogus file.
*/
XLogFileCopy(endLogSegNo, endTLI, endLogSegNo,
- endOfLog % XLOG_SEG_SIZE);
+ XLogSegmentOffset(endOfLog, wal_segment_size));
}
else
{
* Let's just make real sure there are not .ready or .done flags posted
* for the new segment.
*/
- XLogFileName(xlogfname, ThisTimeLineID, startLogSegNo);
+ XLogFileName(xlogfname, ThisTimeLineID, startLogSegNo, wal_segment_size);
XLogArchiveCleanup(xlogfname);
/*
/* Set up XLOG reader facility */
MemSet(&private, 0, sizeof(XLogPageReadPrivate));
- xlogreader = XLogReaderAllocate(&XLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(wal_segment_size, &XLogPageRead, &private);
if (!xlogreader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
XLogRecPtr pageBeginPtr;
pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ);
- Assert(readOff == pageBeginPtr % XLogSegSize);
+ Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size));
firstIdx = XLogRecPtrToBufIdx(EndOfLog);
* restored from the archive to begin with, it's expected to have a
* .done file).
*/
- if (EndOfLog % XLOG_SEG_SIZE != 0 && XLogArchivingActive())
+ if (XLogSegmentOffset(EndOfLog, wal_segment_size) != 0 &&
+ XLogArchivingActive())
{
char origfname[MAXFNAMELEN];
XLogSegNo endLogSegNo;
- XLByteToPrevSeg(EndOfLog, endLogSegNo);
- XLogFileName(origfname, EndOfLogTLI, endLogSegNo);
+ XLByteToPrevSeg(EndOfLog, endLogSegNo, wal_segment_size);
+ XLogFileName(origfname, EndOfLogTLI, endLogSegNo, wal_segment_size);
if (!XLogArchiveIsReadyOrDone(origfname))
{
char partialfname[MAXFNAMELEN];
char partialpath[MAXPGPATH];
- XLogFilePath(origpath, EndOfLogTLI, endLogSegNo);
+ XLogFilePath(origpath, EndOfLogTLI, endLogSegNo, wal_segment_size);
snprintf(partialfname, MAXFNAMELEN, "%s.partial", origfname);
snprintf(partialpath, MAXPGPATH, "%s.partial", origpath);
ThisTimeLineID = XLogCtl->ThisTimeLineID;
Assert(ThisTimeLineID != 0 || IsBootstrapProcessingMode());
+ /* set wal_segment_size */
+ wal_segment_size = ControlFile->xlog_seg_size;
+
/* Use GetRedoRecPtr to copy the RedoRecPtr safely */
(void) GetRedoRecPtr();
/* Also update our copy of doPageWrites. */
* more.
*
* When checkpoints are triggered by max_wal_size, this should converge to
- * CheckpointSegments * XLOG_SEG_SIZE,
+ * CheckpointSegments * wal_segment_size,
*
* Note: This doesn't pay any attention to what caused the checkpoint.
* Checkpoints triggered manually with CHECKPOINT command, or by e.g.
freespace = INSERT_FREESPACE(curInsert);
if (freespace == 0)
{
- if (curInsert % XLogSegSize == 0)
+ if (XLogSegmentOffset(curInsert, wal_segment_size) == 0)
curInsert += SizeOfXLogLongPHD;
else
curInsert += SizeOfXLogShortPHD;
/* Update the average distance between checkpoints. */
UpdateCheckPointDistanceEstimate(RedoRecPtr - PriorRedoPtr);
- XLByteToSeg(PriorRedoPtr, _logSegNo);
+ XLByteToSeg(PriorRedoPtr, _logSegNo, wal_segment_size);
KeepLogSeg(recptr, &_logSegNo);
_logSegNo--;
RemoveOldXlogFiles(_logSegNo, PriorRedoPtr, recptr);
/* Update the average distance between checkpoints/restartpoints. */
UpdateCheckPointDistanceEstimate(RedoRecPtr - PriorRedoPtr);
- XLByteToSeg(PriorRedoPtr, _logSegNo);
+ XLByteToSeg(PriorRedoPtr, _logSegNo, wal_segment_size);
/*
* Get the current end of xlog replayed or received, whichever is
XLogSegNo segno;
XLogRecPtr keep;
- XLByteToSeg(recptr, segno);
+ XLByteToSeg(recptr, segno, wal_segment_size);
keep = XLogGetReplicationSlotMinimumLSN();
/* compute limit for wal_keep_segments first */
{
XLogSegNo slotSegNo;
- XLByteToSeg(keep, slotSegNo);
+ XLByteToSeg(keep, slotSegNo, wal_segment_size);
if (slotSegNo <= 0)
segno = 1;
{
char *result = palloc(MAXFNAMELEN);
- XLogFileName(result, tli, segno);
+ XLogFileName(result, tli, segno, wal_segment_size);
return result;
}
WALInsertLockRelease();
} while (!gotUniqueStartpoint);
- XLByteToSeg(startpoint, _logSegNo);
- XLogFileName(xlogfilename, starttli, _logSegNo);
+ XLByteToSeg(startpoint, _logSegNo, wal_segment_size);
+ XLogFileName(xlogfilename, starttli, _logSegNo, wal_segment_size);
/*
* Construct tablespace_map file
*/
RequestXLogSwitch(false);
- XLByteToPrevSeg(stoppoint, _logSegNo);
- XLogFileName(stopxlogfilename, stoptli, _logSegNo);
+ XLByteToPrevSeg(stoppoint, _logSegNo, wal_segment_size);
+ XLogFileName(stopxlogfilename, stoptli, _logSegNo, wal_segment_size);
/* Use the log timezone here, not the session timezone */
stamp_time = (pg_time_t) time(NULL);
/*
* Write the backup history file
*/
- XLByteToSeg(startpoint, _logSegNo);
+ XLByteToSeg(startpoint, _logSegNo, wal_segment_size);
BackupHistoryFilePath(histfilepath, stoptli, _logSegNo,
- (uint32) (startpoint % XLogSegSize));
+ startpoint, wal_segment_size);
fp = AllocateFile(histfilepath, "w");
if (!fp)
ereport(ERROR,
((!backup_started_in_recovery && XLogArchivingActive()) ||
(backup_started_in_recovery && XLogArchivingAlways())))
{
- XLByteToPrevSeg(stoppoint, _logSegNo);
- XLogFileName(lastxlogfilename, stoptli, _logSegNo);
+ XLByteToPrevSeg(stoppoint, _logSegNo, wal_segment_size);
+ XLogFileName(lastxlogfilename, stoptli, _logSegNo, wal_segment_size);
- XLByteToSeg(startpoint, _logSegNo);
+ XLByteToSeg(startpoint, _logSegNo, wal_segment_size);
BackupHistoryFileName(histfilename, stoptli, _logSegNo,
- (uint32) (startpoint % XLogSegSize));
+ startpoint, wal_segment_size);
seconds_before_warning = 60;
waits = 0;
uint32 targetPageOff;
XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY;
- XLByteToSeg(targetPagePtr, targetSegNo);
- targetPageOff = targetPagePtr % XLogSegSize;
+ XLByteToSeg(targetPagePtr, targetSegNo, wal_segment_size);
+ targetPageOff = XLogSegmentOffset(targetPagePtr, wal_segment_size);
/*
* See if we need to switch to a new segment because the requested record
* is not in the currently open one.
*/
- if (readFile >= 0 && !XLByteInSeg(targetPagePtr, readSegNo))
+ if (readFile >= 0 &&
+ !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size))
{
/*
* Request a restartpoint if we've replayed too much xlog since the
readSource = 0;
}
- XLByteToSeg(targetPagePtr, readSegNo);
+ XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size);
retry:
/* See if we need to retrieve more data */
if (((targetPagePtr) / XLOG_BLCKSZ) != (receivedUpto / XLOG_BLCKSZ))
readLen = XLOG_BLCKSZ;
else
- readLen = receivedUpto % XLogSegSize - targetPageOff;
+ readLen = XLogSegmentOffset(receivedUpto, wal_segment_size) -
+ targetPageOff;
}
else
readLen = XLOG_BLCKSZ;
{
char fname[MAXFNAMELEN];
- XLogFileName(fname, curFileTLI, readSegNo);
+ XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not seek in log segment %s to offset %u: %m",
char fname[MAXFNAMELEN];
pgstat_report_wait_end();
- XLogFileName(fname, curFileTLI, readSegNo);
+ XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size);
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
errmsg("could not read from log segment %s, offset %u: %m",
if (cleanupEnabled)
{
GetOldestRestartPoint(&restartRedoPtr, &restartTli);
- XLByteToSeg(restartRedoPtr, restartSegNo);
- XLogFileName(lastRestartPointFname, restartTli, restartSegNo);
+ XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size);
+ XLogFileName(lastRestartPointFname, restartTli, restartSegNo,
+ wal_segment_size);
/* we shouldn't need anything earlier than last restart point */
Assert(strcmp(lastRestartPointFname, xlogfname) <= 0);
}
else
- XLogFileName(lastRestartPointFname, 0, 0L);
+ XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size);
/*
* construct the command to be executed
* archive, though there is no requirement to do so.
*/
GetOldestRestartPoint(&restartRedoPtr, &restartTli);
- XLByteToSeg(restartRedoPtr, restartSegNo);
- XLogFileName(lastRestartPointFname, restartTli, restartSegNo);
+ XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size);
+ XLogFileName(lastRestartPointFname, restartTli, restartSegNo,
+ wal_segment_size);
/*
* construct the command to be executed
{
char xlog[MAXFNAMELEN];
- XLogFileName(xlog, ThisTimeLineID, segno);
+ XLogFileName(xlog, ThisTimeLineID, segno, wal_segment_size);
XLogArchiveNotify(xlog);
}
/*
* xlogfilename
*/
- XLByteToPrevSeg(locationpoint, xlogsegno);
- XLogFileName(xlogfilename, ThisTimeLineID, xlogsegno);
+ XLByteToPrevSeg(locationpoint, xlogsegno, wal_segment_size);
+ XLogFileName(xlogfilename, ThisTimeLineID, xlogsegno, wal_segment_size);
values[0] = CStringGetTextDatum(xlogfilename);
isnull[0] = false;
/*
* offset
*/
- xrecoff = locationpoint % XLogSegSize;
+ xrecoff = XLogSegmentOffset(locationpoint, wal_segment_size);
values[1] = UInt32GetDatum(xrecoff);
isnull[1] = false;
errmsg("recovery is in progress"),
errhint("pg_walfile_name() cannot be executed during recovery.")));
- XLByteToPrevSeg(locationpoint, xlogsegno);
- XLogFileName(xlogfilename, ThisTimeLineID, xlogsegno);
+ XLByteToPrevSeg(locationpoint, xlogsegno, wal_segment_size);
+ XLogFileName(xlogfilename, ThisTimeLineID, xlogsegno, wal_segment_size);
PG_RETURN_TEXT_P(cstring_to_text(xlogfilename));
}
* Returns NULL if the xlogreader couldn't be allocated.
*/
XLogReaderState *
-XLogReaderAllocate(XLogPageReadCB pagereadfunc, void *private_data)
+XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
+ void *private_data)
{
XLogReaderState *state;
return NULL;
}
+ state->wal_segment_size = wal_segment_size;
state->read_page = pagereadfunc;
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
(record->xl_info & ~XLR_INFO_MASK) == XLOG_SWITCH)
{
/* Pretend it extends to end of segment */
- state->EndRecPtr += XLogSegSize - 1;
- state->EndRecPtr -= state->EndRecPtr % XLogSegSize;
+ state->EndRecPtr += state->wal_segment_size - 1;
+ state->EndRecPtr -= XLogSegmentOffset(state->EndRecPtr, state->wal_segment_size);
}
if (DecodeXLogRecord(state, record, errormsg))
Assert((pageptr % XLOG_BLCKSZ) == 0);
- XLByteToSeg(pageptr, targetSegNo);
- targetPageOff = (pageptr % XLogSegSize);
+ XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
+ targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
/* check whether we have all the requested data already */
if (targetSegNo == state->readSegNo && targetPageOff == state->readOff &&
Assert((recptr % XLOG_BLCKSZ) == 0);
- XLByteToSeg(recptr, segno);
- offset = recptr % XLogSegSize;
+ XLByteToSeg(recptr, segno, state->wal_segment_size);
+ offset = XLogSegmentOffset(recptr, state->wal_segment_size);
- XLogSegNoOffsetToRecPtr(segno, offset, recaddr);
+ XLogSegNoOffsetToRecPtr(segno, offset, recaddr, state->wal_segment_size);
if (hdr->xlp_magic != XLOG_PAGE_MAGIC)
{
char fname[MAXFNAMELEN];
- XLogFileName(fname, state->readPageTLI, segno);
+ XLogFileName(fname, state->readPageTLI, segno, state->wal_segment_size);
report_invalid_record(state,
"invalid magic number %04X in log segment %s, offset %u",
{
char fname[MAXFNAMELEN];
- XLogFileName(fname, state->readPageTLI, segno);
+ XLogFileName(fname, state->readPageTLI, segno, state->wal_segment_size);
report_invalid_record(state,
"invalid info bits %04X in log segment %s, offset %u",
fhdrident_str, sysident_str);
return false;
}
- else if (longhdr->xlp_seg_size != XLogSegSize)
+ else if (longhdr->xlp_seg_size != state->wal_segment_size)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_SEG_SIZE in page header");
+ "WAL file is from different database system: incorrect segment size in page header");
return false;
}
else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
{
char fname[MAXFNAMELEN];
- XLogFileName(fname, state->readPageTLI, segno);
+ XLogFileName(fname, state->readPageTLI, segno, state->wal_segment_size);
/* hmm, first page of file doesn't have a long header? */
report_invalid_record(state,
{
char fname[MAXFNAMELEN];
- XLogFileName(fname, state->readPageTLI, segno);
+ XLogFileName(fname, state->readPageTLI, segno, state->wal_segment_size);
report_invalid_record(state,
"unexpected pageaddr %X/%X in log segment %s, offset %u",
{
char fname[MAXFNAMELEN];
- XLogFileName(fname, state->readPageTLI, segno);
+ XLogFileName(fname, state->readPageTLI, segno, state->wal_segment_size);
report_invalid_record(state,
"out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u",
* frontend). Probably these should be merged at some point.
*/
static void
-XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count)
+XLogRead(char *buf, int segsize, TimeLineID tli, XLogRecPtr startptr,
+ Size count)
{
char *p;
XLogRecPtr recptr;
static TimeLineID sendTLI = 0;
static uint32 sendOff = 0;
+ Assert(segsize == wal_segment_size);
+
p = buf;
recptr = startptr;
nbytes = count;
int segbytes;
int readbytes;
- startoff = recptr % XLogSegSize;
+ startoff = XLogSegmentOffset(recptr, segsize);
/* Do we need to switch to a different xlog segment? */
- if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo) ||
+ if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo, segsize) ||
sendTLI != tli)
{
char path[MAXPGPATH];
if (sendFile >= 0)
close(sendFile);
- XLByteToSeg(recptr, sendSegNo);
+ XLByteToSeg(recptr, sendSegNo, segsize);
- XLogFilePath(path, tli, sendSegNo);
+ XLogFilePath(path, tli, sendSegNo, segsize);
sendFile = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0);
{
char path[MAXPGPATH];
- XLogFilePath(path, tli, sendSegNo);
+ XLogFilePath(path, tli, sendSegNo, segsize);
ereport(ERROR,
(errcode_for_file_access(),
}
/* How many bytes are within this segment? */
- if (nbytes > (XLogSegSize - startoff))
- segbytes = XLogSegSize - startoff;
+ if (nbytes > (segsize - startoff))
+ segbytes = segsize - startoff;
else
segbytes = nbytes;
{
char path[MAXPGPATH];
- XLogFilePath(path, tli, sendSegNo);
+ XLogFilePath(path, tli, sendSegNo, segsize);
ereport(ERROR,
(errcode_for_file_access(),
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = state->readSegNo * XLogSegSize + state->readOff;
+ const XLogRecPtr lastReadPage = state->readSegNo *
+ state->wal_segment_size + state->readOff;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
if (state->currTLIValidUntil != InvalidXLogRecPtr &&
state->currTLI != ThisTimeLineID &&
state->currTLI != 0 &&
- (wantPage + wantLength) / XLogSegSize < state->currTLIValidUntil / XLogSegSize)
+ ((wantPage + wantLength) / state->wal_segment_size) <
+ (state->currTLIValidUntil / state->wal_segment_size))
return;
/*
*/
List *timelineHistory = readTimeLineHistory(ThisTimeLineID);
- XLogRecPtr endOfSegment = (((wantPage / XLogSegSize) + 1) * XLogSegSize) - 1;
+ XLogRecPtr endOfSegment = (((wantPage / state->wal_segment_size) + 1)
+ * state->wal_segment_size) - 1;
- Assert(wantPage / XLogSegSize == endOfSegment / XLogSegSize);
+ Assert(wantPage / state->wal_segment_size ==
+ endOfSegment / state->wal_segment_size);
/*
* Find the timeline of the last LSN on the segment containing
* as 'count', read the whole page anyway. It's guaranteed to be
* zero-padded up to the page boundary if it's incomplete.
*/
- XLogRead(cur_page, *pageTLI, targetPagePtr, XLOG_BLCKSZ);
+ XLogRead(cur_page, state->wal_segment_size, *pageTLI, targetPagePtr,
+ XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
return count;
#include "access/htup_details.h"
#include "access/xact.h"
+#include "access/xlog_internal.h"
#include "bootstrap/bootstrap.h"
#include "catalog/index.h"
#include "catalog/pg_collation.h"
/* If no -x argument, we are a CheckerProcess */
MyAuxProcType = CheckerProcess;
- while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:x:-:")) != -1)
+ while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:x:X:-:")) != -1)
{
switch (flag)
{
case 'x':
MyAuxProcType = atoi(optarg);
break;
+ case 'X':
+ {
+ int WalSegSz = strtoul(optarg, NULL, 0);
+
+ if (!IsValidWalSegSize(WalSegSz))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("-X requires a power of 2 value between 1MB and 1GB")));
+ SetConfigOption("wal_segment_size", optarg, PGC_INTERNAL,
+ PGC_S_OVERRIDE);
+ }
+ break;
case 'c':
case '-':
{
* If the returned pointer points exactly to a segment boundary,
* assume nothing happened.
*/
- if ((switchpoint % XLogSegSize) != 0)
+ if (XLogSegmentOffset(switchpoint, wal_segment_size) != 0)
elog(DEBUG1, "write-ahead log switch forced (archive_timeout=%d)",
XLogArchiveTimeout);
}
recptr = GetXLogReplayRecPtr(NULL);
else
recptr = GetInsertRecPtr();
- elapsed_xlogs = (((double) (recptr - ckpt_start_recptr)) / XLogSegSize) / CheckPointSegments;
+ elapsed_xlogs = (((double) (recptr - ckpt_start_recptr)) /
+ wal_segment_size) / CheckPointSegments;
if (progress < elapsed_xlogs)
{
* shouldn't be such files, but if there are, there's little harm in
* including them.
*/
- XLByteToSeg(startptr, startsegno);
- XLogFileName(firstoff, ThisTimeLineID, startsegno);
- XLByteToPrevSeg(endptr, endsegno);
- XLogFileName(lastoff, ThisTimeLineID, endsegno);
+ XLByteToSeg(startptr, startsegno, wal_segment_size);
+ XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
+ XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
+ XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
dir = AllocateDir("pg_wal");
if (!dir)
* Sanity check: the first and last segment should cover startptr and
* endptr, with no gaps in between.
*/
- XLogFromFileName(walFiles[0], &tli, &segno);
+ XLogFromFileName(walFiles[0], &tli, &segno, wal_segment_size);
if (segno != startsegno)
{
char startfname[MAXFNAMELEN];
- XLogFileName(startfname, ThisTimeLineID, startsegno);
+ XLogFileName(startfname, ThisTimeLineID, startsegno,
+ wal_segment_size);
ereport(ERROR,
(errmsg("could not find WAL file \"%s\"", startfname)));
}
XLogSegNo currsegno = segno;
XLogSegNo nextsegno = segno + 1;
- XLogFromFileName(walFiles[i], &tli, &segno);
+ XLogFromFileName(walFiles[i], &tli, &segno, wal_segment_size);
if (!(nextsegno == segno || currsegno == segno))
{
char nextfname[MAXFNAMELEN];
- XLogFileName(nextfname, ThisTimeLineID, nextsegno);
+ XLogFileName(nextfname, ThisTimeLineID, nextsegno,
+ wal_segment_size);
ereport(ERROR,
(errmsg("could not find WAL file \"%s\"", nextfname)));
}
{
char endfname[MAXFNAMELEN];
- XLogFileName(endfname, ThisTimeLineID, endsegno);
+ XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
ereport(ERROR,
(errmsg("could not find WAL file \"%s\"", endfname)));
}
pgoff_t len = 0;
snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
- XLogFromFileName(walFiles[i], &tli, &segno);
+ XLogFromFileName(walFiles[i], &tli, &segno, wal_segment_size);
fp = AllocateFile(pathbuf, "rb");
if (fp == NULL)
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m",
pathbuf)));
- if (statbuf.st_size != XLogSegSize)
+ if (statbuf.st_size != wal_segment_size)
{
CheckXLogRemoved(segno, tli);
ereport(ERROR,
/* send the WAL file itself */
_tarWriteHeader(pathbuf, NULL, &statbuf, false);
- while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0)
+ while ((cnt = fread(buf, 1,
+ Min(sizeof(buf), wal_segment_size - len),
+ fp)) > 0)
{
CheckXLogRemoved(segno, tli);
/* Send the chunk as a CopyData message */
len += cnt;
throttle(cnt);
- if (len == XLogSegSize)
+ if (len == wal_segment_size)
break;
}
- if (len != XLogSegSize)
+ if (len != wal_segment_size)
{
CheckXLogRemoved(segno, tli);
ereport(ERROR,
errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
}
- /* XLogSegSize is a multiple of 512, so no need for padding */
+ /* wal_segment_size is a multiple of 512, so no need for padding */
FreeFile(fp);
ctx->slot = slot;
- ctx->reader = XLogReaderAllocate(read_page, ctx);
+ ctx->reader = XLogReaderAllocate(wal_segment_size, read_page, ctx);
if (!ctx->reader)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
* store in segment in which it belongs by start lsn, don't split over
* multiple segments tho
*/
- if (fd == -1 || !XLByteInSeg(change->lsn, curOpenSegNo))
+ if (fd == -1 ||
+ !XLByteInSeg(change->lsn, curOpenSegNo, wal_segment_size))
{
XLogRecPtr recptr;
if (fd != -1)
CloseTransientFile(fd);
- XLByteToSeg(change->lsn, curOpenSegNo);
- XLogSegNoOffsetToRecPtr(curOpenSegNo, 0, recptr);
+ XLByteToSeg(change->lsn, curOpenSegNo, wal_segment_size);
+ XLogSegNoOffsetToRecPtr(curOpenSegNo, 0, recptr, wal_segment_size);
/*
* No need to care about TLIs here, only used during a single run,
txn->nentries_mem = 0;
Assert(dlist_is_empty(&txn->changes));
- XLByteToSeg(txn->final_lsn, last_segno);
+ XLByteToSeg(txn->final_lsn, last_segno, wal_segment_size);
while (restored < max_changes_in_memory && *segno <= last_segno)
{
/* first time in */
if (*segno == 0)
{
- XLByteToSeg(txn->first_lsn, *segno);
+ XLByteToSeg(txn->first_lsn, *segno, wal_segment_size);
}
Assert(*segno != 0 || dlist_is_empty(&txn->changes));
- XLogSegNoOffsetToRecPtr(*segno, 0, recptr);
+ XLogSegNoOffsetToRecPtr(*segno, 0, recptr, wal_segment_size);
/*
* No need to care about TLIs here, only used during a single run,
Assert(txn->first_lsn != InvalidXLogRecPtr);
Assert(txn->final_lsn != InvalidXLogRecPtr);
- XLByteToSeg(txn->first_lsn, first);
- XLByteToSeg(txn->final_lsn, last);
+ XLByteToSeg(txn->first_lsn, first, wal_segment_size);
+ XLByteToSeg(txn->final_lsn, last, wal_segment_size);
/* iterate over all possible filenames, and delete them */
for (cur = first; cur <= last; cur++)
char path[MAXPGPATH];
XLogRecPtr recptr;
- XLogSegNoOffsetToRecPtr(cur, 0, recptr);
+ XLogSegNoOffsetToRecPtr(cur, 0, recptr, wal_segment_size);
sprintf(path, "pg_replslot/%s/xid-%u-lsn-%X-%X.snap",
NameStr(MyReplicationSlot->data.name), txn->xid,
* the new restart_lsn above, so normally we should never need to loop
* more than twice.
*/
- XLByteToSeg(slot->data.restart_lsn, segno);
+ XLByteToSeg(slot->data.restart_lsn, segno, wal_segment_size);
if (XLogGetLastRemovedSegno() < segno)
break;
}
* Create .done file forcibly to prevent the streamed segment from
* being archived later.
*/
- XLogFileName(xlogfname, recvFileTLI, recvSegNo);
+ XLogFileName(xlogfname, recvFileTLI, recvSegNo, wal_segment_size);
if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS)
XLogArchiveForceDone(xlogfname);
else
{
int segbytes;
- if (recvFile < 0 || !XLByteInSeg(recptr, recvSegNo))
+ if (recvFile < 0 || !XLByteInSeg(recptr, recvSegNo, wal_segment_size))
{
bool use_existent;
* Create .done file forcibly to prevent the streamed segment
* from being archived later.
*/
- XLogFileName(xlogfname, recvFileTLI, recvSegNo);
+ XLogFileName(xlogfname, recvFileTLI, recvSegNo, wal_segment_size);
if (XLogArchiveMode != ARCHIVE_MODE_ALWAYS)
XLogArchiveForceDone(xlogfname);
else
recvFile = -1;
/* Create/use new log file */
- XLByteToSeg(recptr, recvSegNo);
+ XLByteToSeg(recptr, recvSegNo, wal_segment_size);
use_existent = true;
recvFile = XLogFileInit(recvSegNo, &use_existent, true);
recvFileTLI = ThisTimeLineID;
}
/* Calculate the start offset of the received logs */
- startoff = recptr % XLogSegSize;
+ startoff = XLogSegmentOffset(recptr, wal_segment_size);
- if (startoff + nbytes > XLogSegSize)
- segbytes = XLogSegSize - startoff;
+ if (startoff + nbytes > wal_segment_size)
+ segbytes = wal_segment_size - startoff;
else
segbytes = nbytes;
* being created by XLOG streaming, which might cause trouble later on if
* the segment is e.g archived.
*/
- if (recptr % XLogSegSize != 0)
- recptr -= recptr % XLogSegSize;
+ if (XLogSegmentOffset(recptr, wal_segment_size) != 0)
+ recptr -= XLogSegmentOffset(recptr, wal_segment_size);
SpinLockAcquire(&walrcv->mutex);
int segbytes;
int readbytes;
- startoff = recptr % XLogSegSize;
+ startoff = XLogSegmentOffset(recptr, wal_segment_size);
- if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo))
+ if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo, wal_segment_size))
{
char path[MAXPGPATH];
if (sendFile >= 0)
close(sendFile);
- XLByteToSeg(recptr, sendSegNo);
+ XLByteToSeg(recptr, sendSegNo, wal_segment_size);
/*-------
* When reading from a historic timeline, and there is a timeline
{
XLogSegNo endSegNo;
- XLByteToSeg(sendTimeLineValidUpto, endSegNo);
+ XLByteToSeg(sendTimeLineValidUpto, endSegNo, wal_segment_size);
if (sendSegNo == endSegNo)
curFileTimeLine = sendTimeLineNextTLI;
}
- XLogFilePath(path, curFileTimeLine, sendSegNo);
+ XLogFilePath(path, curFileTimeLine, sendSegNo, wal_segment_size);
sendFile = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0);
if (sendFile < 0)
}
/* How many bytes are within this segment? */
- if (nbytes > (XLogSegSize - startoff))
- segbytes = XLogSegSize - startoff;
+ if (nbytes > (wal_segment_size - startoff))
+ segbytes = wal_segment_size - startoff;
else
segbytes = nbytes;
* read() succeeds in that case, but the data we tried to read might
* already have been overwritten with new WAL records.
*/
- XLByteToSeg(startptr, segno);
+ XLByteToSeg(startptr, segno, wal_segment_size);
CheckXLogRemoved(segno, ThisTimeLineID);
/*
static int segment_size;
static int wal_block_size;
static bool data_checksums;
-static int wal_segment_size;
static bool integer_datetimes;
static bool assert_enabled;
#if XLOG_BLCKSZ < 1024 || XLOG_BLCKSZ > (1024*1024)
#error XLOG_BLCKSZ must be between 1KB and 1MB
#endif
-#if XLOG_SEG_SIZE < (1024*1024) || XLOG_SEG_SIZE > (1024*1024*1024)
-#error XLOG_SEG_SIZE must be between 1MB and 1GB
-#endif
static const char *memory_units_hint = gettext_noop("Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\".");
GUC_UNIT_MB
},
&min_wal_size_mb,
- 5 * (XLOG_SEG_SIZE / (1024 * 1024)), 2, MAX_KILOBYTES,
+ DEFAULT_MIN_WAL_SEGS * (DEFAULT_XLOG_SEG_SIZE / (1024 * 1024)),
+ 2, MAX_KILOBYTES,
NULL, NULL, NULL
},
GUC_UNIT_MB
},
&max_wal_size_mb,
- 64 * (XLOG_SEG_SIZE / (1024 * 1024)), 2, MAX_KILOBYTES,
+ DEFAULT_MAX_WAL_SEGS * (DEFAULT_XLOG_SEG_SIZE / (1024 * 1024)),
+ 2, MAX_KILOBYTES,
NULL, assign_max_wal_size, NULL
},
{
{"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS,
- gettext_noop("Shows the number of pages per write ahead log segment."),
+ gettext_noop("Shows the size of write ahead log segments."),
NULL,
- GUC_UNIT_XBLOCKS | GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
+ GUC_UNIT_BYTE | GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
},
&wal_segment_size,
- (XLOG_SEG_SIZE / XLOG_BLCKSZ),
- (XLOG_SEG_SIZE / XLOG_BLCKSZ),
- (XLOG_SEG_SIZE / XLOG_BLCKSZ),
+ DEFAULT_XLOG_SEG_SIZE,
+ WalSegMinSize,
+ WalSegMaxSize,
NULL, NULL, NULL
},
* Calculate name of the WAL file containing the latest checkpoint's REDO
* start point.
*/
- XLByteToSeg(ControlFile->checkPointCopy.redo, segno);
- XLogFileName(xlogfilename, ControlFile->checkPointCopy.ThisTimeLineID, segno);
+ XLByteToSeg(ControlFile->checkPointCopy.redo, segno, wal_segment_size);
+ XLogFileName(xlogfilename, ControlFile->checkPointCopy.ThisTimeLineID,
+ segno, wal_segment_size);
/* Populate the values and null arrays */
values[0] = LSNGetDatum(ControlFile->checkPoint);
#max_wal_senders = 10 # max number of walsender processes
# (change requires restart)
-#wal_keep_segments = 0 # in logfile segments, 16MB each; 0 disables
+#wal_keep_segments = 0 # in logfile segments; 0 disables
#wal_sender_timeout = 60s # in milliseconds; 0 disables
#max_replication_slots = 10 # max number of replication slots
#include "sys/mman.h"
#endif
+#include "access/xlog_internal.h"
#include "catalog/catalog.h"
#include "catalog/pg_authid.h"
#include "catalog/pg_class.h"
static bool show_setting = false;
static bool data_checksums = false;
static char *xlog_dir = NULL;
+static char *str_wal_segment_size_mb = NULL;
+static int wal_segment_size_mb;
/* internal vars */
printf("%s\n", dynamic_shared_memory_type);
}
+/*
+ * Calculate the default wal_size with a "pretty" unit.
+ */
+static char *
+pretty_wal_size(int segment_count)
+{
+ int sz = wal_segment_size_mb * segment_count;
+ char *result = pg_malloc(10);
+
+ if ((sz % 1024) == 0)
+ snprintf(result, 10, "%dGB", sz / 1024);
+ else
+ snprintf(result, 10, "%dMB", sz);
+
+ return result;
+}
+
/*
* set up all the config files
*/
conflines = replace_token(conflines, "#port = 5432", repltok);
#endif
+ /* set default max_wal_size and min_wal_size */
+ snprintf(repltok, sizeof(repltok), "min_wal_size = %s",
+ pretty_wal_size(DEFAULT_MIN_WAL_SEGS));
+ conflines = replace_token(conflines, "#min_wal_size = 80MB", repltok);
+
+ snprintf(repltok, sizeof(repltok), "max_wal_size = %s",
+ pretty_wal_size(DEFAULT_MAX_WAL_SEGS));
+ conflines = replace_token(conflines, "#max_wal_size = 1GB", repltok);
+
snprintf(repltok, sizeof(repltok), "lc_messages = '%s'",
escape_quotes(lc_messages));
conflines = replace_token(conflines, "#lc_messages = 'C'", repltok);
unsetenv("PGCLIENTENCODING");
snprintf(cmd, sizeof(cmd),
- "\"%s\" --boot -x1 %s %s %s",
+ "\"%s\" --boot -x1 -X %u %s %s %s",
backend_exec,
+ wal_segment_size_mb * (1024 * 1024),
data_checksums ? "-k" : "",
boot_options,
debug ? "-d 5" : "");
printf(_(" -U, --username=NAME database superuser name\n"));
printf(_(" -W, --pwprompt prompt for a password for the new superuser\n"));
printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n"));
+ printf(_(" --wal-segsize=SIZE size of wal segment size\n"));
printf(_("\nLess commonly used options:\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
printf(_(" -k, --data-checksums use data page checksums\n"));
{"no-sync", no_argument, NULL, 'N'},
{"sync-only", no_argument, NULL, 'S'},
{"waldir", required_argument, NULL, 'X'},
+ {"wal-segsize", required_argument, NULL, 12},
{"data-checksums", no_argument, NULL, 'k'},
{NULL, 0, NULL, 0}
};
case 'X':
xlog_dir = pg_strdup(optarg);
break;
+ case 12:
+ str_wal_segment_size_mb = pg_strdup(optarg);
+ break;
default:
/* getopt_long already emitted a complaint */
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
check_need_password(authmethodlocal, authmethodhost);
+ /* set wal segment size */
+ if (str_wal_segment_size_mb == NULL)
+ wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024);
+ else
+ {
+ char *endptr;
+
+ /* check that the argument is a number */
+ wal_segment_size_mb = strtol(str_wal_segment_size_mb, &endptr, 10);
+
+ /* verify that wal segment size is valid */
+ if (*endptr != '\0' ||
+ !IsValidWalSegSize(wal_segment_size_mb * 1024 * 1024))
+ {
+ fprintf(stderr,
+ _("%s: --wal-segsize must be a power of two between 1 and 1024\n"),
+ progname);
+ exit(1);
+ }
+ }
+
get_restricted_token(progname);
setup_pgdata();
#include <zlib.h>
#endif
+#include "access/xlog_internal.h"
#include "common/file_utils.h"
#include "common/string.h"
#include "fe_utils/string_utils.h"
}
param->startptr = ((uint64) hi) << 32 | lo;
/* Round off to even segment position */
- param->startptr -= param->startptr % XLOG_SEG_SIZE;
+ param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
#ifndef WIN32
/* Create our background pipe */
exit(1);
}
+ /* determine remote server's xlog segment size */
+ if (!RetrieveWalSegSize(conn))
+ disconnect_and_exit(1);
+
/* Create pg_wal symlink, if required */
if (xlog_dir)
{
/*
* Determine starting location for streaming, based on any existing xlog
* segments in the directory. We start at the end of the last one that is
- * complete (size matches XLogSegSize), on the timeline with highest ID.
+ * complete (size matches wal segment size), on the timeline with highest ID.
*
* If there are no WAL files in the directory, returns InvalidXLogRecPtr.
*/
/*
* Looks like an xlog file. Parse its position.
*/
- XLogFromFileName(dirent->d_name, &tli, &segno);
+ XLogFromFileName(dirent->d_name, &tli, &segno, WalSegSz);
/*
* Check that the segment has the right size, if it's supposed to be
disconnect_and_exit(1);
}
- if (statbuf.st_size != XLOG_SEG_SIZE)
+ if (statbuf.st_size != WalSegSz)
{
fprintf(stderr,
_("%s: segment file \"%s\" has incorrect size %d, skipping\n"),
bytes_out = (buf[3] << 24) | (buf[2] << 16) |
(buf[1] << 8) | buf[0];
- if (bytes_out != XLOG_SEG_SIZE)
+ if (bytes_out != WalSegSz)
{
fprintf(stderr,
_("%s: compressed segment file \"%s\" has incorrect uncompressed size %d, skipping\n"),
if (!high_ispartial)
high_segno++;
- XLogSegNoOffsetToRecPtr(high_segno, 0, high_ptr);
+ XLogSegNoOffsetToRecPtr(high_segno, 0, high_ptr, WalSegSz);
*tli = high_tli;
return high_ptr;
/*
* Always start streaming at the beginning of a segment
*/
- stream.startpos -= stream.startpos % XLOG_SEG_SIZE;
+ stream.startpos -= XLogSegmentOffset(stream.startpos, WalSegSz);
/*
* Start the replication
if (!RunIdentifySystem(conn, NULL, NULL, NULL, &db_name))
disconnect_and_exit(1);
+ /* determine remote server's xlog segment size */
+ if (!RetrieveWalSegSize(conn))
+ disconnect_and_exit(1);
+
/*
* Check that there is a database associated with connection, none should
* be defined in this context.
ssize_t size;
XLogSegNo segno;
- XLByteToSeg(startpoint, segno);
- XLogFileName(current_walfile_name, stream->timeline, segno);
+ XLByteToSeg(startpoint, segno, WalSegSz);
+ XLogFileName(current_walfile_name, stream->timeline, segno, WalSegSz);
snprintf(fn, sizeof(fn), "%s%s", current_walfile_name,
stream->partial_suffix ? stream->partial_suffix : "");
/*
* When streaming to files, if an existing file exists we verify that it's
- * either empty (just created), or a complete XLogSegSize segment (in
- * which case it has been created and padded). Anything else indicates a
- * corrupt file.
+ * either empty (just created), or a complete WalSegSz segment (in which
+ * case it has been created and padded). Anything else indicates a corrupt
+ * file.
*
* When streaming to tar, no file with this name will exist before, so we
* never have to verify a size.
progname, fn, stream->walmethod->getlasterror());
return false;
}
- if (size == XLogSegSize)
+ if (size == WalSegSz)
{
/* Already padded file. Open it for use */
f = stream->walmethod->open_for_write(current_walfile_name, stream->partial_suffix, 0);
ngettext("%s: write-ahead log file \"%s\" has %d byte, should be 0 or %d\n",
"%s: write-ahead log file \"%s\" has %d bytes, should be 0 or %d\n",
size),
- progname, fn, (int) size, XLogSegSize);
+ progname, fn, (int) size, WalSegSz);
return false;
}
/* File existed and was empty, so fall through and open */
/* No file existed, so create one */
- f = stream->walmethod->open_for_write(current_walfile_name, stream->partial_suffix, XLogSegSize);
+ f = stream->walmethod->open_for_write(current_walfile_name,
+ stream->partial_suffix, WalSegSz);
if (f == NULL)
{
fprintf(stderr,
if (stream->partial_suffix)
{
- if (currpos == XLOG_SEG_SIZE)
+ if (currpos == WalSegSz)
r = stream->walmethod->close(walfile, CLOSE_NORMAL);
else
{
* new node. This is in line with walreceiver.c always doing a
* XLogArchiveForceDone() after a complete segment.
*/
- if (currpos == XLOG_SEG_SIZE && stream->mark_done)
+ if (currpos == WalSegSz && stream->mark_done)
{
/* writes error message if failed */
if (!mark_file_as_archived(stream, current_walfile_name))
* start streaming at the beginning of a segment.
*/
stream->timeline = newtimeline;
- stream->startpos = stream->startpos - (stream->startpos % XLOG_SEG_SIZE);
+ stream->startpos = stream->startpos -
+ XLogSegmentOffset(stream->startpos, WalSegSz);
continue;
}
else if (PQresultStatus(res) == PGRES_COMMAND_OK)
*blockpos = fe_recvint64(©buf[1]);
/* Extract WAL location for this block */
- xlogoff = *blockpos % XLOG_SEG_SIZE;
+ xlogoff = XLogSegmentOffset(*blockpos, WalSegSz);
/*
* Verify that the initial location in the stream matches where we think
int bytes_to_write;
/*
- * If crossing a WAL boundary, only write up until we reach
- * XLOG_SEG_SIZE.
+ * If crossing a WAL boundary, only write up until we reach wal
+ * segment size.
*/
- if (xlogoff + bytes_left > XLOG_SEG_SIZE)
- bytes_to_write = XLOG_SEG_SIZE - xlogoff;
+ if (xlogoff + bytes_left > WalSegSz)
+ bytes_to_write = WalSegSz - xlogoff;
else
bytes_to_write = bytes_left;
xlogoff += bytes_to_write;
/* Did we reach the end of a WAL segment? */
- if (*blockpos % XLOG_SEG_SIZE == 0)
+ if (XLogSegmentOffset(*blockpos, WalSegSz) == 0)
{
if (!close_walfile(stream, *blockpos))
/* Error message written in close_walfile() */
#include "receivelog.h"
#include "streamutil.h"
+#include "access/xlog_internal.h"
#include "pqexpbuffer.h"
#include "common/fe_memutils.h"
#include "datatype/timestamp.h"
#define ERRCODE_DUPLICATE_OBJECT "42710"
+uint32 WalSegSz;
+
+/* SHOW command for replication connection was introduced in version 10 */
+#define MINIMUM_VERSION_FOR_SHOW_CMD 100000
+
const char *progname;
char *connection_string = NULL;
char *dbhost = NULL;
return tmpconn;
}
+/*
+ * From version 10, explicitly set wal segment size using SHOW wal_segment_size
+ * since ControlFile is not accessible here.
+ */
+bool
+RetrieveWalSegSize(PGconn *conn)
+{
+ PGresult *res;
+ char xlog_unit[3];
+ int xlog_val,
+ multiplier = 1;
+
+ /* check connection existence */
+ Assert(conn != NULL);
+
+ /* for previous versions set the default xlog seg size */
+ if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_SHOW_CMD)
+ {
+ WalSegSz = DEFAULT_XLOG_SEG_SIZE;
+ return true;
+ }
+
+ res = PQexec(conn, "SHOW wal_segment_size");
+ if (PQresultStatus(res) != PGRES_TUPLES_OK)
+ {
+ fprintf(stderr, _("%s: could not send replication command \"%s\": %s\n"),
+ progname, "SHOW wal_segment_size", PQerrorMessage(conn));
+
+ PQclear(res);
+ return false;
+ }
+ if (PQntuples(res) != 1 || PQnfields(res) < 1)
+ {
+ fprintf(stderr,
+ _("%s: could not fetch WAL segment size: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
+ progname, PQntuples(res), PQnfields(res), 1, 1);
+
+ PQclear(res);
+ return false;
+ }
+
+ /* fetch xlog value and unit from the result */
+ if (sscanf(PQgetvalue(res, 0, 0), "%d%s", &xlog_val, xlog_unit) != 2)
+ {
+ fprintf(stderr, _("%s: WAL segment size could not be parsed\n"),
+ progname);
+ return false;
+ }
+
+ /* set the multiplier based on unit to convert xlog_val to bytes */
+ if (strcmp(xlog_unit, "MB") == 0)
+ multiplier = 1024 * 1024;
+ else if (strcmp(xlog_unit, "GB") == 0)
+ multiplier = 1024 * 1024 * 1024;
+
+ /* convert and set WalSegSz */
+ WalSegSz = xlog_val * multiplier;
+
+ if (!IsValidWalSegSize(WalSegSz))
+ {
+ fprintf(stderr,
+ _("%s: WAL segment size must be a power of two between 1MB and 1GB, but the remote server reported a value of %d bytes\n"),
+ progname, WalSegSz);
+ return false;
+ }
+
+ PQclear(res);
+ return true;
+}
+
/*
* Run IDENTIFY_SYSTEM through a given connection and give back to caller
* some result information if requested:
extern char *dbport;
extern char *dbname;
extern int dbgetpassword;
+extern uint32 WalSegSz;
/* Connection kept global so we can disconnect easily */
extern PGconn *conn;
TimeLineID *starttli,
XLogRecPtr *startpos,
char **db_name);
+extern bool RetrieveWalSegSize(PGconn *conn);
extern TimestampTz feGetCurrentTimestamp(void);
extern void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time,
long *secs, int *microsecs);
char xlogfilename[MAXFNAMELEN];
int c;
int i;
+ int WalSegSz;
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_controldata"));
"Either the file is corrupt, or it has a different layout than this program\n"
"is expecting. The results below are untrustworthy.\n\n"));
+ /* set wal segment size */
+ WalSegSz = ControlFile->xlog_seg_size;
+
+ if (!IsValidWalSegSize(WalSegSz))
+ fprintf(stderr,
+ _("WARNING: WAL segment size specified, %d bytes, is not a power of two between 1MB and 1GB.\n"
+ "The file is corrupt and the results below are untrustworthy.\n"),
+ WalSegSz);
+
/*
* This slightly-chintzy coding will work as long as the control file
* timestamps are within the range of time_t; that should be the case in
* Calculate name of the WAL file containing the latest checkpoint's REDO
* start point.
*/
- XLByteToSeg(ControlFile->checkPointCopy.redo, segno);
- XLogFileName(xlogfilename, ControlFile->checkPointCopy.ThisTimeLineID, segno);
+ XLByteToSeg(ControlFile->checkPointCopy.redo, segno, WalSegSz);
+ XLogFileName(xlogfilename, ControlFile->checkPointCopy.ThisTimeLineID,
+ segno, WalSegSz);
/*
* Format system_identifier and mock_authentication_nonce separately to
static MultiXactOffset set_mxoff = (MultiXactOffset) -1;
static uint32 minXlogTli = 0;
static XLogSegNo minXlogSegNo = 0;
+static int WalSegSz;
static void CheckDataVersion(void);
static bool ReadControlFile(void);
char *endptr;
char *endptr2;
char *DataDir = NULL;
+ char *log_fname = NULL;
int fd;
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_resetwal"));
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
}
- XLogFromFileName(optarg, &minXlogTli, &minXlogSegNo);
+
+ /*
+ * XLogFromFileName requires wal segment size which is not yet
+ * set. Hence wal details are set later on.
+ */
+ log_fname = pg_strdup(optarg);
break;
default:
if (!ReadControlFile())
GuessControlValues();
+ if (log_fname != NULL)
+ XLogFromFileName(log_fname, &minXlogTli, &minXlogSegNo, WalSegSz);
+
/*
* Also look at existing segment files to set up newXlogSegNo
*/
offsetof(ControlFileData, crc));
FIN_CRC32C(crc);
- if (EQ_CRC32C(crc, ((ControlFileData *) buffer)->crc))
+ if (!EQ_CRC32C(crc, ((ControlFileData *) buffer)->crc))
{
- /* Valid data... */
- memcpy(&ControlFile, buffer, sizeof(ControlFile));
- return true;
+ /* We will use the data but treat it as guessed. */
+ fprintf(stderr,
+ _("%s: pg_control exists but has invalid CRC; proceed with caution\n"),
+ progname);
+ guessed = true;
}
- fprintf(stderr, _("%s: pg_control exists but has invalid CRC; proceed with caution\n"),
- progname);
- /* We will use the data anyway, but treat it as guessed. */
memcpy(&ControlFile, buffer, sizeof(ControlFile));
- guessed = true;
+ WalSegSz = ControlFile.xlog_seg_size;
+
+ /* return false if WalSegSz is not valid */
+ if (!IsValidWalSegSize(WalSegSz))
+ {
+ fprintf(stderr,
+ _("%s: pg_control specifies invalid WAL segment size (%d bytes); proceed with caution \n"),
+ progname, WalSegSz);
+ guessed = true;
+ }
+
return true;
}
ControlFile.blcksz = BLCKSZ;
ControlFile.relseg_size = RELSEG_SIZE;
ControlFile.xlog_blcksz = XLOG_BLCKSZ;
- ControlFile.xlog_seg_size = XLOG_SEG_SIZE;
+ ControlFile.xlog_seg_size = DEFAULT_XLOG_SEG_SIZE;
ControlFile.nameDataLen = NAMEDATALEN;
ControlFile.indexMaxKeys = INDEX_MAX_KEYS;
ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
/* This will be always printed in order to keep format same. */
printf(_("\n\nValues to be changed:\n\n"));
- XLogFileName(fname, ControlFile.checkPointCopy.ThisTimeLineID, newXlogSegNo);
+ XLogFileName(fname, ControlFile.checkPointCopy.ThisTimeLineID,
+ newXlogSegNo, WalSegSz);
printf(_("First log segment after reset: %s\n"), fname);
if (set_mxid != 0)
* newXlogSegNo.
*/
XLogSegNoOffsetToRecPtr(newXlogSegNo, SizeOfXLogLongPHD,
- ControlFile.checkPointCopy.redo);
+ ControlFile.checkPointCopy.redo, WalSegSz);
ControlFile.checkPointCopy.time = (pg_time_t) time(NULL);
ControlFile.state = DB_SHUTDOWNED;
ControlFile.max_locks_per_xact = 64;
/* Now we can force the recorded xlog seg size to the right thing. */
- ControlFile.xlog_seg_size = XLogSegSize;
+ ControlFile.xlog_seg_size = WalSegSz;
/* Contents are protected with a CRC */
INIT_CRC32C(ControlFile.crc);
* are in virgin territory.
*/
xlogbytepos = newXlogSegNo * ControlFile.xlog_seg_size;
- newXlogSegNo = (xlogbytepos + XLogSegSize - 1) / XLogSegSize;
+ newXlogSegNo = (xlogbytepos + WalSegSz - 1) / WalSegSz;
newXlogSegNo++;
}
page->xlp_pageaddr = ControlFile.checkPointCopy.redo - SizeOfXLogLongPHD;
longpage = (XLogLongPageHeader) page;
longpage->xlp_sysid = ControlFile.system_identifier;
- longpage->xlp_seg_size = XLogSegSize;
+ longpage->xlp_seg_size = WalSegSz;
longpage->xlp_xlog_blcksz = XLOG_BLCKSZ;
/* Insert the initial checkpoint record */
record->xl_crc = crc;
/* Write the first page */
- XLogFilePath(path, ControlFile.checkPointCopy.ThisTimeLineID, newXlogSegNo);
+ XLogFilePath(path, ControlFile.checkPointCopy.ThisTimeLineID,
+ newXlogSegNo, WalSegSz);
unlink(path);
/* Fill the rest of the file with zeroes */
memset(buffer, 0, XLOG_BLCKSZ);
- for (nbytes = XLOG_BLCKSZ; nbytes < XLogSegSize; nbytes += XLOG_BLCKSZ)
+ for (nbytes = XLOG_BLCKSZ; nbytes < WalSegSz; nbytes += XLOG_BLCKSZ)
{
errno = 0;
if (write(fd, buffer, XLOG_BLCKSZ) != XLOG_BLCKSZ)
private.datadir = datadir;
private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
+ &private);
if (xlogreader == NULL)
pg_fatal("out of memory\n");
private.datadir = datadir;
private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
+ &private);
if (xlogreader == NULL)
pg_fatal("out of memory\n");
* header in that case to find the next record.
*/
if (forkptr % XLOG_BLCKSZ == 0)
- forkptr += (forkptr % XLogSegSize == 0) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD;
+ {
+ if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
+ forkptr += SizeOfXLogLongPHD;
+ else
+ forkptr += SizeOfXLogShortPHD;
+ }
private.datadir = datadir;
private.tliIndex = tliIndex;
- xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private);
+ xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
+ &private);
if (xlogreader == NULL)
pg_fatal("out of memory\n");
XLogRecPtr targetSegEnd;
XLogSegNo targetSegNo;
- XLByteToSeg(targetPagePtr, targetSegNo);
- XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, targetSegEnd);
- targetPageOff = targetPagePtr % XLogSegSize;
+ XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
+ XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, targetSegEnd, WalSegSz);
+ targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
/*
* See if we need to switch to a new segment because the requested record
* is not in the currently open one.
*/
- if (xlogreadfd >= 0 && !XLByteInSeg(targetPagePtr, xlogreadsegno))
+ if (xlogreadfd >= 0 &&
+ !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
{
close(xlogreadfd);
xlogreadfd = -1;
}
- XLByteToSeg(targetPagePtr, xlogreadsegno);
+ XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
if (xlogreadfd < 0)
{
targetHistory[private->tliIndex].begin >= targetSegEnd)
private->tliIndex--;
- XLogFileName(xlogfname, targetHistory[private->tliIndex].tli, xlogreadsegno);
+ XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
+ xlogreadsegno, WalSegSz);
snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
static ControlFileData ControlFile_source;
const char *progname;
+int WalSegSz;
/* Configuration options */
char *datadir_target = NULL;
char buf[1000];
int len;
- XLByteToSeg(startpoint, startsegno);
- XLogFileName(xlogfilename, starttli, startsegno);
+ XLByteToSeg(startpoint, startsegno, WalSegSz);
+ XLogFileName(xlogfilename, starttli, startsegno, WalSegSz);
/*
* Construct backup label file
memcpy(ControlFile, src, sizeof(ControlFileData));
+ /* set and validate WalSegSz */
+ WalSegSz = ControlFile->xlog_seg_size;
+
+ if (!IsValidWalSegSize(WalSegSz))
+ pg_fatal("WAL segment size must be a power of two between 1MB and 1GB, but the control file specifies %d bytes\n",
+ WalSegSz);
+
/* Additional checks on control file */
checkControlFile(ControlFile);
}
extern bool debug;
extern bool showprogress;
extern bool dry_run;
+extern int WalSegSz;
/* Target history */
extern TimeLineHistoryEntry *targetHistory;
static int secs_per_test = 5;
static int needs_unlink = 0;
-static char full_buf[XLOG_SEG_SIZE],
+static char full_buf[DEFAULT_XLOG_SEG_SIZE],
*buf,
*filename = FSYNC_FILENAME;
static struct timeval start_t,
int ops;
/* write random data into buffer */
- for (ops = 0; ops < XLOG_SEG_SIZE; ops++)
+ for (ops = 0; ops < DEFAULT_XLOG_SEG_SIZE; ops++)
full_buf[ops] = random();
buf = (char *) TYPEALIGN(XLOG_BLCKSZ, full_buf);
if ((tmpfile = open(filename, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR)) == -1)
die("could not open output file");
needs_unlink = 1;
- if (write(tmpfile, full_buf, XLOG_SEG_SIZE) != XLOG_SEG_SIZE)
+ if (write(tmpfile, full_buf, DEFAULT_XLOG_SEG_SIZE) !=
+ DEFAULT_XLOG_SEG_SIZE)
die("write failed");
/* fsync now so that dirty buffers don't skew later tests */
# Run a given "initdb" binary and overlay the regression testing
# authentication configuration.
standard_initdb() {
- "$1" -N
+ # To increase coverage of non-standard segment size without
+ # increase test runtime, run these tests with a lower setting.
+ "$1" -N --wal-segsize 1
if [ -n "$TEMP_CONFIG" -a -r "$TEMP_CONFIG" ]
then
cat "$TEMP_CONFIG" >> "$PGDATA/postgresql.conf"
#include "postgres.h"
#include <dirent.h>
+#include <sys/stat.h>
#include <unistd.h>
#include "access/xlogreader.h"
static const char *progname;
+static int WalSegSz;
+
typedef struct XLogDumpPrivate
{
TimeLineID timeline;
}
/*
- * Try to find the file in several places:
- * if directory == NULL:
- * fname
- * XLOGDIR / fname
- * $PGDATA / XLOGDIR / fname
- * else
- * directory / fname
- * directory / XLOGDIR / fname
+ * Open the file in the valid target directory.
*
* return a read only fd
*/
static int
-fuzzy_open_file(const char *directory, const char *fname)
+open_file_in_directory(const char *directory, const char *fname)
{
int fd = -1;
char fpath[MAXPGPATH];
- if (directory == NULL)
+ Assert(directory != NULL);
+
+ snprintf(fpath, MAXPGPATH, "%s/%s", directory, fname);
+ fd = open(fpath, O_RDONLY | PG_BINARY, 0);
+
+ if (fd < 0 && errno != ENOENT)
+ fatal_error("could not open file \"%s\": %s",
+ fname, strerror(errno));
+ return fd;
+}
+
+/*
+ * Try to find fname in the given directory. Returns true if it is found,
+ * false otherwise. If fname is NULL, search the complete directory for any
+ * file with a valid WAL file name. If file is successfully opened, set the
+ * wal segment size.
+ */
+static bool
+search_directory(char *directory, char *fname)
+{
+ int fd = -1;
+ DIR *xldir;
+
+ /* open file if valid filename is provided */
+ if (fname != NULL)
+ fd = open_file_in_directory(directory, fname);
+
+ /*
+ * A valid file name is not passed, so search the complete directory. If
+ * we find any file whose name is a valid WAL file name then try to open
+ * it. If we cannot open it, bail out.
+ */
+ else if ((xldir = opendir(directory)) != NULL)
+ {
+ struct dirent *xlde;
+
+ while ((xlde = readdir(xldir)) != NULL)
+ {
+ if (IsXLogFileName(xlde->d_name))
+ {
+ fd = open_file_in_directory(directory, xlde->d_name);
+ fname = xlde->d_name;
+ break;
+ }
+ }
+
+ closedir(xldir);
+ }
+
+ /* set WalSegSz if file is successfully opened */
+ if (fd >= 0)
+ {
+ char buf[XLOG_BLCKSZ];
+
+ if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ)
+ {
+ XLogLongPageHeader longhdr = (XLogLongPageHeader) buf;
+
+ WalSegSz = longhdr->xlp_seg_size;
+
+ if (!IsValidWalSegSize(WalSegSz))
+ fatal_error("WAL segment size must be a power of two between 1MB and 1GB, but the WAL file \"%s\" header specifies %d bytes",
+ fname, WalSegSz);
+ }
+ else
+ {
+ if (errno != 0)
+ fatal_error("could not read file \"%s\": %s",
+ fname, strerror(errno));
+ else
+ fatal_error("not enough data in file \"%s\"", fname);
+ }
+ close(fd);
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * Identify the target directory and set WalSegSz.
+ *
+ * Try to find the file in several places:
+ * if directory != NULL:
+ * directory /
+ * directory / XLOGDIR /
+ * else
+ * .
+ * XLOGDIR /
+ * $PGDATA / XLOGDIR /
+ *
+ * Set the valid target directory in private->inpath.
+ */
+static void
+identify_target_directory(XLogDumpPrivate *private, char *directory,
+ char *fname)
+{
+ char fpath[MAXPGPATH];
+
+ if (directory != NULL)
+ {
+ if (search_directory(directory, fname))
+ {
+ private->inpath = strdup(directory);
+ return;
+ }
+
+ /* directory / XLOGDIR */
+ snprintf(fpath, MAXPGPATH, "%s/%s", directory, XLOGDIR);
+ if (search_directory(fpath, fname))
+ {
+ private->inpath = strdup(fpath);
+ return;
+ }
+ }
+ else
{
const char *datadir;
- /* fname */
- fd = open(fname, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
-
- /* XLOGDIR / fname */
- snprintf(fpath, MAXPGPATH, "%s/%s",
- XLOGDIR, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
+ /* current directory */
+ if (search_directory(".", fname))
+ {
+ private->inpath = strdup(".");
+ return;
+ }
+ /* XLOGDIR */
+ if (search_directory(XLOGDIR, fname))
+ {
+ private->inpath = strdup(XLOGDIR);
+ return;
+ }
datadir = getenv("PGDATA");
- /* $PGDATA / XLOGDIR / fname */
+ /* $PGDATA / XLOGDIR */
if (datadir != NULL)
{
- snprintf(fpath, MAXPGPATH, "%s/%s/%s",
- datadir, XLOGDIR, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
+ snprintf(fpath, MAXPGPATH, "%s/%s", datadir, XLOGDIR);
+ if (search_directory(fpath, fname))
+ {
+ private->inpath = strdup(fpath);
+ return;
+ }
}
}
+
+ /* could not locate WAL file */
+ if (fname)
+ fatal_error("could not locate WAL file \"%s\"", fname);
else
- {
- /* directory / fname */
- snprintf(fpath, MAXPGPATH, "%s/%s",
- directory, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
-
- /* directory / XLOGDIR / fname */
- snprintf(fpath, MAXPGPATH, "%s/%s/%s",
- directory, XLOGDIR, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
- }
- return -1;
+ fatal_error("could not find any WAL file");
}
/*
int segbytes;
int readbytes;
- startoff = recptr % XLogSegSize;
+ startoff = XLogSegmentOffset(recptr, WalSegSz);
- if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo))
+ if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo, WalSegSz))
{
char fname[MAXFNAMELEN];
int tries;
if (sendFile >= 0)
close(sendFile);
- XLByteToSeg(recptr, sendSegNo);
+ XLByteToSeg(recptr, sendSegNo, WalSegSz);
- XLogFileName(fname, timeline_id, sendSegNo);
+ XLogFileName(fname, timeline_id, sendSegNo, WalSegSz);
/*
* In follow mode there is a short period of time after the server
*/
for (tries = 0; tries < 10; tries++)
{
- sendFile = fuzzy_open_file(directory, fname);
+ sendFile = open_file_in_directory(directory, fname);
if (sendFile >= 0)
break;
if (errno == ENOENT)
int err = errno;
char fname[MAXPGPATH];
- XLogFileName(fname, timeline_id, sendSegNo);
+ XLogFileName(fname, timeline_id, sendSegNo, WalSegSz);
fatal_error("could not seek in log file %s to offset %u: %s",
fname, startoff, strerror(err));
}
/* How many bytes are within this segment? */
- if (nbytes > (XLogSegSize - startoff))
- segbytes = XLogSegSize - startoff;
+ if (nbytes > (WalSegSz - startoff))
+ segbytes = WalSegSz - startoff;
else
segbytes = nbytes;
int err = errno;
char fname[MAXPGPATH];
- XLogFileName(fname, timeline_id, sendSegNo);
+ XLogFileName(fname, timeline_id, sendSegNo, WalSegSz);
fatal_error("could not read from log file %s, offset %u, length %d: %s",
fname, sendOff, segbytes, strerror(err));
private.inpath, strerror(errno));
}
- fd = fuzzy_open_file(private.inpath, fname);
+ identify_target_directory(&private, private.inpath, fname);
+ fd = open_file_in_directory(private.inpath, fname);
if (fd < 0)
fatal_error("could not open file \"%s\"", fname);
close(fd);
/* parse position from file */
- XLogFromFileName(fname, &private.timeline, &segno);
+ XLogFromFileName(fname, &private.timeline, &segno, WalSegSz);
if (XLogRecPtrIsInvalid(private.startptr))
- XLogSegNoOffsetToRecPtr(segno, 0, private.startptr);
- else if (!XLByteInSeg(private.startptr, segno))
+ XLogSegNoOffsetToRecPtr(segno, 0, private.startptr, WalSegSz);
+ else if (!XLByteInSeg(private.startptr, segno, WalSegSz))
{
fprintf(stderr,
_("%s: start WAL location %X/%X is not inside file \"%s\"\n"),
/* no second file specified, set end position */
if (!(optind + 1 < argc) && XLogRecPtrIsInvalid(private.endptr))
- XLogSegNoOffsetToRecPtr(segno + 1, 0, private.endptr);
+ XLogSegNoOffsetToRecPtr(segno + 1, 0, private.endptr, WalSegSz);
/* parse ENDSEG if passed */
if (optind + 1 < argc)
/* ignore directory, already have that */
split_path(argv[optind + 1], &directory, &fname);
- fd = fuzzy_open_file(private.inpath, fname);
+ fd = open_file_in_directory(private.inpath, fname);
if (fd < 0)
fatal_error("could not open file \"%s\"", fname);
close(fd);
/* parse position from file */
- XLogFromFileName(fname, &private.timeline, &endsegno);
+ XLogFromFileName(fname, &private.timeline, &endsegno, WalSegSz);
if (endsegno < segno)
fatal_error("ENDSEG %s is before STARTSEG %s",
argv[optind + 1], argv[optind]);
if (XLogRecPtrIsInvalid(private.endptr))
- XLogSegNoOffsetToRecPtr(endsegno + 1, 0, private.endptr);
+ XLogSegNoOffsetToRecPtr(endsegno + 1, 0, private.endptr,
+ WalSegSz);
/* set segno to endsegno for check of --end */
segno = endsegno;
}
- if (!XLByteInSeg(private.endptr, segno) &&
- private.endptr != (segno + 1) * XLogSegSize)
+ if (!XLByteInSeg(private.endptr, segno, WalSegSz) &&
+ private.endptr != (segno + 1) * WalSegSz)
{
fprintf(stderr,
_("%s: end WAL location %X/%X is not inside file \"%s\"\n"),
goto bad_argument;
}
}
+ else
+ identify_target_directory(&private, private.inpath, NULL);
/* we don't know what to print */
if (XLogRecPtrIsInvalid(private.startptr))
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(XLogDumpReadPage, &private);
+ xlogreader_state = XLogReaderAllocate(WalSegSz, XLogDumpReadPage,
+ &private);
if (!xlogreader_state)
fatal_error("out of memory");
* to the start of a record and also wasn't a pointer to the beginning of
* a segment (e.g. we were used in file mode).
*/
- if (first_record != private.startptr && (private.startptr % XLogSegSize) != 0)
+ if (first_record != private.startptr &&
+ XLogSegmentOffset(private.startptr, WalSegSz) != 0)
printf(ngettext("first record is after %X/%X, at %X/%X, skipping over %u byte\n",
"first record is after %X/%X, at %X/%X, skipping over %u bytes\n",
(first_record - private.startptr)),
extern bool reachedConsistency;
/* these variables are GUC parameters related to XLOG */
+extern int wal_segment_size;
extern int min_wal_size_mb;
extern int max_wal_size_mb;
extern int wal_keep_segments;
#define XLogPageHeaderSize(hdr) \
(((hdr)->xlp_info & XLP_LONG_HEADER) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD)
-/*
- * The XLOG is split into WAL segments (physical files) of the size indicated
- * by XLOG_SEG_SIZE.
- */
-#define XLogSegSize ((uint32) XLOG_SEG_SIZE)
-#define XLogSegmentsPerXLogId (UINT64CONST(0x100000000) / XLOG_SEG_SIZE)
+/* wal_segment_size can range from 1MB to 1GB */
+#define WalSegMinSize 1024 * 1024
+#define WalSegMaxSize 1024 * 1024 * 1024
+/* default number of min and max wal segments */
+#define DEFAULT_MIN_WAL_SEGS 5
+#define DEFAULT_MAX_WAL_SEGS 64
+
+/* check that the given size is a valid wal_segment_size */
+#define IsPowerOf2(x) (x > 0 && ((x) & ((x)-1)) == 0)
+#define IsValidWalSegSize(size) \
+ (IsPowerOf2(size) && \
+ ((size) >= WalSegMinSize && (size) <= WalSegMaxSize))
+
+#define XLogSegmentsPerXLogId(wal_segsz_bytes) \
+ (UINT64CONST(0x100000000) / (wal_segsz_bytes))
+
+#define XLogSegNoOffsetToRecPtr(segno, offset, dest, wal_segsz_bytes) \
+ (dest) = (segno) * (wal_segsz_bytes) + (offset)
-#define XLogSegNoOffsetToRecPtr(segno, offset, dest) \
- (dest) = (segno) * XLOG_SEG_SIZE + (offset)
+#define XLogSegmentOffset(xlogptr, wal_segsz_bytes) \
+ ((xlogptr) & ((wal_segsz_bytes) - 1))
/*
* Compute a segment number from an XLogRecPtr.
* for deciding which segment to write given a pointer to a record end,
* for example.
*/
-#define XLByteToSeg(xlrp, logSegNo) \
- logSegNo = (xlrp) / XLogSegSize
+#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) \
+ logSegNo = (xlrp) / (wal_segsz_bytes)
-#define XLByteToPrevSeg(xlrp, logSegNo) \
- logSegNo = ((xlrp) - 1) / XLogSegSize
+#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes) \
+ logSegNo = ((xlrp) - 1) / (wal_segsz_bytes)
/*
* Is an XLogRecPtr within a particular XLOG segment?
* For XLByteInSeg, do the computation at face value. For XLByteInPrevSeg,
* a boundary byte is taken to be in the previous segment.
*/
-#define XLByteInSeg(xlrp, logSegNo) \
- (((xlrp) / XLogSegSize) == (logSegNo))
+#define XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) \
+ (((xlrp) / (wal_segsz_bytes)) == (logSegNo))
-#define XLByteInPrevSeg(xlrp, logSegNo) \
- ((((xlrp) - 1) / XLogSegSize) == (logSegNo))
+#define XLByteInPrevSeg(xlrp, logSegNo, wal_segsz_bytes) \
+ ((((xlrp) - 1) / (wal_segsz_bytes)) == (logSegNo))
/* Check if an XLogRecPtr value is in a plausible range */
#define XRecOffIsValid(xlrp) \
/* Length of XLog file name */
#define XLOG_FNAME_LEN 24
-#define XLogFileName(fname, tli, logSegNo) \
+#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \
snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli, \
- (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \
- (uint32) ((logSegNo) % XLogSegmentsPerXLogId))
+ (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+ (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)))
#define XLogFileNameById(fname, tli, log, seg) \
snprintf(fname, MAXFNAMELEN, "%08X%08X%08X", tli, log, seg)
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + XLOG_FNAME_LEN, ".partial") == 0)
-#define XLogFromFileName(fname, tli, logSegNo) \
+#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \
do { \
uint32 log; \
uint32 seg; \
sscanf(fname, "%08X%08X%08X", tli, &log, &seg); \
- *logSegNo = (uint64) log * XLogSegmentsPerXLogId + seg; \
+ *logSegNo = (uint64) log * XLogSegmentsPerXLogId(wal_segsz_bytes) + seg; \
} while (0)
-#define XLogFilePath(path, tli, logSegNo) \
- snprintf(path, MAXPGPATH, XLOGDIR "/%08X%08X%08X", tli, \
- (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \
- (uint32) ((logSegNo) % XLogSegmentsPerXLogId))
+#define XLogFilePath(path, tli, logSegNo, wal_segsz_bytes) \
+ snprintf(path, MAXPGPATH, XLOGDIR "/%08X%08X%08X", tli, \
+ (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+ (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)))
#define TLHistoryFileName(fname, tli) \
snprintf(fname, MAXFNAMELEN, "%08X.history", tli)
#define StatusFilePath(path, xlog, suffix) \
snprintf(path, MAXPGPATH, XLOGDIR "/archive_status/%s%s", xlog, suffix)
-#define BackupHistoryFileName(fname, tli, logSegNo, offset) \
+#define BackupHistoryFileName(fname, tli, logSegNo, startpoint, wal_segsz_bytes) \
snprintf(fname, MAXFNAMELEN, "%08X%08X%08X.%08X.backup", tli, \
- (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \
- (uint32) ((logSegNo) % XLogSegmentsPerXLogId), offset)
+ (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+ (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+ (uint32) (XLogSegmentOffset(startpoint, wal_segsz_bytes)))
#define IsBackupHistoryFileName(fname) \
(strlen(fname) > XLOG_FNAME_LEN && \
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + strlen(fname) - strlen(".backup"), ".backup") == 0)
-#define BackupHistoryFilePath(path, tli, logSegNo, offset) \
+#define BackupHistoryFilePath(path, tli, logSegNo, startpoint, wal_segsz_bytes) \
snprintf(path, MAXPGPATH, XLOGDIR "/%08X%08X%08X.%08X.backup", tli, \
- (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \
- (uint32) ((logSegNo) % XLogSegmentsPerXLogId), offset)
+ (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+ (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)), \
+ (uint32) (XLogSegmentOffset((startpoint), wal_segsz_bytes)))
/*
* Information logged when we detect a change in one of the parameters
* ----------------------------------------
*/
+ /*
+ * Segment size of the to-be-parsed data (mandatory).
+ */
+ int wal_segment_size;
+
/*
* Data input callback (mandatory).
*
};
/* Get a new XLogReader */
-extern XLogReaderState *XLogReaderAllocate(XLogPageReadCB pagereadfunc,
+extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
+ XLogPageReadCB pagereadfunc,
void *private_data);
/* Free an XLogReader */
/* Version identifier for this pg_control format */
-#define PG_CONTROL_VERSION 1002
+#define PG_CONTROL_VERSION 1003
/* Nonce key length, see below */
#define MOCK_AUTH_NONCE_LEN 32
*/
#undef XLOG_BLCKSZ
-/* XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2
- and larger than XLOG_BLCKSZ (preferably, a great deal larger than
- XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
-#undef XLOG_SEG_SIZE
-
/* Number of bits in a file offset, on hosts where this is settable. */
*------------------------------------------------------------------------
*/
+/*
+ * This is default value for wal_segment_size to be used at intidb when run
+ * without --walsegsize option. Must be a valid segment size.
+ */
+#define DEFAULT_XLOG_SEG_SIZE (16*1024*1024)
+
/*
* Maximum length for identifiers (e.g. table names, column names,
* function names). Names actually are limited to one less byte than this,
1024, "\n";
print $o "#define XLOG_BLCKSZ ",
1024 * $self->{options}->{wal_blocksize}, "\n";
- print $o "#define XLOG_SEG_SIZE (", $self->{options}->{wal_segsize},
- " * 1024 * 1024)\n";
if ($self->{options}->{float4byval})
{
projects / postgresql / commitdiff