* buffile.c
* Management of large buffered files, primarily temporary files.
*
- * Portions Copyright (c) 1996-2000, PostgreSQL, Inc
+ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
- * $Header: /cvsroot/pgsql/src/backend/storage/file/buffile.c,v 1.4 2000/01/26 05:56:55 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/storage/file/buffile.c,v 1.36 2010/01/02 16:57:51 momjian Exp $
*
* NOTES:
*
* will go away automatically at transaction end. If the underlying
* virtual File is made with OpenTemporaryFile, then all resources for
* the file are certain to be cleaned up even if processing is aborted
- * by elog(ERROR). To avoid confusion, the caller should take care that
+ * by ereport(ERROR). To avoid confusion, the caller should take care that
* all calls for a single BufFile are made in the same palloc context.
*
* BufFile also supports temporary files that exceed the OS file size limit
- * (by opening multiple fd.c temporary files). This is an essential feature
+ * (by opening multiple fd.c temporary files). This is an essential feature
* for sorts and hashjoins on large amounts of data.
*-------------------------------------------------------------------------
*/
-#include <errno.h>
-
#include "postgres.h"
+#include "executor/instrument.h"
+#include "storage/fd.h"
#include "storage/buffile.h"
+#include "storage/buf_internals.h"
/*
- * The maximum safe file size is presumed to be RELSEG_SIZE * BLCKSZ.
- * Note we adhere to this limit whether or not LET_OS_MANAGE_FILESIZE
- * is defined, although md.c ignores it when that symbol is defined.
+ * We break BufFiles into gigabyte-sized segments, regardless of RELSEG_SIZE.
+ * The reason is that we'd like large temporary BufFiles to be spread across
+ * multiple tablespaces when available.
*/
-#define MAX_PHYSICAL_FILESIZE (RELSEG_SIZE * BLCKSZ)
+#define MAX_PHYSICAL_FILESIZE 0x40000000
+#define BUFFILE_SEG_SIZE (MAX_PHYSICAL_FILESIZE / BLCKSZ)
/*
* This data structure represents a buffered file that consists of one or
int numFiles; /* number of physical files in set */
/* all files except the last have length exactly MAX_PHYSICAL_FILESIZE */
File *files; /* palloc'd array with numFiles entries */
- long *offsets; /* palloc'd array with numFiles entries */
- /* offsets[i] is the current seek position of files[i]. We use this
- * to avoid making redundant FileSeek calls.
+ off_t *offsets; /* palloc'd array with numFiles entries */
+
+ /*
+ * offsets[i] is the current seek position of files[i]. We use this to
+ * avoid making redundant FileSeek calls.
*/
bool isTemp; /* can only add files if this is TRUE */
+ bool isInterXact; /* keep open over transactions? */
bool dirty; /* does buffer need to be written? */
+
/*
* "current pos" is position of start of buffer within the logical file.
* Position as seen by user of BufFile is (curFile, curOffset + pos).
*/
int curFile; /* file index (0..n) part of current pos */
- int curOffset; /* offset part of current pos */
+ off_t curOffset; /* offset part of current pos */
int pos; /* next read/write position in buffer */
int nbytes; /* total # of valid bytes in buffer */
char buffer[BLCKSZ];
/*
* Create a BufFile given the first underlying physical file.
- * NOTE: caller must set isTemp true if appropriate.
+ * NOTE: caller must set isTemp and isInterXact if appropriate.
*/
static BufFile *
makeBufFile(File firstfile)
{
- BufFile *file = (BufFile *) palloc(sizeof(BufFile));
+ BufFile *file = (BufFile *) palloc(sizeof(BufFile));
file->numFiles = 1;
file->files = (File *) palloc(sizeof(File));
file->files[0] = firstfile;
- file->offsets = (long *) palloc(sizeof(long));
+ file->offsets = (off_t *) palloc(sizeof(off_t));
file->offsets[0] = 0L;
file->isTemp = false;
+ file->isInterXact = false;
file->dirty = false;
file->curFile = 0;
file->curOffset = 0L;
File pfile;
Assert(file->isTemp);
- pfile = OpenTemporaryFile();
+ pfile = OpenTemporaryFile(file->isInterXact);
Assert(pfile >= 0);
file->files = (File *) repalloc(file->files,
- (file->numFiles+1) * sizeof(File));
- file->offsets = (long *) repalloc(file->offsets,
- (file->numFiles+1) * sizeof(long));
+ (file->numFiles + 1) * sizeof(File));
+ file->offsets = (off_t *) repalloc(file->offsets,
+ (file->numFiles + 1) * sizeof(off_t));
file->files[file->numFiles] = pfile;
file->offsets[file->numFiles] = 0L;
file->numFiles++;
* Create a BufFile for a new temporary file (which will expand to become
* multiple temporary files if more than MAX_PHYSICAL_FILESIZE bytes are
* written to it).
+ *
+ * If interXact is true, the temp file will not be automatically deleted
+ * at end of transaction.
+ *
+ * Note: if interXact is true, the caller had better be calling us in a
+ * memory context that will survive across transaction boundaries.
*/
BufFile *
-BufFileCreateTemp(void)
+BufFileCreateTemp(bool interXact)
{
BufFile *file;
File pfile;
- pfile = OpenTemporaryFile();
+ pfile = OpenTemporaryFile(interXact);
Assert(pfile >= 0);
file = makeBufFile(pfile);
file->isTemp = true;
+ file->isInterXact = interXact;
return file;
}
+#ifdef NOT_USED
/*
* Create a BufFile and attach it to an already-opened virtual File.
*
{
return makeBufFile(file);
}
+#endif
/*
* Close a BufFile
void
BufFileClose(BufFile *file)
{
- int i;
+ int i;
/* flush any unwritten data */
BufFileFlush(file);
static void
BufFileLoadBuffer(BufFile *file)
{
- File thisfile;
+ File thisfile;
/*
* Advance to next component file if necessary and possible.
*
- * This path can only be taken if there is more than one component,
- * so it won't interfere with reading a non-temp file that is over
+ * This path can only be taken if there is more than one component, so it
+ * won't interfere with reading a non-temp file that is over
* MAX_PHYSICAL_FILESIZE.
*/
if (file->curOffset >= MAX_PHYSICAL_FILESIZE &&
- file->curFile+1 < file->numFiles)
+ file->curFile + 1 < file->numFiles)
{
file->curFile++;
file->curOffset = 0L;
}
+
/*
* May need to reposition physical file.
*/
return; /* seek failed, read nothing */
file->offsets[file->curFile] = file->curOffset;
}
+
/*
* Read whatever we can get, up to a full bufferload.
*/
file->nbytes = 0;
file->offsets[file->curFile] += file->nbytes;
/* we choose not to advance curOffset here */
+
+ pgBufferUsage.temp_blks_read++;
}
/*
File thisfile;
/*
- * Unlike BufFileLoadBuffer, we must dump the whole buffer even if
- * it crosses a component-file boundary; so we need a loop.
+ * Unlike BufFileLoadBuffer, we must dump the whole buffer even if it
+ * crosses a component-file boundary; so we need a loop.
*/
while (wpos < file->nbytes)
{
*/
if (file->curOffset >= MAX_PHYSICAL_FILESIZE && file->isTemp)
{
- while (file->curFile+1 >= file->numFiles)
+ while (file->curFile + 1 >= file->numFiles)
extendBufFile(file);
file->curFile++;
file->curOffset = 0L;
}
+
/*
- * Enforce per-file size limit only for temp files, else just try
- * to write as much as asked...
+ * Enforce per-file size limit only for temp files, else just try to
+ * write as much as asked...
*/
bytestowrite = file->nbytes - wpos;
if (file->isTemp)
{
- long availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset;
+ off_t availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset;
- if ((long) bytestowrite > availbytes)
+ if ((off_t) bytestowrite > availbytes)
bytestowrite = (int) availbytes;
}
+
/*
* May need to reposition physical file.
*/
return; /* seek failed, give up */
file->offsets[file->curFile] = file->curOffset;
}
- bytestowrite = FileWrite(thisfile, file->buffer, bytestowrite);
+ bytestowrite = FileWrite(thisfile, file->buffer + wpos, bytestowrite);
if (bytestowrite <= 0)
return; /* failed to write */
file->offsets[file->curFile] += bytestowrite;
file->curOffset += bytestowrite;
wpos += bytestowrite;
+
+ pgBufferUsage.temp_blks_written++;
}
file->dirty = false;
+
/*
* At this point, curOffset has been advanced to the end of the buffer,
* ie, its original value + nbytes. We need to make it point to the
Assert(file->curFile >= 0);
file->curOffset += MAX_PHYSICAL_FILESIZE;
}
- /* Now we can set the buffer empty without changing the logical position */
+
+ /*
+ * Now we can set the buffer empty without changing the logical position
+ */
file->pos = 0;
file->nbytes = 0;
}
{
if (BufFileFlush(file) != 0)
return 0; /* could not flush... */
- Assert(! file->dirty);
+ Assert(!file->dirty);
}
while (size > 0)
* impossible seek is attempted.
*/
int
-BufFileSeek(BufFile *file, int fileno, long offset, int whence)
+BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
{
- int newFile;
- long newOffset;
+ int newFile;
+ off_t newOffset;
+
switch (whence)
{
case SEEK_SET:
newOffset = offset;
break;
case SEEK_CUR:
+
/*
- * Relative seek considers only the signed offset, ignoring fileno.
- * Note that large offsets (> 1 gig) risk overflow in this add...
+ * Relative seek considers only the signed offset, ignoring
+ * fileno. Note that large offsets (> 1 gig) risk overflow in this
+ * add, unless we have 64-bit off_t.
*/
newFile = file->curFile;
newOffset = (file->curOffset + file->pos) + offset;
break;
#endif
default:
- elog(ERROR, "BufFileSeek: invalid whence: %d", whence);
+ elog(ERROR, "invalid whence: %d", whence);
return EOF;
}
while (newOffset < 0)
{
/*
* Seek is to a point within existing buffer; we can just adjust
- * pos-within-buffer, without flushing buffer. Note this is OK
- * whether reading or writing, but buffer remains dirty if we
- * were writing.
+ * pos-within-buffer, without flushing buffer. Note this is OK
+ * whether reading or writing, but buffer remains dirty if we were
+ * writing.
*/
file->pos = (int) (newOffset - file->curOffset);
return 0;
/* Otherwise, must reposition buffer, so flush any dirty data */
if (BufFileFlush(file) != 0)
return EOF;
+
/*
- * At this point and no sooner, check for seek past last segment.
- * The above flush could have created a new segment, so
- * checking sooner would not work (at least not with this code).
+ * At this point and no sooner, check for seek past last segment. The
+ * above flush could have created a new segment, so checking sooner would
+ * not work (at least not with this code).
*/
if (file->isTemp)
{
}
void
-BufFileTell(BufFile *file, int *fileno, long *offset)
+BufFileTell(BufFile *file, int *fileno, off_t *offset)
{
*fileno = file->curFile;
*offset = file->curOffset + file->pos;
BufFileSeekBlock(BufFile *file, long blknum)
{
return BufFileSeek(file,
- (int) (blknum / RELSEG_SIZE),
- (blknum % RELSEG_SIZE) * BLCKSZ,
+ (int) (blknum / BUFFILE_SEG_SIZE),
+ (off_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ,
SEEK_SET);
}
+#ifdef NOT_USED
/*
* BufFileTellBlock --- block-oriented tell
*
long
BufFileTellBlock(BufFile *file)
{
- long blknum;
+ long blknum;
blknum = (file->curOffset + file->pos) / BLCKSZ;
- blknum += file->curFile * RELSEG_SIZE;
+ blknum += file->curFile * BUFFILE_SEG_SIZE;
return blknum;
}
+
+#endif
projects / postgresql / blobdiff