From a9ec978eb730e0ee9a0aa329009b15f1ead78386 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 9 Jun 2013 13:47:00 -0400 Subject: [PATCH] Remove fixed limit on the number of concurrent AllocateFile() requests. AllocateFile(), AllocateDir(), and some sister routines share a small array for remembering requests, so that the files can be closed on transaction failure. Previously that array had a fixed size, MAX_ALLOCATED_DESCS (32). While historically that had seemed sufficient, Steve Toutant pointed out that this meant you couldn't scan more than 32 file_fdw foreign tables in one query, because file_fdw depends on the COPY code which uses AllocateFile(). There are probably other cases, or will be in the future, where this nonconfigurable limit impedes users. We can't completely remove any such limit, at least not without a lot of work, since each such request requires a kernel file descriptor and most platforms limit the number we can have. (In principle we could "virtualize" these descriptors, as fd.c already does for the main VFD pool, but not without an additional layer of overhead and a lot of notational impact on the calling code.) But we can at least let the array size be configurable. Hence, change the code to allow up to max_safe_fds/2 allocated file requests. On modern platforms this should allow several hundred concurrent file_fdw scans, or more if one increases the value of max_files_per_process. To go much further than that, we'd need to do some more work on the data structure, since the current code for closing requests has potentially O(N^2) runtime; but it should still be all right for request counts in this range. Back-patch to 9.1 where contrib/file_fdw was introduced. --- src/backend/storage/file/fd.c | 144 +++++++++++++++++++++++++--------- 1 file changed, 106 insertions(+), 38 deletions(-) diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index b858cfc261..8410b7a528 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -179,13 +179,7 @@ static uint64 temporary_files_size = 0; /* * List of stdio FILEs and DIRs opened with AllocateFile * and AllocateDir. - * - * Since we don't want to encourage heavy use of AllocateFile or AllocateDir, - * it seems OK to put a pretty small maximum limit on the number of - * simultaneously allocated descs. */ -#define MAX_ALLOCATED_DESCS 32 - typedef enum { AllocateDescFile, @@ -195,16 +189,17 @@ typedef enum typedef struct { AllocateDescKind kind; + SubTransactionId create_subid; union { FILE *file; DIR *dir; } desc; - SubTransactionId create_subid; } AllocateDesc; static int numAllocatedDescs = 0; -static AllocateDesc allocatedDescs[MAX_ALLOCATED_DESCS]; +static int maxAllocatedDescs = 0; +static AllocateDesc *allocatedDescs = NULL; /* * Number of temporary files opened during the current session; @@ -230,6 +225,7 @@ static int nextTempTableSpace = 0; * Insert - put a file at the front of the Lru ring * LruInsert - put a file at the front of the Lru ring and open it * ReleaseLruFile - Release an fd by closing the last entry in the Lru ring + * ReleaseLruFiles - Release fd(s) until we're under the max_safe_fds limit * AllocateVfd - grab a free (or new) file record (from VfdArray) * FreeVfd - free a file record * @@ -257,11 +253,14 @@ static void LruDelete(File file); static void Insert(File file); static int LruInsert(File file); static bool ReleaseLruFile(void); +static void ReleaseLruFiles(void); static File AllocateVfd(void); static void FreeVfd(File file); static int FileAccess(File file); static File OpenTemporaryFileInTablespace(Oid tblspcOid, bool rejectError); +static bool reserveAllocatedDesc(void); +static int FreeDesc(AllocateDesc *desc); static void AtProcExit_Files(int code, Datum arg); static void CleanupTempFiles(bool isProcExit); static void RemovePgTempFilesInDir(const char *tmpdirname); @@ -664,11 +663,8 @@ LruInsert(File file) if (FileIsNotOpen(file)) { - while (nfile + numAllocatedDescs >= max_safe_fds) - { - if (!ReleaseLruFile()) - break; - } + /* Close excess kernel FDs. */ + ReleaseLruFiles(); /* * The open could still fail for lack of file descriptors, eg due to @@ -707,6 +703,9 @@ LruInsert(File file) return 0; } +/* + * Release one kernel FD by closing the least-recently-used VFD. + */ static bool ReleaseLruFile(void) { @@ -725,6 +724,20 @@ ReleaseLruFile(void) return false; /* no files available to free */ } +/* + * Release kernel FDs as needed to get under the max_safe_fds limit. + * After calling this, it's OK to try to open another file. + */ +static void +ReleaseLruFiles(void) +{ + while (nfile + numAllocatedDescs >= max_safe_fds) + { + if (!ReleaseLruFile()) + break; + } +} + static File AllocateVfd(void) { @@ -878,11 +891,8 @@ PathNameOpenFile(FileName fileName, int fileFlags, int fileMode) file = AllocateVfd(); vfdP = &VfdCache[file]; - while (nfile + numAllocatedDescs >= max_safe_fds) - { - if (!ReleaseLruFile()) - break; - } + /* Close excess kernel FDs. */ + ReleaseLruFiles(); vfdP->fd = BasicOpenFile(fileName, fileFlags, fileMode); @@ -1461,6 +1471,66 @@ FilePathName(File file) } +/* + * Make room for another allocatedDescs[] array entry if needed and possible. + * Returns true if an array element is available. + */ +static bool +reserveAllocatedDesc(void) +{ + AllocateDesc *newDescs; + int newMax; + + /* Quick out if array already has a free slot. */ + if (numAllocatedDescs < maxAllocatedDescs) + return true; + + /* + * If the array hasn't yet been created in the current process, initialize + * it with FD_MINFREE / 2 elements. In many scenarios this is as many as + * we will ever need, anyway. We don't want to look at max_safe_fds + * immediately because set_max_safe_fds() may not have run yet. + */ + if (allocatedDescs == NULL) + { + newMax = FD_MINFREE / 2; + newDescs = (AllocateDesc *) malloc(newMax * sizeof(AllocateDesc)); + /* Out of memory already? Treat as fatal error. */ + if (newDescs == NULL) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); + allocatedDescs = newDescs; + maxAllocatedDescs = newMax; + return true; + } + + /* + * Consider enlarging the array beyond the initial allocation used above. + * By the time this happens, max_safe_fds should be known accurately. + * + * We mustn't let allocated descriptors hog all the available FDs, and in + * practice we'd better leave a reasonable number of FDs for VFD use. So + * set the maximum to max_safe_fds / 2. (This should certainly be at + * least as large as the initial size, FD_MINFREE / 2.) + */ + newMax = max_safe_fds / 2; + if (newMax > maxAllocatedDescs) + { + newDescs = (AllocateDesc *) realloc(allocatedDescs, + newMax * sizeof(AllocateDesc)); + /* Treat out-of-memory as a non-fatal error. */ + if (newDescs == NULL) + return false; + allocatedDescs = newDescs; + maxAllocatedDescs = newMax; + return true; + } + + /* Can't enlarge allocatedDescs[] any more. */ + return false; +} + /* * Routines that want to use stdio (ie, FILE*) should use AllocateFile * rather than plain fopen(). This lets fd.c deal with freeing FDs if @@ -1486,16 +1556,15 @@ AllocateFile(const char *name, const char *mode) DO_DB(elog(LOG, "AllocateFile: Allocated %d (%s)", numAllocatedDescs, name)); - /* - * The test against MAX_ALLOCATED_DESCS prevents us from overflowing - * allocatedFiles[]; the test against max_safe_fds prevents AllocateFile - * from hogging every one of the available FDs, which'd lead to infinite - * looping. - */ - if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || - numAllocatedDescs >= max_safe_fds - 1) - elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open file \"%s\"", - name); + /* Can we allocate another non-virtual FD? */ + if (!reserveAllocatedDesc()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("exceeded maxAllocatedDescs (%d) while trying to open file \"%s\"", + maxAllocatedDescs, name))); + + /* Close excess kernel FDs. */ + ReleaseLruFiles(); TryAgain: if ((file = fopen(name, mode)) != NULL) @@ -1602,16 +1671,15 @@ AllocateDir(const char *dirname) DO_DB(elog(LOG, "AllocateDir: Allocated %d (%s)", numAllocatedDescs, dirname)); - /* - * The test against MAX_ALLOCATED_DESCS prevents us from overflowing - * allocatedDescs[]; the test against max_safe_fds prevents AllocateDir - * from hogging every one of the available FDs, which'd lead to infinite - * looping. - */ - if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || - numAllocatedDescs >= max_safe_fds - 1) - elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open directory \"%s\"", - dirname); + /* Can we allocate another non-virtual FD? */ + if (!reserveAllocatedDesc()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("exceeded maxAllocatedDescs (%d) while trying to open directory \"%s\"", + maxAllocatedDescs, dirname))); + + /* Close excess kernel FDs. */ + ReleaseLruFiles(); TryAgain: if ((dir = opendir(dirname)) != NULL) -- 2.40.0