Create a GUC parameter max_files_per_process that is a configurable

upper limit on what we will believe from sysconf(_SC_OPEN_MAX).  The
default value is 1000, so that under ordinary conditions it won't
affect the behavior.  But on platforms where the kernel promises far
more than it can deliver, this can be used to prevent running out of
file descriptors.  See numerous past discussions, eg, pgsql-hackers
around 23-Dec-2000.

diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml
index 8cb1eb364ae3a964080c022e1efbdb3e39b6b3ad..6afdd9c9637e2fe2814a399e6b2ce0ce16d466ab 100644 (file)
--- a/doc/src/sgml/runtime.sgml
+++ b/doc/src/sgml/runtime.sgml
@@ -1,5 +1,5 @@
 <!--
-$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.85 2001/09/23 21:52:36 petere Exp $
+$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.86 2001/09/30 18:57:45 tgl Exp $
 -->
 
 <Chapter Id="runtime">
@@ -1218,6 +1218,26 @@ dynamic_library_path = '/usr/local/lib/postgresql:/home/my_project/lib:$libdir'
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><varname>MAX_FILES_PER_PROCESS</varname> (<type>integer</type>)</term>
+      <listitem>
+       <para>
+        Sets the maximum number of simultaneously open files in each server
+       process.  The default is 1000.  The limit actually used by the code
+       is the smaller of this setting and the result of
+       <literal>sysconf(_SC_OPEN_MAX)</literal>.
+       Therefore, on systems where sysconf returns a reasonable limit,
+       you don't need to worry about this setting.  But on some platforms
+       (notably, most BSD systems), sysconf returns a value that is much
+       larger than the system can really support when a large number of
+       processes all try to open that many files.  If you find yourself
+       seeing <quote>Too many open files</> failures, try reducing this
+       setting.
+        This option can only be set at server start.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><varname>MAX_FSM_RELATIONS</varname> (<type>integer</type>)</term>
       <listitem>

diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index a696f1fa1fb5d5a0b2b7da2deffa238650d2abc9..c2db3c8dea8328dc408ab6d9d450a761b89c1784 100644 (file)
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *       $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.83 2001/08/04 19:42:34 momjian Exp $
+ *       $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.84 2001/09/30 18:57:45 tgl Exp $
  *
  * NOTES:
  *
@@ -69,7 +69,7 @@
  *
  * (Even though most dynamic loaders now use dlopen(3) or the
  * equivalent, the OS must still open several files to perform the
- * dynamic loading.  Keep this here.)
+ * dynamic loading.  And stdin/stdout/stderr count too.  Keep this here.)
  */
 #ifndef RESERVE_FOR_LD
 #define RESERVE_FOR_LD 10
@@ -87,6 +87,14 @@
 #define FD_MINFREE 10
 #endif
 
+/*
+ * A number of platforms return values for sysconf(_SC_OPEN_MAX) that are
+ * far beyond what they can really support.  This GUC parameter limits what
+ * we will believe.
+ */
+int max_files_per_process = 1000;
+
+
 /* Debugging.... */
 
 #ifdef FDDEBUG
@@ -281,29 +289,46 @@ pg_nofile(void)
 {
        static long no_files = 0;
 
+       /* need do this calculation only once */
        if (no_files == 0)
        {
-               /* need do this calculation only once */
-#ifndef HAVE_SYSCONF
-               no_files = (long) NOFILE;
-#else
+               /*
+                * Ask the system what its files-per-process limit is.
+                */
+#ifdef HAVE_SYSCONF
                no_files = sysconf(_SC_OPEN_MAX);
-               if (no_files == -1)
+               if (no_files <= 0)
                {
-/* tweak for Hurd, which does not support NOFILE */
 #ifdef NOFILE
-                       elog(DEBUG, "pg_nofile: Unable to get _SC_OPEN_MAX using sysconf(); using %d", NOFILE);
                        no_files = (long) NOFILE;
 #else
-                       elog(FATAL, "pg_nofile: Unable to get _SC_OPEN_MAX using sysconf() and NOFILE is undefined");
+                       no_files = (long) max_files_per_process;
 #endif
+                       elog(DEBUG, "pg_nofile: sysconf(_SC_OPEN_MAX) failed; using %ld",
+                                no_files);
                }
+#else /* !HAVE_SYSCONF */
+#ifdef NOFILE
+               no_files = (long) NOFILE;
+#else
+               no_files = (long) max_files_per_process;
 #endif
+#endif /* HAVE_SYSCONF */
+
+               /*
+                * Some platforms return hopelessly optimistic values.  Apply a
+                * configurable upper limit.
+                */
+               if (no_files > (long) max_files_per_process)
+                       no_files = (long) max_files_per_process;
 
+               /*
+                * Make sure we have enough to get by after reserving some for LD.
+                */
                if ((no_files - RESERVE_FOR_LD) < FD_MINFREE)
-                       elog(FATAL, "pg_nofile: insufficient File Descriptors in postmaster to start backend (%ld).\n"
-                                "                   O/S allows %ld, Postmaster reserves %d, We need %d (MIN) after that.",
-                                no_files - RESERVE_FOR_LD, no_files, RESERVE_FOR_LD, FD_MINFREE);
+                       elog(FATAL, "pg_nofile: insufficient file descriptors available to start backend.\n"
+                                "\tSystem allows %ld, we need at least %d.",
+                                no_files, RESERVE_FOR_LD + FD_MINFREE);
 
                no_files -= RESERVE_FOR_LD;
        }

diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 6efd41c2a8e857445c101de4ff5dd4321202d778..c8a9a0afc28362e58555617945f43a6f7d40d693 100644 (file)
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4,7 +4,7 @@
  * Support for grand unified configuration scheme, including SET
  * command, configuration file, and command line options.
  *
- * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.53 2001/09/29 04:02:25 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.54 2001/09/30 18:57:45 tgl Exp $
  *
  * Copyright 2000 by PostgreSQL Global Development Group
  * Written by Peter Eisentraut <peter_e@gmx.net>.
@@ -31,6 +31,7 @@
 #include "optimizer/paths.h"
 #include "optimizer/planmain.h"
 #include "parser/parse_expr.h"
+#include "storage/fd.h"
 #include "storage/freespace.h"
 #include "storage/lock.h"
 #include "storage/proc.h"
@@ -302,6 +303,9 @@ static struct config_int
        {"vacuum_mem", PGC_USERSET, &VacuumMem,
        8192, 1024, INT_MAX, NULL, NULL},
 
+       {"max_files_per_process", PGC_BACKEND, &max_files_per_process,
+       1000, 25, INT_MAX, NULL, NULL},
+
        {"debug_level", PGC_USERSET, &DebugLvl,
        0, 0, 16, NULL, NULL},
 

diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 789f1a1d08d3911267dab58f5c50c29afe19e961..fbf555304e32f7c76c11dea529f4b85b62ffffa1 100644 (file)
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -176,10 +176,11 @@
 #
 #dynamic_library_path = '$libdir'
 #australian_timezones = false
-#authentication_timeout = 60  # min 1, max 600
+#authentication_timeout = 60    # min 1, max 600
 #deadlock_timeout = 1000
 #default_transaction_isolation = 'read committed'
-#max_expr_depth = 10000 # min 10
+#max_expr_depth = 10000         # min 10
+#max_files_per_process = 1000   # min 25
 #password_encryption = false
 #sql_inheritance = true
 #transform_null_equals = false

diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h
index f3e06978a67ed5c4825851bf905d3e3078e3a398..3526be1a1ef6ee1519bd50450106b28a00b74ab4 100644 (file)
--- a/src/include/storage/fd.h
+++ b/src/include/storage/fd.h
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: fd.h,v 1.30 2001/06/11 04:12:29 tgl Exp $
+ * $Id: fd.h,v 1.31 2001/09/30 18:57:45 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -43,6 +43,11 @@ typedef char *FileName;
 
 typedef int File;
 
+
+/* GUC parameter */
+extern int max_files_per_process;
+
+
 /*
  * prototypes for functions in fd.c
  */