Need for speed: Patch #921466 : sys.path_importer_cache is now used to cache valid and

  invalid file paths for the built-in import machinery which leads to
  fewer open calls on startup.

  Also fix issue with PEP 302 style import hooks which lead to more open()
  calls than necessary.

diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
index ddf2a727615e912bbd495cf16c1d6bd7095e3de4..26c797f0f6775611c37a6559de6a26513c997bdf 100644 (file)
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -340,11 +340,13 @@ def get_importer(path_item):
             importer = None
         sys.path_importer_cache.setdefault(path_item, importer)
 
-    if importer is None:
+    # The boolean values are used for caching valid and invalid
+    # file paths for the built-in import machinery
+    if importer in (None, True, False):
         try:
             importer = ImpImporter(path_item)
         except ImportError:
-            pass
+            importer = None
     return importer
 
 

diff --git a/Misc/NEWS b/Misc/NEWS
index be5e4eaa8a2bd072be6e7c721af506cbd0787bc7..a18e9e7376fc262aa225a7a3613c6c7fd38e782c 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 2.5 alpha 3?
 Core and builtins
 -----------------
 
+- Patch #921466: sys.path_importer_cache is now used to cache valid and
+  invalid file paths for the built-in import machinery which leads to
+  fewer open calls on startup.
+
 - Patch #1442927: ``long(str, base)`` is now up to 6x faster for non-power-
   of-2 bases.  The largest speedup is for inputs with about 1000 decimal
   digits.  Conversion from non-power-of-2 bases remains quadratic-time in

diff --git a/Python/import.c b/Python/import.c
index 862f33c70396e220e6e20f8028bace1af670b2fb..e09365b8c995c02276a39f10881ec2de13af4733 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -1240,7 +1240,33 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
                        if (importer == NULL)
                                return NULL;
                        /* Note: importer is a borrowed reference */
-                       if (importer != Py_None) {
+                       if (importer == Py_False) {
+                               /* Cached as not being a valid dir. */
+                               Py_XDECREF(copy);
+                               continue;
+                       }
+                       else if (importer == Py_True) {
+                               /* Cached as being a valid dir, so just
+                                * continue below. */
+                       }
+                       else if (importer == Py_None) {
+                               /* No importer was found, so it has to be a file.
+                                * Check if the directory is valid. */
+#ifdef HAVE_STAT
+                               if (stat(buf, &statbuf) != 0) {
+                                       /* Directory does not exist. */
+                                       PyDict_SetItem(path_importer_cache,
+                                                      v, Py_False);
+                                       Py_XDECREF(copy);
+                                       continue;
+                               } else {
+                                       PyDict_SetItem(path_importer_cache,
+                                                      v, Py_True);
+                               }
+#endif
+                       }
+                       else {
+                               /* A real import hook importer was found. */
                                PyObject *loader;
                                loader = PyObject_CallMethod(importer,
                                                             "find_module",
@@ -1253,9 +1279,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
                                        return &importhookdescr;
                                }
                                Py_DECREF(loader);
+                               Py_XDECREF(copy);
+                               continue;
                        }
-                       /* no hook was successful, use builtin import */
                }
+               /* no hook was found, use builtin import */
 
                if (len > 0 && buf[len-1] != SEP
 #ifdef ALTSEP