]> granicus.if.org Git - python/commitdiff
Fixes issue #19081: When a zipimport .zip file in sys.path being imported from
authorGregory P. Smith <greg@krypto.org>
Wed, 8 Jan 2014 02:34:23 +0000 (18:34 -0800)
committerGregory P. Smith <greg@krypto.org>
Wed, 8 Jan 2014 02:34:23 +0000 (18:34 -0800)
is modified during the lifetime of the Python process after zipimport has
already cached the zip's table of contents we detect this and recover
rather than read bad data from the .zip (causing odd import errors).

1  2 
Lib/test/test_zipimport.py
Misc/NEWS
Modules/zipimport.c

index 1e351c8c8a16e30cfdc5e0ebb18e33f766b234ea,0459596b2a60321f8d45dd19b54649fc8d1ee159..bf058bcc5668f1afe460a499028b182eeaf29492
@@@ -54,23 -46,27 +54,44 @@@ pyc_file = importlib.util.cache_from_so
  pyc_ext = ('.pyc' if __debug__ else '.pyo')
  
  
 +class ImportHooksBaseTestCase(unittest.TestCase):
 +
 +    def setUp(self):
 +        self.path = sys.path[:]
 +        self.meta_path = sys.meta_path[:]
 +        self.path_hooks = sys.path_hooks[:]
 +        sys.path_importer_cache.clear()
 +        self.modules_before = support.modules_setup()
 +
 +    def tearDown(self):
 +        sys.path[:] = self.path
 +        sys.meta_path[:] = self.meta_path
 +        sys.path_hooks[:] = self.path_hooks
 +        sys.path_importer_cache.clear()
 +        support.modules_cleanup(*self.modules_before)
 +
 +
+ def _write_zip_package(zipname, files,
+                        data_to_prepend=b"", compression=ZIP_STORED):
+     z = ZipFile(zipname, "w")
+     try:
+         for name, (mtime, data) in files.items():
+             zinfo = ZipInfo(name, time.localtime(mtime))
+             zinfo.compress_type = compression
+             z.writestr(zinfo, data)
+     finally:
+         z.close()
+     if data_to_prepend:
+         # Prepend data to the start of the zipfile
+         with open(zipname, "rb") as f:
+             zip_data = f.read()
+         with open(zipname, "wb") as f:
+             f.write(data_to_prepend)
+             f.write(zip_data)
  class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
  
      compression = ZIP_STORED
diff --cc Misc/NEWS
index fa6d0d530dd6695db8d9962557379f38d6189644,c17e493f76f8889011f2fbcd808b6772ea108b70..778e7df0f4487103892bf0e6e4bcff0aa0bdf3a2
+++ b/Misc/NEWS
@@@ -10,58 -10,13 +10,63 @@@ Release date: 2014-01-1
  Core and Builtins
  -----------------
  
+ - Issue #19081: When a zipimport .zip file in sys.path being imported from
+   is modified during the lifetime of the Python process after zipimport has
+   already cached the zip's table of contents we detect this and recover
+   rather than read bad data from the .zip (causing odd import errors).
 +Library
 +-------
 +
 +- Issue #19719: Make importlib.abc.MetaPathFinder.find_module(),
 +  PathEntryFinder.find_loader(), and Loader.load_module() use PEP 451 APIs to
 +  help with backwards-compatibility.
 +
 +- Issue #20144: inspect.Signature now supports parsing simple symbolic
 +  constants as parameter default values in __text_signature__.
 +
 +- Issue #20072: Fixed multiple errors in tkinter with wantobjects is False.
 +
 +Tools/Demos
 +-----------
 +
 +- Issue #19723: The marker comments Argument Clinic uses have been changed
 +  to improve readability.
 +
 +- Issue #20157: When Argument Clinic renames a parameter because its name
 +  collides with a C keyword, it no longer exposes that rename to PyArg_Parse.
 +
 +- Issue #20141: Improved Argument Clinic's support for the PyArg_Parse "O!"
 +  format unit.
 +
 +- Issue #20144: Argument Clinic now supports simple symbolic constants
 +  as parameter default values.
 +
 +- Issue #20143: The line numbers reported in Argument Clinic errors are
 +  now more accurate.
 +
 +- Issue #20142: Py_buffer variables generated by Argument Clinic are now
 +  initialized with a default value.
 +
 +Build
 +-----
 +
 +- Issue #12837: Silence a tautological comparison warning on OS X under Clang in
 +  socketmodule.c.
 +
 +What's New in Python 3.4.0 Beta 2?
 +==================================
 +
 +Release date: 2014-01-05
 +
 +Core and Builtins
 +-----------------
 +
  - Issue #17432: Drop UCS2 from names of Unicode functions in python3.def.
  
 +- Issue #19526: Exclude all new API from the stable ABI. Exceptions can be
 +  made if a need is demonstrated.
 +
  - Issue #19969: PyBytes_FromFormatV() now raises an OverflowError if "%c"
    argument is not in range [0; 255].
  
index 8fe919539fdfffd7b1e13be5002c443cfdac4f8e,bfac46233db06bc0b1d7ec69672807cf274d9d8e..02bdb28dfa2a0feca23447c6b4da1e77ea717125
@@@ -560,7 -588,11 +594,8 @@@ zipimporter_get_data(PyObject *obj, PyO
  {
      ZipImporter *self = (ZipImporter *)obj;
      PyObject *path, *key;
-     PyObject *toc_entry;
+     FILE *fp;
 -#ifdef ALTSEP
 -    _Py_IDENTIFIER(replace);
 -#endif
+     PyObject *toc_entry, *data;
      Py_ssize_t path_start, path_len, len;
  
      if (!PyArg_ParseTuple(args, "U:zipimporter.get_data", &path))
@@@ -831,10 -879,135 +882,135 @@@ get_long(unsigned char *buf) 
      return x;
  }
  
+ /* Return 1 if objects a and b fail a Py_EQ test for an attr. */
+ static int
+ compare_obj_attr_strings(PyObject *obj_a, PyObject *obj_b, char *attr_name)
+ {
+     int problem = 0;
+     PyObject *attr_a = PyObject_GetAttrString(obj_a, attr_name);
+     PyObject *attr_b = PyObject_GetAttrString(obj_b, attr_name);
+     if (attr_a == NULL || attr_b == NULL)
+         problem = 1;
+     else
+         problem = (PyObject_RichCompareBool(attr_a, attr_b, Py_EQ) != 1);
+     Py_XDECREF(attr_a);
+     Py_XDECREF(attr_b);
+     return problem;
+ }
  /*
-    read_directory(archive) -> files dict (new reference)
+  * Returns an open FILE * on success.
+  * Returns NULL on error with the Python error context set.
+  */
+ static FILE *
+ safely_reopen_archive(ZipImporter *self)
+ {
+     FILE *fp;
+     PyObject *stat_now = NULL;
+     fp = fopen_rb_and_stat(self->archive, &stat_now);
+     if (!fp) {
+         PyErr_Format(ZipImportError,
+                      "zipimport: can not open file %U", self->archive);
+         Py_XDECREF(stat_now);
+         return NULL;
+     }
  
-    Given a path to a Zip archive, build a dict, mapping file names
+     if (stat_now != NULL) {
+         int problem = 0;
+         PyObject *files;
+         PyObject *prev_stat = PyDict_GetItem(zip_stat_cache, self->archive);
+         /* Test stat_now vs the old cached stat on some key attributes. */
+         if (prev_stat != NULL) {
+             problem = compare_obj_attr_strings(prev_stat, stat_now,
+                                                "st_ino");
+             problem |= compare_obj_attr_strings(prev_stat, stat_now,
+                                                 "st_size");
+             problem |= compare_obj_attr_strings(prev_stat, stat_now,
+                                                 "st_mtime");
+         } else {
+             if (Py_VerboseFlag)
+                 PySys_FormatStderr("# zipimport: no stat data for %U!\n",
+                                    self->archive);
+             problem = 1;
+         }
+         if (problem) {
+             if (Py_VerboseFlag)
+                 PySys_FormatStderr("# zipimport: %U modified since last"
+                                    " import, rereading TOC.\n", self->archive);
+             files = read_directory(fp, self->archive);
+             if (files == NULL) {
+                 Py_DECREF(stat_now);
+                 fclose(fp);
+                 return NULL;
+             }
+             if (PyDict_SetItem(zip_directory_cache, self->archive,
+                                files) != 0) {
+                 Py_DECREF(files);
+                 Py_DECREF(stat_now);
+                 fclose(fp);
+                 return NULL;
+             }
+             if (stat_now && PyDict_SetItem(zip_stat_cache, self->archive,
+                                            stat_now) != 0) {
+                 Py_DECREF(files);
+                 Py_DECREF(stat_now);
+                 fclose(fp);
+                 return NULL;
+             }
+             Py_XDECREF(self->files);  /* free the old value. */
+             self->files = files;
+         } else {
+             /* No problem, discard the new stat data. */
+             Py_DECREF(stat_now);
+         }
+     }  /* stat succeeded */
+     return fp;
+ }
+ /*
+    fopen_rb_and_stat(path, &py_stat) -> FILE *
+    Opens path in "rb" mode and populates the Python py_stat stat_result
+    with information about the opened file.  *py_stat may not be changed
+    if there is no fstat_function or if fstat_function fails.
+    Returns NULL and does nothing to *py_stat if the open failed.
+ */
+ static FILE *
+ fopen_rb_and_stat(PyObject *path, PyObject **py_stat_p)
+ {
+     FILE *fp;
+     assert(py_stat_p != NULL);
+     assert(*py_stat_p == NULL);
 -    fp = _Py_fopen(path, "rb");
++    fp = _Py_fopen_obj(path, "rb");
+     if (fp == NULL) {
+         if (!PyErr_Occurred())
+             PyErr_Format(ZipImportError,
+                          "zipimport: can not open file %U", path);
+         return NULL;
+     }
+     if (fstat_function) {
+         PyObject *stat_result = PyObject_CallFunction(fstat_function,
+                                                       "i", fileno(fp));
+         if (stat_result == NULL) {
+             PyErr_Clear();  /* We can function without it. */
+         } else {
+             *py_stat_p = stat_result;
+         }
+     }
+     return fp;
+ }
+ /*
+    read_directory(fp, archive) -> files dict (new reference)
+    Given an open Zip archive, build a dict, mapping file names
     (local to the archive, using SEP as a separator) to toc entries.
  
     A toc_entry is a tuple:
@@@ -1005,8 -1153,7 +1170,7 @@@ read_directory(FILE *fp, PyObject *arch
          PySys_FormatStderr("# zipimport: found %ld names in %R\n",
                             count, archive);
      return files;
 -fseek_error:
 +file_error:
-     fclose(fp);
      Py_XDECREF(files);
      Py_XDECREF(nameobj);
      PyErr_Format(ZipImportError, "can't read Zip file: %R", archive);
@@@ -1090,11 -1226,9 +1243,10 @@@ get_data(FILE *fp, PyObject *archive, P
      l = PyMarshal_ReadLongFromFile(fp);
      if (l != 0x04034B50) {
          /* Bad: Local File Header */
 -        PyErr_Format(ZipImportError,
 -                     "bad local file header in %U",
 -                     archive);
 +        if (!PyErr_Occurred())
 +            PyErr_Format(ZipImportError,
 +                         "bad local file header in %U",
 +                         archive);
-         fclose(fp);
          return NULL;
      }
      if (fseek(fp, file_offset + 26, 0) == -1) {
  
      l = 30 + PyMarshal_ReadShortFromFile(fp) +
          PyMarshal_ReadShortFromFile(fp);        /* local header size */
-         fclose(fp);
 +    if (PyErr_Occurred()) {
 +        return NULL;
 +    }
      file_offset += l;           /* Start of file data */
  
      bytes_size = compress == 0 ? data_size : data_size + 1;