Issue #5604: non-ASCII characters in module name passed to

imp.find_module() were converted to UTF-8 while the path is
converted to the default filesystem encoding, causing nonsense.
Thanks to Andrew Svetlov.

(This time to the right branch.  Will block duplicate merge to 3.0.2.)

diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index 1e04f5ad0c187fbbd1a35d06b68649f816bd1946..cc6792ab46972ad5e1b3f38380416310da1e15ec 100644 (file)
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -1,4 +1,7 @@
 import imp
+import locale
+import os
+import os.path
 import sys
 import unittest
 from test import support
@@ -75,6 +78,74 @@ class ImportTests(unittest.TestCase):
             support.unlink(temp_mod_name + '.pyc')
             support.unlink(temp_mod_name + '.pyo')
 
+    def test_issue5604(self):
+        # Test cannot cover imp.load_compiled function.
+        # Martin von Loewis note what shared library cannot have non-ascii
+        # character because init_xxx function cannot be compiled
+        # and issue never happens for dynamic modules.
+        # But sources modified to follow generic way for processing pathes.
+
+        locale_encoding = locale.getpreferredencoding()
+
+        # covers utf-8 and Windows ANSI code pages
+        # one non-space symbol from every page
+        # (http://en.wikipedia.org/wiki/Code_page)
+        known_locales = {
+            'utf-8' : b'\xe4',
+            'cp1250' : b'\x8C',
+            'cp1251' : b'\xc0',
+            'cp1252' : b'\xc0',
+            'cp1253' : b'\xc1',
+            'cp1254' : b'\xc0',
+            'cp1255' : b'\xe0',
+            'cp1256' : b'\xe0',
+            'cp1257' : b'\xc0',
+            'cp1258' : b'\xc0',
+            }
+
+        special_char = known_locales.get(locale_encoding)
+        if special_char:
+            encoded_char = special_char.decode(locale_encoding)
+            temp_mod_name = 'test_imp_helper_' + encoded_char
+            test_package_name = 'test_imp_helper_package_' + encoded_char
+            init_file_name = os.path.join(test_package_name, '__init__.py')
+            try:
+                with open(temp_mod_name + '.py', 'w') as file:
+                    file.write('a = 1\n')
+                file, filename, info = imp.find_module(temp_mod_name)
+                self.assertNotEquals(None, file)
+                self.assertTrue(filename[:-3].endswith(temp_mod_name))
+                self.assertEquals('.py', info[0])
+                self.assertEquals('U', info[1])
+                self.assertEquals(imp.PY_SOURCE, info[2])
+
+                mod = imp.load_module(temp_mod_name, file, filename, info)
+                self.assertEquals(1, mod.a)
+                file.close()
+
+                mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
+                self.assertEquals(1, mod.a)
+
+                mod = imp.load_compiled(temp_mod_name, temp_mod_name + '.pyc')
+                self.assertEquals(1, mod.a)
+
+                if not os.path.exists(test_package_name):
+                    os.mkdir(test_package_name)
+                with open(init_file_name, 'w') as file:
+                    file.write('b = 2\n')
+                package = imp.load_package(test_package_name, test_package_name)
+                self.assertEquals(2, package.b)
+            finally:
+                support.unlink(temp_mod_name + '.py')
+                support.unlink(temp_mod_name + '.pyc')
+                support.unlink(temp_mod_name + '.pyo')
+
+                support.unlink(init_file_name + '.py')
+                support.unlink(init_file_name + '.pyc')
+                support.unlink(init_file_name + '.pyo')
+                support.rmtree(test_package_name)
+
+
     def test_reload(self):
         import marshal
         imp.reload(marshal)

diff --git a/Misc/ACKS b/Misc/ACKS
index 85a443bba7d4dead46cb755a3bfc2028d201da4f..03ed92de10f6c8cbbb042b0b174b2db3a1f391b4 100644 (file)
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -688,6 +688,7 @@ Nathan Sullivan
 Mark Summerfield
 Hisao Suzuki
 Kalle Svensson
+Andrew Svetlov
 Paul Swartz
 Thenault Sylvain
 Geoff Talvola

diff --git a/Misc/NEWS b/Misc/NEWS
index f2888be1f3b1191ed7b4ca452b3da88658baacb3..a12b8eaffa99a6da8e663bad496621c6e7558da5 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 3.1 alpha 2?
 Core and Builtins
 -----------------
 
+- Issue #5604: non-ASCII characters in module name passed to
+  imp.find_module() were converted to UTF-8 while the path is
+  converted to the default filesystem encoding, causing nonsense.
+
 - Issue #5126: str.isprintable() returned False for space characters.
 
 - Issue #4688: Add a heuristic so that tuples and dicts containing only

diff --git a/Python/import.c b/Python/import.c
index 9c70ed83dfb9aca54598e331ec622132afb6c556..2be33085cac42f70f439d56b504cdce595b548d2 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -3040,15 +3040,20 @@ imp_load_compiled(PyObject *self, PyObject *args)
        PyObject *fob = NULL;
        PyObject *m;
        FILE *fp;
-       if (!PyArg_ParseTuple(args, "ss|O:load_compiled",
-                             &name, &pathname, &fob))
+       if (!PyArg_ParseTuple(args, "ses|O:load_compiled",
+                             &name, 
+                             Py_FileSystemDefaultEncoding, &pathname, 
+                             &fob))
                return NULL;
        fp = get_file(pathname, fob, "rb");
-       if (fp == NULL)
+       if (fp == NULL) {
+               PyMem_Free(pathname);
                return NULL;
+       }
        m = load_compiled_module(name, pathname, fp);
        if (fob == NULL)
                fclose(fp);
+       PyMem_Free(pathname);
        return m;
 }
 
@@ -3062,15 +3067,20 @@ imp_load_dynamic(PyObject *self, PyObject *args)
        PyObject *fob = NULL;
        PyObject *m;
        FILE *fp = NULL;
-       if (!PyArg_ParseTuple(args, "ss|O:load_dynamic",
-                             &name, &pathname, &fob))
+       if (!PyArg_ParseTuple(args, "ses|O:load_dynamic",
+                             &name, 
+                             Py_FileSystemDefaultEncoding, &pathname, 
+                             &fob))
                return NULL;
        if (fob) {
                fp = get_file(pathname, fob, "r");
-               if (fp == NULL)
+               if (fp == NULL) {
+                       PyMem_Free(pathname);
                        return NULL;
+               }
        }
        m = _PyImport_LoadDynamicModule(name, pathname, fp);
+       PyMem_Free(pathname);
        return m;
 }
 
@@ -3084,12 +3094,16 @@ imp_load_source(PyObject *self, PyObject *args)
        PyObject *fob = NULL;
        PyObject *m;
        FILE *fp;
-       if (!PyArg_ParseTuple(args, "ss|O:load_source",
-                             &name, &pathname, &fob))
+       if (!PyArg_ParseTuple(args, "ses|O:load_source",
+                             &name, 
+                             Py_FileSystemDefaultEncoding, &pathname,
+                             &fob))
                return NULL;
        fp = get_file(pathname, fob, "r");
-       if (fp == NULL)
+       if (fp == NULL) {
+               PyMem_Free(pathname);
                return NULL;
+       }
        m = load_source_module(name, pathname, fp);
        if (fob == NULL)
                fclose(fp);
@@ -3102,13 +3116,15 @@ imp_load_module(PyObject *self, PyObject *args)
        char *name;
        PyObject *fob;
        char *pathname;
+       PyObject * ret;
        char *suffix; /* Unused */
        char *mode;
        int type;
        FILE *fp;
 
-       if (!PyArg_ParseTuple(args, "sOs(ssi):load_module",
-                             &name, &fob, &pathname,
+       if (!PyArg_ParseTuple(args, "sOes(ssi):load_module",
+                             &name, &fob, 
+                             Py_FileSystemDefaultEncoding, &pathname,
                              &suffix, &mode, &type))
                return NULL;
        if (*mode) {
@@ -3119,6 +3135,7 @@ imp_load_module(PyObject *self, PyObject *args)
                if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) {
                        PyErr_Format(PyExc_ValueError,
                                     "invalid file open mode %.200s", mode);
+                       PyMem_Free(pathname);
                        return NULL;
                }
        }
@@ -3126,10 +3143,14 @@ imp_load_module(PyObject *self, PyObject *args)
                fp = NULL;
        else {
                fp = get_file(NULL, fob, mode);
-               if (fp == NULL)
+               if (fp == NULL) {
+                       PyMem_Free(pathname);
                        return NULL;
-       }
-       return load_module(name, fp, pathname, type, NULL);
+               }
+       } 
+       ret = load_module(name, fp, pathname, type, NULL);
+       PyMem_Free(pathname);
+       return ret;
 }
 
 static PyObject *
@@ -3137,9 +3158,13 @@ imp_load_package(PyObject *self, PyObject *args)
 {
        char *name;
        char *pathname;
-       if (!PyArg_ParseTuple(args, "ss:load_package", &name, &pathname))
+       PyObject * ret;
+       if (!PyArg_ParseTuple(args, "ses:load_package", 
+                             &name, Py_FileSystemDefaultEncoding, &pathname))
                return NULL;
-       return load_package(name, pathname);
+       ret = load_package(name, pathname);
+       PyMem_Free(pathname);
+       return ret;
 }
 
 static PyObject *