Issue #5604: non-ASCII characters in module name passed to

imp.find_module() were converted to UTF-8 while the path is
converted to the default filesystem encoding, causing nonsense.
Thanks Andrew Svetlov!

diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index 1e04f5ad0c187fbbd1a35d06b68649f816bd1946..cc6792ab46972ad5e1b3f38380416310da1e15ec 100644 (file)
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -1,4 +1,7 @@
 import imp
+import locale
+import os
+import os.path
 import sys
 import unittest
 from test import support
@@ -75,6 +78,74 @@ class ImportTests(unittest.TestCase):
             support.unlink(temp_mod_name + '.pyc')
             support.unlink(temp_mod_name + '.pyo')
 
+    def test_issue5604(self):
+        # Test cannot cover imp.load_compiled function.
+        # Martin von Loewis note what shared library cannot have non-ascii
+        # character because init_xxx function cannot be compiled
+        # and issue never happens for dynamic modules.
+        # But sources modified to follow generic way for processing pathes.
+
+        locale_encoding = locale.getpreferredencoding()
+
+        # covers utf-8 and Windows ANSI code pages
+        # one non-space symbol from every page
+        # (http://en.wikipedia.org/wiki/Code_page)
+        known_locales = {
+            'utf-8' : b'\xe4',
+            'cp1250' : b'\x8C',
+            'cp1251' : b'\xc0',
+            'cp1252' : b'\xc0',
+            'cp1253' : b'\xc1',
+            'cp1254' : b'\xc0',
+            'cp1255' : b'\xe0',
+            'cp1256' : b'\xe0',
+            'cp1257' : b'\xc0',
+            'cp1258' : b'\xc0',
+            }
+
+        special_char = known_locales.get(locale_encoding)
+        if special_char:
+            encoded_char = special_char.decode(locale_encoding)
+            temp_mod_name = 'test_imp_helper_' + encoded_char
+            test_package_name = 'test_imp_helper_package_' + encoded_char
+            init_file_name = os.path.join(test_package_name, '__init__.py')
+            try:
+                with open(temp_mod_name + '.py', 'w') as file:
+                    file.write('a = 1\n')
+                file, filename, info = imp.find_module(temp_mod_name)
+                self.assertNotEquals(None, file)
+                self.assertTrue(filename[:-3].endswith(temp_mod_name))
+                self.assertEquals('.py', info[0])
+                self.assertEquals('U', info[1])
+                self.assertEquals(imp.PY_SOURCE, info[2])
+
+                mod = imp.load_module(temp_mod_name, file, filename, info)
+                self.assertEquals(1, mod.a)
+                file.close()
+
+                mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
+                self.assertEquals(1, mod.a)
+
+                mod = imp.load_compiled(temp_mod_name, temp_mod_name + '.pyc')
+                self.assertEquals(1, mod.a)
+
+                if not os.path.exists(test_package_name):
+                    os.mkdir(test_package_name)
+                with open(init_file_name, 'w') as file:
+                    file.write('b = 2\n')
+                package = imp.load_package(test_package_name, test_package_name)
+                self.assertEquals(2, package.b)
+            finally:
+                support.unlink(temp_mod_name + '.py')
+                support.unlink(temp_mod_name + '.pyc')
+                support.unlink(temp_mod_name + '.pyo')
+
+                support.unlink(init_file_name + '.py')
+                support.unlink(init_file_name + '.pyc')
+                support.unlink(init_file_name + '.pyo')
+                support.rmtree(test_package_name)
+
+
     def test_reload(self):
         import marshal
         imp.reload(marshal)

diff --git a/Misc/ACKS b/Misc/ACKS
index ca866fff84645241b3657b5b04fcd257b0b04cb7..e48e1df29980aa01203808360ca2d252104cd5b8 100644 (file)
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -675,6 +675,7 @@ Nathan Sullivan
 Mark Summerfield
 Hisao Suzuki
 Kalle Svensson
+Andrew Svetlov
 Paul Swartz
 Thenault Sylvain
 Geoff Talvola
@@ -777,5 +778,5 @@ Artur Zaprzala
 Mike Zarnstorff
 Siebren van der Zee
 Uwe Zessin
-Tarek Ziad\8e
+Tarek ZiadÂ\8e
 Peter Åstrand

diff --git a/Misc/NEWS b/Misc/NEWS
index b1e54e1f2498443358027cb069ba6bbf9ed26c4a..da6626bdada3f6fd0892cb06dd4075b4f491175a 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 3.0.2?
 Core and Builtins
 -----------------
 
+- Issue #5604: non-ASCII characters in module name passed to
+  imp.find_module() were converted to UTF-8 while the path is
+  converted to the default filesystem encoding, causing nonsense.
+
 - Issue #5392: when a very low recursion limit was set, the interpreter would
   abort with a fatal error after the recursion limit was hit twice.
 

diff --git a/Python/import.c b/Python/import.c
index 998a3f1bd47228f1b8f484eb82728c1b455022d7..a460dfea285a4ebc0b3e37d9dcd29160fd697b72 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -3006,15 +3006,20 @@ imp_load_compiled(PyObject *self, PyObject *args)
        PyObject *fob = NULL;
        PyObject *m;
        FILE *fp;
-       if (!PyArg_ParseTuple(args, "ss|O:load_compiled",
-                             &name, &pathname, &fob))
+       if (!PyArg_ParseTuple(args, "ses|O:load_compiled",
+                             &name, 
+                             Py_FileSystemDefaultEncoding, &pathname, 
+                             &fob))
                return NULL;
        fp = get_file(pathname, fob, "rb");
-       if (fp == NULL)
+       if (fp == NULL) {
+               PyMem_Free(pathname);
                return NULL;
+       }
        m = load_compiled_module(name, pathname, fp);
        if (fob == NULL)
                fclose(fp);
+       PyMem_Free(pathname);
        return m;
 }
 
@@ -3028,15 +3033,20 @@ imp_load_dynamic(PyObject *self, PyObject *args)
        PyObject *fob = NULL;
        PyObject *m;
        FILE *fp = NULL;
-       if (!PyArg_ParseTuple(args, "ss|O:load_dynamic",
-                             &name, &pathname, &fob))
+       if (!PyArg_ParseTuple(args, "ses|O:load_dynamic",
+                             &name, 
+                             Py_FileSystemDefaultEncoding, &pathname, 
+                             &fob))
                return NULL;
        if (fob) {
                fp = get_file(pathname, fob, "r");
-               if (fp == NULL)
+               if (fp == NULL) {
+                       PyMem_Free(pathname);
                        return NULL;
+               }
        }
        m = _PyImport_LoadDynamicModule(name, pathname, fp);
+       PyMem_Free(pathname);
        return m;
 }
 
@@ -3050,12 +3060,16 @@ imp_load_source(PyObject *self, PyObject *args)
        PyObject *fob = NULL;
        PyObject *m;
        FILE *fp;
-       if (!PyArg_ParseTuple(args, "ss|O:load_source",
-                             &name, &pathname, &fob))
+       if (!PyArg_ParseTuple(args, "ses|O:load_source",
+                             &name, 
+                             Py_FileSystemDefaultEncoding, &pathname,
+                             &fob))
                return NULL;
        fp = get_file(pathname, fob, "r");
-       if (fp == NULL)
+       if (fp == NULL) {
+               PyMem_Free(pathname);
                return NULL;
+       }
        m = load_source_module(name, pathname, fp);
        if (fob == NULL)
                fclose(fp);
@@ -3068,13 +3082,15 @@ imp_load_module(PyObject *self, PyObject *args)
        char *name;
        PyObject *fob;
        char *pathname;
+       PyObject * ret;
        char *suffix; /* Unused */
        char *mode;
        int type;
        FILE *fp;
 
-       if (!PyArg_ParseTuple(args, "sOs(ssi):load_module",
-                             &name, &fob, &pathname,
+       if (!PyArg_ParseTuple(args, "sOes(ssi):load_module",
+                             &name, &fob, 
+                             Py_FileSystemDefaultEncoding, &pathname,
                              &suffix, &mode, &type))
                return NULL;
        if (*mode) {
@@ -3085,6 +3101,7 @@ imp_load_module(PyObject *self, PyObject *args)
                if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) {
                        PyErr_Format(PyExc_ValueError,
                                     "invalid file open mode %.200s", mode);
+                       PyMem_Free(pathname);
                        return NULL;
                }
        }
@@ -3092,10 +3109,14 @@ imp_load_module(PyObject *self, PyObject *args)
                fp = NULL;
        else {
                fp = get_file(NULL, fob, mode);
-               if (fp == NULL)
+               if (fp == NULL) {
+                       PyMem_Free(pathname);
                        return NULL;
-       }
-       return load_module(name, fp, pathname, type, NULL);
+               }
+       } 
+       ret = load_module(name, fp, pathname, type, NULL);
+       PyMem_Free(pathname);
+       return ret;
 }
 
 static PyObject *
@@ -3103,9 +3124,13 @@ imp_load_package(PyObject *self, PyObject *args)
 {
        char *name;
        char *pathname;
-       if (!PyArg_ParseTuple(args, "ss:load_package", &name, &pathname))
+       PyObject * ret;
+       if (!PyArg_ParseTuple(args, "ses:load_package", 
+                             &name, Py_FileSystemDefaultEncoding, &pathname))
                return NULL;
-       return load_package(name, pathname);
+       ret = load_package(name, pathname);
+       PyMem_Free(pathname);
+       return ret;
 }
 
 static PyObject *