gh-81489: Use Unicode APIs for mmap tagname on Windows (GH-14133)

(cherry picked from commit b4d4aa9e8d61476267951c72321fadffc2d82227)

Co-authored-by: Zackery Spytz <zspytz@gmail.com>
Co-authored-by: Erlend E. Aasland <erlend@python.org>

diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index 4ca7a64451d4c729111e65912a1c09ceb3258361..9572468ddf804cc64035fbf6c507a2658f90cb8d 100644 (file)
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -62,8 +62,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
    the same file.  If you specify the name of an existing tag, that tag is
    opened, otherwise a new tag of this name is created.  If this parameter is
    omitted or ``None``, the mapping is created without a name.  Avoiding the
-   use of the tag parameter will assist in keeping your code portable between
-   Unix and Windows.
+   use of the *tagname* parameter will assist in keeping your code portable
+   between Unix and Windows.
 
    *offset* may be specified as a non-negative integer offset. mmap references
    will be relative to the offset from the beginning of the file. *offset*

diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py
index dfcf3039422af5e0d0328c2b1c5d022a4a3d6a02..1867e8c957f58cf34e5e0f433d8f80ba3a50c759 100644 (file)
--- a/Lib/test/test_mmap.py
+++ b/Lib/test/test_mmap.py
@@ -671,14 +671,16 @@ class MmapTests(unittest.TestCase):
         m2.close()
         m1.close()
 
+        with self.assertRaisesRegex(TypeError, 'tagname'):
+            mmap.mmap(-1, 8, tagname=1)
+
     @cpython_only
     @unittest.skipUnless(os.name == 'nt', 'requires Windows')
     def test_sizeof(self):
         m1 = mmap.mmap(-1, 100)
         tagname = random_tagname()
         m2 = mmap.mmap(-1, 100, tagname=tagname)
-        self.assertEqual(sys.getsizeof(m2),
-                         sys.getsizeof(m1) + len(tagname) + 1)
+        self.assertGreater(sys.getsizeof(m2), sys.getsizeof(m1))
 
     @unittest.skipUnless(os.name == 'nt', 'requires Windows')
     def test_crasher_on_windows(self):

diff --git a/Misc/NEWS.d/next/Windows/2019-06-16-11-27-05.bpo-37308.Iz_NU_.rst b/Misc/NEWS.d/next/Windows/2019-06-16-11-27-05.bpo-37308.Iz_NU_.rst
new file mode 100644 (file)
index 0000000..3ba29d6
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-16-11-27-05.bpo-37308.Iz_NU_.rst
@@ -0,0 +1,2 @@
+Fix mojibake in :class:`mmap.mmap` when using a non-ASCII *tagname* argument
+on Windows.

diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index c5e6c27f73b855e0e7414610696cbf60f2909162..827f619e6eb8b6bdc3113a685f50cc15d8c83c75 100644 (file)
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -29,10 +29,6 @@
 #include "structmember.h"         // PyMemberDef
 #include <stddef.h>               // offsetof()
 
-// to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA
-// need to be replaced with OpenFileMappingW / CreateFileMappingW
-#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES)
-
 #ifndef MS_WINDOWS
 #define UNIX
 # ifdef HAVE_FCNTL_H
@@ -113,7 +109,7 @@ typedef struct {
 #ifdef MS_WINDOWS
     HANDLE      map_handle;
     HANDLE      file_handle;
-    char *      tagname;
+    wchar_t *   tagname;
 #endif
 
 #ifdef UNIX
@@ -531,7 +527,7 @@ mmap_resize_method(mmap_object *self,
         CloseHandle(self->map_handle);
         /* if the file mapping still exists, it cannot be resized. */
         if (self->tagname) {
-            self->map_handle = OpenFileMappingA(FILE_MAP_WRITE, FALSE,
+            self->map_handle = OpenFileMappingW(FILE_MAP_WRITE, FALSE,
                                     self->tagname);
             if (self->map_handle) {
                 PyErr_SetFromWindowsErr(ERROR_USER_MAPPED_FILE);
@@ -560,7 +556,7 @@ mmap_resize_method(mmap_object *self,
 
         /* create a new file mapping and map a new view */
         /* FIXME: call CreateFileMappingW with wchar_t tagname */
-        self->map_handle = CreateFileMappingA(
+        self->map_handle = CreateFileMappingW(
             self->file_handle,
             NULL,
             PAGE_READWRITE,
@@ -836,7 +832,7 @@ mmap__sizeof__method(mmap_object *self, void *unused)
 {
     size_t res = _PyObject_SIZE(Py_TYPE(self));
     if (self->tagname) {
-        res += strlen(self->tagname) + 1;
+        res += (wcslen(self->tagname) + 1) * sizeof(self->tagname[0]);
     }
     return PyLong_FromSize_t(res);
 }
@@ -1390,7 +1386,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
     DWORD off_lo;       /* lower 32 bits of offset */
     DWORD size_hi;      /* upper 32 bits of size */
     DWORD size_lo;      /* lower 32 bits of size */
-    const char *tagname = "";
+    PyObject *tagname = Py_None;
     DWORD dwErr = 0;
     int fileno;
     HANDLE fh = 0;
@@ -1400,7 +1396,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
                                 "tagname",
                                 "access", "offset", NULL };
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|ziL", keywords,
+    if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords,
                                      &fileno, &map_size,
                                      &tagname, &access, &offset)) {
         return NULL;
@@ -1533,17 +1529,19 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
     m_obj->weakreflist = NULL;
     m_obj->exports = 0;
     /* set the tag name */
-    if (tagname != NULL && *tagname != '\0') {
-        m_obj->tagname = PyMem_Malloc(strlen(tagname)+1);
+    if (!Py_IsNone(tagname)) {
+        if (!PyUnicode_Check(tagname)) {
+            Py_DECREF(m_obj);
+            return PyErr_Format(PyExc_TypeError, "expected str or None for "
+                                "'tagname', not %.200s",
+                                Py_TYPE(tagname)->tp_name);
+        }
+        m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL);
         if (m_obj->tagname == NULL) {
-            PyErr_NoMemory();
             Py_DECREF(m_obj);
             return NULL;
         }
-        strcpy(m_obj->tagname, tagname);
     }
-    else
-        m_obj->tagname = NULL;
 
     m_obj->access = (access_mode)access;
     size_hi = (DWORD)(size >> 32);
@@ -1552,7 +1550,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
     off_lo = (DWORD)(offset & 0xFFFFFFFF);
     /* For files, it would be sufficient to pass 0 as size.
        For anonymous maps, we have to pass the size explicitly. */
-    m_obj->map_handle = CreateFileMappingA(m_obj->file_handle,
+    m_obj->map_handle = CreateFileMappingW(m_obj->file_handle,
                                            NULL,
                                            flProtect,
                                            size_hi,
@@ -1763,5 +1761,3 @@ PyInit_mmap(void)
 {
     return PyModuleDef_Init(&mmapmodule);
 }
-
-#endif /* !MS_WINDOWS || MS_WINDOWS_DESKTOP || MS_WINDOWS_GAMES */

diff --git a/PC/config.c b/PC/config.c
index 9d0fe6f87df69a7ef5eb464ad4a7b1fb9d2d90a8..48d4bef27ebb39b30752ed345b1a01e1990867d5 100644 (file)
--- a/PC/config.c
+++ b/PC/config.c
@@ -43,9 +43,7 @@ extern PyObject* PyInit__collections(void);
 extern PyObject* PyInit__heapq(void);
 extern PyObject* PyInit__bisect(void);
 extern PyObject* PyInit__symtable(void);
-#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES)
 extern PyObject* PyInit_mmap(void);
-#endif
 extern PyObject* PyInit__csv(void);
 extern PyObject* PyInit__sre(void);
 #if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)