[3.8] bpo-40330: Fix utf-8 size check in ShareableList (GH-19606) (GH-19625)

The item size must be checked after encoding to bytes, not before.

Automerge-Triggered-By: @pitrou.
(cherry picked from commit eba9f6155df59c9beed97fb5764c9f01dd941af0)

Co-authored-by: Antoine Pitrou <antoine@python.org>

diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py
index 184e36704baaeba95cc36a9030a2ff819a77c2e2..f92eb012c83160e2e10a23e8c7c9741da3e53b9d 100644 (file)
--- a/Lib/multiprocessing/shared_memory.py
+++ b/Lib/multiprocessing/shared_memory.py
@@ -433,9 +433,12 @@ class ShareableList:
 
         if not isinstance(value, (str, bytes)):
             new_format = self._types_mapping[type(value)]
+            encoded_value = value
         else:
-            if len(value) > self._allocated_bytes[position]:
-                raise ValueError("exceeds available storage for existing str")
+            encoded_value = (value.encode(_encoding)
+                             if isinstance(value, str) else value)
+            if len(encoded_value) > self._allocated_bytes[position]:
+                raise ValueError("bytes/str item exceeds available storage")
             if current_format[-1] == "s":
                 new_format = current_format
             else:
@@ -448,8 +451,7 @@ class ShareableList:
             new_format,
             value
         )
-        value = value.encode(_encoding) if isinstance(value, str) else value
-        struct.pack_into(new_format, self.shm.buf, offset, value)
+        struct.pack_into(new_format, self.shm.buf, offset, encoded_value)
 
     def __reduce__(self):
         return partial(self.__class__, name=self.shm.name), ()

diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index e64a8e9b3b87936f9a7cdd09a39e50a8802aebff..d5336e4d7be301dee1c4c9d0d2bcdc575cb0fc69 100644 (file)
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -3964,9 +3964,21 @@ class _TestSharedMemory(BaseTestCase):
         sl[4] = 'some'  # Change type at a given position.
         self.assertEqual(sl[4], 'some')
         self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
-        with self.assertRaises(ValueError):
-            sl[4] = 'far too many'  # Exceeds available storage.
+        with self.assertRaisesRegex(ValueError,
+                                    "exceeds available storage"):
+            sl[4] = 'far too many'
         self.assertEqual(sl[4], 'some')
+        sl[0] = 'encodés'  # Exactly 8 bytes of UTF-8 data
+        self.assertEqual(sl[0], 'encodés')
+        self.assertEqual(sl[1], b'HoWdY')  # no spillage
+        with self.assertRaisesRegex(ValueError,
+                                    "exceeds available storage"):
+            sl[0] = 'encodées'  # Exactly 9 bytes of UTF-8 data
+        self.assertEqual(sl[1], b'HoWdY')
+        with self.assertRaisesRegex(ValueError,
+                                    "exceeds available storage"):
+            sl[1] = b'123456789'
+        self.assertEqual(sl[1], b'HoWdY')
 
         # Exercise count().
         with warnings.catch_warnings():

diff --git a/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst b/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst
new file mode 100644 (file)
index 0000000..98cb62f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst
@@ -0,0 +1,2 @@
+In :meth:`ShareableList.__setitem__`, check the size of a new string item
+after encoding it to utf-8, not before.