Issue #25262. Added support for BINBYTES8 opcode in Python implementation of

unpickler.  Highest 32 bits of 64-bit size for BINUNICODE8 and BINBYTES8
opcodes no longer silently ignored on 32-bit platforms in C implementation.

diff --git a/Lib/pickle.py b/Lib/pickle.py
index 67382aee9219fa89d8f3a9ff421465b67e54214b..c7298af9384ddb39d770f77d704c3d2b29c1ffa9 100644 (file)
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -1204,6 +1204,14 @@ class _Unpickler:
         self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
     dispatch[BINUNICODE8[0]] = load_binunicode8
 
+    def load_binbytes8(self):
+        len, = unpack('<Q', self.read(8))
+        if len > maxsize:
+            raise UnpicklingError("BINBYTES8 exceeds system's maximum size "
+                                  "of %d bytes" % maxsize)
+        self.append(self.read(len))
+    dispatch[BINBYTES8[0]] = load_binbytes8
+
     def load_short_binstring(self):
         len = self.read(1)[0]
         data = self.read(len)

diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index a0ee168fcfd4c28fc91d1e32161bfacc33839b4e..c6f4f6cb43c56e3aead2e039da2ee7b797ef80a7 100644 (file)
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -857,6 +857,26 @@ class AbstractUnpickleTests(unittest.TestCase):
         self.assert_is_copy([(100,), (100,)],
                             self.loads(b'((Kdtp0\nh\x00l.))'))
 
+    def test_binbytes8(self):
+        dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
+        self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
+
+    def test_binunicode8(self):
+        dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
+        self.assertEqual(self.loads(dumped), '\u20ac\x00')
+
+    @requires_32b
+    def test_large_32b_binbytes8(self):
+        dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
+        with self.assertRaises((pickle.UnpicklingError, OverflowError)):
+            self.loads(dumped)
+
+    @requires_32b
+    def test_large_32b_binunicode8(self):
+        dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
+        with self.assertRaises((pickle.UnpicklingError, OverflowError)):
+            self.loads(dumped)
+
     def test_get(self):
         pickled = b'((lp100000\ng100000\nt.'
         unpickled = self.loads(pickled)

diff --git a/Misc/NEWS b/Misc/NEWS
index 48f5c4e436ce1b8b784d6ab762855cf65f883863..a75b6e92cb4206512d6f4e692da24fb8b2335b7e 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -78,6 +78,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #25262. Added support for BINBYTES8 opcode in Python implementation of
+  unpickler.  Highest 32 bits of 64-bit size for BINUNICODE8 and BINBYTES8
+  opcodes no longer silently ignored on 32-bit platforms in C implementation.
+
 - Issue #25034: Fix string.Formatter problem with auto-numbering and
   nested format_specs. Patch by Anthon van der Neut.
 

diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index c6cb8bc7c0e47f679a9bf4a4f191eb51829022ea..830479fe9490d854761567df816adf9452b352d4 100644 (file)
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -4540,7 +4540,17 @@ calc_binsize(char *bytes, int nbytes)
     int i;
     size_t x = 0;
 
-    for (i = 0; i < nbytes && i < sizeof(size_t); i++) {
+    if (nbytes > (int)sizeof(size_t)) {
+        /* Check for integer overflow.  BINBYTES8 and BINUNICODE8 opcodes
+         * have 64-bit size that can't be represented on 32-bit platform.
+         */
+        for (i = (int)sizeof(size_t); i < nbytes; i++) {
+            if (s[i])
+                return -1;
+        }
+        nbytes = (int)sizeof(size_t);
+    }
+    for (i = 0; i < nbytes; i++) {
         x |= (size_t) s[i] << (8 * i);
     }