[3.6] bpo-34922: Fix integer overflow in the digest() and hexdigest() methods (GH...

for the SHAKE algorithm in the hashlib module.
(cherry picked from commit 9b8c2e767643256202bb11456ba8665593b9a500)
(cherry picked from commit 8b040e55395b37bdb8fd4ec85a270cfc9ec95307)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 995fe60a076c4e0209a8641d617a9a8614785b7d..9711856853ded32cafee514a5e183805089f0be6 100644 (file)
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -233,6 +233,20 @@ class HashLibTestCase(unittest.TestCase):
                 self.assertIsInstance(h.digest(), bytes)
                 self.assertEqual(hexstr(h.digest()), h.hexdigest())
 
+    def test_digest_length_overflow(self):
+        # See issue #34922
+        large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10)
+        for cons in self.hash_constructors:
+            h = cons()
+            if h.name not in self.shakes:
+                continue
+            for digest in h.digest, h.hexdigest:
+                with self.assertRaises((ValueError, OverflowError)):
+                    digest(-10)
+                for length in large_sizes:
+                    with self.assertRaises((ValueError, OverflowError)):
+                        digest(length)
+
     def test_name_attribute(self):
         for cons in self.hash_constructors:
             h = cons()

diff --git a/Misc/NEWS.d/next/Library/2018-10-07-21-18-52.bpo-34922.37IdsA.rst b/Misc/NEWS.d/next/Library/2018-10-07-21-18-52.bpo-34922.37IdsA.rst
new file mode 100644 (file)
index 0000000..6463886
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-10-07-21-18-52.bpo-34922.37IdsA.rst
@@ -0,0 +1,3 @@
+Fixed integer overflow in the :meth:`~hashlib.shake.digest()` and
+:meth:`~hashlib.shake.hexdigest()` methods for the SHAKE algorithm
+in the :mod:`hashlib` module.

diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c
index 8d880d0e702690c2b67f4ef6fbb920b5aad4d79a..2c2b2dbc5c7d41e0e2583d5b689eaa66a95b2b01 100644 (file)
--- a/Modules/_sha3/sha3module.c
+++ b/Modules/_sha3/sha3module.c
@@ -609,7 +609,10 @@ _SHAKE_digest(SHA3object *self, PyObject *digestlen_obj, int hex)
     if (digestlen == (unsigned long) -1 && PyErr_Occurred()) {
         return NULL;
     }
-
+    if (digestlen >= (1 << 29)) {
+        PyErr_SetString(PyExc_ValueError, "length is too large");
+        return NULL;
+    }
     /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and
      * SHA3_LANESIZE extra space.
      */