gh-141370: Fix undefined behavior when using Py_ABS() (GH-141548)

Co-authored-by: Sergey B Kirpichev <skirpichev@gmail.com>

diff --git a/Include/pymacro.h b/Include/pymacro.h
index 857cdf12db9bf26b2f2315855289647e731bf62a..7ecce44a0d2a4287652b74fc888bebe168886745 100644 (file)
--- a/Include/pymacro.h
+++ b/Include/pymacro.h
@@ -116,6 +116,12 @@
 
 /* Absolute value of the number x */
 #define Py_ABS(x) ((x) < 0 ? -(x) : (x))
+/* Safer implementation that avoids an undefined behavior for the minimal
+   value of the signed integer type if its absolute value is larger than
+   the maximal value of the signed integer type (in the two's complement
+   representations, which is common).
+ */
+#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u)))
 
 #define _Py_XSTRINGIFY(x) #x
 

diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index a6cf899fa51e759632f510903d353181b3100d97..a55ec6cf3b8353541ca5b4674122e00a97a43951 100644 (file)
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -549,6 +549,17 @@ class BaseBytesTest:
         self.assertEqual(three_bytes.hex(':', 2), 'b9:01ef')
         self.assertEqual(three_bytes.hex(':', 1), 'b9:01:ef')
         self.assertEqual(three_bytes.hex('*', -2), 'b901*ef')
+        self.assertEqual(three_bytes.hex(sep=':', bytes_per_sep=2), 'b9:01ef')
+        self.assertEqual(three_bytes.hex(sep='*', bytes_per_sep=-2), 'b901*ef')
+        for bytes_per_sep in 3, -3, 2**31-1, -(2**31-1):
+            with self.subTest(bytes_per_sep=bytes_per_sep):
+                self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef')
+        for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000:
+            with self.subTest(bytes_per_sep=bytes_per_sep):
+                try:
+                    self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef')
+                except OverflowError:
+                    pass
 
         value = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
         self.assertEqual(value.hex('.', 8), '7b7305000000776f.726c646902000000.730500000068656c.6c6f690100000030')

diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
index 8b1fb0eba1f8b60bd7653c996c0cac1558130bef..662bdfccc79125155f188083622ade6a82875092 100644 (file)
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -43,6 +43,11 @@ class IntTestCase(unittest.TestCase, HelperMixin):
             for expected in (-n, n):
                 self.helper(expected)
             n = n >> 1
+        n = 1 << 100
+        while n:
+            for expected in (-n, -n+1, n-1, n):
+                self.helper(expected)
+            n = n >> 1
 
     def test_int64(self):
         # Simulate int marshaling with TYPE_INT64.

diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py
index 64f440f180bbf0d7963d5f2fa67c827377aec1a6..1bd58eb6408833a28ad947b732a7ce98cedb0fe5 100644 (file)
--- a/Lib/test/test_memoryview.py
+++ b/Lib/test/test_memoryview.py
@@ -600,6 +600,25 @@ class OtherTest(unittest.TestCase):
         m2 = m1[::-1]
         self.assertEqual(m2.hex(), '30' * 200000)
 
+    def test_memoryview_hex_separator(self):
+        x = bytes(range(97, 102))
+        m1 = memoryview(x)
+        m2 = m1[::-1]
+        self.assertEqual(m2.hex(':'), '65:64:63:62:61')
+        self.assertEqual(m2.hex(':', 2), '65:6463:6261')
+        self.assertEqual(m2.hex(':', -2), '6564:6362:61')
+        self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261')
+        self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61')
+        for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1):
+            with self.subTest(bytes_per_sep=bytes_per_sep):
+                self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261')
+        for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000:
+            with self.subTest(bytes_per_sep=bytes_per_sep):
+                try:
+                    self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261')
+                except OverflowError:
+                    pass
+
     def test_copy(self):
         m = memoryview(b'abc')
         with self.assertRaises(TypeError):

diff --git a/Python/marshal.c b/Python/marshal.c
index 8b56de6575559ca7fa916675f7f04eff0af269d2..69d6dd7cf0f802c31df3c80b9b4d41a10b1cc5bf 100644 (file)
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -310,7 +310,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
     }
     if (!long_export.digits) {
         int8_t sign = long_export.value < 0 ? -1 : 1;
-        uint64_t abs_value = Py_ABS(long_export.value);
+        uint64_t abs_value = _Py_ABS_CAST(uint64_t, long_export.value);
         uint64_t d = abs_value;
         long l = 0;
 

diff --git a/Python/pystrhex.c b/Python/pystrhex.c
index 38484f5a7d4227bc1f99a39a3c893ea40df4c1df..af2f5c5dce5fca5c02d6bfdda4cbed67caddf5e8 100644 (file)
--- a/Python/pystrhex.c
+++ b/Python/pystrhex.c
@@ -42,8 +42,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen,
     else {
         bytes_per_sep_group = 0;
     }
-
-    unsigned int abs_bytes_per_sep = Py_ABS(bytes_per_sep_group);
+    unsigned int abs_bytes_per_sep = _Py_ABS_CAST(unsigned int, bytes_per_sep_group);
     Py_ssize_t resultlen = 0;
     if (bytes_per_sep_group && arglen > 0) {
         /* How many sep characters we'll be inserting. */