bpo-35372: Fix the code page decoder for input > 2 GiB. (GH-10848)

(cherry picked from commit 4013c179117754b039957db4730880bf3285919d)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index eb21a3915b938a96664ea68fb56f0e5545bcd673..56485de3f6e9bc8b95be2801f28300bede618e46 100644 (file)
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -3191,6 +3191,24 @@ class CodePageTest(unittest.TestCase):
         finally:
             _bootlocale.getpreferredencoding = old_getpreferredencoding
 
+    @support.bigmemtest(size=2**31, memuse=7, dry_run=False)
+    def test_large_input(self):
+        # Test input longer than INT_MAX.
+        # Input should contain undecodable bytes before and after
+        # the INT_MAX limit.
+        encoded = (b'01234567' * (2**28-1) +
+                   b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff')
+        self.assertEqual(len(encoded), 2**31+2)
+        decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True)
+        self.assertEqual(decoded[1], len(encoded))
+        del encoded
+        self.assertEqual(len(decoded[0]), decoded[1])
+        self.assertEqual(decoded[0][:10], '0123456701')
+        self.assertEqual(decoded[0][-20:],
+                         '6701234567'
+                         '\udc85\udc86\udcea\udceb\udcec'
+                         '\udcef\udcfc\udcfd\udcfe\udcff')
+
 
 class ASCIITest(unittest.TestCase):
     def test_encode(self):

diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst
new file mode 100644 (file)
index 0000000..dc2de44
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst
@@ -0,0 +1,2 @@
+Fixed the code page decoder for input longer than 2 GiB containing
+undecodable bytes.

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 29b019887ac0f9ea2f56743ea468b8622e6a2919..0e64bf943db3e14d3583a6b9cb58f36e73e33e76 100644 (file)
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -7285,7 +7285,7 @@ decode_code_page_errors(UINT code_page,
                          "in the target code page.";
     /* each step cannot decode more than 1 character, but a character can be
        represented as a surrogate pair */
-    wchar_t buffer[2], *startout, *out;
+    wchar_t buffer[2], *out;
     int insize;
     Py_ssize_t outsize;
     PyObject *errorHandler = NULL;
@@ -7322,7 +7322,7 @@ decode_code_page_errors(UINT code_page,
         *v = (PyObject*)_PyUnicode_New(size * Py_ARRAY_LENGTH(buffer));
         if (*v == NULL)
             goto error;
-        startout = PyUnicode_AS_UNICODE(*v);
+        out = PyUnicode_AS_UNICODE(*v);
     }
     else {
         /* Extend unicode object */
@@ -7333,11 +7333,10 @@ decode_code_page_errors(UINT code_page,
         }
         if (unicode_resize(v, n + size * Py_ARRAY_LENGTH(buffer)) < 0)
             goto error;
-        startout = PyUnicode_AS_UNICODE(*v) + n;
+        out = PyUnicode_AS_UNICODE(*v) + n;
     }
 
     /* Decode the byte string character per character */
-    out = startout;
     while (in < endin)
     {
         /* Decode a character */
@@ -7392,7 +7391,7 @@ decode_code_page_errors(UINT code_page,
     *out = 0;
 
     /* Extend unicode object */
-    outsize = out - startout;
+    outsize = out - PyUnicode_AS_UNICODE(*v);
     assert(outsize <= PyUnicode_WSTR_LENGTH(*v));
     if (unicode_resize(v, outsize) < 0)
         goto error;