bpo-24214: Fixed the UTF-8 incremental decoder. (GH-12603) (GH-12627)

The bug occurred when the encoded surrogate character is passed
to the incremental decoder in two chunks.
(cherry picked from commit 7a465cb5ee7e298cae626ace1fc3e7d97df79f2e)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 293dfbc61aba10ecb40598aec5dc8f62510da1e6..5ba2c7bdc5f8d3925779be72891b4f9d39459680 100644 (file)
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -401,6 +401,15 @@ class ReadTest(MixInCheckStateHandling):
             self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"),
                              before + backslashreplace + after)
 
+    def test_incremental_surrogatepass(self):
+        # Test incremental decoder for surrogatepass handler:
+        # see issue #24214
+        data = '\uD901'.encode(self.encoding, 'surrogatepass')
+        for i in range(1, len(data)):
+            dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
+            self.assertEqual(dec.decode(data[:i]), '')
+            self.assertEqual(dec.decode(data[i:], True), '\uD901')
+
 
 class UTF32Test(ReadTest, unittest.TestCase):
     encoding = "utf-32"

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst b/Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst
new file mode 100644 (file)
index 0000000..abb2759
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst
@@ -0,0 +1,2 @@
+Fixed support of the surrogatepass error handler in the UTF-8 incremental
+decoder.

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index adcf69d4e53935630f84a9acc7d36f8695f88a11..e18937981bb763d508aa7aa8811bda6f89df8c67 100644 (file)
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4890,6 +4890,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
         case 2:
         case 3:
         case 4:
+            if (s == end || consumed) {
+                goto End;
+            }
             errmsg = "invalid continuation byte";
             startinpos = s - starts;
             endinpos = startinpos + ch - 1;