Backport from trunk r52223:

Bug #1572832: fix a bug in ISO-2022 codecs which may cause segfault
when encoding non-BMP unicode characters.  (Submitted by Ray Chason)

diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index 9e42f71e797f8241fab7b8cff2a8da385a7a7731..3fcc8ba81a50cf23e7645be42207e7c4bd215e93 100644 (file)
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -7,7 +7,7 @@
 
 from test import test_support
 from test import test_multibytecodec_support
-import unittest, StringIO, codecs
+import unittest, StringIO, codecs, sys
 
 class Test_StreamWriter(unittest.TestCase):
     if len(u'\U00012345') == 2: # UCS2
@@ -87,6 +87,16 @@ class Test_ISO2022(unittest.TestCase):
             e = u'\u3406'.encode(encoding)
             self.failIf(filter(lambda x: x >= '\x80', e))
 
+    def test_bug1572832(self):
+        if sys.maxunicode >= 0x10000:
+            myunichr = unichr
+        else:
+            myunichr = lambda x: unichr(0xD7C0+(x>>10)) + unichr(0xDC00+(x&0x3FF))
+
+        for x in xrange(0x10000, 0x110000):
+            # Any ISO 2022 codec will cause the segfault
+            myunichr(x).encode('iso_2022_jp', 'ignore')
+
 def test_main():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(Test_StreamWriter))

diff --git a/Misc/NEWS b/Misc/NEWS
index 97619ab8a6983239716e9d2de13dd807fe7bd3f6..2eff31b7f1726f5cdae562223b43906c07c5dfa7 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -80,6 +80,9 @@ Core and builtins
 Extension Modules
 -----------------
 
+- Bug #1572832: fix a bug in ISO-2022 codecs which may cause segfault
+  when encoding non-BMP unicode characters.
+
 - Bug #1556784: allow format strings longer than 127 characters in
   datetime's strftime function.
 

diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c
index e785727ac80624663aa191d6c07d4827da17f0ba..f57226343f3895165e20092884ea05317fadb507 100644 (file)
--- a/Modules/cjkcodecs/_codecs_iso2022.c
+++ b/Modules/cjkcodecs/_codecs_iso2022.c
@@ -593,9 +593,11 @@ ksx1001_encoder(const ucs4_t *data, int *length)
 {
        DBCHAR coded;
        assert(*length == 1);
-       TRYMAP_ENC(cp949, coded, *data)
-               if (!(coded & 0x8000))
-                       return coded;
+       if (*data < 0x10000) {
+               TRYMAP_ENC(cp949, coded, *data)
+                       if (!(coded & 0x8000))
+                               return coded;
+       }
        return MAP_UNMAPPABLE;
 }
 
@@ -629,11 +631,13 @@ jisx0208_encoder(const ucs4_t *data, int *length)
 {
        DBCHAR coded;
        assert(*length == 1);
-       if (*data == 0xff3c) /* F/W REVERSE SOLIDUS */
-               return 0x2140;
-       else TRYMAP_ENC(jisxcommon, coded, *data) {
-               if (!(coded & 0x8000))
-                       return coded;
+       if (*data < 0x10000) {
+               if (*data == 0xff3c) /* F/W REVERSE SOLIDUS */
+                       return 0x2140;
+               else TRYMAP_ENC(jisxcommon, coded, *data) {
+                       if (!(coded & 0x8000))
+                               return coded;
+               }
        }
        return MAP_UNMAPPABLE;
 }
@@ -666,9 +670,11 @@ jisx0212_encoder(const ucs4_t *data, int *length)
 {
        DBCHAR coded;
        assert(*length == 1);
-       TRYMAP_ENC(jisxcommon, coded, *data) {
-               if (coded & 0x8000)
-                       return coded & 0x7fff;
+       if (*data < 0x10000) {
+               TRYMAP_ENC(jisxcommon, coded, *data) {
+                       if (coded & 0x8000)
+                               return coded & 0x7fff;
+               }
        }
        return MAP_UNMAPPABLE;
 }
@@ -971,9 +977,11 @@ gb2312_encoder(const ucs4_t *data, int *length)
 {
        DBCHAR coded;
        assert(*length == 1);
-       TRYMAP_ENC(gbcommon, coded, *data) {
-               if (!(coded & 0x8000))
-                       return coded;
+       if (*data < 0x10000) {
+               TRYMAP_ENC(gbcommon, coded, *data) {
+                       if (!(coded & 0x8000))
+                               return coded;
+               }
        }
        return MAP_UNMAPPABLE;
 }