Fix str.translate()

Issue #26464: Fix str.translate() when string is ASCII and first replacements
removes character, but next replacement uses a non-ASCII character or a string
longer than 1 character. Regression introduced in Python 3.5.0.

diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index fac8b7b6022a471c35db2cfe989c103d2eb46789..b258db1ffae5d3c148dcb24eb7c5b23d77159e2c 100644 (file)
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -347,6 +347,10 @@ class UnicodeTest(string_tests.CommonTest,
                          "[a]")
         self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': None})),
                          "[]")
+        self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '123'})),
+                         "x123")
+        self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '\xe9'})),
+                         "x\xe9")
 
         # invalid Unicode characters
         invalid_char = 0x10ffff+1

diff --git a/Misc/NEWS b/Misc/NEWS
index 0ee28e9ccc9ff68f2ab047f28b6e01258654b5d9..303daa21f36a8325429680f6581419399ff31b0d 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@ Release date: tba
 Core and Builtins
 -----------------
 
+- Issue #26464: Fix str.translate() when string is ASCII and first replacements
+  removes character, but next replacement uses a non-ASCII character or a
+  string longer than 1 character. Regression introduced in Python 3.5.0.
+
 - Issue #22836: Ensure exception reports from PyErr_Display() and
   PyErr_WriteUnraisable() are sensible even when formatting them produces
   secondary errors.  This affects the reports produced by

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index d4c9aaee217cb380cf8127bdff3ee012dda59180..a1ee776d45ad5d6bcd73e12cf422d65ca657415c 100644 (file)
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -8574,7 +8574,8 @@ unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch,
    translated into writer, raise an exception and return -1 on error. */
 static int
 unicode_fast_translate(PyObject *input, PyObject *mapping,
-                       _PyUnicodeWriter *writer, int ignore)
+                       _PyUnicodeWriter *writer, int ignore,
+                       Py_ssize_t *input_pos)
 {
     Py_UCS1 ascii_table[128], ch, ch2;
     Py_ssize_t len;
@@ -8621,6 +8622,7 @@ unicode_fast_translate(PyObject *input, PyObject *mapping,
 
 exit:
     writer->pos = out - PyUnicode_1BYTE_DATA(writer->buffer);
+    *input_pos = in - PyUnicode_1BYTE_DATA(input);
     return res;
 }
 
@@ -8666,7 +8668,7 @@ _PyUnicode_TranslateCharmap(PyObject *input,
 
     ignore = (errors != NULL && strcmp(errors, "ignore") == 0);
 
-    res = unicode_fast_translate(input, mapping, &writer, ignore);
+    res = unicode_fast_translate(input, mapping, &writer, ignore, &i);
     if (res < 0) {
         _PyUnicodeWriter_Dealloc(&writer);
         return NULL;
@@ -8674,7 +8676,6 @@ _PyUnicode_TranslateCharmap(PyObject *input,
     if (res == 1)
         return _PyUnicodeWriter_Finish(&writer);
 
-    i = writer.pos;
     while (i<size) {
         /* try to encode it */
         int translate;