From: Serhiy Storchaka Date: Tue, 9 Sep 2025 18:07:21 +0000 (+0300) Subject: gh-88886: Remove excessive encoding name normalization (GH-137167) X-Git-Tag: v3.15.0a1~431 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=af58a6f883108f02996c784de010df62e75f5e13;p=thirdparty%2FPython%2Fcpython.git gh-88886: Remove excessive encoding name normalization (GH-137167) The codecs lookup function now performs only minimal normalization of the encoding name before passing it to the search functions: all ASCII letters are converted to lower case, spaces are replaced with hyphens. Excessive normalization broke third-party codecs providers, like python-iconv. Revert "bpo-37751: Fix codecs.lookup() normalization (GH-15092)" This reverts commit 20f59fe1f7748ae899aceee4cb560e5e1f528a1f. --- diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 5932012c535b..90a695ef937f 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -68,11 +68,21 @@ The full details for each codec can also be looked up directly: Looks up the codec info in the Python codec registry and returns a :class:`CodecInfo` object as defined below. - Encodings are first looked up in the registry's cache. If not found, the list of + This function first normalizes the *encoding*: all ASCII letters are + converted to lower case, spaces are replaced with hyphens. + Then encoding is looked up in the registry's cache. If not found, the list of registered search functions is scanned. If no :class:`CodecInfo` object is found, a :exc:`LookupError` is raised. Otherwise, the :class:`CodecInfo` object is stored in the cache and returned to the caller. + .. versionchanged:: 3.9 + Any characters except ASCII letters and digits and a dot are converted to underscore. + + .. versionchanged:: next + No characters are converted to underscore anymore. + Spaces are converted to hyphens. + + .. class:: CodecInfo(encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None) Codec details when looking up the codec registry. The constructor @@ -167,14 +177,11 @@ function: .. function:: register(search_function, /) Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters with hyphens - and spaces converted to underscores, and return a :class:`CodecInfo` object. + argument, being the encoding name in all lower case letters with spaces + converted to hyphens, and return a :class:`CodecInfo` object. In case a search function cannot find a given encoding, it should return ``None``. - .. versionchanged:: 3.9 - Hyphens and spaces are converted to underscore. - .. function:: unregister(search_function, /) @@ -1065,7 +1072,7 @@ or with dictionaries as mapping tables. The following table lists the codecs by name, together with a few common aliases, and the languages for which the encoding is likely used. Neither the list of aliases nor the list of languages is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases +case or use a space or a hyphen instead of an underscore are also valid aliases because they are equivalent when normalized by :func:`~encodings.normalize_encoding`. For example, ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index a0355c7a388c..1a3f476ed0f3 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -630,7 +630,6 @@ class CAPICodecs(unittest.TestCase): for name in [ encoding_name, encoding_name.upper(), - encoding_name.replace('_', '-'), ]: with self.subTest(name): self.assertTrue(_testcapi.codec_known_encoding(name)) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index fd7769e8c275..c35a45089435 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3873,26 +3873,22 @@ class Rot13UtilTest(unittest.TestCase): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" def test_codecs_lookup(self): - FOUND = (1, 2, 3, 4) - NOT_FOUND = (None, None, None, None) def search_function(encoding): - if encoding == "aaa_8": - return FOUND + if encoding.startswith("test."): + return (encoding, 2, 3, 4) else: - return NOT_FOUND + return None codecs.register(search_function) self.addCleanup(codecs.unregister, search_function) - self.assertEqual(FOUND, codecs.lookup('aaa_8')) - self.assertEqual(FOUND, codecs.lookup('AAA-8')) - self.assertEqual(FOUND, codecs.lookup('AAA---8')) - self.assertEqual(FOUND, codecs.lookup('AAA 8')) - self.assertEqual(FOUND, codecs.lookup('aaa\xe9\u20ac-8')) - self.assertEqual(NOT_FOUND, codecs.lookup('AAA.8')) - self.assertEqual(NOT_FOUND, codecs.lookup('AAA...8')) - self.assertEqual(NOT_FOUND, codecs.lookup('BBB-8')) - self.assertEqual(NOT_FOUND, codecs.lookup('BBB.8')) - self.assertEqual(NOT_FOUND, codecs.lookup('a\xe9\u20ac-8')) + self.assertEqual(codecs.lookup('test.aaa_8'), ('test.aaa_8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA-8'), ('test.aaa-8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa-8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA---8'), ('test.aaa---8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa---8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA.8'), ('test.aaa.8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA...8'), ('test.aaa...8', 2, 3, 4)) def test_encodings_normalize_encoding(self): # encodings.normalize_encoding() ignores non-ASCII characters. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-28-17-01-05.gh-issue-88886.g4XFPb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-28-17-01-05.gh-issue-88886.g4XFPb.rst new file mode 100644 index 000000000000..0d119efc93d2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-28-17-01-05.gh-issue-88886.g4XFPb.rst @@ -0,0 +1,4 @@ +The codecs lookup function now again performs only minimal normalization of +the encoding name before passing it to the search functions: all ASCII +letters are converted to lower case, spaces are replaced with hyphens. +This restores the pre-Python 3.9 behavior. diff --git a/Python/codecs.c b/Python/codecs.c index 8eb9f2db4135..364e07990ba4 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -85,14 +85,15 @@ PyCodec_Unregister(PyObject *search_function) extern int _Py_normalize_encoding(const char *, char *, size_t); -/* Convert a string to a normalized Python string(decoded from UTF-8): all characters are - converted to lower case, spaces and hyphens are replaced with underscores. */ +/* Convert a string to a normalized Python string: all ASCII letters are + converted to lower case, spaces are replaced with hyphens. */ -static -PyObject *normalizestring(const char *string) +static PyObject* +normalizestring(const char *string) { + size_t i; size_t len = strlen(string); - char *encoding; + char *p; PyObject *v; if (len > PY_SSIZE_T_MAX) { @@ -100,28 +101,30 @@ PyObject *normalizestring(const char *string) return NULL; } - encoding = PyMem_Malloc(len + 1); - if (encoding == NULL) + p = PyMem_Malloc(len + 1); + if (p == NULL) return PyErr_NoMemory(); - - if (!_Py_normalize_encoding(string, encoding, len + 1)) - { - PyErr_SetString(PyExc_RuntimeError, "_Py_normalize_encoding() failed"); - PyMem_Free(encoding); - return NULL; - } - - v = PyUnicode_FromString(encoding); - PyMem_Free(encoding); + for (i = 0; i < len; i++) { + char ch = string[i]; + if (ch == ' ') + ch = '-'; + else + ch = Py_TOLOWER(Py_CHARMASK(ch)); + p[i] = ch; + } + p[i] = '\0'; + v = PyUnicode_FromString(p); + PyMem_Free(p); return v; } /* Lookup the given encoding and return a tuple providing the codec facilities. - The encoding string is looked up converted to all lower-case - characters. This makes encodings looked up through this mechanism - effectively case-insensitive. + ASCII letters in the encoding string is looked up converted to all + lower case. This makes encodings looked up through this mechanism + effectively case-insensitive. Spaces are replaced with hyphens for + names like "US ASCII" and "ISO 8859-1". If no codec is found, a LookupError is set and NULL returned. @@ -142,8 +145,8 @@ PyObject *_PyCodec_Lookup(const char *encoding) assert(interp->codecs.initialized); /* Convert the encoding to a normalized Python string: all - characters are converted to lower case, spaces and hyphens are - replaced with underscores. */ + ASCII letters are converted to lower case, spaces are + replaced with hyphens. */ PyObject *v = normalizestring(encoding); if (v == NULL) { return NULL;