gh-91156: Fix `encoding="locale"` in UTF-8 mode (GH-70056)

diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index 80107d539505c0f15c7cfd6f2cb85e668fe568c0..53dad99fa1dbc3c8a6a64332023c4bb281ad519e 100644 (file)
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -112,7 +112,7 @@ Text Encoding
 -------------
 
 The default encoding of :class:`TextIOWrapper` and :func:`open` is
-locale-specific (:func:`locale.getpreferredencoding(False) <locale.getpreferredencoding>`).
+locale-specific (:func:`locale.getencoding`).
 
 However, many developers forget to specify the encoding when opening text files
 encoded in UTF-8 (e.g. JSON, TOML, Markdown, etc...) since most Unix
@@ -948,8 +948,7 @@ Text I/O
    :class:`TextIOBase`.
 
    *encoding* gives the name of the encoding that the stream will be decoded or
-   encoded with.  It defaults to
-   :func:`locale.getpreferredencoding(False) <locale.getpreferredencoding>`.
+   encoded with.  It defaults to :func:`locale.getencoding()`.
    ``encoding="locale"`` can be used to specify the current locale's encoding
    explicitly. See :ref:`io-text-encoding` for more information.
 

diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst
index 83eee281d4e5c829921199a9827cb679a2be2d1a..88dcb002e2c2494f2ffe0ef0f1b021ac195c4745 100644 (file)
--- a/Doc/using/windows.rst
+++ b/Doc/using/windows.rst
@@ -618,7 +618,7 @@ UTF-8 mode
 
 Windows still uses legacy encodings for the system encoding (the ANSI Code
 Page).  Python uses it for the default encoding of text files (e.g.
-:func:`locale.getpreferredencoding`).
+:func:`locale.getencoding`).
 
 This may cause issues because UTF-8 is widely used on the internet
 and most Unix systems, including WSL (Windows Subsystem for Linux).

diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index e3ff59eb1adb1953fed329744295f8449f8f512d..0f33ed59492e71181e1f701e9e1de2bdf6cf546f 100644 (file)
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -1988,7 +1988,7 @@ class TextIOWrapper(TextIOBase):
     r"""Character and line based layer over a BufferedIOBase object, buffer.
 
     encoding gives the name of the encoding that the stream will be
-    decoded or encoded with. It defaults to locale.getpreferredencoding(False).
+    decoded or encoded with. It defaults to locale.getencoding().
 
     errors determines the strictness of encoding and decoding (see the
     codecs.register) and defaults to "strict".
@@ -2021,7 +2021,9 @@ class TextIOWrapper(TextIOBase):
         self._check_newline(newline)
         encoding = text_encoding(encoding)
 
-        if encoding == "locale":
+        if encoding == "locale" and sys.platform == "win32":
+            # On Unix, os.device_encoding() returns "utf-8" instead of locale encoding
+            # in the UTF-8 mode. So we use os.device_encoding() only on Windows.
             try:
                 encoding = os.device_encoding(buffer.fileno()) or "locale"
             except (AttributeError, UnsupportedOperation):
@@ -2034,7 +2036,7 @@ class TextIOWrapper(TextIOBase):
                 # Importing locale may fail if Python is being built
                 encoding = "utf-8"
             else:
-                encoding = locale.getpreferredencoding(False)
+                encoding = locale.getencoding()
 
         if not isinstance(encoding, str):
             raise ValueError("invalid encoding: %r" % encoding)

diff --git a/Lib/locale.py b/Lib/locale.py
index 496cc803c88f7c97fd473c812497cf28ea2e6d44..170e5eea45b8ca5a5983601d3545f72da2747f2c 100644 (file)
--- a/Lib/locale.py
+++ b/Lib/locale.py
@@ -557,7 +557,7 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')):
 
     import warnings
     warnings.warn(
-        "Use setlocale(), getpreferredencoding(False) and getlocale() instead",
+        "Use setlocale(), getencoding() and getlocale() instead",
         DeprecationWarning, stacklevel=2
     )
 

diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index 29fe287550b2d0f3909aca85a1eb97fa674ec342..c86251dfe5734c832c95a81a4df3af288651df49 100644 (file)
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -2737,6 +2737,7 @@ class TextIOWrapperTest(unittest.TestCase):
             os.environ.update(old_environ)
 
     @support.cpython_only
+    @unittest.skipIf(sys.platform != "win32", "Windows-only test")
     @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled")
     def test_device_encoding(self):
         # Issue 15989

diff --git a/Misc/NEWS.d/next/Library/2022-04-10-17-50-18.bpo-47000.JlQkFx.rst b/Misc/NEWS.d/next/Library/2022-04-10-17-50-18.bpo-47000.JlQkFx.rst
new file mode 100644 (file)
index 0000000..77d5b8f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-04-10-17-50-18.bpo-47000.JlQkFx.rst
@@ -0,0 +1,2 @@
+Make :class:`TextIOWrapper` uses locale encoding when ``encoding="locale"``
+is specified even in UTF-8 mode.

diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
index 065f5e29c315bf48d6084c0120044dfc645226fa..38ef24637b73183d5493816da9990922f9d52a3d 100644 (file)
--- a/Modules/_io/_iomodule.c
+++ b/Modules/_io/_iomodule.c
@@ -92,9 +92,9 @@ it already exists), 'x' for creating and writing to a new file, and
 'a' for appending (which on some Unix systems, means that all writes
 append to the end of the file regardless of the current seek position).
 In text mode, if encoding is not specified the encoding used is platform
-dependent: locale.getpreferredencoding(False) is called to get the
-current locale encoding. (For reading and writing raw bytes use binary
-mode and leave encoding unspecified.) The available modes are:
+dependent: locale.getencoding() is called to get the current locale encoding.
+(For reading and writing raw bytes use binary mode and leave encoding
+unspecified.) The available modes are:
 
 ========= ===============================================================
 Character Meaning
@@ -196,7 +196,7 @@ static PyObject *
 _io_open_impl(PyObject *module, PyObject *file, const char *mode,
               int buffering, const char *encoding, const char *errors,
               const char *newline, int closefd, PyObject *opener)
-/*[clinic end generated code: output=aefafc4ce2b46dc0 input=1543f4511d2356a5]*/
+/*[clinic end generated code: output=aefafc4ce2b46dc0 input=5bb37f174cb2fb11]*/
 {
     unsigned i;
 

diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h
index e4a6b8c42e1d849d829517faa0a620a772f34036..1fdbe6835c71751d4e7cda8c9eb9f3bb7b80a802 100644 (file)
--- a/Modules/_io/clinic/_iomodule.c.h
+++ b/Modules/_io/clinic/_iomodule.c.h
@@ -22,9 +22,9 @@ PyDoc_STRVAR(_io_open__doc__,
 "\'a\' for appending (which on some Unix systems, means that all writes\n"
 "append to the end of the file regardless of the current seek position).\n"
 "In text mode, if encoding is not specified the encoding used is platform\n"
-"dependent: locale.getpreferredencoding(False) is called to get the\n"
-"current locale encoding. (For reading and writing raw bytes use binary\n"
-"mode and leave encoding unspecified.) The available modes are:\n"
+"dependent: locale.getencoding() is called to get the current locale encoding.\n"
+"(For reading and writing raw bytes use binary mode and leave encoding\n"
+"unspecified.) The available modes are:\n"
 "\n"
 "========= ===============================================================\n"
 "Character Meaning\n"
@@ -355,4 +355,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=1a7fd7755c9a9609 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=e562f29e3c2533a6 input=a9049054013a1b77]*/

diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h
index 0b047ac0aab4eb5a10f6cf37359f77dbda68d49a..7e81eb370fab40c31d637319d3de215c0e0d2364 100644 (file)
--- a/Modules/_io/clinic/textio.c.h
+++ b/Modules/_io/clinic/textio.c.h
@@ -146,7 +146,7 @@ PyDoc_STRVAR(_io_TextIOWrapper___init____doc__,
 "Character and line based layer over a BufferedIOBase object, buffer.\n"
 "\n"
 "encoding gives the name of the encoding that the stream will be\n"
-"decoded or encoded with. It defaults to locale.getpreferredencoding(False).\n"
+"decoded or encoded with. It defaults to locale.getencoding().\n"
 "\n"
 "errors determines the strictness of encoding and decoding (see\n"
 "help(codecs.Codec) or the documentation for codecs.register) and\n"
@@ -671,4 +671,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored))
 {
     return _io_TextIOWrapper_close_impl(self);
 }
-/*[clinic end generated code: output=2604c8f3a45b9a03 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=e88abad34e31c0cb input=a9049054013a1b77]*/

diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c
index 0e207413257f49806ce169b9b6038f25f7320fc1..6ba7393c3a6a309cea57bcb2910409fa4d4e62fa 100644 (file)
--- a/Modules/_io/textio.c
+++ b/Modules/_io/textio.c
@@ -1023,7 +1023,7 @@ _io.TextIOWrapper.__init__
 Character and line based layer over a BufferedIOBase object, buffer.
 
 encoding gives the name of the encoding that the stream will be
-decoded or encoded with. It defaults to locale.getpreferredencoding(False).
+decoded or encoded with. It defaults to locale.getencoding().
 
 errors determines the strictness of encoding and decoding (see
 help(codecs.Codec) or the documentation for codecs.register) and
@@ -1055,12 +1055,12 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
                                 const char *encoding, PyObject *errors,
                                 const char *newline, int line_buffering,
                                 int write_through)
-/*[clinic end generated code: output=72267c0c01032ed2 input=77d8696d1a1f460b]*/
+/*[clinic end generated code: output=72267c0c01032ed2 input=72590963698f289b]*/
 {
     PyObject *raw, *codec_info = NULL;
-    _PyIO_State *state = NULL;
     PyObject *res;
     int r;
+    int use_locale_encoding = 0; // Use locale encoding even in UTF-8 mode.
 
     self->ok = 0;
     self->detached = 0;
@@ -1076,6 +1076,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
     }
     else if (strcmp(encoding, "locale") == 0) {
         encoding = NULL;
+        use_locale_encoding = 1;
     }
 
     if (errors == Py_None) {
@@ -1113,10 +1114,15 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
     self->encodefunc = NULL;
     self->b2cratio = 0.0;
 
+#ifdef MS_WINDOWS
+    // os.device_encoding() on Unix is the locale encoding or UTF-8
+    // according to UTF-8 Mode.
+    // Since UTF-8 mode shouldn't affect `encoding="locale"`, we call
+    // os.device_encoding() only on Windows.
     if (encoding == NULL) {
         /* Try os.device_encoding(fileno) */
         PyObject *fileno;
-        state = IO_STATE();
+        _PyIO_State *state = IO_STATE();
         if (state == NULL)
             goto error;
         fileno = PyObject_CallMethodNoArgs(buffer, &_Py_ID(fileno));
@@ -1144,8 +1150,10 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
                 Py_CLEAR(self->encoding);
         }
     }
+#endif
+
     if (encoding == NULL && self->encoding == NULL) {
-        if (_PyRuntime.preconfig.utf8_mode) {
+        if (_PyRuntime.preconfig.utf8_mode && !use_locale_encoding) {
             _Py_DECLARE_STR(utf_8, "utf-8");
             self->encoding = Py_NewRef(&_Py_STR(utf_8));
         }

diff --git a/Tools/c-analyzer/TODO b/Tools/c-analyzer/TODO
index 55338ebc855d096d1318298fc8d283e4aa17d522..6683df5993b8c0b4f832adebd1f482704ed66626 100644 (file)
--- a/Tools/c-analyzer/TODO
+++ b/Tools/c-analyzer/TODO
@@ -251,7 +251,6 @@ Modules/_io/textio.c:PyId_close                                  _Py_IDENTIFIER(
 Modules/_io/textio.c:PyId_decode                                 _Py_IDENTIFIER(decode)
 Modules/_io/textio.c:PyId_fileno                                 _Py_IDENTIFIER(fileno)
 Modules/_io/textio.c:PyId_flush                                  _Py_IDENTIFIER(flush)
-Modules/_io/textio.c:PyId_getpreferredencoding                   _Py_IDENTIFIER(getpreferredencoding)
 Modules/_io/textio.c:PyId_isatty                                 _Py_IDENTIFIER(isatty)
 Modules/_io/textio.c:PyId_mode                                   _Py_IDENTIFIER(mode)
 Modules/_io/textio.c:PyId_name                                   _Py_IDENTIFIER(name)