* Set the :attr:`~sys.flags.dev_mode` attribute of :attr:`sys.flags` to
``True``
- * ``-X utf8`` enables UTF-8 mode for operating system interfaces, overriding
+ * ``-X utf8`` enables UTF-8 mode (:pep:`540`) for operating system interfaces, overriding
the default locale-aware mode. ``-X utf8=0`` explicitly disables UTF-8
mode (even when it would otherwise activate automatically).
See :envvar:`PYTHONUTF8` for more details.
+ * ``-X coerce_c_locale`` or ``-X coerce_c_locale=1`` tries to coerce the C
+ locale (:pep:`538`).
+ ``-X coerce_c_locale=0`` skips coercing the legacy ASCII-based C and POSIX
+ locales to a more capable UTF-8 based alternative.
+ ``-X coerce_c_locale=warn`` will cause Python to emit warning messages on
+ ``stderr`` if either the locale coercion activates, or else if a locale
+ that *would* have triggered coercion is still active when the Python
+ runtime is initialized.
+ See :envvar:`PYTHONCOERCECLOCALE` for more details.
It also allows passing arbitrary values and retrieving them through the
:data:`sys._xoptions` dictionary.
.. versionadded:: 3.7
The ``-X importtime``, ``-X dev`` and ``-X utf8`` options.
+ .. versionadded:: 3.7.1
+ The ``-X coerce_c_locale`` option.
+
Options you shouldn't use
~~~~~~~~~~~~~~~~~~~~~~~~~
order to force the interpreter to use ``ASCII`` instead of ``UTF-8`` for
system interfaces.
+ Also available as the :option:`-X` ``coerce_c_locale`` option.
+
Availability: \*nix
.. versionadded:: 3.7
while in Python 3.7.0 it didn't read any of them due to :issue:`34247`). If
this behavior is unwanted, set :c:data:`Py_IgnoreEnvironmentFlag` to 1 before
calling :c:func:`Py_Initialize`.
+
+:c:func:`Py_Initialize` and :c:func:`Py_Main` cannot enable the C locale
+coercion (:pep:`538`) anymore: it is always disabled. It can now only be
+enabled by the Python program ("python3).
+
+New :option:`-X` ``coerce_c_locale`` command line option to control C locale
+coercion (:pep:`538`).
/* Bootstrap __main__ (defined in Modules/main.c) */
PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
#ifdef Py_BUILD_CORE
+# ifdef MS_WINDOWS
+PyAPI_FUNC(int) _Py_WindowsMain(int argc, wchar_t **argv);
+# else
PyAPI_FUNC(int) _Py_UnixMain(int argc, char **argv);
+# endif
#endif
/* In getpath.c */
int show_alloc_count; /* -X showalloccount */
int dump_refs; /* PYTHONDUMPREFS */
int malloc_stats; /* PYTHONMALLOCSTATS */
- int coerce_c_locale; /* PYTHONCOERCECLOCALE, -1 means unknown */
- int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
int utf8_mode; /* PYTHONUTF8, -X utf8; -1 means unknown */
wchar_t *program_name; /* Program name, see also Py_GetProgramName() */
/* Private fields */
int _disable_importlib; /* Needed by freeze_importlib */
+ int _coerce_c_locale; /* PYTHONCOERCECLOCALE, -1 means unknown */
+ int _coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
} _PyCoreConfig;
#define _PyCoreConfig_INIT \
.install_signal_handlers = -1, \
.ignore_environment = -1, \
.use_hash_seed = -1, \
- .coerce_c_locale = -1, \
+ ._coerce_c_locale = 0, \
+ ._coerce_c_locale_warn = 0, \
.faulthandler = -1, \
.tracemalloc = -1, \
.utf8_mode = -1, \
return data
@classmethod
- def get_child_details(cls, env_vars):
+ def get_child_details(cls, env_vars, xoption=None):
"""Retrieves fsencoding and standard stream details from a child process
Returns (encoding_details, stderr_lines):
The child is run in isolated mode if the current interpreter supports
that.
"""
- result, py_cmd = run_python_until_end(
- "-X", "utf8=0", "-c", cls.CHILD_PROCESS_SCRIPT,
- **env_vars
- )
+ args = []
+ if xoption:
+ args.extend(("-X", f"coerce_c_locale={xoption}"))
+ args.extend(("-X", "utf8=0", "-c", cls.CHILD_PROCESS_SCRIPT))
+ result, py_cmd = run_python_until_end(*args, **env_vars)
if not result.rc == 0:
result.fail(py_cmd)
# All subprocess outputs in this test case should be pure ASCII
expected_fs_encoding,
expected_stream_encoding,
expected_warnings,
- coercion_expected):
+ coercion_expected,
+ xoption=None):
"""Check the C locale handling for the given process environment
Parameters:
expected_stream_encoding: expected encoding for standard streams
expected_warning: stderr output to expect (if any)
"""
- result = EncodingDetails.get_child_details(env_vars)
+ result = EncodingDetails.get_child_details(env_vars, xoption)
encoding_details, stderr_lines = result
expected_details = EncodingDetails.get_expected_details(
coercion_expected,
coerce_c_locale,
expected_warnings=None,
coercion_expected=True,
+ use_xoption=False,
**extra_vars):
"""Check the C locale handling for various configurations
"PYTHONCOERCECLOCALE": "",
}
base_var_dict.update(extra_vars)
+ xoption = None
if coerce_c_locale is not None:
- base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
+ if use_xoption:
+ xoption = coerce_c_locale
+ else:
+ base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
# Check behaviour for the default locale
with self.subTest(default_locale=True,
fs_encoding,
stream_encoding,
_expected_warnings,
- _coercion_expected)
+ _coercion_expected,
+ xoption=xoption)
# Check behaviour for explicitly configured locales
for locale_to_set in EXPECTED_C_LOCALE_EQUIVALENTS:
fs_encoding,
stream_encoding,
expected_warnings,
- coercion_expected)
+ coercion_expected,
+ xoption=xoption)
def test_PYTHONCOERCECLOCALE_not_set(self):
# This should coerce to the first available target locale by default
expected_warnings=[LEGACY_LOCALE_WARNING],
coercion_expected=False)
+ def test_xoption_set_to_1(self):
+ self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale="1",
+ use_xoption=True)
+
+ def test_xoption_set_to_zero(self):
+ # The setting "0" should result in the locale coercion being disabled
+ self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
+ EXPECTED_C_LOCALE_STREAM_ENCODING,
+ coerce_c_locale="0",
+ coercion_expected=False,
+ use_xoption=True)
+ # Setting LC_ALL=C shouldn't make any difference to the behaviour
+ self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
+ EXPECTED_C_LOCALE_STREAM_ENCODING,
+ coerce_c_locale="0",
+ LC_ALL="C",
+ coercion_expected=False,
+ use_xoption=True)
+
+ def test_xoption_set_to_warn(self):
+ # -X coerce_c_locale=warn enables runtime warnings for legacy locales
+ self._check_c_locale_coercion("utf-8", "utf-8",
+ coerce_c_locale="warn",
+ expected_warnings=[CLI_COERCION_WARNING],
+ use_xoption=True)
+
def test_main():
test.support.run_unittest(
LocaleConfigurationTests,
env = os.environ.copy()
# Use C locale to get ascii for the locale encoding
env['LC_ALL'] = 'C'
- env['PYTHONCOERCECLOCALE'] = '0'
code = (
b'import locale; '
b'print(ascii("' + undecodable + b'"), '
b'locale.getpreferredencoding())')
p = subprocess.Popen(
- [sys.executable, "-c", code],
+ [sys.executable,
+ # Disable C locale coercion and UTF-8 Mode to not use UTF-8
+ "-X", "coerce_c_locale=0",
+ "-X", "utf8=0",
+ "-c", code],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
env=env)
stdout, stderr = p.communicate()
'malloc_stats': 0,
'utf8_mode': 0,
- 'coerce_c_locale': 0,
- 'coerce_c_locale_warn': 0,
-
'program_name': './_testembed',
'argc': 0,
'argv': '[]',
'_disable_importlib': 0,
'Py_FrozenFlag': 0,
+ '_coerce_c_locale': 0,
+ '_coerce_c_locale_warn': 0,
}
def check_config(self, testname, expected):
def c_locale_get_error_handler(self, locale, isolated=False, encoding=None):
# Force the POSIX locale
- env = os.environ.copy()
+ env = dict(os.environ)
env["LC_ALL"] = locale
- env["PYTHONCOERCECLOCALE"] = "0"
code = '\n'.join((
'import sys',
'def dump(name):',
'dump("stdout")',
'dump("stderr")',
))
- args = [sys.executable, "-c", code]
+ args = [sys.executable,
+ "-X", "utf8=0",
+ "-X", "coerce_c_locale=0",
+ "-c", code]
if isolated:
args.append("-I")
if encoding is not None:
return (loc in POSIX_LOCALES)
def get_output(self, *args, failure=False, **kw):
+ # Always disable the C locale coercion (PEP 538)
+ args = ('-X', 'coerce_c_locale=0', *args)
kw = dict(self.DEFAULT_ENV, **kw)
if failure:
out = assert_python_failure(*args, **kw)
# PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode
# and has the priority over -X utf8 and PYTHONUTF8
out = self.get_output('-X', 'utf8', '-c', code,
- PYTHONUTF8='strict',
PYTHONLEGACYWINDOWSFSENCODING='1')
self.assertEqual(out, 'mbcs/replace')
--- /dev/null
+Py_Initialize() and Py_Main() cannot enable the C locale coercion (PEP 538)
+anymore: it is always disabled. It can now only be enabled by the Python
+program ("python3).
--- /dev/null
+Add a new :option:`-X` ``coerce_c_locale`` command line option to control C
+locale coercion (:pep:`538`).
return _Py_INIT_OK();
}
+#ifndef MS_WINDOWS
+ /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */
+ const char *ctype_loc = setlocale(LC_CTYPE, NULL);
+ if (ctype_loc != NULL
+ && (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0))
+ {
+ config->utf8_mode = 1;
+ return _Py_INIT_OK();
+ }
+#endif
+
return _Py_INIT_OK();
}
const char *env = config_get_env_var("PYTHONCOERCECLOCALE");
if (env) {
if (strcmp(env, "0") == 0) {
- if (config->coerce_c_locale < 0) {
- config->coerce_c_locale = 0;
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 0;
}
}
else if (strcmp(env, "warn") == 0) {
- config->coerce_c_locale_warn = 1;
+ if (config->_coerce_c_locale_warn < 0) {
+ config->_coerce_c_locale_warn = 1;
+ }
}
else {
- if (config->coerce_c_locale < 0) {
- config->coerce_c_locale = 1;
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 1;
}
}
}
* See the documentation of the PYTHONCOERCECLOCALE setting for more
* details.
*/
- if (config->coerce_c_locale && !locale_coerced) {
+ if (config->_coerce_c_locale && !locale_coerced) {
locale_coerced = 1;
_Py_CoerceLegacyLocale(config);
encoding_changed = 1;
pymain_read_conf_impl(). Reset Py_IsolatedFlag and Py_NoSiteFlag
modified by _PyCoreConfig_Read(). */
int new_utf8_mode = config->utf8_mode;
- int new_coerce_c_locale = config->coerce_c_locale;
+ int new_coerce_c_locale = config->_coerce_c_locale;
Py_IgnoreEnvironmentFlag = init_ignore_env;
if (_PyCoreConfig_Copy(config, &save_config) < 0) {
pymain->err = _Py_INIT_NO_MEMORY();
cmdline_get_global_config(cmdline);
_PyCoreConfig_GetGlobalConfig(config);
config->utf8_mode = new_utf8_mode;
- config->coerce_c_locale = new_coerce_c_locale;
+ config->_coerce_c_locale = new_coerce_c_locale;
/* The encoding changed: read again the configuration
with the new encoding */
}
-static void
-config_init_locale(_PyCoreConfig *config)
+static _PyInitError
+config_init_coerce_c_locale(_PyCoreConfig *config)
{
- if (config->coerce_c_locale < 0) {
- /* The C locale enables the C locale coercion (PEP 538) */
- if (_Py_LegacyLocaleDetected()) {
- config->coerce_c_locale = 1;
+ const wchar_t *xopt = config_get_xoption(config, L"coerce_c_locale");
+ if (xopt) {
+ wchar_t *sep = wcschr(xopt, L'=');
+ if (sep) {
+ xopt = sep + 1;
+ if (wcscmp(xopt, L"1") == 0) {
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 1;
+ }
+ }
+ else if (wcscmp(xopt, L"0") == 0) {
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 0;
+ }
+ }
+ else if (wcscmp(xopt, L"warn") == 0) {
+ if (config->_coerce_c_locale_warn < 0) {
+ config->_coerce_c_locale_warn = 1;
+ }
+ }
+ else {
+ return _Py_INIT_USER_ERR("invalid -X coerce_c_locale option value");
+ }
+ }
+ else {
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 1;
+ }
+ }
+
+ if (config->_coerce_c_locale_warn < 0) {
+ config->_coerce_c_locale_warn = 0;
}
}
-#ifndef MS_WINDOWS
- if (config->utf8_mode < 0) {
- /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */
- const char *ctype_loc = setlocale(LC_CTYPE, NULL);
- if (ctype_loc != NULL
- && (strcmp(ctype_loc, "C") == 0
- || strcmp(ctype_loc, "POSIX") == 0))
- {
- config->utf8_mode = 1;
+ const char *env = config_get_env_var("PYTHONCOERCECLOCALE");
+ if (env) {
+ if (strcmp(env, "0") == 0) {
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 0;
+ }
+ }
+ else if (strcmp(env, "warn") == 0) {
+ if (config->_coerce_c_locale_warn < 0) {
+ config->_coerce_c_locale_warn = 1;
+ }
+ }
+ else {
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 1;
+ }
+ }
+
+ if (config->_coerce_c_locale_warn < 0) {
+ config->_coerce_c_locale_warn = 0;
}
}
-#endif
+
+ if (config->_coerce_c_locale < 0) {
+ /* The C locale enables the C locale coercion (PEP 538) */
+ if (_Py_LegacyLocaleDetected()) {
+ config->_coerce_c_locale = 1;
+ return _Py_INIT_OK();
+ }
+ }
+
+ return _Py_INIT_OK();
}
}
}
- if (config->utf8_mode < 0 || config->coerce_c_locale < 0) {
- config_init_locale(config);
+ if (config->_coerce_c_locale < 0 || config->_coerce_c_locale_warn < 0) {
+ err = config_init_coerce_c_locale(config);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
}
if (!config->_disable_importlib) {
if (config->tracemalloc < 0) {
config->tracemalloc = 0;
}
- if (config->coerce_c_locale < 0) {
- config->coerce_c_locale = 0;
+ if (config->_coerce_c_locale < 0) {
+ config->_coerce_c_locale = 0;
+ }
+ if (config->_coerce_c_locale_warn < 0) {
+ config->_coerce_c_locale_warn = 0;
}
if (config->utf8_mode < 0) {
config->utf8_mode = 0;
config->argc = 0;
}
+ assert(config->_coerce_c_locale >= 0);
+ assert(config->_coerce_c_locale_warn >= 0);
+ assert(config->ignore_environment >= 0);
+
return _Py_INIT_OK();
}
COPY_ATTR(dump_refs);
COPY_ATTR(malloc_stats);
- COPY_ATTR(coerce_c_locale);
- COPY_ATTR(coerce_c_locale_warn);
+ COPY_ATTR(_coerce_c_locale);
+ COPY_ATTR(_coerce_c_locale_warn);
COPY_ATTR(utf8_mode);
COPY_STR_ATTR(module_search_path_env);
static void
-pymain_init(_PyMain *pymain)
+pymain_init(_PyMain *pymain, int use_c_locale_coercion)
{
/* 754 requires that FP exceptions run in "no stop" mode by default,
* and until C vendors implement C99's ways to control FP exceptions,
pymain->config._disable_importlib = 0;
pymain->config.install_signal_handlers = 1;
+ if (use_c_locale_coercion) {
+ /* set to -1 to be able to enable the feature */
+ pymain->config._coerce_c_locale = -1;
+ pymain->config._coerce_c_locale_warn = -1;
+ }
}
static int
-pymain_main(_PyMain *pymain)
+pymain_main(_PyMain *pymain, int use_c_locale_coercion)
{
- pymain_init(pymain);
+ pymain_init(pymain, use_c_locale_coercion);
int res = pymain_cmdline(pymain);
if (res < 0) {
pymain.argc = argc;
pymain.wchar_argv = argv;
- return pymain_main(&pymain);
+ return pymain_main(&pymain, 0);
}
+#ifdef MS_WINDOWS
+int
+_Py_WindowsMain(int argc, wchar_t **argv)
+{
+ _PyMain pymain = _PyMain_INIT;
+ pymain.use_bytes_argv = 0;
+ pymain.argc = argc;
+ pymain.wchar_argv = argv;
+
+ return pymain_main(&pymain, 1);
+}
+#else
int
_Py_UnixMain(int argc, char **argv)
{
pymain.argc = argc;
pymain.bytes_argv = argv;
- return pymain_main(&pymain);
+ return pymain_main(&pymain, 1);
}
+#endif
/* this is gonna seem *real weird*, but if you put some other code between
printf("dump_refs = %i\n", config->dump_refs);
printf("malloc_stats = %i\n", config->malloc_stats);
- printf("coerce_c_locale = %i\n", config->coerce_c_locale);
- printf("coerce_c_locale_warn = %i\n", config->coerce_c_locale_warn);
+ printf("_coerce_c_locale = %i\n", config->_coerce_c_locale);
+ printf("_coerce_c_locale_warn = %i\n", config->_coerce_c_locale_warn);
printf("utf8_mode = %i\n", config->utf8_mode);
printf("program_name = %ls\n", config->program_name);
putenv("PYTHONMALLOCSTATS=0");
config.malloc_stats = 1;
- /* FIXME: test coerce_c_locale and coerce_c_locale_warn */
-
putenv("PYTHONUTF8=0");
Py_UTF8Mode = 0;
config.utf8_mode = 1;
/* Test _PyCoreConfig.isolated=1 */
_PyCoreConfig config = _PyCoreConfig_INIT;
- /* Set coerce_c_locale and utf8_mode to not depend on the locale */
- config.coerce_c_locale = 0;
+ /* Set utf8_mode to not depend on the locale */
config.utf8_mode = 0;
/* Use path starting with "./" avoids a search along the PATH */
config.program_name = L"./_testembed";
int
wmain(int argc, wchar_t **argv)
{
- return Py_Main(argc, argv);
+ return _Py_WindowsMain(argc, argv);
}
#else
int
static void
_emit_stderr_warning_for_legacy_locale(const _PyCoreConfig *core_config)
{
- if (core_config->coerce_c_locale_warn) {
+ if (core_config->_coerce_c_locale_warn) {
if (_Py_LegacyLocaleDetected()) {
fprintf(stderr, "%s", _C_LOCALE_WARNING);
}
"Error setting LC_CTYPE, skipping C locale coercion\n");
return;
}
- if (config->coerce_c_locale_warn) {
+ if (config->_coerce_c_locale_warn) {
fprintf(stderr, C_LOCALE_COERCION_WARNING, newloc);
}
projects / thirdparty / Python / cpython.git / commitdiff
