+import _locale
from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error)
try:
from _locale import (RADIXCHAR, THOUSEP, nl_langinfo)
import locale
import sys
+import unicodedata
import unittest
-from platform import uname
+from platform import uname, libc_ver
from test import support
if not tested:
self.skipTest('no suitable locales')
+ @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available")
+ @unittest.skipUnless(libc_ver()[0] == 'glibc',
+ "wide nl_langinfo variants are glibc-specific")
+ def test_nl_langinfo_encoding_independent(self):
+ # gh-152905: The LC_TIME text items are decoded independently of the
+ # LC_CTYPE encoding (on glibc via the wide nl_langinfo variants), so
+ # the same locale in different encodings yields identical strings.
+ # ERA has no wide variant and is not tested.
+ self.addCleanup(setlocale, LC_TIME, setlocale(LC_TIME))
+
+ names = [f'MON_{i}' for i in range(1, 13)]
+ names += [f'ABMON_{i}' for i in range(1, 13)]
+ names += [f'DAY_{i}' for i in range(1, 8)]
+ names += [f'ABDAY_{i}' for i in range(1, 8)]
+ names += ['AM_STR', 'PM_STR', 'D_T_FMT', 'D_FMT', 'T_FMT']
+ names += [name for name in ('T_FMT_AMPM', 'ERA_D_FMT', 'ERA_D_T_FMT',
+ 'ERA_T_FMT', 'ALT_DIGITS', '_DATE_FMT')
+ if hasattr(_locale, name)]
+ items = [(name, getattr(_locale, name)) for name in names]
+
+ # Legacy (non-UTF-8) locales, compared against their UTF-8 variant.
+ legacy_locales = [
+ 'en_US.ISO8859-1',
+ 'es_ES.ISO8859-1',
+ 'fr_FR.ISO8859-1',
+ 'de_DE.ISO8859-1',
+ 'pl_PL.ISO8859-2',
+ 'mt_MT.ISO8859-3',
+ 'ar_SA.ISO8859-6',
+ 'el_GR.ISO8859-7',
+ 'he_IL.ISO8859-8',
+ 'tr_TR.ISO8859-9',
+ 'lt_LT.ISO8859-13',
+ 'cy_GB.ISO8859-14',
+ 'et_EE.ISO8859-15',
+ 'uk_UA.KOI8-U',
+ 'bg_BG.CP1251',
+ 'ja_JP.EUC-JP',
+ 'ko_KR.EUC-KR',
+ 'zh_CN.GB2312',
+ 'zh_TW.BIG5',
+ 'th_TH.TIS-620',
+ ]
+
+ # An 8-bit locale substitutes an equivalent for a space it cannot
+ # encode (e.g. es_ES has U+202F in UTF-8 but U+00A0 in ISO-8859-1),
+ # so fold spaces before comparing.
+ def fold_spaces(s):
+ return ''.join(' ' if unicodedata.category(c) == 'Zs' else c
+ for c in s)
+
+ tested = False
+ for legacy_locale in legacy_locales:
+ locs = (legacy_locale.partition('.')[0] + '.UTF-8', legacy_locale)
+ values = []
+ for loc in locs:
+ try:
+ setlocale(LC_TIME, loc)
+ except Error:
+ break
+ values.append({name: nl_langinfo(item) for name, item in items})
+ if len(values) < 2:
+ continue
+ tested = True
+
+ # The result must not depend on the locale encoding.
+ for name, item in items:
+ with self.subTest(locales=locs, name=name):
+ self.assertEqual(fold_spaces(values[0][name]),
+ fold_spaces(values[1][name]))
+ if not tested:
+ self.skipTest('no suitable locale pairs')
+
def test_float_parsing(self):
# Bug #1391872: Test whether float parsing is okay on European
# locales.
#endif
#ifdef HAVE_LANGINFO_H
-#define LANGINFO(X, Y) {#X, X, Y}
+/* On glibc, LC_TIME text items also have a wide (_NL_W*) form, named
+ _NL_W<narrow name>, that nl_langinfo() returns as wchar_t*. LANGINFO_NW
+ marks the items that have no wide counterpart. */
+#ifdef __GLIBC__
+# define LANGINFO(X, Y) {#X, X, Y, _NL_W ## X}
+# define LANGINFO_NW(X, Y) {#X, X, Y, 0}
+#else
+# define LANGINFO(X, Y) {#X, X, Y}
+# define LANGINFO_NW(X, Y) {#X, X, Y}
+#endif
static struct langinfo_constant{
const char *name;
int value;
int category;
+#ifdef __GLIBC__
+ nl_item wide; /* wide _NL_W* counterpart, or 0 if none */
+#endif
} langinfo_constants[] =
{
/* These constants should exist on any langinfo implementation */
#ifdef RADIXCHAR
/* The following are not available with glibc 2.0 */
- LANGINFO(RADIXCHAR, LC_NUMERIC),
- LANGINFO(THOUSEP, LC_NUMERIC),
+ LANGINFO_NW(RADIXCHAR, LC_NUMERIC),
+ LANGINFO_NW(THOUSEP, LC_NUMERIC),
/* YESSTR and NOSTR are deprecated in glibc, since they are
a special case of message translation, which should be rather
done using gettext. So we don't expose it to Python in the
LANGINFO(YESSTR, LC_MESSAGES),
LANGINFO(NOSTR, LC_MESSAGES),
*/
- LANGINFO(CRNCYSTR, LC_MONETARY),
+ LANGINFO_NW(CRNCYSTR, LC_MONETARY),
#endif
LANGINFO(D_T_FMT, LC_TIME),
a few of the others.
Solution: ifdef-test them all. */
#ifdef CODESET
- LANGINFO(CODESET, LC_CTYPE),
+ LANGINFO_NW(CODESET, LC_CTYPE),
#endif
#ifdef T_FMT_AMPM
LANGINFO(T_FMT_AMPM, LC_TIME),
#endif
#ifdef ERA
- LANGINFO(ERA, LC_TIME),
+ LANGINFO_NW(ERA, LC_TIME),
#endif
#ifdef ERA_D_FMT
LANGINFO(ERA_D_FMT, LC_TIME),
LANGINFO(ALT_DIGITS, LC_TIME),
#endif
#ifdef YESEXPR
- LANGINFO(YESEXPR, LC_MESSAGES),
+ LANGINFO_NW(YESEXPR, LC_MESSAGES),
#endif
#ifdef NOEXPR
- LANGINFO(NOEXPR, LC_MESSAGES),
+ LANGINFO_NW(NOEXPR, LC_MESSAGES),
#endif
#ifdef _DATE_FMT
/* This is not available in all glibc versions that have CODESET. */
}
#ifdef __GLIBC__
-#if defined(ALT_DIGITS) || defined(ERA)
static PyObject *
-decode_strings(const char *result, size_t max_count)
+decode_strings(const char *result)
{
/* Convert a sequence of NUL-separated C strings to a Python string
* containing semicolon separated items. */
size_t i = 0;
size_t count = 0;
- for (; count < max_count && result[i]; count++) {
+ for (; result[i]; count++) {
i += strlen(result + i) + 1;
}
char *buf = PyMem_Malloc(i);
return pyresult;
}
#endif
-#endif
+
+#ifdef __GLIBC__
+static PyObject *
+decode_wide_strings(const wchar_t *result, size_t max_count)
+{
+ /* Wide-character counterpart of decode_strings(): convert a sequence of
+ * NUL-separated wchar_t strings to a str with semicolon-separated items. */
+ size_t i = 0;
+ size_t count = 0;
+ for (; count < max_count && result[i]; count++) {
+ i += wcslen(result + i) + 1;
+ }
+ wchar_t *buf = PyMem_New(wchar_t, i);
+ if (buf == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ memcpy(buf, result, i * sizeof(wchar_t));
+ /* Replace all NULs with semicolons. */
+ i = 0;
+ while (--count) {
+ i += wcslen(buf + i);
+ buf[i++] = L';';
+ }
+ PyObject *pyresult = PyUnicode_FromWideChar(buf, -1);
+ PyMem_Free(buf);
+ return pyresult;
+}
+#endif /* __GLIBC__ */
/*[clinic input]
_locale.nl_langinfo
crash PyUnicode_FromString. */
for (i = 0; langinfo_constants[i].name; i++) {
if (langinfo_constants[i].value == item) {
+#ifdef __GLIBC__
+ /* Prefer the wide variant: it is decoded independently of the
+ LC_CTYPE encoding. */
+ nl_item wide = langinfo_constants[i].wide;
+ if (wide) {
+ const wchar_t *wresult = (const wchar_t *)nl_langinfo(wide);
+ if (wresult == NULL) {
+ wresult = L"";
+ }
+ /* ALT_DIGITS is a sequence of NUL-separated strings. */
+ if (item == ALT_DIGITS && *wresult) {
+ return decode_wide_strings(wresult, 100);
+ }
+ return PyUnicode_FromWideChar(wresult, -1);
+ }
+#endif
/* Check NULL as a workaround for GNU libc's returning NULL
instead of an empty string for nl_langinfo(ERA). */
const char *result = nl_langinfo(item);
if (langinfo_constants[i].category != LC_CTYPE
&& *result && (
#ifdef __GLIBC__
- // gh-133740: Always change the locale for ALT_DIGITS and ERA
-# ifdef ALT_DIGITS
- item == ALT_DIGITS ||
-# endif
-# ifdef ERA
+ // gh-133740: Always change the locale for ERA
item == ERA ||
-# endif
#endif
!is_all_ascii(result))
&& change_locale(langinfo_constants[i].category, &oldloc) < 0)
/* According to the POSIX specification the result must be
* a sequence of semicolon-separated strings.
* But in Glibc they are NUL-separated. */
-#ifdef ALT_DIGITS
- if (item == ALT_DIGITS && *result) {
- pyresult = decode_strings(result, 100);
- }
- else
-#endif
-#ifdef ERA
if (item == ERA && *result) {
- pyresult = decode_strings(result, SIZE_MAX);
+ pyresult = decode_strings(result);
}
else
-#endif
#endif
{
pyresult = PyUnicode_DecodeLocale(result, NULL);
projects / thirdparty / Python / cpython.git / commitdiff
