tests: fix UTF-8 detection, per-test `LC_*` settings, CI coverage

- runtests: fix `codeset-utf8` feature detection. Before this patch it
  detected if the calling environment had UTF-8 enabled. If not, UTF-8
  tests were all skipped. After this patch, it detects if UTF-8 is
  supported by the calling environment regardless of what's currently
  enabled.
  Follow-up to 0b70b23ef4d007031bc2ae4fc63d5ed9136bc2b5 #15039

- GHA/linux: sync `codeset-test` to also reset `LC_CTYPE` and
  `LC_NUMBER`. To give it more spin.
  Follow-up to c221c0ee5935497168c52686a9d8cc87b45bbca9 #17938

- GHA/macos: fix to actually enable `codeset-test`. Also set `LC_ALL`,
  which seems necessary to trigger issues.
  Follow-up to c221c0ee5935497168c52686a9d8cc87b45bbca9 #17938

- tests/data: replace `LC_CTYPE` env with `LC_ALL` in all tests
  requiring a locale. Also to avoid potential issues with a blank or
  unset `LC_ALL`, as seen earlier. And to ensure that the override works
  on all platforms (as tested in CI.)
  Slight downside is that this now resets the language/culture to `C`.
  Ref: b4c9982382469398115cc0e3e0747e79db083455 #4743
  Ref: 23208e330ac0c2164d59971baf79e87c45da1840 #4738

- replace `en_US.UTF-8` with `C.UTF-8` to be language/culture-agnostic.

- TEST-SUITE.md: drop `UTF-8` as a requirement for tests.
  Tests shall work (or least be skipped) without UTF-8 support.

Tests requiring UTF-8 locale:
165, 962, 963, 964, 965, 966, 967, 1448, 1560, 2046, 2047
Tests requiring UTF-8 locale, but passing without one anyway:
955, 956, 957, 958, 959, 960, 961, 968, 1034, 1035

Spec 1997: https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
Spec 2008: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html

Ref: c221c0ee5935497168c52686a9d8cc87b45bbca9 #17938
Ref: 7cf8414fabc3063cc3d2121eacec4a6daa4164a8
Ref: 4c140a56283703161e5f26ae022bad694a481603
Ref: 28faaacee287b019bcf2961da3bf2f91d331bcbd #2436
Ref: ecd1d020abdae3c3ce3643ddab3106501e62e7c0

Closes #17988

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index dbb4d5d2d3be4b950fa5151ee034265d4cf67638..973516abde71666a412d42e7305193a37d7ef976 100644 (file)
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -685,7 +685,12 @@ jobs:
             fi
           fi
           [ -x ~/venv/bin/activate ] && source ~/venv/bin/activate
-          [[ "${MATRIX_INSTALL_STEPS}" = *'codeset-test'* ]] && export LC_ALL=C
+          if [[ "${MATRIX_INSTALL_STEPS}" = *'codeset-test'* ]]; then
+            locale || true
+            export LC_ALL=C
+            export LC_CTYPE=C
+            export LC_NUMERIC=fr_FR.UTF-8
+          fi
           if [ "${MATRIX_BUILD}" = 'cmake' ]; then
             cmake --build bld --verbose --target "${TEST_TARGET}"
           else

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index bb1b376b27e3b9cbfda4a9c2f8ef293b095b14e6..b35c9a08c4bdabd2e19ff2be8ede82bfaf5b37a5 100644 (file)
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -214,6 +214,7 @@ jobs:
       MATRIX_BUILD: ${{ matrix.build.generate && 'cmake' || 'autotools' }}
       MATRIX_COMPILER: '${{ matrix.compiler }}'
       MATRIX_INSTALL: '${{ matrix.build.install }}'
+      MATRIX_INSTALL_STEPS: '${{ matrix.build.install_steps }}'
       MATRIX_MACOS_VERSION_MIN: '${{ matrix.build.macos-version-min }}'
     strategy:
       fail-fast: false
@@ -381,7 +382,6 @@ jobs:
           MATRIX_CHKPREFILL: '${{ matrix.build.chkprefill }}'
           MATRIX_CONFIGURE: '${{ matrix.build.configure }}'
           MATRIX_GENERATE: '${{ matrix.build.generate }}'
-          MATRIX_INSTALL_STEPS: '${{ matrix.build.install_steps }}'
         run: |
           if [[ "${MATRIX_COMPILER}" = 'gcc'* ]]; then
             sysroot="$("${CC}" --print-sysroot)"  # Must match the SDK gcc was built for
@@ -481,6 +481,8 @@ jobs:
           TFLAGS="-j20 ${TFLAGS}"
           source ~/venv/bin/activate
           if [[ "${MATRIX_INSTALL_STEPS}" = *'codeset-test'* ]]; then
+            locale || true
+            export LC_ALL=C
             export LC_CTYPE=C
             export LC_NUMERIC=fr_FR.UTF-8
           fi

diff --git a/docs/tests/TEST-SUITE.md b/docs/tests/TEST-SUITE.md
index 1e387898f722a4477885d0107eec9c609ebd43d7..2110cfb362ff4d160552705fa08dd2dc38ee0415 100644 (file)
--- a/docs/tests/TEST-SUITE.md
+++ b/docs/tests/TEST-SUITE.md
@@ -53,7 +53,6 @@ SPDX-License-Identifier: curl
   - `openssl` (the command line tool, for generating test server certificates)
   - `openssh` or `SunSSH` (for SCP and SFTP tests)
   - `nghttpx` (for HTTP/2 and HTTP/3 tests)
-  - An available `en_US.UTF-8` locale
 
 ### Installation of impacket
 

diff --git a/tests/data/test1034 b/tests/data/test1034
index ef671dfe2e0463fbac98bb24e07b6072cfbf9f6d..eda13bb23e9e13c4d5b86d16d5216583741682c1 100644 (file)
--- a/tests/data/test1034
+++ b/tests/data/test1034
@@ -29,8 +29,7 @@ proxy
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 HTTP over proxy with malformatted IDN host name

diff --git a/tests/data/test1035 b/tests/data/test1035
index b6f30ae56b9e18fc81f61c9a983fa7a917b84b6e..c8eb282aff1bdfa1163f948b0f4d8ffb34d47fd5 100644 (file)
--- a/tests/data/test1035
+++ b/tests/data/test1035
@@ -27,8 +27,7 @@ proxy
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 HTTP over proxy with too long IDN host name

diff --git a/tests/data/test1448 b/tests/data/test1448
index 96d64b85a59b54b8222af3118727bc5fd55ed705..f7ff09f952c02feec70d615f442ef0951083ccc9 100644 (file)
--- a/tests/data/test1448
+++ b/tests/data/test1448
@@ -43,8 +43,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 Redirect following to UTF-8 IDN host name

diff --git a/tests/data/test1560 b/tests/data/test1560
index 976cc98ec2ecbc0b63560d7893011bba973f7dad..ce8c7c42a795005b23f97000a01a1c05cbbe304e 100644 (file)
--- a/tests/data/test1560
+++ b/tests/data/test1560
@@ -13,7 +13,7 @@ urlapi
 none
 </server>
 <setenv>
-LC_ALL=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <features>
 file

diff --git a/tests/data/test165 b/tests/data/test165
index 0b5cfd2346f2bd878c5e5e1899606457931f04ff..4996a3564f5b9e0e725851826eee51b5c1d41f4d 100644 (file)
--- a/tests/data/test165
+++ b/tests/data/test165
@@ -33,8 +33,7 @@ proxy
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 HTTP over proxy with IDN host name

diff --git a/tests/data/test2046 b/tests/data/test2046
index cf25d0c57b2e9bf021d35a55b52ed036f87d828c..555515aa4a31af9d7b9628559f0ada919b384b11 100644 (file)
--- a/tests/data/test2046
+++ b/tests/data/test2046
@@ -43,8 +43,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 Connection reuse with IDN host name

diff --git a/tests/data/test2047 b/tests/data/test2047
index c32e6aabc714f914a3f4dbafa3b70c405df71837..9c671b178536585fa5eac65ff17e77699fd8ea49 100644 (file)
--- a/tests/data/test2047
+++ b/tests/data/test2047
@@ -44,8 +44,7 @@ proxy
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 Connection reuse with IDN host name over HTTP proxy

diff --git a/tests/data/test955 b/tests/data/test955
index 817b47a67e7a7d9137d38a614ac8b95f78cfe51c..633c2e369d3577eb65b9814d31745d852cf57b07 100644 (file)
--- a/tests/data/test955
+++ b/tests/data/test955
@@ -24,8 +24,7 @@ smtp
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP without SMTPUTF8 support - UTF-8 based sender (local part only)

diff --git a/tests/data/test956 b/tests/data/test956
index 4546f2ee61b82eb12b3f1df6371ef592eb98b4ef..9472e96e1596a9b501edad996e911e33fde969b1 100644 (file)
--- a/tests/data/test956
+++ b/tests/data/test956
@@ -21,8 +21,7 @@ smtp
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP without SMTPUTF8 support - UTF-8 based recipient (local part only)

diff --git a/tests/data/test957 b/tests/data/test957
index 9548627b98c6b7eebec312b65e2d73edacfa904a..671d73bbf21864100a8c5b9bb0f58d010170957e 100644 (file)
--- a/tests/data/test957
+++ b/tests/data/test957
@@ -22,8 +22,7 @@ smtp
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP VRFY without SMTPUTF8 support - UTF-8 recipient (local part only)

diff --git a/tests/data/test958 b/tests/data/test958
index f1006a0a66e0a3fc6c23f1879a69a3201cd4e0b9..203254bfc2d2c4a8759af5d6382bf9456b01ec47 100644 (file)
--- a/tests/data/test958
+++ b/tests/data/test958
@@ -22,8 +22,7 @@ smtp
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP external VRFY without SMTPUTF8 - UTF-8 recipient (local part only)

diff --git a/tests/data/test959 b/tests/data/test959
index 2154600da2fd6af072c945728f51013a6906b9e8..5693e349449ef32c9873b8e4e3482d569e99629d 100644 (file)
--- a/tests/data/test959
+++ b/tests/data/test959
@@ -25,8 +25,7 @@ smtp
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP without SMTPUTF8 support - UTF-8 based sender (host part only)

diff --git a/tests/data/test960 b/tests/data/test960
index fb50a326b28890e549f137d290636b90a28d57a9..ced2ba0b54f59a6c0889da34afbcdfa3559399ee 100644 (file)
--- a/tests/data/test960
+++ b/tests/data/test960
@@ -22,8 +22,7 @@ smtp
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP without SMTPUTF8 support - UTF-8 based recipient (host part only)

diff --git a/tests/data/test961 b/tests/data/test961
index 2068d8fc843e1e4841fffb60cd148cb998fcab90..c52d5df344500b3226ea31927a8b3f8007920844 100644 (file)
--- a/tests/data/test961
+++ b/tests/data/test961
@@ -23,8 +23,7 @@ smtp
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP external VRFY without SMTPUTF8 - UTF-8 recipient (host part only)

diff --git a/tests/data/test962 b/tests/data/test962
index 80292862ba744edeca11ecda96a3fc1479236ed1..3f737b65ea8dae7d3064affc228165c594e2f6d6 100644 (file)
--- a/tests/data/test962
+++ b/tests/data/test962
@@ -23,8 +23,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP without SMTPUTF8 support - UTF-8 based sender (host part only)

diff --git a/tests/data/test963 b/tests/data/test963
index 5d3bae7b566c515c36b24a01165d2e8e27c531d8..bce626c5e30e11465b2ecdb23a433775d3c965b0 100644 (file)
--- a/tests/data/test963
+++ b/tests/data/test963
@@ -23,8 +23,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP without SMTPUTF8 support (IDN) - UTF-8 recipient (host part only)

diff --git a/tests/data/test964 b/tests/data/test964
index 12c312eccda4c25d239962bce7e16794d7b76c5e..04e5b1c152358baf365c299caa6b6d4ce319c2ce 100644 (file)
--- a/tests/data/test964
+++ b/tests/data/test964
@@ -24,8 +24,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP external VRFY without SMTPUTF8 (IDN) - UTF-8 recipient (host part)

diff --git a/tests/data/test965 b/tests/data/test965
index 0d29c0984932354ab611e8670973d2317e724769..601db62fe09af7647d7441b5e83106194c4d0182 100644 (file)
--- a/tests/data/test965
+++ b/tests/data/test965
@@ -26,8 +26,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP with SMTPUTF8 support - UTF-8 based sender

diff --git a/tests/data/test966 b/tests/data/test966
index b642e19a9d2dde6e0b1e25954b027f2fe9244263..646e80758979e108f48677000a7e8a51be2b71d1 100644 (file)
--- a/tests/data/test966
+++ b/tests/data/test966
@@ -26,8 +26,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP with SMTPUTF8 support - UTF-8 based recipient

diff --git a/tests/data/test967 b/tests/data/test967
index f6e2e32382de6a4bf769df8af9c59127cae02da7..7cacbcbdd2006ef16934c62b6c07eca4d41bcb76 100644 (file)
--- a/tests/data/test967
+++ b/tests/data/test967
@@ -30,8 +30,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP external VRFY with SMTPUTF8 support

diff --git a/tests/data/test968 b/tests/data/test968
index 8532c35e7db883ef81af6e94fbc664fc9be5b2ee..4df600483be65a6a426d82926051c3fd320b2570 100644 (file)
--- a/tests/data/test968
+++ b/tests/data/test968
@@ -27,8 +27,7 @@ IDN
 codeset-utf8
 </features>
 <setenv>
-LC_ALL=en_US.UTF-8
-LC_CTYPE=en_US.UTF-8
+LC_ALL=C.UTF-8
 </setenv>
 <name>
 SMTP VRFY with SMTPUTF8 support

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 23736f2ac56606bdc438a874e95058bfb3c342f0..43718fa58ba69a7064b6d0ee7110cb7b5b4590a8 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -83,6 +83,7 @@ BEGIN {
 use Digest::MD5 qw(md5);
 use List::Util 'sum';
 use I18N::Langinfo qw(langinfo CODESET);
+use POSIX qw(setlocale LC_ALL);
 
 use serverhelp qw(
     server_exe
@@ -484,6 +485,25 @@ sub parseprotocols {
     push @protocols, 'none';
 }
 
+#######################################################################
+# Check if the operating environment supports UTF-8.
+sub is_utf8_supported {
+    my $result;
+    my $old_LC_ALL;
+    my $was_defined = defined $ENV{'LC_ALL'};
+    if($was_defined) {
+        $old_LC_ALL = $ENV{'LC_ALL'};
+    }
+    setlocale(LC_ALL, $ENV{'LC_ALL'} = "C.UTF-8");
+    $result = lc(langinfo(CODESET())) eq "utf-8";
+    if($was_defined) {
+        $ENV{'LC_ALL'} = $old_LC_ALL;
+    }
+    else {
+        delete $ENV{'LC_ALL'};
+    }
+    return $result;
+}
 
 #######################################################################
 # Check & display information about curl and the host the test suite runs on.
@@ -808,7 +828,7 @@ sub checksystemfeatures {
     # Use this as a proxy for any cryptographic authentication
     $feature{"crypto"} = $feature{"NTLM"} || $feature{"Kerberos"} || $feature{"SPNEGO"};
     $feature{"local-http"} = servers::localhttp();
-    $feature{"codeset-utf8"} = lc(langinfo(CODESET())) eq "utf-8";
+    $feature{"codeset-utf8"} = is_utf8_supported();
     if($feature{"codeset-utf8"}) {
         $ENV{'CURL_TEST_HAVE_CODESET_UTF8'} = 1;
     }