tests: provide all non-ascii data hex encoded

- make the scanner not whitelist anything for test cases making
  everything non-ascii forced to be hex encoded

- update all tests using non-ascii bytes to use %hex[] sequences

Closes #17331

diff --git a/.github/scripts/spacecheck.pl b/.github/scripts/spacecheck.pl
index b47dbc6a8c6ef215f477c389e285f885c74e6ada..992667e48be79f2347a2af67c26068a990f91f9c 100755 (executable)
--- a/.github/scripts/spacecheck.pl
+++ b/.github/scripts/spacecheck.pl
@@ -158,11 +158,19 @@ while(my $filename = <$git_ls_files>) {
         push @err, "content: has binary contents";
     }
 
-    $content =~ s/[$non_ascii_allowed]//g;
+    if($filename !~ /tests\/data/) {
+        # the tests have no allowed UTF bytes
+        $content =~ s/[$non_ascii_allowed]//g;
+    }
 
     if(!fn_match($filename, @non_ascii) &&
-       ($content =~ /([\x80-\xff]+)/ && $content !~ /^(codeset-utf8|Unicode|non-ascii)/m)) {
-        push @err, "content: has non-ASCII: '$1'";
+       ($content =~ /([\x80-\xff]+)/)) {
+        my $non = $1;
+        my $hex;
+        for my $e (split(//, $non)) {
+            $hex .= sprintf("%s%02x", $hex ? " ": "", ord($e));
+        }
+        push @err, "content: has non-ASCII: '$non' ($hex)";
     }
 
     if(@err) {

diff --git a/tests/FILEFORMAT.md b/tests/FILEFORMAT.md
index 0a0aee0fec85e54562c33dd11da8161ed1e561df..72bdb747a978c478814cd91a9f3f2d56825cb37d 100644 (file)
--- a/tests/FILEFORMAT.md
+++ b/tests/FILEFORMAT.md
@@ -228,9 +228,8 @@ Tests that have strict timing dependencies have the `timing-dependent` keyword.
 These are intended to eventually be treated specially on CI builds which are
 often run on overloaded machines with unpredictable timing.
 
-Tests using non-7-bit-ASCII characters, and not using features `Unicode` or
-`codeset-utf8`, need to add the `non-ascii` keyword to tell the code checker
-to allow these characters.
+Tests using non-7-bit-ASCII characters must provide them with `%hex[]` or
+similar.
 
 ## `<reply>`
 

diff --git a/tests/data/test1034 b/tests/data/test1034
index 7e62f6d63b412194b1e1c048803258d5fcbb1727..ef671dfe2e0463fbac98bb24e07b6072cfbf9f6d 100644 (file)
--- a/tests/data/test1034
+++ b/tests/data/test1034
@@ -39,7 +39,7 @@ HTTP over proxy with malformatted IDN host name
 # This host name contains an invalid UTF-8 byte sequence that can't be
 # converted into an IDN name
 <stdin>
-url = "http://invalid-utf8-â\90.local/page/%TESTNUMBER"
+url = "http://invalid-utf8-%hex[%e2%90]hex%.local/page/%TESTNUMBER"
 </stdin>
 <command>
 -K - -x %HOSTIP:%NOLISTENPORT

diff --git a/tests/data/test1035 b/tests/data/test1035
index 7c4f557241b46ecc9e52a4d25825b21f80a4d276..b6f30ae56b9e18fc81f61c9a983fa7a917b84b6e 100644 (file)
--- a/tests/data/test1035
+++ b/tests/data/test1035
@@ -34,7 +34,7 @@ LC_CTYPE=en_US.UTF-8
 HTTP over proxy with too long IDN host name
 </name>
 <command>
-http://too-long-IDN-name-cürl-rüleß-la-la-la-dee-da-flooby-nooby.local/page/%TESTNUMBER -x %HOSTIP:%NOLISTENPORT
+http://too-long-IDN-name-c%hex[%c3%bc]hex%rl-r%hex[%c3%bc]hex%le%hex[%c3%9f]hex%-la-la-la-dee-da-flooby-nooby.local/page/%TESTNUMBER -x %HOSTIP:%NOLISTENPORT
 </command>
 </client>
 

diff --git a/tests/data/test1138 b/tests/data/test1138
index 74648184962a9b2dc1c015cd093f25df02aa0512..1f125f5844627ab924d7d622615935dc232c366b 100644 (file)
--- a/tests/data/test1138
+++ b/tests/data/test1138
@@ -4,38 +4,37 @@
 HTTP
 HTTP GET
 followlocation
-non-ascii
 </keywords>
 </info>
 #
 # Server-side
 <reply>
-<data>
-HTTP/1.1 302 OK swsclose\r
-Location: ../moo.html/?name=آغاز-سم-زدایی-از-بازار-پول&testcase=/%TESTNUMBER0002\r
-Date: Tue, 09 Nov 2010 14:49:00 GMT\r
-Connection: close\r
-\r
+<data crlf="yes">
+HTTP/1.1 302 OK swsclose
+Location: ../moo.html/?name=%hex[%d8%a2%d8%ba%d8%a7%d8%b2%2d%d8%b3%d9%85%2d%d8%b2%d8%af%d8%a7%db%8c%db%8c%2d%d8%a7%d8%b2%2d%d8%a8%d8%a7%d8%b2%d8%a7%d8%b1%2d%d9%be%d9%88%d9%84]hex%&testcase=/%TESTNUMBER0002
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Connection: close
+
 </data>
-<data2>
-HTTP/1.1 200 OK swsclose\r
-Location: this should be ignored\r
-Date: Tue, 09 Nov 2010 14:49:00 GMT\r
-Connection: close\r
-\r
+<data2 crlf="yes">
+HTTP/1.1 200 OK swsclose
+Location: this should be ignored
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Connection: close
+
 body
 </data2>
-<datacheck>
-HTTP/1.1 302 OK swsclose\r
-Location: ../moo.html/?name=آغاز-سم-زدایی-از-بازار-پول&testcase=/%TESTNUMBER0002\r
-Date: Tue, 09 Nov 2010 14:49:00 GMT\r
-Connection: close\r
-\r
-HTTP/1.1 200 OK swsclose\r
-Location: this should be ignored\r
-Date: Tue, 09 Nov 2010 14:49:00 GMT\r
-Connection: close\r
-\r
+<datacheck crlf="yes">
+HTTP/1.1 302 OK swsclose
+Location: ../moo.html/?name=%hex[%d8%a2%d8%ba%d8%a7%d8%b2%2d%d8%b3%d9%85%2d%d8%b2%d8%af%d8%a7%db%8c%db%8c%2d%d8%a7%d8%b2%2d%d8%a8%d8%a7%d8%b2%d8%a7%d8%b1%2d%d9%be%d9%88%d9%84]hex%&testcase=/%TESTNUMBER0002
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Connection: close
+
+HTTP/1.1 200 OK swsclose
+Location: this should be ignored
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Connection: close
+
 body
 </datacheck>
 </reply>
@@ -57,17 +56,17 @@ http://%HOSTIP:%HTTPPORT/we/are/all/twits/%TESTNUMBER -L
 #
 # Verify data after the test has been "shot"
 <verify>
-<protocol>
-GET /we/are/all/twits/%TESTNUMBER HTTP/1.1\r
-Host: %HOSTIP:%HTTPPORT\r
-User-Agent: curl/%VERSION\r
-Accept: */*\r
-\r
-GET /we/are/all/moo.html/?name=%d8%a2%d8%ba%d8%a7%d8%b2-%d8%b3%d9%85-%d8%b2%d8%af%d8%a7%db%8c%db%8c-%d8%a7%d8%b2-%d8%a8%d8%a7%d8%b2%d8%a7%d8%b1-%d9%be%d9%88%d9%84&testcase=/%TESTNUMBER0002 HTTP/1.1\r
-Host: %HOSTIP:%HTTPPORT\r
-User-Agent: curl/%VERSION\r
-Accept: */*\r
-\r
+<protocol crlf="yes">
+GET /we/are/all/twits/%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+GET /we/are/all/moo.html/?name=%d8%a2%d8%ba%d8%a7%d8%b2-%d8%b3%d9%85-%d8%b2%d8%af%d8%a7%db%8c%db%8c-%d8%a7%d8%b2-%d8%a8%d8%a7%d8%b2%d8%a7%d8%b1-%d9%be%d9%88%d9%84&testcase=/%TESTNUMBER0002 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
 </protocol>
 </verify>
 </testcase>

diff --git a/tests/data/test1160 b/tests/data/test1160
index 77ea92a042a07056a51fbd8831f01054e74480f5..c24decedca7d518b60d9378a02918f2669e9500f 100644 (file)
--- a/tests/data/test1160
+++ b/tests/data/test1160
@@ -5,7 +5,6 @@
 HTTP
 HTTP GET
 cookies
-non-ascii
 </keywords>
 </info>
 
@@ -16,7 +15,7 @@ non-ascii
 HTTP/1.1 200 OK\r
 Date: Tue, 09 Nov 2010 14:49:00 GMT\r
 Content-Length: 0\r
-Set-Cookie: ____________ÿ=         ;                                                                                                                     ÿ                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\86\85\85\80zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzúzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzó \81\96zzzzzzzzzzzz~zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz¶zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\r
+Set-Cookie: ____________%hex[%ff]hex%=         ;                                                                                                                     %hex[%ff]hex%                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%hex[%86%85%85%80]hex%zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%hex[%fa]hex%zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%hex[%f3%a0%81%96]hex%zzzzzzzzzzzz~zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%hex[%b6]hex%zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\r
 \r
 </data>
 </reply>

diff --git a/tests/data/test1448 b/tests/data/test1448
index 33aedc4f011a5a3a9e5d6527ab7c9e88e9f28c9f..96d64b85a59b54b8222af3118727bc5fd55ed705 100644 (file)
--- a/tests/data/test1448
+++ b/tests/data/test1448
@@ -18,7 +18,7 @@ HTTP/1.1 302 OK swsbounce
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Content-Length: 9
 Content-Type: text/plain
-Location: http://åäö.se:%HTTPPORT/%TESTNUMBER0001
+Location: http://%hex[%c3%a5%c3%a4%c3%b6]hex%.se:%HTTPPORT/%TESTNUMBER0001
 
 redirect
 </data>
@@ -51,7 +51,7 @@ Redirect following to UTF-8 IDN host name
 </name>
 
 <command>
-http://åäö.se:%HTTPPORT/%TESTNUMBER --resolve xn--4cab6c.se:%HTTPPORT:%HOSTIP -L --connect-to %HOSTIP:%HTTPPORT:%HOSTIP:%HTTPPORT
+http://%hex[%c3%a5%c3%a4%c3%b6]hex%.se:%HTTPPORT/%TESTNUMBER --resolve xn--4cab6c.se:%HTTPPORT:%HOSTIP -L --connect-to %HOSTIP:%HTTPPORT:%HOSTIP:%HTTPPORT
 </command>
 </client>
 
@@ -76,7 +76,7 @@ HTTP/1.1 302 OK swsbounce
 Date: Tue, 09 Nov 2010 14:49:00 GMT
 Content-Length: 9
 Content-Type: text/plain
-Location: http://åäö.se:%HTTPPORT/%TESTNUMBER0001
+Location: http://%hex[%c3%a5%c3%a4%c3%b6]hex%.se:%HTTPPORT/%TESTNUMBER0001
 
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT

diff --git a/tests/data/test1631 b/tests/data/test1631
index db7d3912332f21dbd0f963a1c871140624ef8f81..7c470b3c84646f4b7f462d053016015a2a1c3d3f 100644 (file)
--- a/tests/data/test1631
+++ b/tests/data/test1631
@@ -4,7 +4,6 @@
 FTP
 HTTPS proxy
 flaky
-non-ascii
 </keywords>
 </info>
 
@@ -30,7 +29,7 @@ Hull.  He got a good estate by merchandise,and leaving off his trade, lived
 afterwards at York, from whence he had married my mother, whoserelations were
 named Robinson, a very good family in that country, and from whom I was
 calledRobinson Kreutznaer; but, by the usual corruption of words in England,
-we are now called—nay wecall ourselves and write our name—Crusoe; and so my
+we are now called-nay wecall ourselves and write our name-Crusoe; and so my
 companions always called me.
 </data>
 </reply>

diff --git a/tests/data/test1632 b/tests/data/test1632
index c72296bd2f36c632ed28452fa51ea6b790b18a45..18d400fd4b57955ccb72434aea19b3654c0e4496 100644 (file)
--- a/tests/data/test1632
+++ b/tests/data/test1632
@@ -4,7 +4,6 @@
 FTP
 HTTPS proxy
 flaky
-non-ascii
 </keywords>
 </info>
 
@@ -30,7 +29,7 @@ Hull.  He got a good estate by merchandise,and leaving off his trade, lived
 afterwards at York, from whence he had married my mother, whoserelations were
 named Robinson, a very good family in that country, and from whom I was
 calledRobinson Kreutznaer; but, by the usual corruption of words in England,
-we are now called—nay wecall ourselves and write our name—Crusoe; and so my
+we are now called-nay wecall ourselves and write our name-Crusoe; and so my
 companions always called me.
 </data>
 

diff --git a/tests/data/test165 b/tests/data/test165
index 39787db87e593abf785ff9140d0b85d543b78434..0b5cfd2346f2bd878c5e5e1899606457931f04ff 100644 (file)
--- a/tests/data/test165
+++ b/tests/data/test165
@@ -40,7 +40,7 @@ LC_CTYPE=en_US.UTF-8
 HTTP over proxy with IDN host name
 </name>
 <command>
-http://www.åäö.se/page/%TESTNUMBER -x %HOSTIP:%HTTPPORT http://www.große.de/page/%TESTNUMBER
+http://www.%hex[%c3%a5%c3%a4%c3%b6]hex%.se/page/%TESTNUMBER -x %HOSTIP:%HTTPPORT http://www.gro%hex[%c3%9f]hex%e.de/page/%TESTNUMBER
 </command>
 </client>
 

diff --git a/tests/data/test1703 b/tests/data/test1703
index bdc647632380272ad3650309a339656a4be6e2b2..677414850960c528980776213857faf504fc3bb2 100644 (file)
--- a/tests/data/test1703
+++ b/tests/data/test1703
@@ -30,7 +30,7 @@ basic file:// file with UTF-8 characters
 <command option="no-include">
 file://localhost%FILE_PWD/%LOGDIR/%E3%83%86%E3%82%B9%E3%83%88%TESTNUMBER.txt
 </command>
-<file name="%LOGDIR/テスト%TESTNUMBER.txt">
+<file name="%LOGDIR/%hex[%E3%83%86%E3%82%B9%E3%83%88]hex%%TESTNUMBER.txt">
 foo
    bar
 bar

diff --git a/tests/data/test2046 b/tests/data/test2046
index c0d3a0e82503369c456193589cdd33b5be51cd36..cf25d0c57b2e9bf021d35a55b52ed036f87d828c 100644 (file)
--- a/tests/data/test2046
+++ b/tests/data/test2046
@@ -51,7 +51,7 @@ Connection reuse with IDN host name
 </name>
 
 <command>
-http://åäö.se:%HTTPPORT/%TESTNUMBER --resolve xn--4cab6c.se:%HTTPPORT:%HOSTIP -w "%{num_connects}\n%{num_redirects}\n%{size_download}\n%{url_effective}\n%{content_type}\n%{response_code}\n" -L
+http://%hex[%c3%a5%c3%a4%c3%b6]hex%.se:%HTTPPORT/%TESTNUMBER --resolve xn--4cab6c.se:%HTTPPORT:%HOSTIP -w "%{num_connects}\n%{num_redirects}\n%{size_download}\n%{url_effective}\n%{content_type}\n%{response_code}\n" -L
 </command>
 </client>
 
@@ -87,7 +87,7 @@ OK
 1
 1
 3
-http://åäö.se:%HTTPPORT/%TESTNUMBER0001
+http://%hex[%c3%a5%c3%a4%c3%b6]hex%.se:%HTTPPORT/%TESTNUMBER0001
 text/plain; charset=us-ascii
 200
 </stdout>

diff --git a/tests/data/test2047 b/tests/data/test2047
index 1d7be6db364a848e3823feb0fc5ae838af1c5e71..c32e6aabc714f914a3f4dbafa3b70c405df71837 100644 (file)
--- a/tests/data/test2047
+++ b/tests/data/test2047
@@ -52,7 +52,7 @@ Connection reuse with IDN host name over HTTP proxy
 </name>
 
 <command>
-http://åäö.se/%TESTNUMBER -x %HOSTIP:%HTTPPORT -w "%{num_connects}\n%{num_redirects}\n%{size_download}\n%{url_effective}\n%{content_type}\n%{response_code}\n" -L
+http://%hex[%c3%a5%c3%a4%c3%b6]hex%.se/%TESTNUMBER -x %HOSTIP:%HTTPPORT -w "%{num_connects}\n%{num_redirects}\n%{size_download}\n%{url_effective}\n%{content_type}\n%{response_code}\n" -L
 </command>
 </client>
 
@@ -90,7 +90,7 @@ OK
 1
 1
 3
-http://åäö.se/%TESTNUMBER0001
+http://%hex[%c3%a5%c3%a4%c3%b6]hex%.se/%TESTNUMBER0001
 text/plain; charset=us-ascii
 200
 </stdout>

diff --git a/tests/data/test31 b/tests/data/test31
index 0392207f65a127be6e7eca7cf2f24c6aa5b13474..8de7adc7836aaf17f6b3014cc1699a9e8e78eee3 100644 (file)
--- a/tests/data/test31
+++ b/tests/data/test31
@@ -5,7 +5,6 @@ HTTP
 HTTP GET
 cookies
 cookiejar
-non-ascii
 </keywords>
 </info>
 # Server-side
@@ -69,7 +68,7 @@ Set-Cookie: withspaces=  yes  within and around    ;
 Set-Cookie: withspaces2 =before equals;\r
 Set-Cookie: prespace=  yes before;\r
 Set-Cookie: securewithspace=after    ; secure =\r
-Set-Cookie: ³Ã\x92Ã\x9aÃ\x8dÃ\x97=%96%A6g%9Ay%B0%A5g%A7tm%7C%95%9A
+Set-Cookie: %hex[%c3%82%c2%b3%c3%83%5c%78%39%32%c3%83%5c%78%39%61%c3%83%5c%78%38%64%c3%83%5c%78%39%37]hex%=%96%A6g%9Ay%B0%A5g%A7tm%7C%95%9A
 \r
 boo
 </data>
@@ -112,7 +111,7 @@ Accept: */*
 # https://curl.se/docs/http-cookies.html
 # This file was generated by libcurl! Edit at your own risk.
 
-test31.curl    FALSE   /we/want/       FALSE   0       ³Ã\x92Ã\x9aÃ\x8dÃ\x97    %96%A6g%9Ay%B0%A5g%A7tm%7C%95%9A
+test31.curl    FALSE   /we/want/       FALSE   0       %hex[%c3%82%c2%b3%c3%83%5c%78%39%32%c3%83%5c%78%39%61%c3%83%5c%78%38%64%c3%83%5c%78%39%37]hex%  %96%A6g%9Ay%B0%A5g%A7tm%7C%95%9A
 test31.curl    FALSE   /we/want/       FALSE   0       prespace        yes before
 test31.curl    FALSE   /we/want/       FALSE   0       withspaces2     before equals
 test31.curl    FALSE   /we/want/       FALSE   0       withspaces      yes  within and around

diff --git a/tests/data/test469 b/tests/data/test469
index ace2694933d573d9a22cf54e86a2995f8f0caf31..1e394c9f260cbbfb6ff47cbc11fc57a46baab2e3 100644 (file)
--- a/tests/data/test469
+++ b/tests/data/test469
@@ -2,7 +2,6 @@
 <info>
 <keywords>
 HTTP
-non-ascii
 </keywords>
 </info>
 
@@ -38,7 +37,7 @@ http
 warn about Unicode quote character
 </name>
 <command>
--H “host: %HOSTIP:%HTTPPORT/” -s
+-H %hex[%e2%80%9c]hex%host: %HOSTIP:%HTTPPORT/%hex[%e2%80%9c]hex% -s
 </command>
 </client>
 

diff --git a/tests/data/test470 b/tests/data/test470
index f99d461b773c1ee2e18d619bb7129e026be95767..5667bbb590f65a8586cf3af732e3ab1740be1ff3 100644 (file)
--- a/tests/data/test470
+++ b/tests/data/test470
@@ -2,7 +2,6 @@
 <info>
 <keywords>
 HTTP
-non-ascii
 </keywords>
 </info>
 
@@ -35,7 +34,7 @@ http
 warn about Unicode quote character read from config file
 </name>
 <file name="%LOGDIR/input%TESTNUMBER">
--H “host:fake”
+-H %hex[%e2%80%9c]hex%host:fake%hex[%e2%80%9d]hex%
 </file>
 <command>
 %HOSTIP:%HTTPPORT --no-progress-meter -K "%LOGDIR/input%TESTNUMBER"

diff --git a/tests/data/test472 b/tests/data/test472
index 88cc78f937926fedab240213288dc41d212d8d7e..d0abb5ce77493ed7f548a6996cc87b3c30401351 100644 (file)
--- a/tests/data/test472
+++ b/tests/data/test472
@@ -39,7 +39,7 @@ Unicode
 aws-sigv4 with query
 </name>
 <command>
-"http://fake.fake.fake:8000/%TESTNUMBER/a=あ" -u user:secret --aws-sigv4 "aws:amz:us-east-2:es" --connect-to fake.fake.fake:8000:%HOSTIP:%HTTPPORT
+"http://fake.fake.fake:8000/%TESTNUMBER/a=%hex[%e3%81%82]hex%" -u user:secret --aws-sigv4 "aws:amz:us-east-2:es" --connect-to fake.fake.fake:8000:%HOSTIP:%HTTPPORT
 </command>
 </client>
 

diff --git a/tests/data/test497 b/tests/data/test497
index 300201b1fc7149b4cec08fa30dd947f5636c5f10..dec1808f9b666b822691bd8c422514c1300a03c8 100644 (file)
--- a/tests/data/test497
+++ b/tests/data/test497
@@ -3,7 +3,6 @@
 <keywords>
 HTTP
 HTTP GET
-non-ascii
 </keywords>
 </info>
 
@@ -20,7 +19,7 @@ Accept-Ranges: bytes
 Content-Length: 6
 Connection: close
 Content-Type: text/html
-%repeat[1750 x Repeat-this-Header-a-large-number-of-times: Dorothy lived in the midst of the great Kansas prairies, with Uncle Henry, who was a farmer, and Aunt Em, who was the farmer’s wife.%0a]%
+%repeat[1750 x Repeat-this-Header-a-large-number-of-times: Dorothy lived in the midst of the great Kansas prairies, with Uncle Henry, who was a farmer, and Aunt Em, who was the farmer's wife.%0a]%
 -foo-
 </data>
 </reply>

diff --git a/tests/data/test498 b/tests/data/test498
index 4426f7a29ca6aa95c54f177e6664779b8ead690b..6c9f8d239e0ba7371a2713b49451e59db2dde34d 100644 (file)
--- a/tests/data/test498
+++ b/tests/data/test498
@@ -3,7 +3,6 @@
 <keywords>
 HTTP
 HTTP GET
-non-ascii
 </keywords>
 </info>
 

diff --git a/tests/data/test649 b/tests/data/test649
index 25128aee67f9dab5f0e23957b8735e390aa0619e..371f00bfdc5cff93876f7d0614e301e77ffd62ab 100644 (file)
--- a/tests/data/test649
+++ b/tests/data/test649
@@ -3,7 +3,6 @@
 <keywords>
 SMTP
 MULTIPART
-non-ascii
 </keywords>
 </info>
 
@@ -34,7 +33,7 @@ body
 smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from sender@example.com -F '=This is valid;encoder=7bit' -F "=@%LOGDIR/test%TESTNUMBER.txt;encoder=7bit" -H "From: different" -H "To: another"
 </command>
 <file name="%LOGDIR/test%TESTNUMBER.txt">
-This is an attached file (in french: pièce jointe).
+This is an attached file (in french: pi%hex[%c3%a8]hex%ce jointe).
 
 It contains at least an 8-bit byte value.
 </file>

diff --git a/tests/data/test955 b/tests/data/test955
index 982afc352ee6edb22ba686533793a892d9350d06..817b47a67e7a7d9137d38a614ac8b95f78cfe51c 100644 (file)
--- a/tests/data/test955
+++ b/tests/data/test955
@@ -37,7 +37,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from Avsändaren@example.com -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from Avs%hex[%c3%a4]hex%ndaren@example.com -T -
 </command>
 </client>
 
@@ -50,7 +50,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-fr
 </errorcode>
 <protocol>
 EHLO %TESTNUMBER\r
-MAIL FROM:<Avsändaren@example.com>\r
+MAIL FROM:<Avs%hex[%c3%a4]hex%ndaren@example.com>\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test956 b/tests/data/test956
index 22bf3a14fa172dcf6cef521f69368e812fa2cbf9..4546f2ee61b82eb12b3f1df6371ef592eb98b4ef 100644 (file)
--- a/tests/data/test956
+++ b/tests/data/test956
@@ -34,7 +34,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Stödmottagaren@example.com --mail-from sender@example.com -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt St%hex[%c3%b6]hex%dmottagaren@example.com --mail-from sender@example.com -T -
 </command>
 </client>
 
@@ -48,7 +48,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Stödmottagaren@example.com --m
 <protocol>
 EHLO %TESTNUMBER\r
 MAIL FROM:<sender@example.com>\r
-RCPT TO:<Stödmottagaren@example.com>\r
+RCPT TO:<St%hex[%c3%b6]hex%dmottagaren@example.com>\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test957 b/tests/data/test957
index d49739d84f0a7c04749fc5b03f59c76808ad4268..fbe07a0fd7f7b3f8dd0f56f5642b871e4f8f9f1b 100644 (file)
--- a/tests/data/test957
+++ b/tests/data/test957
@@ -29,7 +29,7 @@ LC_CTYPE=en_US.UTF-8
 SMTP VRFY without SMTPUTF8 support - UTF-8 based recipient (local part only)
 </name>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Anv%hex[%c3%a4]hex%ndaren
 </command>
 </client>
 
@@ -42,7 +42,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren
 </errorcode>
 <protocol>
 EHLO %TESTNUMBER\r
-VRFY Användaren\r
+VRFY Anv%hex[%c3%a4]hex%ndaren\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test958 b/tests/data/test958
index ea5a2dbae64dc574dce3e4ced09b1ccb48d1205a..1687ebbf05f63769fd90e61e84d935cd22e63d7f 100644 (file)
--- a/tests/data/test958
+++ b/tests/data/test958
@@ -29,7 +29,7 @@ LC_CTYPE=en_US.UTF-8
 SMTP external VRFY without SMTPUTF8 support - UTF-8 based recipient (local part only)
 </name>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren@example.com
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Anv%hex[%c3%a4]hex%ndaren@example.com
 </command>
 </client>
 
@@ -42,7 +42,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren@example.com
 </errorcode>
 <protocol>
 EHLO %TESTNUMBER\r
-VRFY Användaren@example.com\r
+VRFY Anv%hex[%c3%a4]hex%ndaren@example.com\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test959 b/tests/data/test959
index 178e0f13c40706b0e5fb30fadfabe2bc31ede64b..2154600da2fd6af072c945728f51013a6906b9e8 100644 (file)
--- a/tests/data/test959
+++ b/tests/data/test959
@@ -38,7 +38,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from sender@åäö.se -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from sender@%hex[%c3%a5%c3%a4%c3%b6]hex%.se -T -
 </command>
 </client>
 
@@ -51,7 +51,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-fr
 </errorcode>
 <protocol>
 EHLO %TESTNUMBER\r
-MAIL FROM:<sender@åäö.se>\r
+MAIL FROM:<sender@%hex[%c3%a5%c3%a4%c3%b6]hex%.se>\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test960 b/tests/data/test960
index 2cf348eb53dce49a4fcd98c2d39d9ec769252d86..fb50a326b28890e549f137d290636b90a28d57a9 100644 (file)
--- a/tests/data/test960
+++ b/tests/data/test960
@@ -35,7 +35,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@åäö.se --mail-from sender@example.com -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@%hex[%c3%a5%c3%a4%c3%b6]hex%.se --mail-from sender@example.com -T -
 </command>
 </client>
 
@@ -49,7 +49,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@åäö.se --mail-from
 <protocol>
 EHLO %TESTNUMBER\r
 MAIL FROM:<sender@example.com>\r
-RCPT TO:<recipient@åäö.se>\r
+RCPT TO:<recipient@%hex[%c3%a5%c3%a4%c3%b6]hex%.se>\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test961 b/tests/data/test961
index 57958a4eb0afc01ee258284b27603ff76a0d8623..f6dd8ce75a94f24ee840aa176b099a9f20b366d9 100644 (file)
--- a/tests/data/test961
+++ b/tests/data/test961
@@ -30,7 +30,7 @@ LC_CTYPE=en_US.UTF-8
 SMTP external VRFY without SMTPUTF8 support - UTF-8 based recipient (host part only)
 </name>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt user@åäö.se
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt user@%hex[%c3%a5%c3%a4%c3%b6]hex%.se
 </command>
 </client>
 
@@ -43,7 +43,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt user@åäö.se
 </errorcode>
 <protocol>
 EHLO %TESTNUMBER\r
-VRFY user@åäö.se\r
+VRFY user@%hex[%c3%a5%c3%a4%c3%b6]hex%.se\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test962 b/tests/data/test962
index 6ac4ab08c2d85cb85c276ad2bc3482a85b08b5e9..80292862ba744edeca11ecda96a3fc1479236ed1 100644 (file)
--- a/tests/data/test962
+++ b/tests/data/test962
@@ -36,7 +36,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from sender@åäö.se -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from sender@%hex[%c3%a5%c3%a4%c3%b6]hex%.se -T -
 </command>
 </client>
 

diff --git a/tests/data/test963 b/tests/data/test963
index 9fe94f894cef2b846ee09ca3739f7023d7bdcc25..06dbf3201aec6c5e7d3abeba208fd89d32b83b6c 100644 (file)
--- a/tests/data/test963
+++ b/tests/data/test963
@@ -36,7 +36,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@åäö.se --mail-from sender@example.com -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@%hex[%c3%a5%c3%a4%c3%b6]hex%.se --mail-from sender@example.com -T -
 </command>
 </client>
 

diff --git a/tests/data/test964 b/tests/data/test964
index 05dbda9319bd5ebe0fb01f97dd9043f2c7fcc175..8963856485ce12bf434d016fe5dbb679e343b10f 100644 (file)
--- a/tests/data/test964
+++ b/tests/data/test964
@@ -31,7 +31,7 @@ LC_CTYPE=en_US.UTF-8
 SMTP external VRFY without SMTPUTF8 support (IDN Enabled) - UTF-8 based recipient (host part only)
 </name>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt user@åäö.se
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt user@%hex[%c3%a5%c3%a4%c3%b6]hex%.se
 </command>
 </client>
 

diff --git a/tests/data/test965 b/tests/data/test965
index f44239e784c692d27136caaba6f98ce25e93be31..0d29c0984932354ab611e8670973d2317e724769 100644 (file)
--- a/tests/data/test965
+++ b/tests/data/test965
@@ -39,7 +39,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from Avsändaren@åäö.se -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from Avs%hex[%c3%a4]hex%ndaren@%hex[%c3%a5%c3%a4%c3%b6]hex%.se -T -
 </command>
 </client>
 
@@ -48,7 +48,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-fr
 <verify>
 <protocol>
 EHLO %TESTNUMBER\r
-MAIL FROM:<Avsändaren@xn--4cab6c.se> SMTPUTF8\r
+MAIL FROM:<Avs%hex[%c3%a4]hex%ndaren@xn--4cab6c.se> SMTPUTF8\r
 RCPT TO:<recipient@example.com>\r
 DATA\r
 QUIT\r

diff --git a/tests/data/test966 b/tests/data/test966
index 31b6c288f1c34aed7e14bec32377e84bf0f69b41..b642e19a9d2dde6e0b1e25954b027f2fe9244263 100644 (file)
--- a/tests/data/test966
+++ b/tests/data/test966
@@ -39,7 +39,7 @@ To: another
 body\r
 </stdin>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Stödmottagaren@åäö.se --mail-from sender@example.com -T -
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt St%hex[%c3%b6]hex%dmottagaren@%hex[%c3%a5%c3%a4%c3%b6]hex%.se --mail-from sender@example.com -T -
 </command>
 </client>
 
@@ -49,7 +49,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Stödmottagaren@åäö.se --mai
 <protocol>
 EHLO %TESTNUMBER\r
 MAIL FROM:<sender@example.com> SMTPUTF8\r
-RCPT TO:<Stödmottagaren@xn--4cab6c.se>\r
+RCPT TO:<St%hex[%c3%b6]hex%dmottagaren@xn--4cab6c.se>\r
 DATA\r
 QUIT\r
 </protocol>

diff --git a/tests/data/test967 b/tests/data/test967
index d823b4839e9e1cdb63eb76b522209864e84d0278..f6e2e32382de6a4bf769df8af9c59127cae02da7 100644 (file)
--- a/tests/data/test967
+++ b/tests/data/test967
@@ -37,7 +37,7 @@ LC_CTYPE=en_US.UTF-8
 SMTP external VRFY with SMTPUTF8 support
 </name>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren@åäö.se
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Anv%hex[%c3%a4]hex%ndaren@%hex[%c3%a5%c3%a4%c3%b6]hex%.se
 </command>
 </client>
 
@@ -46,7 +46,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren@åäö.se
 <verify>
 <protocol>
 EHLO %TESTNUMBER\r
-VRFY Användaren@xn--4cab6c.se SMTPUTF8\r
+VRFY Anv%hex[%c3%a4]hex%ndaren@xn--4cab6c.se SMTPUTF8\r
 QUIT\r
 </protocol>
 </verify>

diff --git a/tests/data/test968 b/tests/data/test968
index b057be62917c63c7b94331ec38122e7db5e892ad..8532c35e7db883ef81af6e94fbc664fc9be5b2ee 100644 (file)
--- a/tests/data/test968
+++ b/tests/data/test968
@@ -34,7 +34,7 @@ LC_CTYPE=en_US.UTF-8
 SMTP VRFY with SMTPUTF8 support
 </name>
 <command>
-smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Anv%hex[%c3%a4]hex%ndaren
 </command>
 </client>
 
@@ -43,7 +43,7 @@ smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt Användaren
 <verify>
 <protocol>
 EHLO %TESTNUMBER\r
-VRFY Användaren SMTPUTF8\r
+VRFY Anv%hex[%c3%a4]hex%ndaren SMTPUTF8\r
 QUIT\r
 </protocol>
 </verify>