tests/data: support using native newlines on disk, drop `.gitattributes`

Data files no longer depend on mixed newline styles. Before this
patch the harness still assumed data files to use LF newlines,
ensured by `.gitattribute` and distributing sources with LF newlines.

To allow using platform native newlines (CRLF on Windows typically),
update the test harness to support data files with any newline style
on disk. And delete `.gitattributes`.

Fix to:
- load original data files (from test/data) so that their newline-style
  doesn't matter on the checked out source repo, meaning it works
  when its CRLF on Windows, just like any other file.
  (if a BOM slips in, it's caught by `spacecheck.pl` as binary content.)
- do the same in `util.py` used by `smbserver.py` (for test 1451).
- also fix `util.py` to use us-ascii encoding for data files, replacing utf-8.

Also:
- runtests: rework the stray CR checker to allow full CRLF data files,
  and keep warning for mixed newlines.

Follow-up to 904e7ecb66519951681377758fe6b07dde28ce36 #19347

Closes #19398

diff --git a/tests/data/.gitattributes b/tests/data/.gitattributes
deleted file mode 100644 (file)
index bb1b928..0000000
--- a/tests/data/.gitattributes
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
-#
-# SPDX-License-Identifier: curl
-
-test* -crlf

diff --git a/tests/devtest.pl b/tests/devtest.pl
index 0bfc71b07a53be06521abb5701523864b6f5589a..30124a9f91d84605dd23f32c01e7e388584a95c9 100755 (executable)
--- a/tests/devtest.pl
+++ b/tests/devtest.pl
@@ -176,7 +176,7 @@ while(@ARGV) {
     }
     elsif($ARGV[0] eq "preprocess") {
         shift @ARGV;
-        loadtest("${TESTDIR}/test${ARGV[0]}");
+        loadtest("${TESTDIR}/test${ARGV[0]}", 1);
         readtestkeywords();
         singletest_preprocess($ARGV[0]);
     }

diff --git a/tests/getpart.pm b/tests/getpart.pm
index b555038cf7242c27c44bf97f0aca679df743e52e..2b6d2a63c4bd97202d007576121c7152ab0ede77 100644 (file)
--- a/tests/getpart.pm
+++ b/tests/getpart.pm
@@ -217,7 +217,7 @@ sub partexists {
 # memoize('partexists', NORMALIZER => 'normalize_part');  # cache each result
 
 sub loadtest {
-    my ($file)=@_;
+    my ($file, $original)=@_;
 
     if(defined $xmlfile && $file eq $xmlfile) {
         # This test is already loaded
@@ -228,7 +228,12 @@ sub loadtest {
     $xmlfile = "";
 
     if(open(my $xmlh, "<", "$file")) {
-        binmode $xmlh; # for crapage systems, use binary
+        if($original) {
+            binmode $xmlh, ':crlf'
+        }
+        else {
+            binmode $xmlh; # for crapage systems, use binary
+        }
         while(<$xmlh>) {
             push @xml, $_;
         }
@@ -251,16 +256,44 @@ sub fulltest {
     return @xml;
 }
 
+sub eol_detect {
+    my ($content) = @_;
+
+    my $cr = () = $content =~ /\r/g;
+    my $lf = () = $content =~ /\n/g;
+
+    if($cr > 0 && $lf == 0) {
+        return "cr";
+    }
+    elsif($cr == 0 && $lf > 0) {
+        return "lf";
+    }
+    elsif($cr == 0 && $lf == 0) {
+        return "bin";
+    }
+    elsif($cr == $lf) {
+        return "crlf";
+    }
+
+    return "";
+}
+
 sub checktest {
-    my $anyerr = 0;
+    my ($file) = @_;
 
-    for my $i (0 .. $#xml) {
-        if(index($xml[$i], "\r") >= 0) {
-            print STDERR "*** getpart.pm: $xmlfile:$i: 0x0d carriage return found. Use %CR macro instead.\n";
-            $anyerr = 1;
+    if(open(my $xmlh, '<', $file)) {
+        binmode $xmlh; # we want the raw data to check original newlines
+        my $content = do { local $/; <$xmlh> };
+        close($xmlh);
+
+        my $eol = eol_detect($content);
+        if($eol eq '') {
+            print STDERR "*** getpart.pm: $xmlfile has mixed newlines. Replace significant carriage return with %CR macro, or convert to consistent newlines.\n";
+            return 1;
         }
     }
-    return $anyerr;
+
+    return 0;
 }
 
 # write the test to the given file

diff --git a/tests/runner.pm b/tests/runner.pm
index 8b9c0c3f2e513343cec07d814629fdcef32581af..c711834e01495ae0d02a949feb6f107da79c56ae 100644 (file)
--- a/tests/runner.pm
+++ b/tests/runner.pm
@@ -1149,12 +1149,9 @@ sub singletest_postcheck {
         }
     }
 
-    if($checktests) {
-        loadtest("${TESTDIR}/test${testnum}");  # load the raw original data
-        if(checktest()) {
-            logmsg " $testnum: postcheck FAILED: issue(s) found in test data\n";
-            return -1;
-        }
+    if($checktests && checktest("${TESTDIR}/test${testnum}")) {
+        logmsg " $testnum: postcheck FAILED: issue(s) found in test data\n";
+        return -1;
     }
 
     return 0;
@@ -1181,7 +1178,7 @@ sub runner_test_preprocess {
     # ignore any error here--if there were one, it would have been
     # caught during the selection phase and this test would not be
     # running now
-    loadtest("${TESTDIR}/test${testnum}");
+    loadtest("${TESTDIR}/test${testnum}", 1);
     readtestkeywords();
 
     ###################################################################

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 761056a8ed6c50a5c86c924f5f680f87e6dd81cb..226573941d354d5e55187df9bc399e717225eb3f 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -1078,7 +1078,7 @@ sub singletest_shouldrun {
         $errorreturncode = 2;
     }
 
-    if(loadtest("${TESTDIR}/test${testnum}")) {
+    if(loadtest("${TESTDIR}/test${testnum}", 1)) {
         if($verbose) {
             # this is not a test
             logmsg "RUN: $testnum doesn't look like a test case\n";
@@ -1191,7 +1191,7 @@ sub singletest_shouldrun {
         }
     }
 
-    if($why && $checktests && checktest()) {
+    if($why && $checktests && checktest("${TESTDIR}/test${testnum}")) {
         logmsg "Warning: issue(s) found in test data: ${TESTDIR}/test${testnum}\n";
     }
 
@@ -1980,7 +1980,7 @@ sub singletest {
         ###################################################################
         # Load test file so CI registration can get the right data before the
         # runner is called
-        loadtest("${TESTDIR}/test${testnum}");
+        loadtest("${TESTDIR}/test${testnum}", 1);
 
         ###################################################################
         # Register the test case with the CI environment
@@ -3281,7 +3281,7 @@ if(%skipped && !$short) {
 sub testnumdetails {
     my ($desc, $numlist) = @_;
     foreach my $testnum (split(' ', $numlist)) {
-        if(!loadtest("${TESTDIR}/test${testnum}")) {
+        if(!loadtest("${TESTDIR}/test${testnum}", 1)) {
             my @info_keywords = getpart("info", "keywords");
             my $testname = (getpart("client", "name"))[0];
             chomp $testname;

diff --git a/tests/util.py b/tests/util.py
index fa4ca8272386194f94a9110fe016b64e2ec91630..8ed8c892676f84fa48ace47889ef894e72b101ec 100755 (executable)
--- a/tests/util.py
+++ b/tests/util.py
@@ -75,8 +75,8 @@ class TestData(object):
 
         log.debug("Parsing file %s", filename)
 
-        with open(filename, "rb") as f:
-            contents = f.read().decode("utf-8")
+        with open(filename, "r", encoding='us-ascii') as f:
+            contents = f.read()
 
         m = REPLY_DATA.search(contents)
         if not m:
@@ -88,5 +88,5 @@ class TestData(object):
 
 if __name__ == '__main__':
     td = TestData("./data")
-    data = td.get_test_data(1)
+    data = td.get_test_data(1451)
     print(data)