ci/tests: Make it possible to still run but ignore failing tests

This enables the development of a solution for the failing tests by
running them on CI while ignoring their result for the overall status.

Closes #4994

diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
index f7ca699a9e745541c050e0927810c700ae170fa5..9e2d74e993fa0bc025adba0e510787a6f8104515 100644 (file)
--- a/.azure-pipelines.yml
+++ b/.azure-pipelines.yml
@@ -219,7 +219,7 @@ stages:
       env:
         MSYSTEM: MINGW32
         MSYS2_PATH_TYPE: inherit
-        TFLAGS: "!323 !1056 !1299"
+        TFLAGS: "~323 ~1056 ~1299"
     steps:
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --enable-debug --enable-werror"
       displayName: 'configure debug'
@@ -242,7 +242,7 @@ stages:
       env:
         MSYSTEM: MINGW64
         MSYS2_PATH_TYPE: inherit
-        TFLAGS: "!323 !1056 !1299"
+        TFLAGS: "~323 ~1056 ~1299"
     steps:
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --enable-debug --enable-werror"
       displayName: 'configure debug'
@@ -263,7 +263,7 @@ stages:
     container:
       image: mback2k/curl-docker-winbuildenv-msys1-mingw:ltsc2019
       env:
-        TFLAGS: "!203 !1056 !1143"
+        TFLAGS: "~203 ~1056 ~1143"
     steps:
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-pc-mingw32 --build=i686-pc-mingw32 --prefix=/mingw --enable-debug"
       displayName: 'configure debug'
@@ -284,7 +284,7 @@ stages:
     container:
       image: mback2k/curl-docker-winbuildenv-msys1-mingw32:ltsc2019
       env:
-        TFLAGS: "!203 !1056 !1143 !1299"
+        TFLAGS: "~203 ~1056 ~1143 ~1299"
     steps:
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-debug --enable-werror --without-zlib"
       displayName: 'configure debug without zlib'
@@ -305,7 +305,7 @@ stages:
     container:
       image: mback2k/curl-docker-winbuildenv-msys1-mingw64:ltsc2019
       env:
-        TFLAGS: "!203 !1056 !1143 !1299"
+        TFLAGS: "~203 ~1056 ~1143 ~1299"
     steps:
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64 --enable-debug --enable-werror --without-zlib"
       displayName: 'configure debug without zlib'
@@ -328,7 +328,7 @@ stages:
       env:
         MSYSTEM: MINGW32
         MSYS2_PATH_TYPE: inherit
-        TFLAGS: "!165 !310 !1013 !1056 !1299 !1448 !2034 !2037 !2041 !2046 !2047 !3000 !3001"
+        TFLAGS: "~165 ~310 ~1013 ~1056 ~1299 ~1448 ~2034 ~2037 ~2041 ~2046 ~2047 ~3000 ~3001"
     steps:
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn"
       displayName: 'configure debug with sspi/schannel/winidn'
@@ -351,7 +351,7 @@ stages:
       env:
         MSYSTEM: MINGW64
         MSYS2_PATH_TYPE: inherit
-        TFLAGS: "!165 !310 !1013 !1056 !1299 !1448 !2034 !2037 !2041 !2046 !2047 !3000 !3001"
+        TFLAGS: "~165 ~310 ~1013 ~1056 ~1299 ~1448 ~2034 ~2037 ~2041 ~2046 ~2047 ~3000 ~3001"
     steps:
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn"
       displayName: 'configure debug with sspi/schannel/winidn'
@@ -372,7 +372,7 @@ stages:
     container:
       image: mback2k/curl-docker-winbuildenv-msys1-mingw:ltsc2019
       env:
-        TFLAGS: "!203 !305 !310 !311 !312 !313 !404 !1013 !1056 !1143 !2034 !2035 !2037 !2038 !2041 !2042 !2048 !3000 !3001"
+        TFLAGS: "~203 ~305 ~310 ~311 ~312 ~313 ~404 ~1013 ~1056 ~1143 ~2034 ~2035 ~2037 ~2038 ~2041 ~2042 ~2048 ~3000 ~3001"
     steps:
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-pc-mingw32 --build=i686-pc-mingw32 --prefix=/mingw --enable-debug --enable-sspi --without-ssl --with-schannel --with-winidn"
       displayName: 'configure debug with sspi/schannel/winidn'
@@ -393,7 +393,7 @@ stages:
     container:
       image: mback2k/curl-docker-winbuildenv-msys1-mingw32:ltsc2019
       env:
-        TFLAGS: "!203 !310 !1013 !1056 !1143 !1299 !2034 !2037 !2041 !3000 !3001"
+        TFLAGS: "~203 ~310 ~1013 ~1056 ~1143 ~1299 ~2034 ~2037 ~2041 ~3000 ~3001"
     steps:
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn --without-zlib"
       displayName: 'configure debug with sspi/schannel/winidn without zlib'
@@ -414,7 +414,7 @@ stages:
     container:
       image: mback2k/curl-docker-winbuildenv-msys1-mingw64:ltsc2019
       env:
-        TFLAGS: "!203 !310 !1013 !1056 !1143 !1299 !2034 !2037 !2041 !3000 !3001"
+        TFLAGS: "~203 ~310 ~1013 ~1056 ~1143 ~1299 ~2034 ~2037 ~2041 ~3000 ~3001"
     steps:
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64  --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn --without-zlib"
       displayName: 'configure debug with sspi/schannel/winidn without zlib'

diff --git a/tests/appveyor.pm b/tests/appveyor.pm
index abfb18523174c1e50d8565949ef470c7bf20ec47..6ed83dd22b65da94a34259f578b9972c891bb22e 100644 (file)
--- a/tests/appveyor.pm
+++ b/tests/appveyor.pm
@@ -63,7 +63,11 @@ sub appveyor_update_test_result {
     my $appveyor_duration = sprintf("%.0f", ($stop-$start)*1000);
     my $appveyor_outcome;
     my $appveyor_category;
-    if($error < 0) {
+    if($error == 2) {
+        $appveyor_outcome = 'Ignored';
+        $appveyor_category = 'Warning';
+    }
+    elsif($error < 0) {
         $appveyor_outcome = 'NotRunnable';
         $appveyor_category = 'Warning';
     }

diff --git a/tests/azure.pm b/tests/azure.pm
index 2fc11ae21eb2776d2f52c28222e37d21033c486f..94fe995d984bc5158e6fdb5e959769741367d62c 100644 (file)
--- a/tests/azure.pm
+++ b/tests/azure.pm
@@ -88,7 +88,10 @@ sub azure_update_test_result {
     my $azure_complete = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $stop;
     my $azure_duration = sprintf("%.0f", ($stop-$start)*1000);
     my $azure_outcome;
-    if($error < 0) {
+    if($error == 2) {
+        $azure_outcome = 'Not applicable';
+    }
+    elsif($error < 0) {
         $azure_outcome = 'Not executed';
     }
     elsif(!$error) {

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 51a84f09b93a828e36cefa03b9c82e627198ce57..3306de397a0d5b82d3e7fe1d10c09eeb9c0c7b0e 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -276,8 +276,10 @@ my $skipped=0;  # number of tests skipped; reported in main loop
 my %skipped;    # skipped{reason}=counter, reasons for skip
 my @teststat;   # teststat[testnum]=reason, reasons for skip
 my %disabled_keywords;  # key words of tests to skip
+my %ignored_keywords;   # key words of tests to ignore results
 my %enabled_keywords;   # key words of tests to run
 my %disabled;           # disabled test cases
+my %ignored;            # ignored results of test cases
 
 my $sshdid;      # for socks server, ssh daemon version id
 my $sshdvernum;  # for socks server, ssh daemon version number
@@ -3250,6 +3252,7 @@ sub singletest {
     my $why;
     my $cmd;
     my $disablevalgrind;
+    my $errorreturncode = 1; # 1 means normal error, 2 means ignored error
 
     # fist, remove all lingering log files
     cleardir($LOGDIR);
@@ -3267,6 +3270,10 @@ sub singletest {
     if($disabled{$testnum}) {
         logmsg "Warning: test$testnum is explicitly disabled\n";
     }
+    if($ignored{$testnum}) {
+        logmsg "Warning: test$testnum result is ignored\n";
+        $errorreturncode = 2;
+    }
 
     # load the test case file definition
     if(loadtest("${TESTDIR}/test${testnum}")) {
@@ -3334,6 +3341,10 @@ sub singletest {
             } elsif ($enabled_keywords{lc($k)}) {
                 $match = 1;
             }
+            if ($ignored_keywords{lc($k)}) {
+                logmsg "Warning: test$testnum result is ignored due to $k\n";
+                $errorreturncode = 2;
+            }
 
             $keywords{$k} = 1;
         }
@@ -3918,7 +3929,7 @@ sub singletest {
                 logmsg " postcheck FAILED\n";
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return 1;
+                return $errorreturncode;
             }
         }
     }
@@ -3990,7 +4001,7 @@ sub singletest {
 
         $res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
         if($res) {
-            return 1;
+            return $errorreturncode;
         }
         $ok .= "s";
     }
@@ -4041,7 +4052,7 @@ sub singletest {
 
         $res = compare($testnum, $testname, "stderr", \@actual, \@validstderr);
         if($res) {
-            return 1;
+            return $errorreturncode;
         }
         $ok .= "r";
     }
@@ -4087,7 +4098,7 @@ sub singletest {
 
         $res = compare($testnum, $testname, "protocol", \@out, \@protstrip);
         if($res) {
-            return 1;
+            return $errorreturncode;
         }
 
         $ok .= "p";
@@ -4102,7 +4113,7 @@ sub singletest {
         my @out = loadarray($CURLOUT);
         $res = compare($testnum, $testname, "data", \@out, \@reply);
         if ($res) {
-            return 1;
+            return $errorreturncode;
         }
         $ok .= "d";
     }
@@ -4126,7 +4137,7 @@ sub singletest {
 
         $res = compare($testnum, $testname, "upload", \@out, \@upload);
         if ($res) {
-            return 1;
+            return $errorreturncode;
         }
         $ok .= "u";
     }
@@ -4172,7 +4183,7 @@ sub singletest {
 
         $res = compare($testnum, $testname, "proxy", \@out, \@protstrip);
         if($res) {
-            return 1;
+            return $errorreturncode;
         }
 
         $ok .= "P";
@@ -4230,7 +4241,7 @@ sub singletest {
             $res = compare($testnum, $testname, "output ($filename)",
                            \@generated, \@outfile);
             if($res) {
-                return 1;
+                return $errorreturncode;
             }
 
             $outputok = 1; # output checked
@@ -4260,7 +4271,7 @@ sub singletest {
         logmsg " exit FAILED\n";
         # timestamp test result verification end
         $timevrfyend{$testnum} = Time::HiRes::time();
-        return 1;
+        return $errorreturncode;
     }
 
     if($has_memory_tracking) {
@@ -4283,7 +4294,7 @@ sub singletest {
                 logmsg @memdata;
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return 1;
+                return $errorreturncode;
             }
             else {
                 $ok .= "m";
@@ -4300,7 +4311,7 @@ sub singletest {
                 logmsg "ERROR: unable to read $LOGDIR\n";
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return 1;
+                return $errorreturncode;
             }
             my @files = readdir(DIR);
             closedir(DIR);
@@ -4315,7 +4326,7 @@ sub singletest {
                 logmsg "ERROR: valgrind log file missing for test $testnum\n";
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return 1;
+                return $errorreturncode;
             }
             my @e = valgrindparse("$LOGDIR/$vgfile");
             if(@e && $e[0]) {
@@ -4328,7 +4339,7 @@ sub singletest {
                 }
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return 1;
+                return $errorreturncode;
             }
             $ok .= "v";
         }
@@ -4367,6 +4378,10 @@ sub singletest {
         logmsg "PASS: $testnum - $testname\n";
     }
 
+    if($errorreturncode==2) {
+        logmsg "Warning: test$testnum result is ignored, but passed!\n";
+    }
+
     return 0;
 }
 
@@ -5164,8 +5179,10 @@ Usage: runtests.pl [options] [test selection(s)]
   -vc path use this curl only to verify the existing servers
   [num]    like "5 6 9" or " 5 to 22 " to run those tests only
   [!num]   like "!5 !6 !9" to disable those tests
+  [~num]   like "~5 ~6 ~9" to ignore the result of those tests
   [keyword] like "IPv6" to select only tests containing the key word
   [!keyword] like "!cookies" to disable any tests containing the key word
+  [~keyword] like "~cookies" to ignore results of tests containing key word
 EOHELP
     ;
         exit;
@@ -5198,9 +5215,16 @@ EOHELP
         $fromnum = -1;
         $disabled{$1}=$1;
     }
+    elsif($ARGV[0] =~ /^~(\d+)/) {
+        $fromnum = -1;
+        $ignored{$1}=$1;
+    }
     elsif($ARGV[0] =~ /^!(.+)/) {
         $disabled_keywords{lc($1)}=$1;
     }
+    elsif($ARGV[0] =~ /^~(.+)/) {
+        $ignored_keywords{lc($1)}=$1;
+    }
     elsif($ARGV[0] =~ /^([-[{a-zA-Z].*)/) {
         $enabled_keywords{lc($1)}=$1;
     }
@@ -5532,6 +5556,7 @@ if(azure_check_environment()) {
 my $failed;
 my $testnum;
 my $ok=0;
+my $ign=0;
 my $total=0;
 my $lasttest=0;
 my @at = split(" ", $TESTCASES);
@@ -5568,7 +5593,10 @@ foreach $testnum (@at) {
             # display all files in log/ in a nice way
             displaylogs($testnum);
         }
-        if(!$anyway) {
+        if($error==2) {
+            $ign++; # ignored test result counter
+        }
+        elsif(!$anyway) {
             # a test failed, abort
             logmsg "\n - abort tests\n";
             last;
@@ -5644,6 +5672,6 @@ if($skipped && !$short) {
     }
 }
 
-if($total && ($ok != $total)) {
+if($total && (($ok+$ign) != $total)) {
     exit 1;
 }