runtests: also ignore test file problems when ignoring results

This simplifies error handling in the test verification code and makes
it more consistent.

Ref: #10818

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 6a566fc351ceff13c97b2e5904fc07143d06ed76..56abac152e0107629aa4db5137fb3d7d64df0cdb 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -4454,7 +4454,7 @@ sub singletest_check {
                 logmsg " postcheck FAILED\n";
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return -3;
+                return -1;
             }
         }
     }
@@ -4523,7 +4523,7 @@ sub singletest_check {
 
         $res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
         if($res) {
-            return -3;
+            return -1;
         }
         $ok .= "s";
     }
@@ -4575,7 +4575,7 @@ sub singletest_check {
 
         $res = compare($testnum, $testname, "stderr", \@actual, \@validstderr);
         if($res) {
-            return -3;
+            return -1;
         }
         $ok .= "r";
     }
@@ -4628,12 +4628,12 @@ sub singletest_check {
             logmsg "\n $testnum: protocol FAILED!\n".
                 " There was no content at all in the file $SERVERIN.\n".
                 " Server glitch? Total curl failure? Returned: $cmdres\n";
-            return -3;
+            return -1;
         }
 
         $res = compare($testnum, $testname, "protocol", \@out, \@protocol);
         if($res) {
-            return -3;
+            return -1;
         }
 
         $ok .= "p";
@@ -4699,7 +4699,7 @@ sub singletest_check {
         my @out = loadarray($CURLOUT);
         $res = compare($testnum, $testname, "data", \@out, \@reply);
         if ($res) {
-            return -3;
+            return -1;
         }
         $ok .= "d";
     }
@@ -4728,7 +4728,7 @@ sub singletest_check {
 
         $res = compare($testnum, $testname, "upload", \@out, \@upload);
         if ($res) {
-            return -3;
+            return -1;
         }
         $ok .= "u";
     }
@@ -4772,7 +4772,7 @@ sub singletest_check {
 
         $res = compare($testnum, $testname, "proxy", \@out, \@proxyprot);
         if($res) {
-            return -3;
+            return -1;
         }
 
         $ok .= "P";
@@ -4833,7 +4833,7 @@ sub singletest_check {
             $res = compare($testnum, $testname, "output ($filename)",
                            \@generated, \@outfile);
             if($res) {
-                return -3;
+                return -1;
             }
 
             $outputok = 1; # output checked
@@ -4848,7 +4848,7 @@ sub singletest_check {
         my @out = loadarray($SOCKSIN);
         $res = compare($testnum, $testname, "socks", \@out, \@socksprot);
         if($res) {
-            return -3;
+            return -1;
         }
     }
 
@@ -4874,7 +4874,7 @@ sub singletest_check {
         logmsg " exit FAILED\n";
         # timestamp test result verification end
         $timevrfyend{$testnum} = Time::HiRes::time();
-        return -3;
+        return -1;
     }
 
     if($has_memory_tracking) {
@@ -4899,7 +4899,7 @@ sub singletest_check {
                 logmsg @memdata;
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return -3;
+                return -1;
             }
             else {
                 $ok .= "m";
@@ -4916,7 +4916,7 @@ sub singletest_check {
                 logmsg "ERROR: unable to read $LOGDIR\n";
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return -3;
+                return -1;
             }
             my @files = readdir(DIR);
             closedir(DIR);
@@ -4931,7 +4931,7 @@ sub singletest_check {
                 logmsg "ERROR: valgrind log file missing for test $testnum\n";
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return -3;
+                return -1;
             }
             my @e = valgrindparse("$LOGDIR/$vgfile");
             if(@e && $e[0]) {
@@ -4944,7 +4944,7 @@ sub singletest_check {
                 }
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time();
-                return -3;
+                return -1;
             }
             $ok .= "v";
         }
@@ -5087,16 +5087,15 @@ sub singletest {
     #######################################################################
     # Verify that the test succeeded
     $error = singletest_check($testnum, $cmdres, $CURLOUT, $tool, $disablevalgrind);
-    # TODO: try to simplify the return codes
     if($error == -1) {
-      return $error;
+      # return a test failure, either to be reported or to be ignored
+      return $errorreturncode;
     }
     elsif($error == -2) {
+      # torture test; there is no verification, so the run result holds the
+      # test success code
       return $cmdres;
     }
-    elsif($error == -3) {
-      return $errorreturncode;
-    }
 
     #######################################################################
     # Report a successful test