tests: testrunner reliability improvements

- perform torture tests with '-j2' for shorter runtime
- when waiting on test results overly long, log the tests
  waited for and eventually log the test log directories
  for easier analysis what is wrong in CI jobs.
- sockfilt.c: treat the windows errno 109 (ERROR_BROKEN_PIPE)
  as a socket closed by the client and do not exit.
- when verifying https server, do not in addition check
  the http server behind it also
- when tearing down the stunnel of a non-responsive https
  server, tear down the http server with it

Closes #14960

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 178aefb6c4e64435683a9b9dab970ed694bd5777..b4a406a97b48d84202276c06c7dafd9678a571cf 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -125,7 +125,7 @@ TEST = srcdir=$(srcdir) $(PERL) $(PERLFLAGS) $(srcdir)/runtests.pl
 TEST_Q = -a -s
 TEST_AM = -a -am
 TEST_F = -a -p -r
-TEST_T = -a -t
+TEST_T = -a -t -j2
 TEST_E = -a -e
 
 # ~<keyword> means that it will run all tests matching the keyword, but will

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 1f39e8faad42a134ab3a5e5a76c66104dbf18341..dfeb8fdbd57a17e7d1fa5f0ad6fd3a291d3770b9 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -2886,6 +2886,7 @@ createrunners($numrunners);
 #   - if a runner has a response for us, process the response
 
 # run through each candidate test and execute it
+my $runner_wait_cnt = 0;
 while () {
     # check the abort flag
     if($globalabort) {
@@ -2943,7 +2944,7 @@ while () {
     # If we could be running more tests, don't wait so we can schedule a new
     # one immediately. If all runners are busy, wait a fraction of a second
     # for one to finish so we can still loop around to check the abort flag.
-    my $runnerwait = scalar(@runnersidle) && scalar(@runtests) ? 0 : 0.5;
+    my $runnerwait = scalar(@runnersidle) && scalar(@runtests) ? 0 : 1.0;
     my ($ridready, $riderror) = runnerar_ready($runnerwait);
     if($ridready && ! defined $runnersrunning{$ridready}) {
         # On Linux, a closed pipe still shows up as ready instead of error.
@@ -2964,6 +2965,7 @@ while () {
             $runnersrunning{$ridready} = $testnum;
         } else {
             # Test is complete
+            $runner_wait_cnt = 0;
             runnerready($ridready);
 
             if($error < 0) {
@@ -2999,6 +3001,26 @@ while () {
             }
         }
     }
+    if(!$ridready && $runnerwait && !$torture && scalar(%runnersrunning)) {
+        $runner_wait_cnt++;
+        if($runner_wait_cnt >= 5) {
+            my $msg = "waiting for " . scalar(%runnersrunning) . " results:";
+            my $sep = " ";
+            foreach my $rid (keys %runnersrunning) {
+                $msg .= $sep . $runnersrunning{$rid} . "[$rid]";
+                $sep = ", "
+            }
+            logmsg "$msg\n";
+        }
+        if($runner_wait_cnt >= 10) {
+            $runner_wait_cnt = 0;
+            foreach my $rid (keys %runnersrunning) {
+                my $testnum = $runnersrunning{$rid};
+                logmsg "current state of test $testnum in [$rid]:\n";
+                displaylogs($rid, $testnum);
+            }
+        }
+    }
     if($riderror) {
         logmsg "ERROR: runner $riderror is dead! aborting test run\n";
         delete $runnersrunning{$riderror} if(defined $runnersrunning{$riderror});

diff --git a/tests/server/sockfilt.c b/tests/server/sockfilt.c
index 8450a1189c6b85ceb9fb686521f10f22455cad32..033d6630381e930cd22909cf177c336599b07438 100644 (file)
--- a/tests/server/sockfilt.c
+++ b/tests/server/sockfilt.c
@@ -218,6 +218,10 @@ static ssize_t write_wincon(int fd, const void *buf, size_t count)
 #define write(a,b,c) write_wincon(a,b,c)
 #endif
 
+/* On Windows, we sometimes get this for a broken pipe, seemingly
+ * when the client just closed stdin? */
+#define CURL_WIN32_EPIPE      109
+
 /*
  * fullread is a wrapper around the read() function. This will repeat the call
  * to read() until it actually has read the complete number of bytes indicated
@@ -243,6 +247,11 @@ static ssize_t fullread(int filedes, void *buffer, size_t nbytes)
       error = errno;
       if((error == EINTR) || (error == EAGAIN))
         continue;
+      if(error == CURL_WIN32_EPIPE) {
+        logmsg("got Windows ERROR_BROKEN_PIPE on fd=%d, treating as close",
+               filedes);
+        return 0;
+      }
       logmsg("reading from file descriptor: %d,", filedes);
       logmsg("unrecoverable read() failure: (%d) %s",
              error, strerror(error));

diff --git a/tests/servers.pm b/tests/servers.pm
index 195d663fa3b6057791cb62d0790ae1a601317ba5..88ec10b4845724c5f53ac85aa5680bdf9495bf2f 100644 (file)
--- a/tests/servers.pm
+++ b/tests/servers.pm
@@ -2587,15 +2587,24 @@ sub startservers {
                 if(stopserver('https')) {
                     return ("failed stopping HTTPS server with different cert", 3);
                 }
+                # also stop http server, we do not know which state it is in
+                if($run{'http'} && stopserver('http')) {
+                    return ("failed stopping HTTP server", 3);
+                }
             }
             if($run{'https'} &&
                !responsive_http_server("https", $verbose, 0,
                                        protoport('https'))) {
-               if(stopserver('https')) {
-                   return ("failed stopping unresponsive HTTPS server", 3);
-               }
+                if(stopserver('https')) {
+                    return ("failed stopping unresponsive HTTPS server", 3);
+                }
+                # also stop http server, we do not know which state it is in
+                if($run{'http'} && stopserver('http')) {
+                    return ("failed stopping unresponsive HTTP server", 3);
+                }
             }
-            if($run{'http'} &&
+            # check a running http server if we not already checked https
+            if($run{'http'} && !$run{'https'} &&
                !responsive_http_server("http", $verbose, 0,
                                        protoport('http'))) {
                 if(stopserver('http')) {