runtests: create multiple test runners when requested

Parallel testing is enabled by using a nonzero value for the -j option
to runtests.pl. Performant values seem to be about 7*num CPU cores, or
1.3*num CPU cores if Valgrind is in use.

Flaky tests due to improper log locking (bug #11231) are exacerbated
while parallel testing, so it is not enabled by default yet.

Fixes #10818
Closes #11246

diff --git a/tests/runtests.pl b/tests/runtests.pl
index d00759783194029e5aaa83d41f641cc2d23fddb5..c8ddd9a107282c05c9aa385bc062abc59ec7ea82 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -965,11 +965,27 @@ sub updatetesttimings {
 }
 
 
+#######################################################################
+# Return the log directory for the given test runner
+sub getrunnernumlogdir {
+    my $runnernum = $_[0];
+    return $jobs > 1 ? $LOGDIR . $runnernum : $LOGDIR;
+}
+
 #######################################################################
 # Return the log directory for the given test runner ID
 sub getrunnerlogdir {
     my $runnerid = $_[0];
-    return $LOGDIR;
+    if($jobs <= 1) {
+        return $LOGDIR;
+    }
+    # TODO: speed up this O(n) operation
+    for my $runnernum (keys %runnerids) {
+        if($runnerid eq $runnerids{$runnernum}) {
+            return $LOGDIR . $runnernum;
+        }
+    }
+    die "Internal error: runner ID $runnerid not found";
 }
 
 
@@ -2017,13 +2033,17 @@ sub runnerready {
 #
 sub createrunners {
     my ($numrunners)=@_;
-    # No runners have been created; create one now
-    my $runnernum = 1;
-    cleardir($LOGDIR);
-    mkdir($LOGDIR, 0777);
-    $runnerids{$runnernum} = runner_init($LOGDIR, $jobs);
-    my $dir = getrunnerlogdir($runnerids{$runnernum});
-    runnerready($runnerids{$runnernum});
+    if(! $numrunners) {
+        $numrunners++;
+    }
+    # create $numrunners runners with minimum 1
+    for my $runnernum (1..$numrunners) {
+        my $dir = getrunnernumlogdir($runnernum);
+        cleardir($dir);
+        mkdir($dir, 0777);
+        $runnerids{$runnernum} = runner_init($dir, $jobs);
+        runnerready($runnerids{$runnernum});
+    }
 }
 
 #######################################################################
@@ -2408,6 +2428,8 @@ if ($gdbthis) {
 # clear and create logging directory:
 #
 
+# TODO: figure how to get around this. This dir is needed for checksystemfeatures()
+# Maybe create & use & delete a temporary directory in that function
 cleardir($LOGDIR);
 mkdir($LOGDIR, 0777);