CI: improvements in test reliability and performance

- CI default: raise parallelism to 20
- CI valgind: set parallelism to 6
- CI non-native: adapt parallelism for OS builds
- CI Windows: no longer ignore FTP, TFTP, MQTT and SMTP
- CI Windows: restrict test timeout to 10 minutes
- CI Windows: do not run tests for msh3 build
- tests, various: restrict curl invocation to ipv4 to avoid talking to
  any ipv6 test server running in parallel
- tests: requiring http/2 server no longer needs to mention http server
- test 190: use a fixed timeout of 10 seconds instead of %FTPTIME2, as
  that value rises under parallel load beyond what the server is waiting
  and then produces different results
- test 1540: add debug logging
- testrunner: add verify checks for http/2 and http/3 server

Closes #15040

diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml
index a2653058bb187b90d1a06cac77d698302fa97cd6..4cd6f2f5cc9fd9395ca99ce5ccd85a2110dcedfa 100644 (file)
--- a/.github/workflows/cygwin.yml
+++ b/.github/workflows/cygwin.yml
@@ -122,7 +122,7 @@ jobs:
         if: ${{ matrix.build == 'automake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 15
         run: |
-          export TFLAGS='-j8 ${{ matrix.tflags }} ~615'
+          export TFLAGS='-j20 ${{ matrix.tflags }} ~615'
           if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then
             TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")"
           fi

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index ca6b4c6a08e48fd8d693e9ddbd3be8c062849c79..cd919a3ac8ff7f14cf9bc9252f25dfcd7a8a8548 100644 (file)
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -649,7 +649,7 @@ jobs:
         run: |
           export TFLAGS='${{ matrix.build.tflags }}'
           if [[ '${{ matrix.build.install_packages }}' = *'valgrind'* ]]; then
-            TFLAGS+=' -j4'
+            TFLAGS+=' -j6'
           fi
           if [[ '${{ matrix.build.install_packages }}' = *'heimdal-dev'* ]]; then
             TFLAGS+=' ~2077 ~2078'  # valgrind errors

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index 61aa95d9a1d03f406b594e02517337d5b854b4a4..623617f60506271ed7228f58e47410ab8c79d00c 100644 (file)
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -231,7 +231,7 @@ jobs:
       - name: 'run tests'
         timeout-minutes: 20
         run: |
-          export TFLAGS='-j10 ${{ matrix.tflags }}'
+          export TFLAGS='-j20 ${{ matrix.tflags }}'
           TFLAGS+=' ~2037 ~2041'  # flaky
           if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then
             TFLAGS+=' ~RTSP'  # 567 568 569 570 571 572 577 689 3100
@@ -381,7 +381,7 @@ jobs:
       - name: 'cmake run tests'
         timeout-minutes: ${{ matrix.build.torture && 20 || 10 }}
         run: |
-          export TFLAGS='-j10 ${{ matrix.build.tflags }}'
+          export TFLAGS='-j20 ${{ matrix.build.tflags }}'
           if [ -z '${{ matrix.build.torture }}' ]; then
             TFLAGS+=' ~2037 ~2041'  # flaky
             if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then

diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml
index 050518e3f732d2b1b00837b5345605e5791f5a53..b66808bf9b711593fc52af62decedc18749553a7 100644 (file)
--- a/.github/workflows/non-native.yml
+++ b/.github/workflows/non-native.yml
@@ -66,7 +66,7 @@ jobs:
             bld/src/curl --disable --version
             if [ '${{ matrix.arch }}' = 'x86_64' ]; then  # Slow on emulated CPU
               cmake --build bld --config Debug --parallel 3 --target testdeps
-              export TFLAGS='-j0'  # flakies: ~389 ~392 ~TFTP and more
+              export TFLAGS='-j4'
               cmake --build bld --config Debug --target test-ci
             fi
             echo '::group::build examples'
@@ -146,7 +146,7 @@ jobs:
             src/curl --disable --version
             if [ '${{ matrix.arch }}' = 'x86_64' ]; then  # Slow on emulated CPU
               make -j3 -C tests
-              make test-ci V=1 TFLAGS='-j8 ~FTP'
+              make test-ci V=1 TFLAGS='-j4'
             fi
             echo '::group::build examples'
             make -j3 examples
@@ -176,7 +176,6 @@ jobs:
             bld/src/curl --disable --version
             if [ '${{ matrix.arch }}' = 'x86_64' ]; then  # Slow on emulated CPU
               cmake --build bld --config Debug --parallel 3 --target testdeps
-              export TFLAGS='-j8 ~FTP'
               cmake --build bld --config Debug --target test-ci
             fi
             echo '::group::build examples'
@@ -206,7 +205,7 @@ jobs:
             gmake -j3 install
             src/curl --disable --version
             gmake -j3 -C tests
-            gmake test-ci V=1 TFLAGS='-j12 ~MQTT ~FTP'
+            gmake test-ci V=1
             echo '::group::build examples'
             gmake -j3 examples
             echo '::endgroup::'

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index d015c264cd3cb44b90b348825d39e3fc089ea88d..e1783d9e7a71114e2bb4e8bc70a177aa79fcd6f3 100644 (file)
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -149,11 +149,11 @@ jobs:
 
       - name: 'autotools run tests'
         if: ${{ matrix.build == 'autotools' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
-        timeout-minutes: 14
+        timeout-minutes: 10
         run: |
-          export TFLAGS='-j14 ${{ matrix.tflags }} ~SCP'
+          export TFLAGS='-j8 ${{ matrix.tflags }} ~SCP'
           if [ '${{ matrix.sys }}' != 'msys' ]; then
-            TFLAGS+=' ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP'
+            TFLAGS+=' ~2301 ~2302'  # WebSockets'
             TFLAGS+=' ~612 ~613 ~616 ~618'  # SFTP
           else
             TFLAGS+=' ~SFTP'
@@ -243,11 +243,11 @@ jobs:
 
       - name: 'cmake run tests'
         if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
-        timeout-minutes: 14
+        timeout-minutes: 10
         run: |
-          export TFLAGS='-j14 ${{ matrix.tflags }} ~SCP'
+          export TFLAGS='-j8 ${{ matrix.tflags }} ~SCP'
           if [ '${{ matrix.sys }}' != 'msys' ]; then
-            TFLAGS+=' ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP'
+            TFLAGS+=' ~WebSockets'
             TFLAGS+=' ~612 ~613 ~616 ~618'  # SFTP
           else
             TFLAGS+=' ~SFTP'
@@ -374,10 +374,10 @@ jobs:
 
       - name: 'cmake run tests'
         if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
-        timeout-minutes: 14
+        timeout-minutes: 10
         run: |
           PATH="$(cygpath "${USERPROFILE}")/my-cache/${{ matrix.dir }}/bin:/c/msys64/usr/bin:$PATH"
-          export TFLAGS='-j14 ~TFTP ~MQTT ~WebSockets ~FTP ${{ matrix.tflags }}'
+          export TFLAGS='-j4 ~WebSockets ${{ matrix.tflags }}'
           if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then
             TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")"
           fi
@@ -531,7 +531,7 @@ jobs:
             arch: 'x64'
             plat: 'windows'
             type: 'Debug'
-            tflags: '~1516'
+            tflags: 'skipall'
             config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DUSE_MSH3=ON -DCURL_USE_GSASL=ON'
       fail-fast: false
     steps:
@@ -617,9 +617,9 @@ jobs:
 
       - name: 'cmake run tests'
         if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
-        timeout-minutes: 16
+        timeout-minutes: 10
         run: |
-          export TFLAGS='-j14 ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP ~SCP ~612 ${{ matrix.tflags }}'
+          export TFLAGS='-j8 ~WebSockets ~SCP ~612 ${{ matrix.tflags }}'
           if [[ '${{ matrix.install }}' = *'libssh2[core,zlib]'* ]]; then
             TFLAGS+=' ~SFTP'
           elif [[ '${{ matrix.install }}' = *'libssh '* ]]; then

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index fc22e4e164f0ab412541ffca59bfdfaf89612d2c..9b6a3a42b1c19ae3350df7f7bb87e5a8d59fd371 100644 (file)
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -105,7 +105,7 @@ add_runtests(test-am        "-a -am")
 add_runtests(test-full      "-a -p -r")
 # ~flaky means that it ignores results of tests using the flaky keyword
 add_runtests(test-nonflaky  "-a -p ~flaky ~timing-dependent")
-add_runtests(test-ci        "-a -p ~flaky ~timing-dependent -r -rm -j2")
+add_runtests(test-ci        "-a -p ~flaky ~timing-dependent -r -rm -j20")
 add_runtests(test-torture   "-a -t -j2")
 add_runtests(test-event     "-a -e")
 

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 9638580e53e45e84213912862dde089e75b87408..ae970074b2025ba0bcdfe04cb01b056cc0bdc8f8 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -140,7 +140,7 @@ TEST_E = -a -e
 TEST_NF = -a -p ~flaky ~timing-dependent
 
 # special CI target derived from nonflaky with CI-specific flags
-TEST_CI = $(TEST_NF) -r -rm -j2
+TEST_CI = $(TEST_NF) -r -rm -j20
 
 PYTEST = pytest
 endif

diff --git a/tests/server/util.c b/tests/server/util.c
index afc534d097a58e63e139906c1b12f3f2a098f2e9..91d35513c8f609a8d5dade94aa7403c276b7f167 100644 (file)
--- a/tests/server/util.c
+++ b/tests/server/util.c
@@ -115,7 +115,9 @@ void logmsg(const char *msg, ...)
   mvsnprintf(buffer, sizeof(buffer), msg, ap);
   va_end(ap);
 
-  logfp = fopen(serverlogfile, "ab");
+  do {
+    logfp = fopen(serverlogfile, "ab");
+  } while(!logfp && (errno == EINTR));
   if(logfp) {
     fprintf(logfp, "%s %s\n", timebuf, buffer);
     fclose(logfp);

diff --git a/tests/serverhelp.pm b/tests/serverhelp.pm
index 502b6eb4408e55f1662110bd5c0ccef25224e6a4..22cf30e529c34a6d37fcd3dbadf8c4a346bedb60 100644 (file)
--- a/tests/serverhelp.pm
+++ b/tests/serverhelp.pm
@@ -86,6 +86,9 @@ sub logmsg {
             localtime($seconds);
         $now = sprintf("%02d:%02d:%02d ", $hour, $min, $sec);
     }
+    # we see warnings on Windows run that $logfile is used uninitialized
+    # TODO: not found yet where this comes from
+    $logfile = "serverhelp_uninitialized.log" if(!$logfile);
     if(open(my $logfilefh, ">>", "$logfile")) {
         print $logfilefh $now;
         print $logfilefh @_;