From: Stefan Eissing Date: Wed, 25 Sep 2024 13:14:50 +0000 (+0200) Subject: CI: improvements in test reliability and performance X-Git-Tag: curl-8_11_0~305 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7f3d598276e988a46558fe6a2951b1ae354afcdf;p=thirdparty%2Fcurl.git CI: improvements in test reliability and performance - CI default: raise parallelism to 20 - CI valgind: set parallelism to 6 - CI non-native: adapt parallelism for OS builds - CI Windows: no longer ignore FTP, TFTP, MQTT and SMTP - CI Windows: restrict test timeout to 10 minutes - CI Windows: do not run tests for msh3 build - tests, various: restrict curl invocation to ipv4 to avoid talking to any ipv6 test server running in parallel - tests: requiring http/2 server no longer needs to mention http server - test 190: use a fixed timeout of 10 seconds instead of %FTPTIME2, as that value rises under parallel load beyond what the server is waiting and then produces different results - test 1540: add debug logging - testrunner: add verify checks for http/2 and http/3 server Closes #15040 --- diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml index a2653058bb..4cd6f2f5cc 100644 --- a/.github/workflows/cygwin.yml +++ b/.github/workflows/cygwin.yml @@ -122,7 +122,7 @@ jobs: if: ${{ matrix.build == 'automake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }} timeout-minutes: 15 run: | - export TFLAGS='-j8 ${{ matrix.tflags }} ~615' + export TFLAGS='-j20 ${{ matrix.tflags }} ~615' if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")" fi diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ca6b4c6a08..cd919a3ac8 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -649,7 +649,7 @@ jobs: run: | export TFLAGS='${{ matrix.build.tflags }}' if [[ '${{ matrix.build.install_packages }}' = *'valgrind'* ]]; then - TFLAGS+=' -j4' + TFLAGS+=' -j6' fi if [[ '${{ matrix.build.install_packages }}' = *'heimdal-dev'* ]]; then TFLAGS+=' ~2077 ~2078' # valgrind errors diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 61aa95d9a1..623617f605 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -231,7 +231,7 @@ jobs: - name: 'run tests' timeout-minutes: 20 run: | - export TFLAGS='-j10 ${{ matrix.tflags }}' + export TFLAGS='-j20 ${{ matrix.tflags }}' TFLAGS+=' ~2037 ~2041' # flaky if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100 @@ -381,7 +381,7 @@ jobs: - name: 'cmake run tests' timeout-minutes: ${{ matrix.build.torture && 20 || 10 }} run: | - export TFLAGS='-j10 ${{ matrix.build.tflags }}' + export TFLAGS='-j20 ${{ matrix.build.tflags }}' if [ -z '${{ matrix.build.torture }}' ]; then TFLAGS+=' ~2037 ~2041' # flaky if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml index 050518e3f7..b66808bf9b 100644 --- a/.github/workflows/non-native.yml +++ b/.github/workflows/non-native.yml @@ -66,7 +66,7 @@ jobs: bld/src/curl --disable --version if [ '${{ matrix.arch }}' = 'x86_64' ]; then # Slow on emulated CPU cmake --build bld --config Debug --parallel 3 --target testdeps - export TFLAGS='-j0' # flakies: ~389 ~392 ~TFTP and more + export TFLAGS='-j4' cmake --build bld --config Debug --target test-ci fi echo '::group::build examples' @@ -146,7 +146,7 @@ jobs: src/curl --disable --version if [ '${{ matrix.arch }}' = 'x86_64' ]; then # Slow on emulated CPU make -j3 -C tests - make test-ci V=1 TFLAGS='-j8 ~FTP' + make test-ci V=1 TFLAGS='-j4' fi echo '::group::build examples' make -j3 examples @@ -176,7 +176,6 @@ jobs: bld/src/curl --disable --version if [ '${{ matrix.arch }}' = 'x86_64' ]; then # Slow on emulated CPU cmake --build bld --config Debug --parallel 3 --target testdeps - export TFLAGS='-j8 ~FTP' cmake --build bld --config Debug --target test-ci fi echo '::group::build examples' @@ -206,7 +205,7 @@ jobs: gmake -j3 install src/curl --disable --version gmake -j3 -C tests - gmake test-ci V=1 TFLAGS='-j12 ~MQTT ~FTP' + gmake test-ci V=1 echo '::group::build examples' gmake -j3 examples echo '::endgroup::' diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d015c264cd..e1783d9e7a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -149,11 +149,11 @@ jobs: - name: 'autotools run tests' if: ${{ matrix.build == 'autotools' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }} - timeout-minutes: 14 + timeout-minutes: 10 run: | - export TFLAGS='-j14 ${{ matrix.tflags }} ~SCP' + export TFLAGS='-j8 ${{ matrix.tflags }} ~SCP' if [ '${{ matrix.sys }}' != 'msys' ]; then - TFLAGS+=' ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP' + TFLAGS+=' ~2301 ~2302' # WebSockets' TFLAGS+=' ~612 ~613 ~616 ~618' # SFTP else TFLAGS+=' ~SFTP' @@ -243,11 +243,11 @@ jobs: - name: 'cmake run tests' if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }} - timeout-minutes: 14 + timeout-minutes: 10 run: | - export TFLAGS='-j14 ${{ matrix.tflags }} ~SCP' + export TFLAGS='-j8 ${{ matrix.tflags }} ~SCP' if [ '${{ matrix.sys }}' != 'msys' ]; then - TFLAGS+=' ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP' + TFLAGS+=' ~WebSockets' TFLAGS+=' ~612 ~613 ~616 ~618' # SFTP else TFLAGS+=' ~SFTP' @@ -374,10 +374,10 @@ jobs: - name: 'cmake run tests' if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }} - timeout-minutes: 14 + timeout-minutes: 10 run: | PATH="$(cygpath "${USERPROFILE}")/my-cache/${{ matrix.dir }}/bin:/c/msys64/usr/bin:$PATH" - export TFLAGS='-j14 ~TFTP ~MQTT ~WebSockets ~FTP ${{ matrix.tflags }}' + export TFLAGS='-j4 ~WebSockets ${{ matrix.tflags }}' if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")" fi @@ -531,7 +531,7 @@ jobs: arch: 'x64' plat: 'windows' type: 'Debug' - tflags: '~1516' + tflags: 'skipall' config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DUSE_MSH3=ON -DCURL_USE_GSASL=ON' fail-fast: false steps: @@ -617,9 +617,9 @@ jobs: - name: 'cmake run tests' if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }} - timeout-minutes: 16 + timeout-minutes: 10 run: | - export TFLAGS='-j14 ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP ~SCP ~612 ${{ matrix.tflags }}' + export TFLAGS='-j8 ~WebSockets ~SCP ~612 ${{ matrix.tflags }}' if [[ '${{ matrix.install }}' = *'libssh2[core,zlib]'* ]]; then TFLAGS+=' ~SFTP' elif [[ '${{ matrix.install }}' = *'libssh '* ]]; then diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fc22e4e164..9b6a3a42b1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -105,7 +105,7 @@ add_runtests(test-am "-a -am") add_runtests(test-full "-a -p -r") # ~flaky means that it ignores results of tests using the flaky keyword add_runtests(test-nonflaky "-a -p ~flaky ~timing-dependent") -add_runtests(test-ci "-a -p ~flaky ~timing-dependent -r -rm -j2") +add_runtests(test-ci "-a -p ~flaky ~timing-dependent -r -rm -j20") add_runtests(test-torture "-a -t -j2") add_runtests(test-event "-a -e") diff --git a/tests/Makefile.am b/tests/Makefile.am index 9638580e53..ae970074b2 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -140,7 +140,7 @@ TEST_E = -a -e TEST_NF = -a -p ~flaky ~timing-dependent # special CI target derived from nonflaky with CI-specific flags -TEST_CI = $(TEST_NF) -r -rm -j2 +TEST_CI = $(TEST_NF) -r -rm -j20 PYTEST = pytest endif diff --git a/tests/server/util.c b/tests/server/util.c index afc534d097..91d35513c8 100644 --- a/tests/server/util.c +++ b/tests/server/util.c @@ -115,7 +115,9 @@ void logmsg(const char *msg, ...) mvsnprintf(buffer, sizeof(buffer), msg, ap); va_end(ap); - logfp = fopen(serverlogfile, "ab"); + do { + logfp = fopen(serverlogfile, "ab"); + } while(!logfp && (errno == EINTR)); if(logfp) { fprintf(logfp, "%s %s\n", timebuf, buffer); fclose(logfp); diff --git a/tests/serverhelp.pm b/tests/serverhelp.pm index 502b6eb440..22cf30e529 100644 --- a/tests/serverhelp.pm +++ b/tests/serverhelp.pm @@ -86,6 +86,9 @@ sub logmsg { localtime($seconds); $now = sprintf("%02d:%02d:%02d ", $hour, $min, $sec); } + # we see warnings on Windows run that $logfile is used uninitialized + # TODO: not found yet where this comes from + $logfile = "serverhelp_uninitialized.log" if(!$logfile); if(open(my $logfilefh, ">>", "$logfile")) { print $logfilefh $now; print $logfilefh @_;