From: Viktor Szakats Date: Fri, 12 Dec 2025 00:55:30 +0000 (+0100) Subject: runtests: add options to set minimum number of tests, use them X-Git-Tag: rc-8_18_0-2~26 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3f1cd809eeae05f39fec72fe780f3a69d21972fb;p=thirdparty%2Fcurl.git runtests: add options to set minimum number of tests, use them To detect mistakes made in the runtests framework that reduce the number of test runs. Before this patch it could go undetected with a green CI. The minimum thresholds will need light maintenance going forward (either bumping them periodically, or adjust if some may fell below minimums for justified reasons). We may also make minimums tighter or looser, or more job-specific. Latest number of test runs for each job can be seen at Test Clutch: https://testclutch.curl.se/static/reports/feature-matrix.html Also: - GHA: set minimums. Assisted-by: Dan Fandrich Follow-up to f2a75a14dd95e06cf9121c7d3bd5dc559748f350 Follow-up to bb1391f94394e635c1a5c58253e7a3d3b36bde57 #19510 Closes #19942 --- diff --git a/.github/workflows/distcheck.yml b/.github/workflows/distcheck.yml index 86c05fd0f0..2533f30264 100644 --- a/.github/workflows/distcheck.yml +++ b/.github/workflows/distcheck.yml @@ -20,6 +20,7 @@ concurrency: permissions: {} env: + CURL_TEST_MIN: 1450 MAKEFLAGS: -j 5 jobs: diff --git a/.github/workflows/linux-old.yml b/.github/workflows/linux-old.yml index e282b1c471..d4a3071e64 100644 --- a/.github/workflows/linux-old.yml +++ b/.github/workflows/linux-old.yml @@ -52,6 +52,7 @@ permissions: {} env: MAKEFLAGS: -j 5 CURL_CI: github + CURL_TEST_MIN: 1500 DEBIAN_FRONTEND: noninteractive jobs: diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3a9c35fe38..47bb933099 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -36,6 +36,7 @@ permissions: {} env: MAKEFLAGS: -j 5 CURL_CI: github + CURL_TEST_MIN: 1600 CURL_CLANG_TIDYFLAGS: '-checks=-clang-analyzer-security.insecureAPI.bzero,-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.ArrayBound,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,-clang-analyzer-valist.Uninitialized' # renovate: datasource=github-tags depName=libressl/portable versioning=semver registryUrl=https://github.com LIBRESSL_VERSION: 4.2.1 @@ -184,6 +185,7 @@ jobs: configure: --with-openssl --disable-ipv6 --enable-debug --disable-unity --disable-libcurl-option --disable-digest-auth - name: 'openssl https-only' + tflags: '--min=1150' configure: >- --with-openssl --enable-debug --disable-unity --disable-dict --disable-gopher --disable-ldap --disable-telnet @@ -195,13 +197,13 @@ jobs: - name: 'openssl torture !FTP' install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON - tflags: '-t --shallow=25 !FTP' + tflags: '-t --shallow=25 !FTP --min=1450' torture: true - name: 'openssl torture FTP' install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON - tflags: '-t --shallow=20 FTP' + tflags: '-t --shallow=20 FTP --min=260' torture: true - name: 'openssl i686' @@ -215,6 +217,7 @@ jobs: --with-openssl --with-librtmp --with-libssh2 --with-libidn2 --enable-ares --enable-debug - name: '!ssl !http !smtp !imap' + tflags: '--min=475' configure: --without-ssl --enable-debug --disable-http --disable-smtp --disable-imap --disable-unity - name: 'libressl Fil-C' @@ -303,11 +306,12 @@ jobs: LDFLAGS: -fsanitize=memory LIBS: -ldl configure: CC=clang --without-ssl --without-zlib --without-brotli --without-zstd --without-libpsl --without-nghttp2 --enable-debug + tflags: '--min=1480' - name: 'event-based' install_packages: libssh-dev configure: --enable-debug --disable-shared --disable-threaded-resolver --with-libssh --with-openssl - tflags: '-n --test-event' + tflags: '-n --test-event --min=1350' - name: 'duphandle' install_packages: libssh-dev diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index cae51f5f7e..320689bda8 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -44,6 +44,7 @@ permissions: {} env: CURL_CI: github + CURL_TEST_MIN: 1700 MAKEFLAGS: -j 4 LDFLAGS: -w # suppress 'object file was built for newer macOS version than being linked' warnings @@ -225,16 +226,20 @@ jobs: compiler: clang install: brotli zstd configure: --without-ssl --with-brotli --with-zstd + tflags: '--min=1450' xcode: '' # default Xcode. Set it once to silence actionlint. - name: '!ssl !debug' compiler: gcc-13 configure: --without-ssl + tflags: '--min=950' - name: '!ssl' compiler: clang configure: --enable-debug --without-ssl + tflags: '--min=1500' - name: '!ssl libssh2 AppleIDN' compiler: clang configure: --enable-debug --with-libssh2=/opt/homebrew/opt/libssh2 --without-ssl --with-apple-idn + tflags: '--min=1550' - name: 'OpenSSL libssh c-ares' compiler: clang install: libssh @@ -246,8 +251,10 @@ jobs: - name: '!ssl c-ares' compiler: clang configure: --enable-debug --enable-ares --without-ssl + tflags: '--min=1500' - name: '!ssl HTTP-only' compiler: clang + tflags: '--min=950' configure: >- --enable-debug --disable-alt-svc --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap @@ -277,7 +284,7 @@ jobs: - name: 'OpenSSL event-based' compiler: clang configure: --enable-debug --with-openssl=/opt/homebrew/opt/openssl - tflags: --test-event + tflags: '--test-event --min=1300' # cmake - name: 'OpenSSL gsasl rtmp AppleIDN SecTrust +examples' install: libnghttp3 libngtcp2 gsasl rtmpdump @@ -328,18 +335,19 @@ jobs: compiler: clang install: rustls-ffi generate: -DENABLE_DEBUG=ON -DCURL_USE_RUSTLS=ON -DUSE_ECH=ON -DCURL_DISABLE_LDAP=ON + tflags: '--min=1650' - name: 'OpenSSL torture !FTP' compiler: clang install: libnghttp3 install_steps: torture generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON - tflags: -t --shallow=25 !FTP + tflags: '-t --shallow=25 !FTP --min=1450' - name: 'OpenSSL torture FTP' compiler: clang install: libnghttp3 install_steps: torture generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON - tflags: -t --shallow=20 FTP + tflags: '-t --shallow=20 FTP --min=260' exclude: # opt out jobs from combinations that have the compiler set manually - { compiler: llvm@18, build: { compiler: 'clang' } } diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml index d05a0791a7..282d561d78 100644 --- a/.github/workflows/non-native.yml +++ b/.github/workflows/non-native.yml @@ -37,6 +37,7 @@ permissions: {} env: CURL_CI: github + CURL_TEST_MIN: 1750 jobs: netbsd: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d3d3de8522..364c058fd7 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -37,6 +37,7 @@ permissions: {} env: CURL_CI: github + CURL_TEST_MIN: 1700 jobs: cygwin: @@ -197,26 +198,27 @@ jobs: matrix: include: # MSYS - - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' } - - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' } - - { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' } - - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' } + - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '--min=1550', config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' } + - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' } + - { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' } + - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' } # MinGW - - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' } - - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' } - - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' } - - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' } + - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' } + - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' } + - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '--min=1650', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' } + # MinGW torture + - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP --min=1300', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' } # WARNING: libssh uses hard-coded world-writable paths (/etc/..., ~/.ssh/) to # read its configuration from, making it vulnerable to attacks on # Windows. Do not use this component till there is a fix for these. # https://github.com/curl/curl-for-win/blob/3951808deb04df9489ee17430f236ed54436f81a/libssh.sh#L6-L8 - - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' } - - { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' } - - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' } - - { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } - # { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } - - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' } - - { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' } + - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' } + - { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' } + - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' } + - { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } + # { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' } + - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' } + - { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' } fail-fast: false steps: - uses: msys2/setup-msys2@fb197b72ce45fb24f17bf3f807a388985654d1f2 # v2.29.0 @@ -415,6 +417,7 @@ jobs: run: shell: msys2 {0} env: + CURL_TEST_MIN: 1550 MAKEFLAGS: -j 5 MATRIX_DIR: '${{ matrix.dir }}' strategy: @@ -775,6 +778,7 @@ jobs: type: 'Debug' image: 'windows-11-arm' openssh: 'OpenSSH-Windows' + tflags: '--min=1650' config: >- -DENABLE_DEBUG=ON -DCURL_USE_SCHANNEL=ON diff --git a/docs/runtests.md b/docs/runtests.md index b449b3f411..bb64b412df 100644 --- a/docs/runtests.md +++ b/docs/runtests.md @@ -182,7 +182,12 @@ Lists all test case names. ## `-m=[seconds]` -Set timeout for curl commands in tests +Set timeout for curl commands in tests. + +## `--min=[count]` + +Set the minimum number of tests to run. This overrides the `CURL_TEST_MIN` +environment variable, if any. ## `-n` diff --git a/tests/globalconfig.pm b/tests/globalconfig.pm index 8d5105a36c..0c18a1ea6b 100644 --- a/tests/globalconfig.pm +++ b/tests/globalconfig.pm @@ -50,6 +50,7 @@ BEGIN { $LOCKDIR $LOGDIR $maxtime + $mintotal $memanalyze $MEMDUMP $perlcmd @@ -104,6 +105,7 @@ our $CURLVERSION=""; # curl's reported version number our $CURLVERNUM=""; # curl's reported version number (without -DEV) our $randseed = 0; # random number seed our $maxtime; # curl command timeout override +our $mintotal; # minimum number of tests to run # paths our $pwd = getcwd(); # current working directory diff --git a/tests/runtests.pl b/tests/runtests.pl index ef57dc3ec8..d1a1e40935 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -2403,6 +2403,10 @@ while(@ARGV) { my ($num)=($1); $maxtime=$num; } + elsif($ARGV[0] =~ /--min=(\d+)/) { + my ($num)=($1); + $mintotal=$num; + } elsif($ARGV[0] eq "-n") { # no valgrind undef $valgrind; @@ -2546,6 +2550,7 @@ Usage: runtests.pl [options] [test selection(s)] -L path require an additional perl library file to replace certain functions -l list all test case names/descriptions -m=[seconds] set timeout for curl commands in tests + --min=[count] minimum number of tests to run. -n no valgrind --no-debuginfod disable the valgrind debuginfod functionality -o variable=value set internal variable to the specified value @@ -3332,6 +3337,19 @@ else { } } +if(!$mintotal && $ENV{"CURL_TEST_MIN"}) { + $mintotal = $ENV{"CURL_TEST_MIN"}; +} +if($mintotal) { + if($total < $mintotal) { + logmsg "TESTFAIL: number of tests run was below the minimum of: $mintotal\n"; + exit 1; + } + else { + logmsg "TESTDONE: minimum number of tests was met: $mintotal\n"; + } +} + if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) { exit 1; }