runtests: add options to set minimum number of tests, use them

To detect mistakes made in the runtests framework that reduce
the number of test runs. Before this patch it could go undetected with
a green CI.

The minimum thresholds will need light maintenance going forward (either
bumping them periodically, or adjust if some may fell below minimums for
justified reasons). We may also make minimums tighter or looser, or more
job-specific.

Latest number of test runs for each job can be seen at Test Clutch:
https://testclutch.curl.se/static/reports/feature-matrix.html

Also:
- GHA: set minimums.

Assisted-by: Dan Fandrich
Follow-up to f2a75a14dd95e06cf9121c7d3bd5dc559748f350
Follow-up to bb1391f94394e635c1a5c58253e7a3d3b36bde57 #19510

Closes #19942

diff --git a/.github/workflows/distcheck.yml b/.github/workflows/distcheck.yml
index 86c05fd0f0cc7af596ce185747eeae771edbf288..2533f30264c7ad99ad2a9a069dfae22487339494 100644 (file)
--- a/.github/workflows/distcheck.yml
+++ b/.github/workflows/distcheck.yml
@@ -20,6 +20,7 @@ concurrency:
 permissions: {}
 
 env:
+  CURL_TEST_MIN: 1450
   MAKEFLAGS: -j 5
 
 jobs:

diff --git a/.github/workflows/linux-old.yml b/.github/workflows/linux-old.yml
index e282b1c471964d93df8dad968cab217edb1bb860..d4a3071e6406e827f22dccfce12c597200915c4e 100644 (file)
--- a/.github/workflows/linux-old.yml
+++ b/.github/workflows/linux-old.yml
@@ -52,6 +52,7 @@ permissions: {}
 env:
   MAKEFLAGS: -j 5
   CURL_CI: github
+  CURL_TEST_MIN: 1500
   DEBIAN_FRONTEND: noninteractive
 
 jobs:

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 3a9c35fe385c9f5d99c76204cbd0572eef63a590..47bb933099fef3f4abe0181d1a15a2ffdfde2a0d 100644 (file)
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -36,6 +36,7 @@ permissions: {}
 env:
   MAKEFLAGS: -j 5
   CURL_CI: github
+  CURL_TEST_MIN: 1600
   CURL_CLANG_TIDYFLAGS: '-checks=-clang-analyzer-security.insecureAPI.bzero,-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.ArrayBound,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,-clang-analyzer-valist.Uninitialized'
   # renovate: datasource=github-tags depName=libressl/portable versioning=semver registryUrl=https://github.com
   LIBRESSL_VERSION: 4.2.1
@@ -184,6 +185,7 @@ jobs:
             configure: --with-openssl --disable-ipv6 --enable-debug --disable-unity --disable-libcurl-option --disable-digest-auth
 
           - name: 'openssl https-only'
+            tflags: '--min=1150'
             configure: >-
               --with-openssl --enable-debug --disable-unity
               --disable-dict --disable-gopher --disable-ldap --disable-telnet
@@ -195,13 +197,13 @@ jobs:
           - name: 'openssl torture !FTP'
             install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev
             generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON
-            tflags: '-t --shallow=25 !FTP'
+            tflags: '-t --shallow=25 !FTP --min=1450'
             torture: true
 
           - name: 'openssl torture FTP'
             install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev
             generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON
-            tflags: '-t --shallow=20 FTP'
+            tflags: '-t --shallow=20 FTP --min=260'
             torture: true
 
           - name: 'openssl i686'
@@ -215,6 +217,7 @@ jobs:
               --with-openssl --with-librtmp --with-libssh2 --with-libidn2 --enable-ares --enable-debug
 
           - name: '!ssl !http !smtp !imap'
+            tflags: '--min=475'
             configure: --without-ssl --enable-debug --disable-http --disable-smtp --disable-imap --disable-unity
 
           - name: 'libressl Fil-C'
@@ -303,11 +306,12 @@ jobs:
             LDFLAGS: -fsanitize=memory
             LIBS: -ldl
             configure: CC=clang --without-ssl --without-zlib --without-brotli --without-zstd --without-libpsl --without-nghttp2 --enable-debug
+            tflags: '--min=1480'
 
           - name: 'event-based'
             install_packages: libssh-dev
             configure: --enable-debug --disable-shared --disable-threaded-resolver --with-libssh --with-openssl
-            tflags: '-n --test-event'
+            tflags: '-n --test-event --min=1350'
 
           - name: 'duphandle'
             install_packages: libssh-dev

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index cae51f5f7ec7eb82e170ffebb85b1f815c3b689b..320689bda8754ac41d232b68eb6637c5a3740aea 100644 (file)
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -44,6 +44,7 @@ permissions: {}
 
 env:
   CURL_CI: github
+  CURL_TEST_MIN: 1700
   MAKEFLAGS: -j 4
   LDFLAGS: -w  # suppress 'object file was built for newer macOS version than being linked' warnings
 
@@ -225,16 +226,20 @@ jobs:
             compiler: clang
             install: brotli zstd
             configure: --without-ssl --with-brotli --with-zstd
+            tflags: '--min=1450'
             xcode: ''  # default Xcode. Set it once to silence actionlint.
           - name: '!ssl !debug'
             compiler: gcc-13
             configure: --without-ssl
+            tflags: '--min=950'
           - name: '!ssl'
             compiler: clang
             configure: --enable-debug --without-ssl
+            tflags: '--min=1500'
           - name: '!ssl libssh2 AppleIDN'
             compiler: clang
             configure: --enable-debug --with-libssh2=/opt/homebrew/opt/libssh2 --without-ssl --with-apple-idn
+            tflags: '--min=1550'
           - name: 'OpenSSL libssh c-ares'
             compiler: clang
             install: libssh
@@ -246,8 +251,10 @@ jobs:
           - name: '!ssl c-ares'
             compiler: clang
             configure: --enable-debug --enable-ares --without-ssl
+            tflags: '--min=1500'
           - name: '!ssl HTTP-only'
             compiler: clang
+            tflags: '--min=950'
             configure: >-
               --enable-debug
               --disable-alt-svc --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap
@@ -277,7 +284,7 @@ jobs:
           - name: 'OpenSSL event-based'
             compiler: clang
             configure: --enable-debug --with-openssl=/opt/homebrew/opt/openssl
-            tflags: --test-event
+            tflags: '--test-event --min=1300'
           # cmake
           - name: 'OpenSSL gsasl rtmp AppleIDN SecTrust +examples'
             install: libnghttp3 libngtcp2 gsasl rtmpdump
@@ -328,18 +335,19 @@ jobs:
             compiler: clang
             install: rustls-ffi
             generate: -DENABLE_DEBUG=ON -DCURL_USE_RUSTLS=ON -DUSE_ECH=ON -DCURL_DISABLE_LDAP=ON
+            tflags: '--min=1650'
           - name: 'OpenSSL torture !FTP'
             compiler: clang
             install: libnghttp3
             install_steps: torture
             generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON
-            tflags: -t --shallow=25 !FTP
+            tflags: '-t --shallow=25 !FTP --min=1450'
           - name: 'OpenSSL torture FTP'
             compiler: clang
             install: libnghttp3
             install_steps: torture
             generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON
-            tflags: -t --shallow=20 FTP
+            tflags: '-t --shallow=20 FTP --min=260'
         exclude:
           # opt out jobs from combinations that have the compiler set manually
           - { compiler: llvm@18, build: { compiler: 'clang' } }

diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml
index d05a0791a731560442994848b5761c3e66c264f6..282d561d788b14cbb0c18742e6f75535bfc2b85f 100644 (file)
--- a/.github/workflows/non-native.yml
+++ b/.github/workflows/non-native.yml
@@ -37,6 +37,7 @@ permissions: {}
 
 env:
   CURL_CI: github
+  CURL_TEST_MIN: 1750
 
 jobs:
   netbsd:

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index d3d3de8522da8d4586d9f71b46e52bbddefdd5bd..364c058fd7c53fdae5234192972bfdb9aa75b2ef 100644 (file)
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -37,6 +37,7 @@ permissions: {}
 
 env:
   CURL_CI: github
+  CURL_TEST_MIN: 1700
 
 jobs:
   cygwin:
@@ -197,26 +198,27 @@ jobs:
       matrix:
         include:
           # MSYS
-          - { build: 'autotools', sys: 'msys'      , env: 'x86_64'       , tflags: ''                    , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' }
-          - { build: 'autotools', sys: 'msys'      , env: 'x86_64'       , tflags: 'skiprun'             , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' }
-          - { build: 'cmake'    , sys: 'msys'      , env: 'x86_64'       , tflags: ''                    , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' }
-          - { build: 'autotools', sys: 'msys'      , env: 'x86_64'       , tflags: ''                    , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' }
+          - { build: 'autotools', sys: 'msys'      , env: 'x86_64'       , tflags: '--min=1550', config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' }
+          - { build: 'autotools', sys: 'msys'      , env: 'x86_64'       , tflags: 'skiprun'   , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' }
+          - { build: 'cmake'    , sys: 'msys'      , env: 'x86_64'       , tflags: ''          , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' }
+          - { build: 'autotools', sys: 'msys'      , env: 'x86_64'       , tflags: ''          , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' }
           # MinGW
-          - { build: 'autotools', sys: 'mingw64'   , env: 'x86_64'       , tflags: 'skiprun'             , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' }
-          - { build: 'autotools', sys: 'mingw64'   , env: 'x86_64'       , tflags: ''                    , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' }
-          - { build: 'cmake'    , sys: 'mingw64'   , env: 'x86_64'       , tflags: ''                    , config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' }
-          - { build: 'cmake'    , sys: 'mingw64'   , env: 'x86_64'       , tflags: '-t --shallow=13 !FTP', config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' }
+          - { build: 'autotools', sys: 'mingw64'   , env: 'x86_64'       , tflags: 'skiprun'   , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' }
+          - { build: 'autotools', sys: 'mingw64'   , env: 'x86_64'       , tflags: ''          , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' }
+          - { build: 'cmake'    , sys: 'mingw64'   , env: 'x86_64'       , tflags: '--min=1650', config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' }
+          # MinGW torture
+          - { build: 'cmake'    , sys: 'mingw64'   , env: 'x86_64'       , tflags: '-t --shallow=13 !FTP --min=1300', config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' }
           # WARNING: libssh uses hard-coded world-writable paths (/etc/..., ~/.ssh/) to
           #          read its configuration from, making it vulnerable to attacks on
           #          Windows. Do not use this component till there is a fix for these.
           # https://github.com/curl/curl-for-win/blob/3951808deb04df9489ee17430f236ed54436f81a/libssh.sh#L6-L8
-          - { build: 'cmake'    , sys: 'clang64'   , env: 'clang-x86_64' , tflags: ''                    , config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON   -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' }
-          - { build: 'cmake'    , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun'             , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON  -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' }
-          - { build: 'cmake'    , sys: 'clang64'   , env: 'clang-x86_64' , tflags: 'skiprun'             , config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON  -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' }
-          - { build: 'cmake'    , sys: 'ucrt64'    , env: 'ucrt-x86_64'  , tflags: 'skiprun'             , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON  -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
-          # { build: 'autotools', sys: 'ucrt64'    , env: 'ucrt-x86_64'  , tflags: 'skiprun'             , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
-          - { build: 'cmake'    , sys: 'mingw64'   , env: 'x86_64'       , tflags: 'skiprun'             , config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=ON  -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' }
-          - { build: 'cmake'    , sys: 'mingw32'   , env: 'i686'         , tflags: 'skiprun'             , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON  -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' }
+          - { build: 'cmake'    , sys: 'clang64'   , env: 'clang-x86_64' , tflags: ''          , config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON   -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' }
+          - { build: 'cmake'    , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun'   , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON  -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' }
+          - { build: 'cmake'    , sys: 'clang64'   , env: 'clang-x86_64' , tflags: 'skiprun'   , config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON  -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' }
+          - { build: 'cmake'    , sys: 'ucrt64'    , env: 'ucrt-x86_64'  , tflags: 'skiprun'   , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON  -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
+          # { build: 'autotools', sys: 'ucrt64'    , env: 'ucrt-x86_64'  , tflags: 'skiprun'   , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
+          - { build: 'cmake'    , sys: 'mingw64'   , env: 'x86_64'       , tflags: 'skiprun'   , config: '-DENABLE_DEBUG=ON  -DBUILD_SHARED_LIBS=ON  -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' }
+          - { build: 'cmake'    , sys: 'mingw32'   , env: 'i686'         , tflags: 'skiprun'   , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON  -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' }
       fail-fast: false
     steps:
       - uses: msys2/setup-msys2@fb197b72ce45fb24f17bf3f807a388985654d1f2 # v2.29.0
@@ -415,6 +417,7 @@ jobs:
       run:
         shell: msys2 {0}
     env:
+      CURL_TEST_MIN: 1550
       MAKEFLAGS: -j 5
       MATRIX_DIR: '${{ matrix.dir }}'
     strategy:
@@ -775,6 +778,7 @@ jobs:
             type: 'Debug'
             image: 'windows-11-arm'
             openssh: 'OpenSSH-Windows'
+            tflags: '--min=1650'
             config: >-
               -DENABLE_DEBUG=ON
               -DCURL_USE_SCHANNEL=ON

diff --git a/docs/runtests.md b/docs/runtests.md
index b449b3f4118d833c8e65f2af205b20b379dddc44..bb64b412df2f4b896501b977dfe17763d0932625 100644 (file)
--- a/docs/runtests.md
+++ b/docs/runtests.md
@@ -182,7 +182,12 @@ Lists all test case names.
 
 ## `-m=[seconds]`
 
-Set timeout for curl commands in tests
+Set timeout for curl commands in tests.
+
+## `--min=[count]`
+
+Set the minimum number of tests to run. This overrides the `CURL_TEST_MIN`
+environment variable, if any.
 
 ## `-n`
 

diff --git a/tests/globalconfig.pm b/tests/globalconfig.pm
index 8d5105a36c617f616bbf45c144375cf27be09dad..0c18a1ea6b0106358f7bccbf527d2342f2a517de 100644 (file)
--- a/tests/globalconfig.pm
+++ b/tests/globalconfig.pm
@@ -50,6 +50,7 @@ BEGIN {
         $LOCKDIR
         $LOGDIR
         $maxtime
+        $mintotal
         $memanalyze
         $MEMDUMP
         $perlcmd
@@ -104,6 +105,7 @@ our $CURLVERSION="";  # curl's reported version number
 our $CURLVERNUM="";   # curl's reported version number (without -DEV)
 our $randseed = 0;    # random number seed
 our $maxtime;         # curl command timeout override
+our $mintotal;        # minimum number of tests to run
 
 # paths
 our $pwd = getcwd();  # current working directory

diff --git a/tests/runtests.pl b/tests/runtests.pl
index ef57dc3ec8db8903556f85bdb8c1313bec22b054..d1a1e40935aebd18b8503a127dfc6c5cdcb48786 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -2403,6 +2403,10 @@ while(@ARGV) {
         my ($num)=($1);
         $maxtime=$num;
     }
+    elsif($ARGV[0] =~ /--min=(\d+)/) {
+        my ($num)=($1);
+        $mintotal=$num;
+    }
     elsif($ARGV[0] eq "-n") {
         # no valgrind
         undef $valgrind;
@@ -2546,6 +2550,7 @@ Usage: runtests.pl [options] [test selection(s)]
   -L path  require an additional perl library file to replace certain functions
   -l       list all test case names/descriptions
   -m=[seconds] set timeout for curl commands in tests
+  --min=[count] minimum number of tests to run.
   -n       no valgrind
   --no-debuginfod disable the valgrind debuginfod functionality
   -o variable=value set internal variable to the specified value
@@ -3332,6 +3337,19 @@ else {
     }
 }
 
+if(!$mintotal && $ENV{"CURL_TEST_MIN"}) {
+    $mintotal = $ENV{"CURL_TEST_MIN"};
+}
+if($mintotal) {
+    if($total < $mintotal) {
+        logmsg "TESTFAIL: number of tests run was below the minimum of: $mintotal\n";
+        exit 1;
+    }
+    else {
+        logmsg "TESTDONE: minimum number of tests was met: $mintotal\n";
+    }
+}
+
 if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
     exit 1;
 }