CI: tidy up skipping tests build/run in Windows jobs

Simplify controlling whether to build and/run tests in a CI job.

Apply the TFLAGS='skipall' (do not build nor run tests) or
'skiprun' (build, but do not run) method already used with old-mingw-w64
and msvc jobs to existing Windows jobs in GHA and AppVeyor.

Also:
- add Cygwin/cmake test build and run steps while here.
- replace `DISABLED_TESTS` with `TFLAGS` in AppVeyor.

Closes #13796

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 1201a433f2b23342cb2aa9af30970fbc71a59f7f..ed69df25c65a3439601756d6eb78133d6e00542b 100644 (file)
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -49,8 +49,8 @@ jobs:
     strategy:
       matrix:
         include:
-          - { build: 'automake', platform: 'x86_64', tflags: '', config: '--enable-debug --disable-threaded-resolver' }
-          - { build: 'cmake'   , platform: 'x86_64', tflags: '', config: '-DCURL_USE_OPENSSL=ON' }
+          - { build: 'automake', platform: 'x86_64', tflags: ''       , config: '--enable-debug --disable-threaded-resolver' }
+          - { build: 'cmake'   , platform: 'x86_64', tflags: 'skipall', config: '-DCURL_USE_OPENSSL=ON' }
       fail-fast: false
     steps:
       - run: git config --global core.autocrlf input
@@ -93,14 +93,14 @@ jobs:
           make -C bld -j3 V=1 examples
 
       - name: 'autotools build tests'
-        if: ${{ matrix.build == 'automake' }}
+        if: ${{ matrix.build == 'automake' && matrix.tflags != 'skipall' }}
         timeout-minutes: 15
         shell: C:\cygwin\bin\bash.exe '{0}'
         run: |
           make -C bld -j3 -C tests V=1
 
       - name: 'autotools run tests'
-        if: ${{ matrix.build == 'automake' && matrix.tflags != 'skip' }}
+        if: ${{ matrix.build == 'automake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 40
         shell: C:\cygwin\bin\bash.exe '{0}'
         run: |
@@ -135,6 +135,25 @@ jobs:
           find . -name '*.exe' -o -name '*.dll'
           bld/src/curl.exe --disable --version
 
+      - name: 'cmake build tests'
+        if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' }}
+        timeout-minutes: 15
+        shell: C:\cygwin\bin\bash.exe '{0}'
+        run: |
+          cmake --build bld --config '${{ matrix.type }}' --parallel 3 --target testdeps
+
+      - name: 'cmake run tests'
+        if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
+        timeout-minutes: 40
+        shell: C:\cygwin\bin\bash.exe '{0}'
+        run: |
+          export TFLAGS='-j8 ${{ matrix.tflags }}'
+          if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then
+            TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")"
+          fi
+          ls bld/lib/*.dll >/dev/null 2>&1 && cp -f -p bld/lib/*.dll bld/tests/libtest/
+          cmake --build bld --config '${{ matrix.type }}' --target test-ci
+
   msys2:
     name: 'msys2 (${{ matrix.build }}, ${{ matrix.sys }}, ${{ matrix.env }}, ${{ matrix.config }})'
     runs-on: windows-latest
@@ -145,11 +164,11 @@ jobs:
           - { build: 'autotools', sys: 'msys'   , env: 'x86_64'      , tflags: '!19                !1233', config: '--enable-debug --disable-threaded-resolver --disable-proxy' }
           - { build: 'autotools', sys: 'msys'   , env: 'x86_64'      , tflags: '!19 !504 !704 !705 !1233', config: '--enable-debug --disable-threaded-resolver' }
           - { build: 'autotools', sys: 'msys'   , env: 'x86_64'      , tflags: '!19 !504 !704 !705 !1233', config: '' }
-          - { build: 'autotools', sys: 'mingw64', env: 'x86_64'      , tflags: 'skip'                    , config: '--enable-debug --disable-threaded-resolver --enable-static=no' }
+          - { build: 'autotools', sys: 'mingw64', env: 'x86_64'      , tflags: 'skiprun'                 , config: '--enable-debug --disable-threaded-resolver --enable-static=no' }
           # FIXME: WebSockets test results ignored due to frequent failures on native Windows:
           - { build: 'cmake'    , sys: 'mingw64', env: 'x86_64'      , tflags: '!TFTP ~2301 ~2302'       , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', type: 'Debug' }
-          - { build: 'cmake'    , sys: 'mingw64', env: 'x86_64'      , tflags: 'skip'                    , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', type: 'Release' }
-          - { build: 'cmake'    , sys: 'clang64', env: 'clang-x86_64', tflags: 'skip'                    , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF', type: 'Release' }
+          - { build: 'cmake'    , sys: 'mingw64', env: 'x86_64'      , tflags: 'skiprun'                 , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', type: 'Release' }
+          - { build: 'cmake'    , sys: 'clang64', env: 'clang-x86_64', tflags: 'skiprun'                 , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF', type: 'Release' }
       fail-fast: false
     steps:
       - run: git config --global core.autocrlf input
@@ -200,14 +219,14 @@ jobs:
           make -C bld -j3 V=1 examples
 
       - name: 'autotools build tests'
-        if: ${{ matrix.build == 'autotools' }}
+        if: ${{ matrix.build == 'autotools' && matrix.tflags != 'skipall' }}
         timeout-minutes: 10
         shell: msys2 {0}
         run: |
           make -C bld -j3 -C tests V=1
 
       - name: 'autotools run tests'
-        if: ${{ matrix.build == 'autotools' && matrix.tflags != 'skip' }}
+        if: ${{ matrix.build == 'autotools' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 30
         shell: msys2 {0}
         run: |
@@ -264,14 +283,14 @@ jobs:
           bld/src/curl.exe --disable --version
 
       - name: 'cmake build tests'
-        if: ${{ matrix.build == 'cmake' }}
+        if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' }}
         timeout-minutes: 10
         shell: msys2 {0}
         run: |
           cmake --build bld --config '${{ matrix.type }}' --parallel 3 --target testdeps
 
       - name: 'cmake run tests'
-        if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skip' }}
+        if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 40
         shell: msys2 {0}
         run: |

diff --git a/appveyor.sh b/appveyor.sh
index cb96e58745769ebaae98674664bda82d71bd8bc7..ec1e81031ae32ce700e31b3998ba7c53216d6c8c 100644 (file)
--- a/appveyor.sh
+++ b/appveyor.sh
@@ -53,7 +53,6 @@ if [ "${BUILD_SYSTEM}" = 'CMake' ]; then
     "-DCURL_USE_SCHANNEL=${SCHANNEL}" \
     "-DHTTP_ONLY=${HTTP_ONLY}" \
     "-DBUILD_SHARED_LIBS=${SHARED}" \
-    "-DBUILD_TESTING=${TESTING}" \
     "-DENABLE_WEBSOCKETS=${WEBSOCKETS:-}" \
     "-DCMAKE_UNITY_BUILD=${UNITY}" \
     '-DCURL_WERROR=ON' \
@@ -117,18 +116,23 @@ if false; then
   done
 fi
 
-# test
+# build tests
 
-if [ "${TESTING}" = 'ON' ]; then
-  export TFLAGS=''
+if [[ "${TFLAGS}" != 'skipall' ]] && \
+   [ "${BUILD_SYSTEM}" = 'CMake' ]; then
+  cmake --build _bld --config "${PRJ_CFG}" --parallel 2 --target testdeps
+fi
+
+# run tests
+
+if [[ "${TFLAGS}" != 'skipall' ]] && \
+   [[ "${TFLAGS}" != 'skiprun' ]]; then
   if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then
     TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")"
   elif [ -x "$(cygpath 'C:/msys64/usr/bin/curl.exe')" ]; then
     TFLAGS+=" -ac $(cygpath 'C:/msys64/usr/bin/curl.exe')"
   fi
-  TFLAGS+=" ${DISABLED_TESTS:-}"
   if [ "${BUILD_SYSTEM}" = 'CMake' ]; then
-    cmake --build _bld --config "${PRJ_CFG}" --parallel 2 --target testdeps
     ls _bld/lib/*.dll >/dev/null 2>&1 && cp -f -p _bld/lib/*.dll _bld/tests/libtest/
     cmake --build _bld --config "${PRJ_CFG}" --target test-ci
   else

diff --git a/appveyor.yml b/appveyor.yml
index 786bd933fce50d8a3df90ff0754351340a06e572..36ac9742ff395b224bdaa0374b4e4dc68d9b5c5e 100644 (file)
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -35,8 +35,11 @@ environment:
   OPENSSL: 'OFF'
   DEBUG: 'ON'
   SHARED: 'OFF'
+  TFLAGS: 'skipall'
   matrix:
-    # generated CMake-based Visual Studio Release builds
+
+    # generated CMake-based Visual Studio builds
+
     - job_name: 'CMake, VS2008, Release, x86, Schannel, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
       BUILD_SYSTEM: CMake
@@ -47,8 +50,6 @@ environment:
       ENABLE_UNICODE: 'OFF'
       HTTP_ONLY: 'OFF'
       SHARED: 'ON'
-      TESTING: 'OFF'
-      DISABLED_TESTS: ''
     - job_name: 'CMake, VS2008, Debug, x86, Schannel, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
       BUILD_SYSTEM: CMake
@@ -58,8 +59,6 @@ environment:
       ENABLE_UNICODE: 'OFF'
       HTTP_ONLY: 'OFF'
       SHARED: 'ON'
-      TESTING: 'OFF'
-      DISABLED_TESTS: ''
     - job_name: 'CMake, VS2022, Release, x64, OpenSSL 3.2, WebSockets, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
       BUILD_SYSTEM: CMake
@@ -71,8 +70,6 @@ environment:
       ENABLE_UNICODE: 'OFF'
       HTTP_ONLY: 'OFF'
       SHARED: 'ON'
-      TESTING: 'OFF'
-      DISABLED_TESTS: ''
       WEBSOCKETS: 'ON'
     - job_name: 'CMake, VS2022, Release, arm64, Schannel, Static, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
@@ -83,9 +80,6 @@ environment:
       SCHANNEL: 'ON'
       ENABLE_UNICODE: 'OFF'
       HTTP_ONLY: 'OFF'
-      TESTING: 'OFF'
-      DISABLED_TESTS: ''
-    # generated CMake-based Visual Studio Debug builds
     - job_name: 'CMake, VS2010, Debug, x64, Schannel, Static, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
       BUILD_SYSTEM: CMake
@@ -94,8 +88,6 @@ environment:
       SCHANNEL: 'ON'
       ENABLE_UNICODE: 'OFF'
       HTTP_ONLY: 'OFF'
-      TESTING: 'OFF'
-      DISABLED_TESTS: ''
     - job_name: 'CMake, VS2022, Debug, x64, Schannel, Static, Unicode, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
       BUILD_SYSTEM: CMake
@@ -105,8 +97,6 @@ environment:
       SCHANNEL: 'ON'
       ENABLE_UNICODE: 'ON'
       HTTP_ONLY: 'OFF'
-      TESTING: 'OFF'
-      DISABLED_TESTS: '!1139 !1501 !1177 !1477'
     - job_name: 'CMake, VS2022, Debug, x64, no SSL, Static, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
       BUILD_SYSTEM: CMake
@@ -116,8 +106,6 @@ environment:
       SCHANNEL: 'OFF'
       ENABLE_UNICODE: 'OFF'
       HTTP_ONLY: 'OFF'
-      TESTING: 'OFF'
-      DISABLED_TESTS: '!1139 !1501 !1177 !1477'
     - job_name: 'CMake, VS2022, Debug, x64, no SSL, Static, HTTP only, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2022'
       BUILD_SYSTEM: CMake
@@ -127,8 +115,6 @@ environment:
       SCHANNEL: 'OFF'
       ENABLE_UNICODE: 'OFF'
       HTTP_ONLY: 'ON'
-      TESTING: 'OFF'
-      DISABLED_TESTS: '!1139 !1501 !1177 !1477'
 
     # winbuild-based builds
 
@@ -137,56 +123,48 @@ environment:
       BUILD_SYSTEM: winbuild_vs2015
       DEBUG: 'yes'
       PATHPART: debug
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'no'
     - job_name: 'winbuild, VS2015, Release, x64, OpenSSL 1.1.1, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
       BUILD_SYSTEM: winbuild_vs2015
       DEBUG: 'no'
       PATHPART: release
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'no'
     - job_name: 'winbuild, VS2017, Debug, x64, OpenSSL 1.1.1, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2017'
       BUILD_SYSTEM: winbuild_vs2017
       DEBUG: 'yes'
       PATHPART: debug
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'no'
     - job_name: 'winbuild, VS2017, Release, x64, OpenSSL 1.1.1, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2017'
       BUILD_SYSTEM: winbuild_vs2017
       DEBUG: 'no'
       PATHPART: release
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'no'
     - job_name: 'winbuild, VS2015, Debug, x64, OpenSSL 1.1.1, Unicode, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
       BUILD_SYSTEM: winbuild_vs2015
       DEBUG: 'yes'
       PATHPART: debug
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'yes'
     - job_name: 'winbuild, VS2015, Release, x64, OpenSSL 1.1.1, Unicode, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
       BUILD_SYSTEM: winbuild_vs2015
       DEBUG: 'no'
       PATHPART: release
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'yes'
     - job_name: 'winbuild, VS2017, Debug, x64, OpenSSL 1.1.1, Unicode, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2017'
       BUILD_SYSTEM: winbuild_vs2017
       DEBUG: 'yes'
       PATHPART: debug
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'yes'
     - job_name: 'winbuild, VS2017, Release, x64, OpenSSL 1.1.1, Unicode, Build-only'
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2017'
       BUILD_SYSTEM: winbuild_vs2017
       DEBUG: 'no'
       PATHPART: release
-      TESTING: 'OFF'
       ENABLE_UNICODE: 'yes'
 
     # generated VisualStudioSolution-based builds
@@ -195,7 +173,6 @@ environment:
       APPVEYOR_BUILD_WORKER_IMAGE: 'Visual Studio 2015'
       BUILD_SYSTEM: VisualStudioSolution
       PRJ_CFG: 'DLL Debug - DLL Windows SSPI - DLL WinIDN'
-      TESTING: 'OFF'
       VC_VERSION: VC12
 
 install: