From: Viktor Szakats Date: Thu, 15 May 2025 20:11:06 +0000 (+0200) Subject: pytest: enable in 5 GHA jobs, fix fallouts X-Git-Tag: curl-8_14_0~52 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=42331cb48a1f66efaa0920ee8ccba5a74d67de27;p=thirdparty%2Fcurl.git pytest: enable in 5 GHA jobs, fix fallouts They take about 4.5 minutes of CI time in GHA/macos. Also: - autotools: improve `caddy`, `vsftpd` detection. Bringing it closer to cmake. - autotools: fix `--with-test-caddy=no`, `--with-test-vsftps=no`, `--with-test-nghttpx=no` options. - cmake: sync `nghttpx` default with autotools. - pytest: disable failing mbedTLS tests on macOS CI. - pytest: disable failing earlydata tests on macOS CI. - GHA/macos: keep vsftpd pytests disabled due to lengthy run times. - pytest: fix test_05_04 for LibreSSL. Ref: #17367 Authored-by: Stefan Eissing Remaining issues: - some unidentified tests taking a long time with mbedTLS: `================= 462 passed, 278 skipped in 347.93s (0:05:47) =================` Ref: https://github.com/curl/curl/actions/runs/15073354301/job/42374999041#step:17:1536 Workaround: not enabling pytest for mbedTLS jobs - 17 FTP tests taking a long time (affecting all TLS backends): without vsftpd: `====================== 496 passed, 244 skipped in 56.15s =======================` Ref: https://github.com/curl/curl/actions/runs/15073354301/job/42374998230#step:17:1536 with vsftpd: `================= 513 passed, 227 skipped in 409.37s (0:06:49) =================` Ref: https://github.com/curl/curl/actions/runs/15073678568/job/42376039672?pr=17362#step:17:1537 Workaround: force-disable vsftpd. - 100 tests failing with SecureTransport. Let's ignore that due to imminent deprecation. Ref: https://github.com/curl/curl/actions/runs/15055652333/job/42320873732#step:17:15362 Follow-up to 30ef79ed937ca0fc7592ff73d162398773c6a5aa #17295 Follow-up to 914790336664da0451e2f57906373a7707bd7cc3 #16518 Closes #17362 --- diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 224662a027..0f619cee92 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -116,9 +116,11 @@ jobs: - name: 'LibreSSL +examples' compiler: clang install: libressl + install_steps: pytest configure: --enable-debug --with-openssl=$(brew --prefix libressl) - name: 'OpenSSL' compiler: clang + install_steps: pytest configure: --enable-debug --with-openssl=$(brew --prefix openssl) - name: 'OpenSSL event-based' compiler: clang @@ -149,6 +151,7 @@ jobs: generate: -DOPENSSL_ROOT_DIR=$(brew --prefix libressl) -DENABLE_ARES=ON -DCURL_USE_GSSAPI=ON -DGSS_ROOT_DIR=$(brew --prefix heimdal) -DCURL_DISABLE_LDAP=ON - name: 'wolfSSL !ldap brotli zstd' install: brotli wolfssl zstd + install_steps: pytest generate: -DCURL_USE_WOLFSSL=ON -DCURL_DISABLE_LDAP=ON -DUSE_ECH=ON - name: 'mbedTLS openldap brotli zstd' install: brotli mbedtls zstd openldap @@ -237,7 +240,7 @@ jobs: for _chkprefill in '' ${{ matrix.build.chkprefill }}; do options='' [ -n '${{ matrix.build.macos-version-min }}' ] && options+=' -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ matrix.build.macos-version-min }}' - [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' -DTEST_NGHTTPX= -DHTTPD_NGHTTPX=' + [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' -DVSFTPD=NO' # Skip ~20 tests that stretch run time by 7x on macOS [ "${_chkprefill}" = '_chkprefill' ] && options+=' -D_CURL_PREFILL=OFF' cmake -B "bld${_chkprefill}" -G Ninja -D_CURL_PREFILL=ON \ -DCMAKE_INSTALL_PREFIX="$HOME/curl" \ @@ -260,7 +263,7 @@ jobs: CFLAGS+=" --sysroot=${sysroot}" fi [ -n '${{ matrix.build.macos-version-min }}' ] && CFLAGS+=' -mmacosx-version-min=${{ matrix.build.macos-version-min }}' - [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' --with-test-nghttpx= ac_cv_path_HTTPD_NGHTTPX=' + [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' --with-test-vsftpd=no' # Skip ~20 tests that stretch run time by 7x on macOS mkdir bld && cd bld && ../configure --enable-unity --enable-test-bundles --enable-warnings --enable-werror \ --disable-dependency-tracking \ --with-libpsl=$(brew --prefix libpsl) \ diff --git a/configure.ac b/configure.ac index 50f60eec12..dd88ea5e30 100644 --- a/configure.ac +++ b/configure.ac @@ -299,27 +299,39 @@ TEST_NGHTTPX=nghttpx AC_ARG_WITH(test-nghttpx,dnl AS_HELP_STRING([--with-test-nghttpx=PATH],[where to find nghttpx for testing]), TEST_NGHTTPX=$withval - if test X"$OPT_TEST_NGHTTPX" = "Xno"; then + if test X"$TEST_NGHTTPX" = "Xno"; then TEST_NGHTTPX="" fi ) AC_SUBST(TEST_NGHTTPX) -CADDY=/usr/bin/caddy +if test -x /usr/bin/caddy; then + CADDY=/usr/bin/caddy +elif test -x /usr/local/bin/caddy; then + CADDY=/usr/local/bin/caddy +elif test -x `brew --prefix`/bin/caddy; then + CADDY=`brew --prefix`/bin/caddy +fi AC_ARG_WITH(test-caddy,dnl AS_HELP_STRING([--with-test-caddy=PATH],[where to find caddy for testing]), CADDY=$withval - if test X"$OPT_CADDY" = "Xno"; then + if test X"$CADDY" = "Xno"; then CADDY="" fi ) AC_SUBST(CADDY) -VSFTPD=/usr/sbin/vsftpd +if test -x /usr/sbin/vsftpd; then + VSFTPD=/usr/sbin/vsftpd +elif test -x /usr/local/sbin/vsftpd; then + VSFTPD=/usr/local/sbin/vsftpd +elif test -x `brew --prefix`/sbin/vsftpd; then + VSFTPD=`brew --prefix`/sbin/vsftpd +fi AC_ARG_WITH(test-vsftpd,dnl AS_HELP_STRING([--with-test-vsftpd=PATH],[where to find vsftpd for testing]), VSFTPD=$withval - if test X"$OPT_VSFTPD" = "Xno"; then + if test X"$VSFTPD" = "Xno"; then VSFTPD="" fi ) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index defc1b5eb2..d04d3de4c2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,7 +23,7 @@ ########################################################################### find_program(TEST_NGHTTPX "nghttpx") if(NOT TEST_NGHTTPX) - set(TEST_NGHTTPX "nghttpx") + set(TEST_NGHTTPX "") endif() mark_as_advanced(TEST_NGHTTPX) # Consumed variables: TEST_NGHTTPX diff --git a/tests/http/test_02_download.py b/tests/http/test_02_download.py index d91e20488c..6dfa66c65e 100644 --- a/tests/http/test_02_download.py +++ b/tests/http/test_02_download.py @@ -30,6 +30,7 @@ import logging import math import os import re +import sys from datetime import timedelta import pytest @@ -76,6 +77,9 @@ class TestDownload: def test_02_03_download_sequential(self, env: Env, httpd, nghttpx, proto): if proto == 'h3' and not env.have_h3(): pytest.skip("h3 not supported") + if (proto == 'http/1.1' or proto == 'h2') and env.curl_uses_lib('mbedtls') and \ + sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') count = 10 curl = CurlClient(env=env) urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' @@ -87,6 +91,9 @@ class TestDownload: def test_02_04_download_parallel(self, env: Env, httpd, nghttpx, proto): if proto == 'h3' and not env.have_h3(): pytest.skip("h3 not supported") + if proto == 'h2' and env.curl_uses_lib('mbedtls') and \ + sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') count = 10 max_parallel = 5 curl = CurlClient(env=env) @@ -109,6 +116,9 @@ class TestDownload: pytest.skip("h3 not supported") if proto == 'h3' and env.curl_uses_lib('msh3'): pytest.skip("msh3 shaky here") + if proto == 'h2' and env.curl_uses_lib('mbedtls') and \ + sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') count = 200 curl = CurlClient(env=env) urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' @@ -126,6 +136,9 @@ class TestDownload: def test_02_06_download_many_parallel(self, env: Env, httpd, nghttpx, proto): if proto == 'h3' and not env.have_h3(): pytest.skip("h3 not supported") + if proto == 'h2' and env.curl_uses_lib('mbedtls') and \ + sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') count = 200 max_parallel = 50 curl = CurlClient(env=env) @@ -591,6 +604,8 @@ class TestDownload: if proto == 'h3' and \ (not env.have_h3() or not env.curl_can_h3_early_data()): pytest.skip("h3 not supported") + if proto != 'h3' and sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('failing on macOS CI runners') count = 2 docname = 'data-10k' # we want this test to always connect to nghttpx, since it is @@ -635,6 +650,8 @@ class TestDownload: @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) @pytest.mark.parametrize("max_host_conns", [0, 1, 5]) def test_02_33_max_host_conns(self, env: Env, httpd, nghttpx, proto, max_host_conns): + if not env.curl_is_debug(): + pytest.skip('only works for curl debug builds') if proto == 'h3' and not env.have_h3(): pytest.skip("h3 not supported") count = 50 @@ -671,6 +688,8 @@ class TestDownload: @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) @pytest.mark.parametrize("max_total_conns", [0, 1, 5]) def test_02_34_max_total_conns(self, env: Env, httpd, nghttpx, proto, max_total_conns): + if not env.curl_is_debug(): + pytest.skip('only works for curl debug builds') if proto == 'h3' and not env.have_h3(): pytest.skip("h3 not supported") count = 50 diff --git a/tests/http/test_05_errors.py b/tests/http/test_05_errors.py index 10d8972df2..9122c03a33 100644 --- a/tests/http/test_05_errors.py +++ b/tests/http/test_05_errors.py @@ -118,10 +118,11 @@ class TestErrors: url = f'https://{env.authority_for(env.domain1, proto)}'\ f'/curltest/shutdown_unclean?id=[0-{count-1}]&chunks=4' r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[ - '--parallel', + '--parallel', '--trace-config', 'ssl' ]) if proto == 'http/1.0' and not env.curl_uses_lib('wolfssl') and \ - (env.curl_is_debug() or not env.curl_uses_lib('openssl')): + (env.curl_is_debug() or + not env.curl_uses_any_libs(['openssl', 'libressl'])): # we are inconsistent if we fail or not in missing TLS shutdown # openssl code ignore such errors intentionally in non-debug builds r.check_exit_code(56) diff --git a/tests/http/test_07_upload.py b/tests/http/test_07_upload.py index 9bae7041e7..4f23122ae4 100644 --- a/tests/http/test_07_upload.py +++ b/tests/http/test_07_upload.py @@ -29,6 +29,7 @@ import filecmp import logging import os import re +import sys import pytest from typing import List, Union @@ -708,6 +709,8 @@ class TestUpload: if proto == 'h3' and \ (not env.have_h3() or not env.curl_can_h3_early_data()): pytest.skip("h3 not supported") + if proto != 'h3' and sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('failing on macOS CI runners') count = 2 # we want this test to always connect to nghttpx, since it is # the only server we have that supports TLS earlydata diff --git a/tests/http/test_10_proxy.py b/tests/http/test_10_proxy.py index bf89a44311..59e56d583f 100644 --- a/tests/http/test_10_proxy.py +++ b/tests/http/test_10_proxy.py @@ -28,6 +28,7 @@ import filecmp import logging import os import re +import sys import pytest from testenv import Env, CurlClient, ExecResult @@ -179,6 +180,9 @@ class TestProxy: tunnel, fname, fcount): if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'): pytest.skip('only supported with nghttp2') + if env.curl_uses_lib('mbedtls') and \ + sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') count = fcount curl = CurlClient(env=env) url = f'https://localhost:{env.https_port}/{fname}?[0-{count-1}]' @@ -209,6 +213,9 @@ class TestProxy: tunnel, fname, fcount): if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'): pytest.skip('only supported with nghttp2') + if env.curl_uses_lib('mbedtls') and \ + sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') count = fcount srcfile = os.path.join(httpd.docs_dir, fname) curl = CurlClient(env=env) @@ -255,6 +262,9 @@ class TestProxy: # url twice via https: proxy separated with '--next', will reuse if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'): pytest.skip('only supported with nghttp2') + if env.curl_uses_lib('mbedtls') and \ + sys.platform.startswith('darwin') and env.ci_run: + pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') curl = CurlClient(env=env) url = f'https://localhost:{env.https_port}/data.json' proxy_args = curl.get_proxy_args(tunnel=True, proto=tunnel) diff --git a/tests/http/test_20_websockets.py b/tests/http/test_20_websockets.py index 18aa0341a8..306f418ad5 100644 --- a/tests/http/test_20_websockets.py +++ b/tests/http/test_20_websockets.py @@ -49,7 +49,7 @@ class TestWebsockets: 'ws': socket.SOCK_STREAM, } - def check_alive(self, env, port, timeout=5): + def check_alive(self, env, port, timeout=Env.SERVER_TIMEOUT): curl = CurlClient(env=env) url = f'http://localhost:{port}/' end = datetime.now() + timedelta(seconds=timeout) diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py index c7dd65975c..aa66eab296 100644 --- a/tests/http/testenv/env.py +++ b/tests/http/testenv/env.py @@ -33,7 +33,7 @@ import subprocess import tempfile from configparser import ConfigParser, ExtendedInterpolation from datetime import timedelta -from typing import Optional, Dict +from typing import Optional, Dict, List import pytest from filelock import FileLock @@ -329,6 +329,13 @@ class Env: def curl_uses_lib(libname: str) -> bool: return libname.lower() in Env.CONFIG.curl_props['libs'] + @staticmethod + def curl_uses_any_libs(libs: List[str]) -> bool: + for libname in libs: + if libname.lower() in Env.CONFIG.curl_props['libs']: + return True + return False + @staticmethod def curl_uses_ossl_quic() -> bool: if Env.have_h3_curl(): @@ -389,10 +396,7 @@ class Env: @staticmethod def curl_can_early_data() -> bool: - return Env.curl_uses_lib('gnutls') or \ - Env.curl_uses_lib('wolfssl') or \ - Env.curl_uses_lib('quictls') or \ - Env.curl_uses_lib('openssl') + return Env.curl_uses_any_libs(['gnutls', 'wolfssl', 'quictls', 'openssl']) @staticmethod def curl_can_h3_early_data() -> bool: