From 6b70e8a838ae99d3e3fd1e828c4e96c63b894833 Mon Sep 17 00:00:00 2001 From: Stefan Eissing Date: Tue, 29 Jul 2025 10:15:43 +0200 Subject: [PATCH] pytest: use dante-server in CI - add startup check for 'danted' to avoid fails on low cpu - rename 'sockd' to 'danted' everywhere to clarify what we use - add proper defaults for 'danted' for debian - install 'dante-server' in pytest ci runs Closes #18075 --- .github/workflows/http3-linux.yml | 4 ++-- .github/workflows/linux.yml | 2 +- configure.ac | 40 +++++++++++++++---------------- docs/INSTALL-CMAKE.md | 2 +- docs/internals/SCORECARD.md | 4 ++-- docs/tests/HTTP.md | 2 +- tests/http/CMakeLists.txt | 10 ++++---- tests/http/config.ini.in | 4 ++-- tests/http/test_40_socks.py | 28 +++++++++++----------- tests/http/testenv/dante.py | 27 +++++++++++++++++---- tests/http/testenv/env.py | 30 +++++++++++------------ 11 files changed, 85 insertions(+), 68 deletions(-) diff --git a/.github/workflows/http3-linux.yml b/.github/workflows/http3-linux.yml index 445069e7bc..a919ac90f4 100644 --- a/.github/workflows/http3-linux.yml +++ b/.github/workflows/http3-linux.yml @@ -151,7 +151,7 @@ jobs: nettle-dev libp11-kit-dev libtspi-dev libunistring-dev guile-2.2-dev libtasn1-bin \ libtasn1-6-dev libidn2-0-dev gawk gperf libtss2-dev dns-root-data bison gtk-doc-tools \ texinfo texlive texlive-extra-utils autopoint libev-dev \ - apache2 apache2-dev libnghttp2-dev + apache2 apache2-dev libnghttp2-dev dante-server echo 'CC=gcc-12' >> "$GITHUB_ENV" echo 'CXX=g++-12' >> "$GITHUB_ENV" @@ -343,7 +343,7 @@ jobs: nettle-dev libp11-kit-dev libtspi-dev libunistring-dev guile-2.2-dev libtasn1-bin \ libtasn1-6-dev libidn2-0-dev gawk gperf libtss2-dev dns-root-data bison gtk-doc-tools \ texinfo texlive texlive-extra-utils autopoint libev-dev libuv1-dev \ - apache2 apache2-dev libnghttp2-dev vsftpd + apache2 apache2-dev libnghttp2-dev vsftpd dante-server python3 -m venv ~/venv echo 'CC=gcc-12' >> "$GITHUB_ENV" echo 'CXX=g++-12' >> "$GITHUB_ENV" diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 06f7da51c4..bf8bab3d11 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -288,7 +288,7 @@ jobs: env: INSTALL_PACKAGES: >- ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') && 'stunnel4' || '' }} - ${{ contains(matrix.build.install_steps, 'pytest') && 'apache2 apache2-dev libnghttp2-dev vsftpd' || '' }} + ${{ contains(matrix.build.install_steps, 'pytest') && 'apache2 apache2-dev libnghttp2-dev vsftpd dante-server' || '' }} run: | sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list diff --git a/configure.ac b/configure.ac index d877d0e90d..dae3e9e99e 100644 --- a/configure.ac +++ b/configure.ac @@ -372,35 +372,35 @@ fi AC_SUBST(HTTPD) AC_SUBST(APXS) -dnl we'd like a sockd as test server +dnl we'd like a dante as test socks server dnl -SOCKD_ENABLED="maybe" -AC_ARG_WITH(test-sockd, [AS_HELP_STRING([--with-test-sockd=PATH], - [where to find dante sockd for testing])], - [request_sockd=$withval], [request_sockd=check]) -if test x"$request_sockd" = "xcheck" -o x"$request_sockd" = "xyes"; then - if test -x "/usr/sbin/sockd"; then +DANTED_ENABLED="maybe" +AC_ARG_WITH(test-danted, [AS_HELP_STRING([--with-test-danted=PATH], + [where to find danted socks daemon for testing])], + [request_danted=$withval], [request_danted=check]) +if test x"$request_danted" = "xcheck" -o x"$request_danted" = "xyes"; then + if test -x "/usr/sbin/danted"; then # common location on distros (debian/ubuntu) - SOCKD="/usr/sbin/sockd" + DANTED="/usr/sbin/danted" else - AC_PATH_PROG([SOCKD], [sockd]) - if test "x$SOCKD" = "x"; then - AC_PATH_PROG([SOCKD], [sockd]) + AC_PATH_PROG([DANTED], [danted]) + if test "x$DANTED" = "x"; then + AC_PATH_PROG([DANTED], [danted]) fi fi -elif test x"$request_sockd" != "xno"; then - SOCKD="${request_sockd}" - if test ! -x "${SOCKD}"; then - AC_MSG_NOTICE([sockd not found as ${SOCKD}, sockd tests disabled]) - SOCKD_ENABLED="no" +elif test x"$request_danted" != "xno"; then + DANTED="${request_danted}" + if test ! -x "${DANTED}"; then + AC_MSG_NOTICE([danted not found as ${DANTED}, danted tests disabled]) + DANTED_ENABLED="no" else - AC_MSG_NOTICE([using SOCKD=$SOCKD for tests]) + AC_MSG_NOTICE([using DANTED=$DANTED for tests]) fi fi -if test x"$SOCKD_ENABLED" = "xno"; then - SOCKD="" +if test x"$DANTED_ENABLED" = "xno"; then + DANTED="" fi -AC_SUBST(SOCKD) +AC_SUBST(DANTED) dnl the nghttpx we might use in httpd testing if test "x$TEST_NGHTTPX" != "x" -a "x$TEST_NGHTTPX" != "xnghttpx"; then diff --git a/docs/INSTALL-CMAKE.md b/docs/INSTALL-CMAKE.md index 7220e328e1..cad83d909e 100644 --- a/docs/INSTALL-CMAKE.md +++ b/docs/INSTALL-CMAKE.md @@ -460,7 +460,7 @@ Details via CMake - `CADDY`: Default: `caddy` - `HTTPD_NGHTTPX`: Default: `nghttpx` - `HTTPD`: Default: `apache2` -- `SOCKD`: Default: `sockd` +- `DANTED`: Default: `danted` - `TEST_NGHTTPX`: Default: `nghttpx` - `VSFTPD`: Default: `vsftps` diff --git a/docs/internals/SCORECARD.md b/docs/internals/SCORECARD.md index 265b4aab8c..7839e0b61f 100644 --- a/docs/internals/SCORECARD.md +++ b/docs/internals/SCORECARD.md @@ -54,8 +54,8 @@ Similar options are available for uploads and requests scenarios. ## sockd -If you have configured curl with `--with-test-sockd=` for a -`dante sockd` server installed on your system, you can provide the scorecard +If you have configured curl with `--with-test-danted=` for a +`dante-server` installed on your system, you can provide the scorecard with arguments `--socks4` or `--socks5` to test performance with a SOCKS proxy involved. (Note: this does not work for HTTP/3) diff --git a/docs/tests/HTTP.md b/docs/tests/HTTP.md index e28ea15da8..afbfe9e5bd 100644 --- a/docs/tests/HTTP.md +++ b/docs/tests/HTTP.md @@ -52,7 +52,7 @@ Via curl's `configure` script you may specify: * `--with-test-httpd=` if you have an Apache httpd installed somewhere else. On Debian/Ubuntu it will otherwise look into `/usr/bin` and `/usr/sbin` to find those. * `--with-test-caddy=` if you have a Caddy web server installed somewhere else. * `--with-test-vsftpd=` if you have a vsftpd ftp server installed somewhere else. - * `--with-test-sockd=` if you have `dante sockd` server installed + * `--with-test-danted=` if you have `dante-server` installed ## Usage Tips diff --git a/tests/http/CMakeLists.txt b/tests/http/CMakeLists.txt index 5e415cbe7c..c53a8a32dc 100644 --- a/tests/http/CMakeLists.txt +++ b/tests/http/CMakeLists.txt @@ -52,11 +52,11 @@ if(NOT HTTPD_NGHTTPX) endif() mark_as_advanced(HTTPD_NGHTTPX) -find_program(SOCKD "sockd") -if(NOT SOCKD) - set(SOCKD "") +find_program(DANTED "danted") +if(NOT DANTED) + set(DANTED "") endif() -mark_as_advanced(SOCKD) +mark_as_advanced(DANTED) -# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, SOCKD, VSFTPD +# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD configure_file("config.ini.in" "${CMAKE_CURRENT_BINARY_DIR}/config.ini" @ONLY) diff --git a/tests/http/config.ini.in b/tests/http/config.ini.in index 8026df92a3..5fe51a6465 100644 --- a/tests/http/config.ini.in +++ b/tests/http/config.ini.in @@ -38,5 +38,5 @@ caddy = @CADDY@ [vsftpd] vsftpd = @VSFTPD@ -[sockd] -sockd = @SOCKD@ +[danted] +danted = @DANTED@ diff --git a/tests/http/test_40_socks.py b/tests/http/test_40_socks.py index f119d8926c..6b43f878a6 100644 --- a/tests/http/test_40_socks.py +++ b/tests/http/test_40_socks.py @@ -35,15 +35,15 @@ from testenv import Env, CurlClient, Dante log = logging.getLogger(__name__) -@pytest.mark.skipif(condition=not Env.has_sockd(), reason="missing sockd") +@pytest.mark.skipif(condition=not Env.has_danted(), reason="missing danted") class TestSocks: @pytest.fixture(scope='class') - def sockd(self, env: Env) -> Generator[Dante, None, None]: - sockd = Dante(env=env) - assert sockd.initial_start() - yield sockd - sockd.stop() + def danted(self, env: Env) -> Generator[Dante, None, None]: + danted = Dante(env=env) + assert danted.initial_start() + yield danted + danted.stop() @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env, httpd): @@ -52,9 +52,9 @@ class TestSocks: env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) - def test_40_01_socks_http(self, env: Env, sproto, sockd: Dante, httpd): + def test_40_01_socks_http(self, env: Env, sproto, danted: Dante, httpd): curl = CurlClient(env=env, socks_args=[ - f'--{sproto}', f'127.0.0.1:{sockd.port}' + f'--{sproto}', f'127.0.0.1:{danted.port}' ]) url = f'http://{env.domain1}:{env.http_port}/data.json' r = curl.http_get(url=url) @@ -62,11 +62,11 @@ class TestSocks: @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) - def test_40_02_socks_https(self, env: Env, sproto, proto, sockd: Dante, httpd): + def test_40_02_socks_https(self, env: Env, sproto, proto, danted: Dante, httpd): if proto == 'h3' and not env.have_h3(): pytest.skip("h3 not supported") curl = CurlClient(env=env, socks_args=[ - f'--{sproto}', f'127.0.0.1:{sockd.port}' + f'--{sproto}', f'127.0.0.1:{danted.port}' ]) url = f'https://{env.authority_for(env.domain1, proto)}/data.json' r = curl.http_get(url=url, alpn_proto=proto) @@ -77,22 +77,22 @@ class TestSocks: @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) - def test_40_03_dl_serial(self, env: Env, httpd, sockd, proto, sproto): + def test_40_03_dl_serial(self, env: Env, httpd, danted, proto, sproto): count = 3 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' curl = CurlClient(env=env, socks_args=[ - f'--{sproto}', f'127.0.0.1:{sockd.port}' + f'--{sproto}', f'127.0.0.1:{danted.port}' ]) r = curl.http_download(urls=[urln], alpn_proto=proto) r.check_response(count=count, http_status=200) @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) - def test_40_04_ul_serial(self, env: Env, httpd, sockd, proto, sproto): + def test_40_04_ul_serial(self, env: Env, httpd, danted, proto, sproto): fdata = os.path.join(env.gen_dir, 'data-10m') count = 2 curl = CurlClient(env=env, socks_args=[ - f'--{sproto}', f'127.0.0.1:{sockd.port}' + f'--{sproto}', f'127.0.0.1:{danted.port}' ]) url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) diff --git a/tests/http/testenv/dante.py b/tests/http/testenv/dante.py index 995dce6a61..9e1d63ce00 100644 --- a/tests/http/testenv/dante.py +++ b/tests/http/testenv/dante.py @@ -28,9 +28,12 @@ import logging import os import socket import subprocess +import time +from datetime import timedelta, datetime from typing import Dict +from . import CurlClient from .env import Env from .ports import alloc_ports_and_do @@ -41,12 +44,12 @@ class Dante: def __init__(self, env: Env): self.env = env - self._cmd = env.sockd + self._cmd = env.danted self._port = 0 - self.name = 'sockd' - self._port_skey = 'sockd' + self.name = 'danted' + self._port_skey = 'danted' self._port_specs = { - 'sockd': socket.SOCK_STREAM, + 'danted': socket.SOCK_STREAM, } self._dante_dir = os.path.join(env.gen_dir, self.name) self._run_dir = os.path.join(self._dante_dir, 'run') @@ -124,7 +127,21 @@ class Dante: self._process = subprocess.Popen(args=args, stderr=procerr) if self._process.returncode is not None: return False - return True + return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT)) + + def wait_live(self, timeout: timedelta): + curl = CurlClient(env=self.env, run_dir=self._tmp_dir, + timeout=timeout.total_seconds(), socks_args=[ + '--socks5', f'127.0.0.1:{self._port}' + ]) + try_until = datetime.now() + timeout + while datetime.now() < try_until: + r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') + if r.exit_code == 0: + return True + time.sleep(.1) + log.error(f"Server still not responding after {timeout}") + return False def _rmf(self, path): if os.path.exists(path): diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py index 6142440756..0e1ce389d5 100644 --- a/tests/http/testenv/env.py +++ b/tests/http/testenv/env.py @@ -256,28 +256,28 @@ class EnvConfig: except Exception: self.vsftpd = None - self.sockd = self.config['sockd']['sockd'] - if self.sockd == '': - self.sockd = None - self._sockd_version = None - if self.sockd is not None: + self.danted = self.config['danted']['danted'] + if self.danted == '': + self.danted = None + self._danted_version = None + if self.danted is not None: try: - p = subprocess.run(args=[self.sockd, '-v'], + p = subprocess.run(args=[self.danted, '-v'], capture_output=True, text=True) assert p.returncode == 0 if p.returncode != 0: # not a working vsftpd - self.sockd = None + self.danted = None m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stdout) if not m: m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stderr) if m: - self._sockd_version = m.group(1) + self._danted_version = m.group(1) else: - self.sockd = None - raise Exception(f'Unable to determine sockd version from: {p.stderr}') + self.danted = None + raise Exception(f'Unable to determine danted version from: {p.stderr}') except Exception: - self.sockd = None + self.danted = None self._tcpdump = shutil.which('tcpdump') @@ -508,8 +508,8 @@ class Env: return Env.CONFIG.vsftpd_version @staticmethod - def has_sockd() -> bool: - return Env.CONFIG.sockd is not None + def has_danted() -> bool: + return Env.CONFIG.danted is not None @staticmethod def tcpdump() -> Optional[str]: @@ -673,8 +673,8 @@ class Env: return self.CONFIG.ports['caddy'] @property - def sockd(self) -> str: - return self.CONFIG.sockd + def danted(self) -> str: + return self.CONFIG.danted @property def vsftpd(self) -> str: -- 2.47.2