From: Stefan Eissing Date: Mon, 21 Jul 2025 10:23:06 +0000 (+0200) Subject: pytest: add SOCKS tests and scoring X-Git-Tag: curl-8_16_0~415 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ab5e0bfddca224b4a15ad824a2f1a6343e835f99;p=thirdparty%2Fcurl.git pytest: add SOCKS tests and scoring Configure curl with `--with-test-sockd=` for a locally installed dante sockd server and new `test_40_*` will verify that down- and uploads work via SOCKS. Invoke scorecard.py with `--socks4` or `--socks5` to run performance tests with SOCKS. Note that SOCKS is not supported for HTTP/3. Ref: #17969 Closes #17986 --- diff --git a/.github/scripts/spellcheck.words b/.github/scripts/spellcheck.words index a1d030dd83..f8796eaa1f 100644 --- a/.github/scripts/spellcheck.words +++ b/.github/scripts/spellcheck.words @@ -771,6 +771,7 @@ smtp smtps SMTPS SNI +sockd socketopen socketpair sockopt diff --git a/configure.ac b/configure.ac index 6e33598603..af52dbc433 100644 --- a/configure.ac +++ b/configure.ac @@ -372,6 +372,36 @@ fi AC_SUBST(HTTPD) AC_SUBST(APXS) +dnl we'd like a sockd as test server +dnl +SOCKD_ENABLED="maybe" +AC_ARG_WITH(test-sockd, [AS_HELP_STRING([--with-test-sockd=PATH], + [where to find dante sockd for testing])], + [request_sockd=$withval], [request_sockd=check]) +if test x"$request_sockd" = "xcheck" -o x"$request_sockd" = "xyes"; then + if test -x "/usr/sbin/sockd"; then + # common location on distros (debian/ubuntu) + SOCKD="/usr/sbin/sockd" + else + AC_PATH_PROG([SOCKD], [sockd]) + if test "x$SOCKD" = "x"; then + AC_PATH_PROG([SOCKD], [sockd]) + fi + fi +elif test x"$request_sockd" != "xno"; then + SOCKD="${request_sockd}" + if test ! -x "${SOCKD}"; then + AC_MSG_NOTICE([sockd not found as ${SOCKD}, sockd tests disabled]) + SOCKD_ENABLED="no" + else + AC_MSG_NOTICE([using SOCKD=$SOCKD for tests]) + fi +fi +if test x"$SOCKD_ENABLED" = "xno"; then + SOCKD="" +fi +AC_SUBST(SOCKD) + dnl the nghttpx we might use in httpd testing if test "x$TEST_NGHTTPX" != "x" -a "x$TEST_NGHTTPX" != "xnghttpx"; then HTTPD_NGHTTPX="$TEST_NGHTTPX" diff --git a/docs/INSTALL-CMAKE.md b/docs/INSTALL-CMAKE.md index 6f8a914d73..7fe8d1db2e 100644 --- a/docs/INSTALL-CMAKE.md +++ b/docs/INSTALL-CMAKE.md @@ -463,6 +463,7 @@ Details via CMake - `CADDY`: Default: `caddy` - `HTTPD_NGHTTPX`: Default: `nghttpx` - `HTTPD`: Default: `apache2` +- `SOCKD`: Default: `sockd` - `TEST_NGHTTPX`: Default: `nghttpx` - `VSFTPD`: Default: `vsftps` diff --git a/docs/internals/SCORECARD.md b/docs/internals/SCORECARD.md index a28b42f040..265b4aab8c 100644 --- a/docs/internals/SCORECARD.md +++ b/docs/internals/SCORECARD.md @@ -52,6 +52,13 @@ curl> python3 tests/http/scorecard.py -d --download-sizes=1mb --download-count=1 Similar options are available for uploads and requests scenarios. +## sockd + +If you have configured curl with `--with-test-sockd=` for a +`dante sockd` server installed on your system, you can provide the scorecard +with arguments `--socks4` or `--socks5` to test performance with a SOCKS proxy +involved. (Note: this does not work for HTTP/3) + ## dtrace With the `--dtrace` option, scorecard produces a dtrace sample of the user stacks in `tests/http/gen/curl/curl.user_stacks`. On many platforms, `dtrace` requires **special permissions**. It is therefore invoked via `sudo` and you should make sure that sudo works for the run without prompting for a password. diff --git a/docs/tests/HTTP.md b/docs/tests/HTTP.md index 3728140599..e28ea15da8 100644 --- a/docs/tests/HTTP.md +++ b/docs/tests/HTTP.md @@ -52,6 +52,7 @@ Via curl's `configure` script you may specify: * `--with-test-httpd=` if you have an Apache httpd installed somewhere else. On Debian/Ubuntu it will otherwise look into `/usr/bin` and `/usr/sbin` to find those. * `--with-test-caddy=` if you have a Caddy web server installed somewhere else. * `--with-test-vsftpd=` if you have a vsftpd ftp server installed somewhere else. + * `--with-test-sockd=` if you have `dante sockd` server installed ## Usage Tips diff --git a/lib/socks.c b/lib/socks.c index 023696c461..58132cb42d 100644 --- a/lib/socks.c +++ b/lib/socks.c @@ -141,13 +141,15 @@ int Curl_blockread_all(struct Curl_cfilter *cf, #if defined(DEBUGBUILD) && !defined(CURL_DISABLE_VERBOSE_STRINGS) #define DEBUG_AND_VERBOSE -#define sxstate(x,d,y) socksstate(x,d,y, __LINE__) +#define sxstate(x,c,d,y) socksstate(x,c,d,y, __LINE__) #else -#define sxstate(x,d,y) socksstate(x,d,y) +#define sxstate(x,c,d,y) socksstate(x,c,d,y) #endif /* always use this function to change state, to make debugging easier */ -static void socksstate(struct socks_state *sx, struct Curl_easy *data, +static void socksstate(struct socks_state *sx, + struct Curl_cfilter *cf, + struct Curl_easy *data, enum connect_t state #ifdef DEBUG_AND_VERBOSE , int lineno @@ -179,6 +181,7 @@ static void socksstate(struct socks_state *sx, struct Curl_easy *data, }; #endif + (void)cf; (void)data; if(oldstate == state) /* do not bother when the new state is the same as the old state */ @@ -187,10 +190,8 @@ static void socksstate(struct socks_state *sx, struct Curl_easy *data, sx->state = state; #ifdef DEBUG_AND_VERBOSE - infof(data, - "SXSTATE: %s => %s; line %d", - socks_statename[oldstate], socks_statename[sx->state], - lineno); + CURL_TRC_CF(data, cf, "[%s] -> [%s] (line %d)", + socks_statename[oldstate], socks_statename[sx->state], lineno); #endif } @@ -284,12 +285,10 @@ static CURLproxycode do_SOCKS4(struct Curl_cfilter *cf, case CONNECT_SOCKS_INIT: /* SOCKS4 can only do IPv4, insist! */ conn->ip_version = CURL_IPRESOLVE_V4; - if(conn->bits.httpproxy) - infof(data, "SOCKS4%s: connecting to HTTP proxy %s port %d", - protocol4a ? "a" : "", sx->hostname, sx->remote_port); - - infof(data, "SOCKS4 communication to %s:%d", - sx->hostname, sx->remote_port); + CURL_TRC_CF(data, cf, "SOCKS4%s communication to%s %s:%d", + protocol4a ? "a" : "", + conn->bits.httpproxy ? " HTTP proxy" : "", + sx->hostname, sx->remote_port); /* * Compose socks4 request @@ -313,18 +312,19 @@ static CURLproxycode do_SOCKS4(struct Curl_cfilter *cf, cf->conn->ip_version, TRUE, &dns); if(result == CURLE_AGAIN) { - sxstate(sx, data, CONNECT_RESOLVING); - infof(data, "SOCKS4 non-blocking resolve of %s", sx->hostname); + sxstate(sx, cf, data, CONNECT_RESOLVING); + CURL_TRC_CF(data, cf, "SOCKS4 non-blocking resolve of %s", + sx->hostname); return CURLPX_OK; } else if(result) return CURLPX_RESOLVE_HOST; - sxstate(sx, data, CONNECT_RESOLVED); + sxstate(sx, cf, data, CONNECT_RESOLVED); goto CONNECT_RESOLVED; } /* socks4a does not resolve anything locally */ - sxstate(sx, data, CONNECT_REQ_INIT); + sxstate(sx, cf, data, CONNECT_REQ_INIT); goto CONNECT_REQ_INIT; case CONNECT_RESOLVING: @@ -362,8 +362,8 @@ CONNECT_RESOLVED: socksreq[6] = ((unsigned char *)&saddr_in->sin_addr.s_addr)[2]; socksreq[7] = ((unsigned char *)&saddr_in->sin_addr.s_addr)[3]; - infof(data, "SOCKS4 connect to IPv4 %s (locally resolved)", buf); - + CURL_TRC_CF(data, cf, "SOCKS4 connect to IPv4 %s (locally resolved)", + buf); Curl_resolv_unlink(data, &dns); /* not used anymore from now on */ } else @@ -424,7 +424,7 @@ CONNECT_REQ_INIT: sx->outp = socksreq; DEBUGASSERT(packetsize <= sizeof(sx->buffer)); sx->outstanding = packetsize; - sxstate(sx, data, CONNECT_REQ_SENDING); + sxstate(sx, cf, data, CONNECT_REQ_SENDING); } FALLTHROUGH(); case CONNECT_REQ_SENDING: @@ -440,7 +440,7 @@ CONNECT_REQ_INIT: /* done sending! */ sx->outstanding = 8; /* receive data size */ sx->outp = socksreq; - sxstate(sx, data, CONNECT_SOCKS_READ); + sxstate(sx, cf, data, CONNECT_SOCKS_READ); FALLTHROUGH(); case CONNECT_SOCKS_READ: @@ -453,7 +453,7 @@ CONNECT_REQ_INIT: /* remain in reading state */ return CURLPX_OK; } - sxstate(sx, data, CONNECT_DONE); + sxstate(sx, cf, data, CONNECT_DONE); break; default: /* lots of unused states in SOCKS4 */ break; @@ -488,11 +488,11 @@ CONNECT_REQ_INIT: /* Result */ switch(socksreq[1]) { case 90: - infof(data, "SOCKS4%s request granted.", protocol4a ? "a" : ""); + CURL_TRC_CF(data, cf, "SOCKS4%s request granted.", protocol4a ? "a" : ""); break; case 91: failf(data, - "cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" + "[SOCKS] cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" ", request rejected or failed.", socksreq[4], socksreq[5], socksreq[6], socksreq[7], (((unsigned char)socksreq[2] << 8) | (unsigned char)socksreq[3]), @@ -500,7 +500,7 @@ CONNECT_REQ_INIT: return CURLPX_REQUEST_FAILED; case 92: failf(data, - "cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" + "[SOCKS] cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" ", request rejected because SOCKS server cannot connect to " "identd on the client.", socksreq[4], socksreq[5], socksreq[6], socksreq[7], @@ -509,7 +509,7 @@ CONNECT_REQ_INIT: return CURLPX_IDENTD; case 93: failf(data, - "cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" + "[SOCKS] cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" ", request rejected because the client program and identd " "report different user-ids.", socksreq[4], socksreq[5], socksreq[6], socksreq[7], @@ -518,7 +518,7 @@ CONNECT_REQ_INIT: return CURLPX_IDENTD_DIFFER; default: failf(data, - "cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" + "[SOCKS] cannot complete SOCKS4 connection to %d.%d.%d.%d:%d. (%d)" ", Unknown.", socksreq[4], socksreq[5], socksreq[6], socksreq[7], (((unsigned char)socksreq[2] << 8) | (unsigned char)socksreq[3]), @@ -569,8 +569,8 @@ static CURLproxycode do_SOCKS5(struct Curl_cfilter *cf, switch(sx->state) { case CONNECT_SOCKS_INIT: if(conn->bits.httpproxy) - infof(data, "SOCKS5: connecting to HTTP proxy %s port %d", - sx->hostname, sx->remote_port); + CURL_TRC_CF(data, cf, "SOCKS5: connecting to HTTP proxy %s port %d", + sx->hostname, sx->remote_port); /* RFC1928 chapter 5 specifies max 255 chars for domain name in packet */ if(!socks5_resolve_local && hostname_len > 255) { @@ -580,9 +580,8 @@ static CURLproxycode do_SOCKS5(struct Curl_cfilter *cf, } if(auth & ~(CURLAUTH_BASIC | CURLAUTH_GSSAPI)) - infof(data, - "warning: unsupported value passed to CURLOPT_SOCKS5_AUTH: %u", - auth); + infof(data, "warning: unsupported value passed to " + "CURLOPT_SOCKS5_AUTH: %u", auth); if(!(auth & CURLAUTH_BASIC)) /* disable username/password auth */ sx->proxy_user = NULL; @@ -616,7 +615,7 @@ static CURLproxycode do_SOCKS5(struct Curl_cfilter *cf, /* remain in sending state */ return CURLPX_OK; } - sxstate(sx, data, CONNECT_SOCKS_READ); + sxstate(sx, cf, data, CONNECT_SOCKS_READ); goto CONNECT_SOCKS_READ_INIT; case CONNECT_SOCKS_SEND: presult = socks_state_send(cf, sx, data, CURLPX_SEND_CONNECT, @@ -648,17 +647,17 @@ CONNECT_SOCKS_READ_INIT: } else if(socksreq[1] == 0) { /* DONE! No authentication needed. Send request. */ - sxstate(sx, data, CONNECT_REQ_INIT); + sxstate(sx, cf, data, CONNECT_REQ_INIT); goto CONNECT_REQ_INIT; } else if(socksreq[1] == 2) { /* regular name + password authentication */ - sxstate(sx, data, CONNECT_AUTH_INIT); + sxstate(sx, cf, data, CONNECT_AUTH_INIT); goto CONNECT_AUTH_INIT; } #if defined(HAVE_GSSAPI) || defined(USE_WINDOWS_SSPI) else if(allow_gssapi && (socksreq[1] == 1)) { - sxstate(sx, data, CONNECT_GSSAPI_INIT); + sxstate(sx, cf, data, CONNECT_GSSAPI_INIT); result = Curl_SOCKS5_gssapi_negotiate(cf, data); if(result) { failf(data, "Unable to negotiate SOCKS5 GSS-API context."); @@ -732,7 +731,7 @@ CONNECT_AUTH_INIT: memcpy(socksreq + len, sx->proxy_password, proxy_password_len); } len += proxy_password_len; - sxstate(sx, data, CONNECT_AUTH_SEND); + sxstate(sx, cf, data, CONNECT_AUTH_SEND); DEBUGASSERT(len <= sizeof(sx->buffer)); sx->outstanding = len; sx->outp = socksreq; @@ -749,7 +748,7 @@ CONNECT_AUTH_INIT: } sx->outp = socksreq; sx->outstanding = 2; - sxstate(sx, data, CONNECT_AUTH_READ); + sxstate(sx, cf, data, CONNECT_AUTH_READ); FALLTHROUGH(); case CONNECT_AUTH_READ: presult = socks_state_recv(cf, sx, data, CURLPX_RECV_AUTH, @@ -768,7 +767,7 @@ CONNECT_AUTH_INIT: } /* Everything is good so far, user was authenticated! */ - sxstate(sx, data, CONNECT_REQ_INIT); + sxstate(sx, cf, data, CONNECT_REQ_INIT); FALLTHROUGH(); case CONNECT_REQ_INIT: CONNECT_REQ_INIT: @@ -777,12 +776,12 @@ CONNECT_REQ_INIT: cf->conn->ip_version, TRUE, &dns); if(result == CURLE_AGAIN) { - sxstate(sx, data, CONNECT_RESOLVING); + sxstate(sx, cf, data, CONNECT_RESOLVING); return CURLPX_OK; } else if(result) return CURLPX_RESOLVE_HOST; - sxstate(sx, data, CONNECT_RESOLVED); + sxstate(sx, cf, data, CONNECT_RESOLVED); goto CONNECT_RESOLVED; } goto CONNECT_RESOLVE_REMOTE; @@ -834,8 +833,8 @@ CONNECT_RESOLVED: socksreq[len++] = ((unsigned char *)&saddr_in->sin_addr.s_addr)[i]; } - infof(data, "SOCKS5 connect to %s:%d (locally resolved)", dest, - sx->remote_port); + CURL_TRC_CF(data, cf, "SOCKS5 connect to %s:%d (locally resolved)", + dest, sx->remote_port); } #ifdef USE_IPV6 else if(hp->ai_family == AF_INET6) { @@ -849,8 +848,8 @@ CONNECT_RESOLVED: ((unsigned char *)&saddr_in6->sin6_addr.s6_addr)[i]; } - infof(data, "SOCKS5 connect to [%s]:%d (locally resolved)", dest, - sx->remote_port); + CURL_TRC_CF(data, cf, "SOCKS5 connect to [%s]:%d (locally resolved)", + dest, sx->remote_port); } #endif else { @@ -896,8 +895,8 @@ CONNECT_RESOLVE_REMOTE: memcpy(&socksreq[len], sx->hostname, hostname_len); /* w/o NULL */ len += hostname_len; } - infof(data, "SOCKS5 connect to %s:%d (remotely resolved)", - sx->hostname, sx->remote_port); + CURL_TRC_CF(data, cf, "SOCKS5 connect to %s:%d (remotely resolved)", + sx->hostname, sx->remote_port); } FALLTHROUGH(); @@ -917,7 +916,7 @@ CONNECT_REQ_SEND: sx->outp = socksreq; DEBUGASSERT(len <= sizeof(sx->buffer)); sx->outstanding = len; - sxstate(sx, data, CONNECT_REQ_SENDING); + sxstate(sx, cf, data, CONNECT_REQ_SENDING); FALLTHROUGH(); case CONNECT_REQ_SENDING: presult = socks_state_send(cf, sx, data, CURLPX_SEND_REQUEST, @@ -936,7 +935,7 @@ CONNECT_REQ_SEND: #endif sx->outstanding = 10; /* minimum packet size is 10 */ sx->outp = socksreq; - sxstate(sx, data, CONNECT_REQ_READ); + sxstate(sx, cf, data, CONNECT_REQ_READ); FALLTHROUGH(); case CONNECT_REQ_READ: presult = socks_state_recv(cf, sx, data, CURLPX_RECV_REQACK, @@ -1018,10 +1017,10 @@ CONNECT_REQ_SEND: DEBUGASSERT(len <= sizeof(sx->buffer)); sx->outstanding = len - 10; /* get the rest */ sx->outp = &socksreq[10]; - sxstate(sx, data, CONNECT_REQ_READ_MORE); + sxstate(sx, cf, data, CONNECT_REQ_READ_MORE); } else { - sxstate(sx, data, CONNECT_DONE); + sxstate(sx, cf, data, CONNECT_DONE); break; } #if defined(HAVE_GSSAPI) || defined(USE_WINDOWS_SSPI) @@ -1037,9 +1036,9 @@ CONNECT_REQ_SEND: /* remain in reading state */ return CURLPX_OK; } - sxstate(sx, data, CONNECT_DONE); + sxstate(sx, cf, data, CONNECT_DONE); } - infof(data, "SOCKS5 request granted."); + CURL_TRC_CF(data, cf, "SOCKS5 request granted."); return CURLPX_OK; /* Proxy was successful! */ } @@ -1120,7 +1119,7 @@ static CURLcode socks_proxy_cf_connect(struct Curl_cfilter *cf, /* for the secondary socket (FTP), use the "connect to host" * but ignore the "connect to port" (use the secondary port) */ - sxstate(sx, data, CONNECT_SOCKS_INIT); + sxstate(sx, cf, data, CONNECT_SOCKS_INIT); sx->hostname = conn->bits.httpproxy ? conn->http_proxy.host.name : @@ -1212,7 +1211,7 @@ static CURLcode socks_cf_query(struct Curl_cfilter *cf, } struct Curl_cftype Curl_cft_socks_proxy = { - "SOCKS-PROXYY", + "SOCKS", CF_TYPE_IP_CONNECT|CF_TYPE_PROXY, 0, socks_proxy_cf_destroy, diff --git a/tests/http/CMakeLists.txt b/tests/http/CMakeLists.txt index 0c2d65f17f..5e415cbe7c 100644 --- a/tests/http/CMakeLists.txt +++ b/tests/http/CMakeLists.txt @@ -52,5 +52,11 @@ if(NOT HTTPD_NGHTTPX) endif() mark_as_advanced(HTTPD_NGHTTPX) -# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, VSFTPD +find_program(SOCKD "sockd") +if(NOT SOCKD) + set(SOCKD "") +endif() +mark_as_advanced(SOCKD) + +# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, SOCKD, VSFTPD configure_file("config.ini.in" "${CMAKE_CURRENT_BINARY_DIR}/config.ini" @ONLY) diff --git a/tests/http/Makefile.am b/tests/http/Makefile.am index 2a0c2547f4..03b0fbfb1f 100644 --- a/tests/http/Makefile.am +++ b/tests/http/Makefile.am @@ -28,6 +28,7 @@ testenv/caddy.py \ testenv/certs.py \ testenv/client.py \ testenv/curl.py \ +testenv/dante.py \ testenv/env.py \ testenv/httpd.py \ testenv/mod_curltest/mod_curltest.c \ @@ -64,6 +65,7 @@ test_20_websockets.py \ test_30_vsftpd.py \ test_31_vsftpds.py \ test_32_ftps_vsftpd.py \ +test_40_socks.py \ $(TESTENV) clean-local: diff --git a/tests/http/config.ini.in b/tests/http/config.ini.in index 7f9040d013..8026df92a3 100644 --- a/tests/http/config.ini.in +++ b/tests/http/config.ini.in @@ -37,3 +37,6 @@ caddy = @CADDY@ [vsftpd] vsftpd = @VSFTPD@ + +[sockd] +sockd = @SOCKD@ diff --git a/tests/http/scorecard.py b/tests/http/scorecard.py index 7f887eda4b..8e0e6ce5f8 100644 --- a/tests/http/scorecard.py +++ b/tests/http/scorecard.py @@ -34,7 +34,7 @@ import sys from statistics import mean from typing import Dict, Any, Optional, List -from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile +from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile, Dante log = logging.getLogger(__name__) @@ -187,7 +187,8 @@ class ScoreRunner: download_parallel: int = 0, server_addr: Optional[str] = None, with_dtrace: bool = False, - with_flame: bool = False): + with_flame: bool = False, + socks_args: Optional[List[str]] = None): self.verbose = verbose self.env = env self.protocol = protocol @@ -198,6 +199,7 @@ class ScoreRunner: self._download_parallel = download_parallel self._with_dtrace = with_dtrace self._with_flame = with_flame + self._socks_args = socks_args def info(self, msg): if self.verbose > 0: @@ -208,7 +210,8 @@ class ScoreRunner: return CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr, with_dtrace=self._with_dtrace, - with_flame=self._with_flame) + with_flame=self._with_flame, + socks_args=self._socks_args) def handshakes(self) -> Dict[str, Any]: props = {} @@ -368,9 +371,12 @@ class ScoreRunner: row.append(self.dl_parallel(url=url, count=count, nsamples=nsamples)) rows.append(row) self.info('done.\n') + title = f'Downloads from {meta["server"]}' + if self._socks_args: + title += f' via {self._socks_args}' return { 'meta': { - 'title': f'Downloads from {meta["server"]}', + 'title': title, 'count': count, 'max-parallel': max_parallel, }, @@ -477,9 +483,12 @@ class ScoreRunner: row.append(self.ul_parallel(url=url, fpath=fpath, count=count, nsamples=nsamples)) rows.append(row) self.info('done.\n') + title = f'Uploads to {meta["server"]}' + if self._socks_args: + title += f' via {self._socks_args}' return { 'meta': { - 'title': f'Uploads to {meta["server"]}', + 'title': title, 'count': count, 'max-parallel': max_parallel, }, @@ -538,9 +547,12 @@ class ScoreRunner: for mp in mparallel]) rows.append(row) self.info('done.\n') + title = f'Requests in parallel to {meta["server"]}' + if self._socks_args: + title += f' via {self._socks_args}' return { 'meta': { - 'title': f'Requests in parallel to {meta["server"]}', + 'title': title, 'count': count, }, 'cols': cols, @@ -660,6 +672,20 @@ def run_score(args, protocol): env = Env() env.setup() env.test_timeout = None + + sockd = None + socks_args = None + if args.socks4 and args.socks5: + raise ScoreCardError('unable to run --socks4 and --socks5 together') + elif args.socks4 or args.socks5: + sockd = Dante(env=env) + if sockd: + assert sockd.initial_start() + socks_args = [ + '--socks4' if args.socks4 else '--socks5', + f'127.0.0.1:{sockd.port}', + ] + httpd = None nghttpx = None caddy = None @@ -683,7 +709,8 @@ def run_score(args, protocol): curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, with_dtrace=args.dtrace, - with_flame=args.flame) + with_flame=args.flame, + socks_args=socks_args) cards.append(card) if test_httpd: @@ -709,7 +736,8 @@ def run_score(args, protocol): verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, with_dtrace=args.dtrace, - with_flame=args.flame) + with_flame=args.flame, + socks_args=socks_args) card.setup_resources(server_docs, downloads) cards.append(card) @@ -734,7 +762,8 @@ def run_score(args, protocol): server_port=server_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, - with_dtrace=args.dtrace) + with_dtrace=args.dtrace, + socks_args=socks_args) card.setup_resources(server_docs, downloads) cards.append(card) @@ -774,6 +803,8 @@ def run_score(args, protocol): nghttpx.stop(wait_dead=False) if httpd: httpd.stop() + if sockd: + sockd.stop() return rv @@ -849,6 +880,10 @@ def main(): parser.add_argument("--request-parallels", action='append', type=str, metavar='numberlist', default=None, help="evaluate request with these max-parallel numbers") + parser.add_argument("--socks4", action='store_true', + default=False, help="test with SOCKS4 proxy") + parser.add_argument("--socks5", action='store_true', + default=False, help="test with SOCKS5 proxy") args = parser.parse_args() if args.verbose > 0: diff --git a/tests/http/test_40_socks.py b/tests/http/test_40_socks.py new file mode 100644 index 0000000000..f119d8926c --- /dev/null +++ b/tests/http/test_40_socks.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +# +import logging +import os +from typing import Generator +import pytest + +from testenv import Env, CurlClient, Dante + + +log = logging.getLogger(__name__) + + +@pytest.mark.skipif(condition=not Env.has_sockd(), reason="missing sockd") +class TestSocks: + + @pytest.fixture(scope='class') + def sockd(self, env: Env) -> Generator[Dante, None, None]: + sockd = Dante(env=env) + assert sockd.initial_start() + yield sockd + sockd.stop() + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env, httpd): + indir = httpd.docs_dir + env.make_data_file(indir=indir, fname="data-10m", fsize=10*1024*1024) + env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) + + @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) + def test_40_01_socks_http(self, env: Env, sproto, sockd: Dante, httpd): + curl = CurlClient(env=env, socks_args=[ + f'--{sproto}', f'127.0.0.1:{sockd.port}' + ]) + url = f'http://{env.domain1}:{env.http_port}/data.json' + r = curl.http_get(url=url) + r.check_response(http_status=200) + + @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) + @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) + def test_40_02_socks_https(self, env: Env, sproto, proto, sockd: Dante, httpd): + if proto == 'h3' and not env.have_h3(): + pytest.skip("h3 not supported") + curl = CurlClient(env=env, socks_args=[ + f'--{sproto}', f'127.0.0.1:{sockd.port}' + ]) + url = f'https://{env.authority_for(env.domain1, proto)}/data.json' + r = curl.http_get(url=url, alpn_proto=proto) + if proto == 'h3': + assert r.exit_code == 3 # unsupported combination + else: + r.check_response(http_status=200) + + @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) + @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) + def test_40_03_dl_serial(self, env: Env, httpd, sockd, proto, sproto): + count = 3 + urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' + curl = CurlClient(env=env, socks_args=[ + f'--{sproto}', f'127.0.0.1:{sockd.port}' + ]) + r = curl.http_download(urls=[urln], alpn_proto=proto) + r.check_response(count=count, http_status=200) + + @pytest.mark.parametrize("sproto", ['socks4', 'socks5']) + @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) + def test_40_04_ul_serial(self, env: Env, httpd, sockd, proto, sproto): + fdata = os.path.join(env.gen_dir, 'data-10m') + count = 2 + curl = CurlClient(env=env, socks_args=[ + f'--{sproto}', f'127.0.0.1:{sockd.port}' + ]) + url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' + r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) + r.check_stats(count=count, http_status=200, exitcode=0) + indata = open(fdata).readlines() + for i in range(count): + respdata = open(curl.response_file(i)).readlines() + assert respdata == indata diff --git a/tests/http/testenv/__init__.py b/tests/http/testenv/__init__.py index 539af2aadf..fb2f9e53fe 100644 --- a/tests/http/testenv/__init__.py +++ b/tests/http/testenv/__init__.py @@ -36,3 +36,4 @@ from .curl import CurlClient, ExecResult, RunProfile from .client import LocalClient from .nghttpx import Nghttpx, NghttpxQuic, NghttpxFwd from .vsftpd import VsFTPD +from .dante import Dante diff --git a/tests/http/testenv/curl.py b/tests/http/testenv/curl.py index fb28cd7fa7..2f19eb3744 100644 --- a/tests/http/testenv/curl.py +++ b/tests/http/testenv/curl.py @@ -500,7 +500,8 @@ class CurlClient: run_env: Optional[Dict[str, str]] = None, server_addr: Optional[str] = None, with_dtrace: bool = False, - with_flame: bool = False): + with_flame: bool = False, + socks_args: Optional[List[str]] = None): self.env = env self._timeout = timeout if timeout else env.test_timeout self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl @@ -513,6 +514,7 @@ class CurlClient: self._with_flame = with_flame if self._with_flame: self._with_dtrace = True + self._socks_args = socks_args self._silent = silent self._run_env = run_env self._server_addr = server_addr if server_addr else '127.0.0.1' @@ -898,6 +900,9 @@ class CurlClient: if 'CURL_TEST_EVENT' in os.environ: args.append('--test-event') + if self._socks_args: + args.extend(self._socks_args) + if with_headers: args.extend(["-D", self._headerfile]) if def_tracing is not False and not self._silent: diff --git a/tests/http/testenv/dante.py b/tests/http/testenv/dante.py new file mode 100644 index 0000000000..995dce6a61 --- /dev/null +++ b/tests/http/testenv/dante.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +# +import logging +import os +import socket +import subprocess + +from typing import Dict + +from .env import Env +from .ports import alloc_ports_and_do + +log = logging.getLogger(__name__) + + +class Dante: + + def __init__(self, env: Env): + self.env = env + self._cmd = env.sockd + self._port = 0 + self.name = 'sockd' + self._port_skey = 'sockd' + self._port_specs = { + 'sockd': socket.SOCK_STREAM, + } + self._dante_dir = os.path.join(env.gen_dir, self.name) + self._run_dir = os.path.join(self._dante_dir, 'run') + self._tmp_dir = os.path.join(self._dante_dir, 'tmp') + self._conf_file = os.path.join(self._dante_dir, 'test.conf') + self._dante_log = os.path.join(self._dante_dir, 'dante.log') + self._error_log = os.path.join(self._dante_dir, 'error.log') + self._pid_file = os.path.join(self._dante_dir, 'dante.pid') + self._process = None + + self.clear_logs() + + @property + def port(self) -> int: + return self._port + + def clear_logs(self): + self._rmf(self._error_log) + self._rmf(self._dante_log) + + def exists(self): + return os.path.exists(self._cmd) + + def is_running(self): + if self._process: + self._process.poll() + return self._process.returncode is None + return False + + def start_if_needed(self): + if not self.is_running(): + return self.start() + return True + + def stop(self, wait_dead=True): + self._mkpath(self._tmp_dir) + if self._process: + self._process.terminate() + self._process.wait(timeout=2) + self._process = None + return not wait_dead or True + return True + + def restart(self): + self.stop() + return self.start() + + def initial_start(self): + + def startup(ports: Dict[str, int]) -> bool: + self._port = ports[self._port_skey] + if self.start(): + self.env.update_ports(ports) + return True + self.stop() + self._port = 0 + return False + + return alloc_ports_and_do(self._port_specs, startup, + self.env.gen_root, max_tries=3) + + def start(self, wait_live=True): + assert self._port > 0 + self._mkpath(self._tmp_dir) + if self._process: + self.stop() + self._write_config() + args = [ + self._cmd, + '-f', f'{self._conf_file}', + '-p', f'{self._pid_file}', + '-d', '0', + ] + procerr = open(self._error_log, 'a') + self._process = subprocess.Popen(args=args, stderr=procerr) + if self._process.returncode is not None: + return False + return True + + def _rmf(self, path): + if os.path.exists(path): + return os.remove(path) + + def _mkpath(self, path): + if not os.path.exists(path): + return os.makedirs(path) + + def _write_config(self): + conf = [ + f'errorlog: {self._error_log}', + f'logoutput: {self._dante_log}', + f'internal: 127.0.0.1 port = {self._port}', + 'external: 127.0.0.1', + 'clientmethod: none', + 'socksmethod: none', + 'client pass {', + ' from: 127.0.0.0/24 to: 0.0.0.0/0', + ' log: error', + '}', + 'socks pass {', + ' from: 0.0.0.0/0 to: 0.0.0.0/0', + ' command: bindreply connect udpreply', + ' log: error', + '}', + '\n', + ] + with open(self._conf_file, 'w') as fd: + fd.write("\n".join(conf)) diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py index c12b7d9945..6142440756 100644 --- a/tests/http/testenv/env.py +++ b/tests/http/testenv/env.py @@ -225,6 +225,8 @@ class EnvConfig: self.caddy = None self.vsftpd = self.config['vsftpd']['vsftpd'] + if self.vsftpd == '': + self.vsftpd = None self._vsftpd_version = None if self.vsftpd is not None: try: @@ -254,6 +256,29 @@ class EnvConfig: except Exception: self.vsftpd = None + self.sockd = self.config['sockd']['sockd'] + if self.sockd == '': + self.sockd = None + self._sockd_version = None + if self.sockd is not None: + try: + p = subprocess.run(args=[self.sockd, '-v'], + capture_output=True, text=True) + assert p.returncode == 0 + if p.returncode != 0: + # not a working vsftpd + self.sockd = None + m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stdout) + if not m: + m = re.match(r'^Dante v(\d+\.\d+\.\d+).*', p.stderr) + if m: + self._sockd_version = m.group(1) + else: + self.sockd = None + raise Exception(f'Unable to determine sockd version from: {p.stderr}') + except Exception: + self.sockd = None + self._tcpdump = shutil.which('tcpdump') @property @@ -482,6 +507,10 @@ class Env: def vsftpd_version() -> str: return Env.CONFIG.vsftpd_version + @staticmethod + def has_sockd() -> bool: + return Env.CONFIG.sockd is not None + @staticmethod def tcpdump() -> Optional[str]: return Env.CONFIG.tcpdmp @@ -643,6 +672,10 @@ class Env: def caddy_http_port(self) -> int: return self.CONFIG.ports['caddy'] + @property + def sockd(self) -> str: + return self.CONFIG.sockd + @property def vsftpd(self) -> str: return self.CONFIG.vsftpd