From: Daniel Stenberg Date: Mon, 28 Jul 2025 08:41:20 +0000 (+0200) Subject: curl: add --parallel-max-host to limit concurrent connections per host X-Git-Tag: curl-8_16_0~297 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=4654493fedfccbae18bd9c30903456e348ab937c;p=thirdparty%2Fcurl.git curl: add --parallel-max-host to limit concurrent connections per host Where 'host' is protocol + hostname + portnumber. Closes #18052 --- diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc index c5f99477fa..4696ce1cd3 100644 --- a/docs/cmdline-opts/Makefile.inc +++ b/docs/cmdline-opts/Makefile.inc @@ -184,6 +184,7 @@ DPAGES = \ out-null.md \ output.md \ parallel-immediate.md \ + parallel-max-host.md \ parallel-max.md \ parallel.md \ pass.md \ diff --git a/docs/cmdline-opts/parallel-max-host.md b/docs/cmdline-opts/parallel-max-host.md new file mode 100644 index 0000000000..a21c9acddd --- /dev/null +++ b/docs/cmdline-opts/parallel-max-host.md @@ -0,0 +1,28 @@ +--- +c: Copyright (C) Daniel Stenberg, , et al. +SPDX-License-Identifier: curl +Long: parallel-max-host +Arg: +Help: Maximum connections to a single host +Added: 8.16.0 +Category: connection curl global +Multi: single +Scope: global +See-also: + - parallel + - parallel-max +Example: + - --parallel-max-host 5 -Z $URL ftp://example.com/ +--- + +# `--parallel-max-host` + +When asked to do parallel transfers, using --parallel, this option controls +the maximum amount of concurrent connections curl is allowed to do to the same +protocol + hostname + port number target. + +The limit is enforced by libcurl and queued "internally", which means that +transfers that are waiting for an available connection still look like started +transfers in the progress meter. + +The default is 0 (unlimited). 65535 is the largest supported value. diff --git a/docs/cmdline-opts/parallel-max.md b/docs/cmdline-opts/parallel-max.md index fc5470eb02..a487f4cc9e 100644 --- a/docs/cmdline-opts/parallel-max.md +++ b/docs/cmdline-opts/parallel-max.md @@ -10,6 +10,7 @@ Multi: single Scope: global See-also: - parallel + - parallel-max-host Example: - --parallel-max 100 -Z $URL ftp://example.com/ --- diff --git a/docs/options-in-versions b/docs/options-in-versions index bf34e76f2c..52c5c67aa4 100644 --- a/docs/options-in-versions +++ b/docs/options-in-versions @@ -151,6 +151,7 @@ --parallel (-Z) 7.66.0 --parallel-immediate 7.68.0 --parallel-max 7.66.0 +--parallel-max-host 8.16.0 --pass 7.9.3 --path-as-is 7.42.0 --pinnedpubkey 7.39.0 diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index d23902885e..de59185edf 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -357,6 +357,7 @@ struct GlobalConfig { many milliseconds */ trace tracetype; int progressmode; /* CURL_PROGRESS_BAR / CURL_PROGRESS_STATS */ + unsigned short parallel_host; /* MAX_PARALLEL_HOST is the maximum */ unsigned short parallel_max; /* MAX_PARALLEL is the maximum */ unsigned char verbosity; /* How verbose we should be */ #ifdef DEBUGBUILD diff --git a/src/tool_getparam.c b/src/tool_getparam.c index 4540a59136..78af98f5e2 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -230,6 +230,7 @@ static const struct LongShort aliases[]= { {"parallel", ARG_BOOL, 'Z', C_PARALLEL}, {"parallel-immediate", ARG_BOOL, ' ', C_PARALLEL_IMMEDIATE}, {"parallel-max", ARG_STRG, ' ', C_PARALLEL_MAX}, + {"parallel-max-host", ARG_STRG, ' ', C_PARALLEL_HOST}, {"pass", ARG_STRG|ARG_CLEAR, ' ', C_PASS}, {"path-as-is", ARG_BOOL, ' ', C_PATH_AS_IS}, {"pinnedpubkey", ARG_STRG|ARG_TLS, ' ', C_PINNEDPUBKEY}, @@ -2179,6 +2180,7 @@ static ParameterError opt_filestring(struct OperationConfig *config, { ParameterError err = PARAM_OK; curl_off_t value; + long val; struct GlobalConfig *global = config->global; static const char *redir_protos[] = { "http", @@ -2788,8 +2790,19 @@ static ParameterError opt_filestring(struct OperationConfig *config, if(!err && !config->low_speed_time) config->low_speed_time = 30; break; - case C_PARALLEL_MAX: { /* --parallel-max */ - long val; + case C_PARALLEL_HOST: /* --parallel-max-host */ + err = str2unum(&val, nextarg); + if(err) + break; + if(val > MAX_PARALLEL_HOST) + global->parallel_host = MAX_PARALLEL_HOST; + else if(val < 1) + global->parallel_host = PARALLEL_HOST_DEFAULT; + else + global->parallel_host = (unsigned short)val; + break; + break; + case C_PARALLEL_MAX: /* --parallel-max */ err = str2unum(&val, nextarg); if(err) break; @@ -2800,7 +2813,6 @@ static ParameterError opt_filestring(struct OperationConfig *config, else global->parallel_max = (unsigned short)val; break; - } case C_TIME_COND: /* --time-cond */ err = parse_time_cond(config, nextarg); break; diff --git a/src/tool_getparam.h b/src/tool_getparam.h index 47e87eab92..fede165efb 100644 --- a/src/tool_getparam.h +++ b/src/tool_getparam.h @@ -171,6 +171,7 @@ typedef enum { C_OUTPUT, C_OUTPUT_DIR, C_PARALLEL, + C_PARALLEL_HOST, C_PARALLEL_IMMEDIATE, C_PARALLEL_MAX, C_PASS, diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c index 8572250b5c..83b0519c70 100644 --- a/src/tool_listhelp.c +++ b/src/tool_listhelp.c @@ -460,6 +460,9 @@ const struct helptxt helptext[] = { {" --parallel-max ", "Maximum concurrency for parallel transfers", CURLHELP_CONNECTION | CURLHELP_CURL | CURLHELP_GLOBAL}, + {" --parallel-max-host ", + "Maximum connections to a single host", + CURLHELP_CONNECTION | CURLHELP_CURL | CURLHELP_GLOBAL}, {" --pass ", "Passphrase for the private key", CURLHELP_SSH | CURLHELP_TLS | CURLHELP_AUTH}, diff --git a/src/tool_main.h b/src/tool_main.h index 1e32ee7f8d..53ee9a3702 100644 --- a/src/tool_main.h +++ b/src/tool_main.h @@ -33,4 +33,7 @@ #define MAX_PARALLEL 65535 #define PARALLEL_DEFAULT 50 +#define MAX_PARALLEL_HOST 65535 +#define PARALLEL_HOST_DEFAULT 0 /* means not used */ + #endif /* HEADER_CURL_TOOL_MAIN_H */ diff --git a/src/tool_operate.c b/src/tool_operate.c index 7e09528844..0c605b8024 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -1689,6 +1689,8 @@ static CURLcode parallel_event(struct parastate *s) curl_multi_setopt(s->multi, CURLMOPT_SOCKETDATA, &uv); curl_multi_setopt(s->multi, CURLMOPT_TIMERFUNCTION, cb_timeout); curl_multi_setopt(s->multi, CURLMOPT_TIMERDATA, &uv); + curl_multi_setopt(s->multi, CURLMOPT_MAX_HOST_CONNECTIONS, + s->global->parallel_host); /* kickstart the thing */ curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0,