curl: add --parallel-max-host to limit concurrent connections per host

Where 'host' is protocol + hostname + portnumber.

Closes #18052

diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc
index c5f99477faeaf0e37d294f3c08cdf8678eeda518..4696ce1cd358e8acf08870fa1d4798f080c9a202 100644 (file)
--- a/docs/cmdline-opts/Makefile.inc
+++ b/docs/cmdline-opts/Makefile.inc
@@ -184,6 +184,7 @@ DPAGES = \
   out-null.md \
   output.md \
   parallel-immediate.md \
+  parallel-max-host.md \
   parallel-max.md \
   parallel.md \
   pass.md \

diff --git a/docs/cmdline-opts/parallel-max-host.md b/docs/cmdline-opts/parallel-max-host.md
new file mode 100644 (file)
index 0000000..a21c9ac
--- /dev/null
+++ b/docs/cmdline-opts/parallel-max-host.md
@@ -0,0 +1,28 @@
+---
+c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+SPDX-License-Identifier: curl
+Long: parallel-max-host
+Arg: <num>
+Help: Maximum connections to a single host
+Added: 8.16.0
+Category: connection curl global
+Multi: single
+Scope: global
+See-also:
+  - parallel
+  - parallel-max
+Example:
+  - --parallel-max-host 5 -Z $URL ftp://example.com/
+---
+
+# `--parallel-max-host`
+
+When asked to do parallel transfers, using --parallel, this option controls
+the maximum amount of concurrent connections curl is allowed to do to the same
+protocol + hostname + port number target.
+
+The limit is enforced by libcurl and queued "internally", which means that
+transfers that are waiting for an available connection still look like started
+transfers in the progress meter.
+
+The default is 0 (unlimited). 65535 is the largest supported value.

diff --git a/docs/cmdline-opts/parallel-max.md b/docs/cmdline-opts/parallel-max.md
index fc5470eb02969ea36b1e81ecea83654c681a9577..a487f4cc9ead4ab8049e5f5fffa48333456f294e 100644 (file)
--- a/docs/cmdline-opts/parallel-max.md
+++ b/docs/cmdline-opts/parallel-max.md
@@ -10,6 +10,7 @@ Multi: single
 Scope: global
 See-also:
   - parallel
+  - parallel-max-host
 Example:
   - --parallel-max 100 -Z $URL ftp://example.com/
 ---

diff --git a/docs/options-in-versions b/docs/options-in-versions
index bf34e76f2c34b3350c583aa6a099a065763b795c..52c5c67aa430d3171108de77327e945728c64afa 100644 (file)
--- a/docs/options-in-versions
+++ b/docs/options-in-versions
@@ -151,6 +151,7 @@
 --parallel (-Z)                      7.66.0
 --parallel-immediate                 7.68.0
 --parallel-max                       7.66.0
+--parallel-max-host                  8.16.0
 --pass                               7.9.3
 --path-as-is                         7.42.0
 --pinnedpubkey                       7.39.0

diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index d23902885e4d92ee1579a76015c8098408f60c87..de59185edfa42361880e1dfe5275d7b3220c1963 100644 (file)
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -357,6 +357,7 @@ struct GlobalConfig {
                                      many milliseconds */
   trace tracetype;
   int progressmode;               /* CURL_PROGRESS_BAR / CURL_PROGRESS_STATS */
+  unsigned short parallel_host; /* MAX_PARALLEL_HOST is the maximum */
   unsigned short parallel_max; /* MAX_PARALLEL is the maximum */
   unsigned char verbosity;        /* How verbose we should be */
 #ifdef DEBUGBUILD

diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 4540a5913649419284b6bbe918a2904e1dafce03..78af98f5e2a8e9ca66d2e5bf758c08a9a133f2f5 100644 (file)
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -230,6 +230,7 @@ static const struct LongShort aliases[]= {
   {"parallel",                   ARG_BOOL, 'Z', C_PARALLEL},
   {"parallel-immediate",         ARG_BOOL, ' ', C_PARALLEL_IMMEDIATE},
   {"parallel-max",               ARG_STRG, ' ', C_PARALLEL_MAX},
+  {"parallel-max-host",          ARG_STRG, ' ', C_PARALLEL_HOST},
   {"pass",                       ARG_STRG|ARG_CLEAR, ' ', C_PASS},
   {"path-as-is",                 ARG_BOOL, ' ', C_PATH_AS_IS},
   {"pinnedpubkey",               ARG_STRG|ARG_TLS, ' ', C_PINNEDPUBKEY},
@@ -2179,6 +2180,7 @@ static ParameterError opt_filestring(struct OperationConfig *config,
 {
   ParameterError err = PARAM_OK;
   curl_off_t value;
+  long val;
   struct GlobalConfig *global = config->global;
   static const char *redir_protos[] = {
     "http",
@@ -2788,8 +2790,19 @@ static ParameterError opt_filestring(struct OperationConfig *config,
     if(!err && !config->low_speed_time)
       config->low_speed_time = 30;
     break;
-  case C_PARALLEL_MAX: {  /* --parallel-max */
-    long val;
+  case C_PARALLEL_HOST: /* --parallel-max-host */
+    err = str2unum(&val, nextarg);
+    if(err)
+      break;
+    if(val > MAX_PARALLEL_HOST)
+      global->parallel_host = MAX_PARALLEL_HOST;
+    else if(val < 1)
+      global->parallel_host = PARALLEL_HOST_DEFAULT;
+    else
+      global->parallel_host = (unsigned short)val;
+    break;
+    break;
+  case C_PARALLEL_MAX:  /* --parallel-max */
     err = str2unum(&val, nextarg);
     if(err)
       break;
@@ -2800,7 +2813,6 @@ static ParameterError opt_filestring(struct OperationConfig *config,
     else
       global->parallel_max = (unsigned short)val;
     break;
-  }
   case C_TIME_COND: /* --time-cond */
     err = parse_time_cond(config, nextarg);
     break;

diff --git a/src/tool_getparam.h b/src/tool_getparam.h
index 47e87eab9264d5c6e91cdde6ad48e7f6b3042271..fede165efbd680fb20227e2f4ce590d58f20897b 100644 (file)
--- a/src/tool_getparam.h
+++ b/src/tool_getparam.h
@@ -171,6 +171,7 @@ typedef enum {
   C_OUTPUT,
   C_OUTPUT_DIR,
   C_PARALLEL,
+  C_PARALLEL_HOST,
   C_PARALLEL_IMMEDIATE,
   C_PARALLEL_MAX,
   C_PASS,

diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c
index 8572250b5c3ce9622d6a08e3fea912ec9ef8c3a6..83b0519c70ecd386a8fbfced8e67bf97f8d16487 100644 (file)
--- a/src/tool_listhelp.c
+++ b/src/tool_listhelp.c
@@ -460,6 +460,9 @@ const struct helptxt helptext[] = {
   {"    --parallel-max <num>",
    "Maximum concurrency for parallel transfers",
    CURLHELP_CONNECTION | CURLHELP_CURL | CURLHELP_GLOBAL},
+  {"    --parallel-max-host <num>",
+   "Maximum connections to a single host",
+   CURLHELP_CONNECTION | CURLHELP_CURL | CURLHELP_GLOBAL},
   {"    --pass <phrase>",
    "Passphrase for the private key",
    CURLHELP_SSH | CURLHELP_TLS | CURLHELP_AUTH},

diff --git a/src/tool_main.h b/src/tool_main.h
index 1e32ee7f8d2ce69f1dcdc12bd9103bd3083503e9..53ee9a3702a711e7d52ae2d7c3ec070b93aebd12 100644 (file)
--- a/src/tool_main.h
+++ b/src/tool_main.h
@@ -33,4 +33,7 @@
 #define MAX_PARALLEL 65535
 #define PARALLEL_DEFAULT 50
 
+#define MAX_PARALLEL_HOST 65535
+#define PARALLEL_HOST_DEFAULT 0 /* means not used */
+
 #endif /* HEADER_CURL_TOOL_MAIN_H */

diff --git a/src/tool_operate.c b/src/tool_operate.c
index 7e09528844c135c73ec84dc60723b6730234936d..0c605b80242faf3f51ec9f4ad7c8608e46af548d 100644 (file)
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -1689,6 +1689,8 @@ static CURLcode parallel_event(struct parastate *s)
   curl_multi_setopt(s->multi, CURLMOPT_SOCKETDATA, &uv);
   curl_multi_setopt(s->multi, CURLMOPT_TIMERFUNCTION, cb_timeout);
   curl_multi_setopt(s->multi, CURLMOPT_TIMERDATA, &uv);
+  curl_multi_setopt(s->multi, CURLMOPT_MAX_HOST_CONNECTIONS,
+                    s->global->parallel_host);
 
   /* kickstart the thing */
   curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0,