tool_operate: prevent over-queuing in parallel mode

When doing a huge amount of parallel transfers, we must not add them to
the per_transfer list frivolously since they all use memory after all.
This was previous done without really considering millions or billions
of transfers. Massive parallelism would use a lot of memory for no good
purpose.

The queue is now limited to twice the paralleism number.

This makes the 'Qd' value in the parallel progress meter mostly useless
for users, but works for now for us as a debug display.

Reported-by: justchen1369 on github
Fixes #8933
Closes #9389

diff --git a/src/tool_operate.c b/src/tool_operate.c
index 87981d21eae757043b422abf942d2ad748427345..19e1c7b6b7e6d4761770a42a8066a958733d2606 100644 (file)
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -205,6 +205,7 @@ static curl_off_t VmsSpecialSize(const char *name,
 
 struct per_transfer *transfers; /* first node */
 static struct per_transfer *transfersl; /* last node */
+static long all_pers;
 
 /* add_per_transfer creates a new 'per_transfer' node in the linked
    list of transfers */
@@ -227,6 +228,8 @@ static CURLcode add_per_transfer(struct per_transfer **per)
   }
   *per = p;
   all_xfers++; /* count total number of transfers added */
+  all_pers++;
+
   return CURLE_OK;
 }
 
@@ -254,6 +257,7 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per)
     transfersl = p;
 
   free(per);
+  all_pers--;
 
   return n;
 }
@@ -2194,9 +2198,11 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global,
   bool sleeping = FALSE;
   *addedp = FALSE;
   *morep = FALSE;
-  result = create_transfer(global, share, addedp);
-  if(result)
-    return result;
+  if(all_pers < (global->parallel_max*2)) {
+    result = create_transfer(global, share, addedp);
+    if(result)
+      return result;
+  }
   for(per = transfers; per && (all_added < global->parallel_max);
       per = per->next) {
     bool getadded = FALSE;