tool: fix memory use in parallel mode

The curl tool was creating a new transfer every time it checked if
it needed to add one to reach max parallelism. This led to eventually
all configured transfers to have easy handles created.

Limit the creation again to the ones needed for max parallelism.

scorecard.py: set --out-null only for curl versions that support it

Closes #22277

diff --git a/src/tool_operate.c b/src/tool_operate.c
index 8ccd791bf53fc06a9670a2870f08aa1790a2952b..56e3544ca2bb012f4aedec5848592d4cb2b555b5 100644 (file)
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -1552,7 +1552,13 @@ static CURLcode add_parallel_transfers(CURLM *multi, CURLSH *share,
     return CURLE_UNKNOWN_OPTION;
   }
 
-  if(nxfers < (curl_off_t)(global->parallel_max * 2)) {
+  if(all_added >= global->parallel_max) {
+    /* we are at max parallelism, no need to create more transfers */
+    *morep = TRUE; /* pretend there are, we have not checked */
+    return CURLE_OK;
+  }
+  /* if the list is empty, create one to kickstart the loop */
+  if(!transfers) {
     bool skipped = FALSE;
     do {
       result = create_transfer(share, addedp, &skipped);
@@ -1560,6 +1566,7 @@ static CURLcode add_parallel_transfers(CURLM *multi, CURLSH *share,
         return result;
     } while(skipped);
   }
+  /* add transfers until max parallelism is achieved again */
   for(per = transfers; per && (all_added < global->parallel_max);
       per = per->next) {
     if(per->added || per->skip)

diff --git a/tests/http/testenv/curl.py b/tests/http/testenv/curl.py
index 3071d7dd4aa94f8525f285ee9491f2651c9bf5ed..90b7b288567356015abdf109dab1b3550f32e937 100644 (file)
--- a/tests/http/testenv/curl.py
+++ b/tests/http/testenv/curl.py
@@ -729,7 +729,10 @@ class CurlClient:
         if extra_args is None:
             extra_args = []
         if no_save:
-            extra_args.extend(['--out-null'])
+            if self.env.curl_version_at_least('8.16.0'):
+                extra_args.extend(['--out-null'])
+            else:
+                extra_args.extend(['-o', '/dev/null'])
         else:
             extra_args.extend(['-o', 'download_#1.data'])
         if limit_rate:

diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py
index 4a18d65ffab68776f23c343578b402c36cb19dc6..78ab3abec949ec6451c2056f24b28c54322665de 100644 (file)
--- a/tests/http/testenv/env.py
+++ b/tests/http/testenv/env.py
@@ -512,6 +512,13 @@ class Env:
     def curl_version_string() -> str:
         return Env.CONFIG.curl_props["version_string"]
 
+    @staticmethod
+    def curl_version_at_least(min_version) -> bool:
+        version = Env.curl_version()
+        return Env.CONFIG.versiontuple(min_version) <= Env.CONFIG.versiontuple(
+                version
+            )
+
     @staticmethod
     def curl_features_string() -> str:
         return Env.CONFIG.curl_props["features_string"]