From 1ad2009ad63478ace18977ec5314f597f45ad084 Mon Sep 17 00:00:00 2001 From: Stefan Eissing Date: Wed, 23 Jul 2025 09:18:59 +0200 Subject: [PATCH] multi: add new information extraction method Adds `curl_off_t curl_multi_get_offt(CURLM *multi_handle, CURLMinfo_offt info)` to the multi interface with enums: * CURLMINFO_XFERS_CURRENT: current number of transfers * CURLMINFO_XFERS_RUNNING: number of running transfers * CURLMINFO_XFERS_PENDING: number of pending transfers * CURLMINFO_XFERS_DONE: number of finished transfers to read * CURLMINFO_XFERS_ADDED: total number of transfers added, ever Add documentation for functions and info enums. Add use in the curl command line tool to replace two static variables counting the same "from the outside". refs #17870 Closes #17992 --- .github/scripts/spellcheck.curl | 1 + docs/libcurl/Makefile.inc | 1 + docs/libcurl/curl_multi_get_offt.md | 102 ++++++++++++++++++++++++++++ docs/libcurl/symbols-in-versions | 6 ++ include/curl/multi.h | 30 ++++++++ lib/libcurl.def | 1 + lib/multi.c | 38 +++++++++++ lib/multihandle.h | 1 + lib/uint-bset.c | 4 +- scripts/singleuse.pl | 1 + src/tool_operate.c | 18 +++-- src/tool_progress.c | 14 ++-- src/tool_progress.h | 3 +- tests/data/test1135 | 1 + 14 files changed, 203 insertions(+), 18 deletions(-) create mode 100644 docs/libcurl/curl_multi_get_offt.md diff --git a/.github/scripts/spellcheck.curl b/.github/scripts/spellcheck.curl index 4de9d86596..c24edf2b3b 100644 --- a/.github/scripts/spellcheck.curl +++ b/.github/scripts/spellcheck.curl @@ -132,6 +132,7 @@ curl_multi_timeout curl_multi_setopt curl_multi_assign curl_multi_get_handles +curl_multi_get_offt curl_pushheader_bynum curl_pushheader_byname curl_multi_waitfds diff --git a/docs/libcurl/Makefile.inc b/docs/libcurl/Makefile.inc index 83357cf42b..9142f65fab 100644 --- a/docs/libcurl/Makefile.inc +++ b/docs/libcurl/Makefile.inc @@ -75,6 +75,7 @@ man_MANS = \ curl_multi_cleanup.3 \ curl_multi_fdset.3 \ curl_multi_get_handles.3 \ + curl_multi_get_offt.3 \ curl_multi_info_read.3 \ curl_multi_init.3 \ curl_multi_perform.3 \ diff --git a/docs/libcurl/curl_multi_get_offt.md b/docs/libcurl/curl_multi_get_offt.md new file mode 100644 index 0000000000..cc7873df47 --- /dev/null +++ b/docs/libcurl/curl_multi_get_offt.md @@ -0,0 +1,102 @@ +--- +c: Copyright (C) Daniel Stenberg, , et al. +SPDX-License-Identifier: curl +Title: curl_multi_get_offt +Section: 3 +Source: libcurl +See-also: + - curl_multi_add_handle (3) + - curl_multi_remove_handle (3) +Protocol: + - All +Added-in: 8.16.0 +--- + +# NAME + +curl_multi_get_offt - extract information from a multi handle + +# SYNOPSIS + +~~~c +#include + +CURLMcode curl_multi_get_offt(CURLM *multi_handle, + CURLMinfo_offt info, + curl_off_t *pvalue); +~~~ + +# DESCRIPTION + +Get the *info* kept in the *multi* handle for `CURLMI_OFFT_*`. +If the *info* is not applicable, this function returns CURLM_UNKNOWN_OPTION. + +# OPTIONS + +The following information can be extracted: + +## CURLMINFO_XFERS_CURRENT + +The number of easy handles currently added to the multi. This does not +count handles removed. It does count internal handles that get +added for tasks (like resolving via DoH, for example). + +For the total number of easy handles ever added to the multi, see +*CURLMINFO_XFERS_ADDED*. + +## CURLMINFO_XFERS_RUNNING + +The number of easy handles currently running, e.g. where the transfer +has started but not finished yet. + +## CURLMINFO_XFERS_PENDING + +The number of current easy handles waiting to start. An added transfer +might become pending for various reasons: a connection limit forces it +to wait, resolving DNS is not finished or it is not clear if an existing, +matching connection may allow multiplexing (HTTP/2 or HTTP/3). + +## CURLMINFO_XFERS_DONE + +The number of easy handles currently finished, but not yet processed +via curl_multi_info_read(3). + +## CURLMINFO_XFERS_ADDED + +The cumulative number of all easy handles added to the multi, ever. +This includes internal handles added for tasks (like resolving +via DoH, for example). + +For the current number of easy handles managed by the multi, use +*CURLMINFO_XFERS_CURRENT*. + +# %PROTOCOLS% + +# EXAMPLE + +~~~c +int main(void) +{ + /* init a multi stack */ + CURLM *multi = curl_multi_init(); + CURL *curl = curl_easy_init(); + curl_off_t n; + + if(curl) { + /* add the transfer */ + curl_multi_add_handle(multi, curl); + + curl_multi_get_offt(multi, CURLMINFO_XFERS_ADDED, &n); + /* on successful add, n is 1 */ + } +} +~~~ + +# %AVAILABILITY% + +# RETURN VALUE + +This function returns a CURLMcode indicating success or error. + +CURLM_OK (0) means everything was OK, non-zero means an error occurred, +see libcurl-errors(3). diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index 6387ea3014..d2a4cc598e 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -553,6 +553,12 @@ CURLM_RECURSIVE_API_CALL 7.59.0 CURLM_UNKNOWN_OPTION 7.15.4 CURLM_UNRECOVERABLE_POLL 7.84.0 CURLM_WAKEUP_FAILURE 7.68.0 +CURLMINFO_NONE 8.16.0 +CURLMINFO_XFERS_CURRENT 8.16.0 +CURLMINFO_XFERS_RUNNING 8.16.0 +CURLMINFO_XFERS_PENDING 8.16.0 +CURLMINFO_XFERS_DONE 8.16.0 +CURLMINFO_XFERS_ADDED 8.16.0 CURLMIMEOPT_FORMESCAPE 7.81.0 CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE 7.30.0 CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE 7.30.0 diff --git a/include/curl/multi.h b/include/curl/multi.h index 0fbea88707..e336c94a88 100644 --- a/include/curl/multi.h +++ b/include/curl/multi.h @@ -448,6 +448,36 @@ CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle, */ CURL_EXTERN CURL **curl_multi_get_handles(CURLM *multi_handle); + +typedef enum { + CURLMINFO_NONE, /* first, never use this */ + /* The number of easy handles currently managed by the multi handle, + * e.g. have been added but not yet removed. */ + CURLMINFO_XFERS_CURRENT = 1, + /* The number of easy handles running, e.g. not done and not queueing. */ + CURLMINFO_XFERS_RUNNING = 2, + /* The number of easy handles waiting to start, e.g. for a connection + * to become available due to limits on parallelism, max connections + * or other factors. */ + CURLMINFO_XFERS_PENDING = 3, + /* The number of easy handles finished, waiting for their results to + * be read via `curl_multi_info_read()`. */ + CURLMINFO_XFERS_DONE = 4, + /* The total number of easy handles added to the multi handle, ever. */ + CURLMINFO_XFERS_ADDED = 5 +} CURLMinfo_offt; + +/* + * Name: curl_multi_get_offt() + * + * Desc: Retrieves a numeric value for the `CURLMINFO_*` enums. + * + * Returns: CULRM_OK or error when value could not be obtained. + */ +CURL_EXTERN CURLMcode curl_multi_get_offt(CURLM *multi_handle, + CURLMinfo_offt info, + curl_off_t *pvalue); + /* * Name: curl_push_callback * diff --git a/lib/libcurl.def b/lib/libcurl.def index 43e26f655c..ae64776dc2 100644 --- a/lib/libcurl.def +++ b/lib/libcurl.def @@ -54,6 +54,7 @@ curl_multi_assign curl_multi_cleanup curl_multi_fdset curl_multi_get_handles +curl_multi_get_offt curl_multi_info_read curl_multi_init curl_multi_perform diff --git a/lib/multi.c b/lib/multi.c index b5a6970eee..9a653cf959 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -479,6 +479,7 @@ CURLMcode curl_multi_add_handle(CURLM *m, CURL *d) /* add the easy handle to the process set */ Curl_uint_bset_add(&multi->process, data->mid); ++multi->xfers_alive; + ++multi->xfers_total_ever; Curl_cpool_xfer_init(data); multi_warn_debug(multi, data); @@ -3741,6 +3742,43 @@ CURL **curl_multi_get_handles(CURLM *m) return a; } +CURLMcode curl_multi_get_offt(CURLM *m, + CURLMinfo_offt info, + curl_off_t *pvalue) +{ + struct Curl_multi *multi = m; + + if(!GOOD_MULTI_HANDLE(multi)) + return CURLM_BAD_HANDLE; + if(!pvalue) + return CURLM_BAD_FUNCTION_ARGUMENT; + + switch(info) { + case CURLMINFO_XFERS_CURRENT: { + unsigned int n = Curl_uint_tbl_count(&multi->xfers); + if(n && multi->admin) + --n; + *pvalue = (curl_off_t)n; + return CURLM_OK; + } + case CURLMINFO_XFERS_RUNNING: + *pvalue = (curl_off_t)Curl_uint_bset_count(&multi->process); + return CURLM_OK; + case CURLMINFO_XFERS_PENDING: + *pvalue = (curl_off_t)Curl_uint_bset_count(&multi->pending); + return CURLM_OK; + case CURLMINFO_XFERS_DONE: + *pvalue = (curl_off_t)Curl_uint_bset_count(&multi->msgsent); + return CURLM_OK; + case CURLMINFO_XFERS_ADDED: + *pvalue = multi->xfers_total_ever; + return CURLM_OK; + default: + *pvalue = -1; + return CURLM_UNKNOWN_OPTION; + } +} + CURLcode Curl_multi_xfer_buf_borrow(struct Curl_easy *data, char **pbuf, size_t *pbuflen) { diff --git a/lib/multihandle.h b/lib/multihandle.h index cdedfb08ab..ae41044adc 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -89,6 +89,7 @@ struct Curl_multi { unsigned int xfers_alive; /* amount of added transfers that have not yet reached COMPLETE state */ + curl_off_t xfers_total_ever; /* total of added transfers, ever. */ struct uint_tbl xfers; /* transfers added to this multi */ /* Each transfer's mid may be present in at most one of these */ struct uint_bset process; /* transfer being processed */ diff --git a/lib/uint-bset.c b/lib/uint-bset.c index e612c390a6..1c60f22adf 100644 --- a/lib/uint-bset.c +++ b/lib/uint-bset.c @@ -79,8 +79,9 @@ UNITTEST unsigned int Curl_uint_bset_capacity(struct uint_bset *bset) { return bset->nslots * 64; } +#endif -UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset) +unsigned int Curl_uint_bset_count(struct uint_bset *bset) { unsigned int i; unsigned int n = 0; @@ -90,7 +91,6 @@ UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset) } return n; } -#endif bool Curl_uint_bset_empty(struct uint_bset *bset) { diff --git a/scripts/singleuse.pl b/scripts/singleuse.pl index 6156496019..b47c85c995 100755 --- a/scripts/singleuse.pl +++ b/scripts/singleuse.pl @@ -111,6 +111,7 @@ my %api = ( 'curl_multi_cleanup' => 'API', 'curl_multi_fdset' => 'API', 'curl_multi_get_handles' => 'API', + 'curl_multi_get_offt' => 'API', 'curl_multi_info_read' => 'API', 'curl_multi_init' => 'API', 'curl_multi_perform' => 'API', diff --git a/src/tool_operate.c b/src/tool_operate.c index 0c605b8024..11abc1f9d7 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -207,7 +207,6 @@ static curl_off_t VmsSpecialSize(const char *name, struct per_transfer *transfers; /* first node */ static struct per_transfer *transfersl; /* last node */ -static curl_off_t all_pers; /* add_per_transfer creates a new 'per_transfer' node in the linked list of transfers */ @@ -229,8 +228,6 @@ static CURLcode add_per_transfer(struct per_transfer **per) transfersl = p; } *per = p; - all_xfers++; /* count total number of transfers added */ - all_pers++; return CURLE_OK; } @@ -259,7 +256,6 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per) transfersl = p; free(per); - all_pers--; return n; } @@ -1420,9 +1416,17 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global, CURLMcode mcode; bool sleeping = FALSE; char *errorbuf; + curl_off_t nxfers; + *addedp = FALSE; *morep = FALSE; - if(all_pers < (global->parallel_max*2)) { + mcode = curl_multi_get_offt(multi, CURLMINFO_XFERS_CURRENT, &nxfers); + if(mcode) { + DEBUGASSERT(0); + return CURLE_UNKNOWN_OPTION; + } + + if(nxfers < (curl_off_t)(global->parallel_max*2)) { bool skipped = FALSE; do { result = create_transfer(global, share, addedp, &skipped); @@ -1762,7 +1766,7 @@ static CURLcode check_finished(struct parastate *s) CURLMsg *msg; bool checkmore = FALSE; struct GlobalConfig *global = s->global; - progress_meter(global, &s->start, FALSE); + progress_meter(global, s->multi, &s->start, FALSE); do { msg = curl_multi_info_read(s->multi, &rc); if(msg) { @@ -1887,7 +1891,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global, result = check_finished(s); } - (void)progress_meter(global, &s->start, TRUE); + (void)progress_meter(global, s->multi, &s->start, TRUE); } /* Make sure to return some kind of error if there was a multi problem */ diff --git a/src/tool_progress.c b/src/tool_progress.c index 635f8e25ee..a477dbac26 100644 --- a/src/tool_progress.c +++ b/src/tool_progress.c @@ -134,8 +134,6 @@ static curl_off_t all_ultotal = 0; static curl_off_t all_dlalready = 0; static curl_off_t all_ulalready = 0; -curl_off_t all_xfers = 0; /* current total */ - struct speedcount { curl_off_t dl; curl_off_t ul; @@ -151,6 +149,7 @@ static struct speedcount speedstore[SPEEDCNT]; | 6 -- 9.9G 0 2 2 0:00:40 0:00:02 0:00:37 4087M */ bool progress_meter(struct GlobalConfig *global, + CURLM *multi, struct curltime *start, bool final) { @@ -182,9 +181,10 @@ bool progress_meter(struct GlobalConfig *global, struct per_transfer *per; curl_off_t all_dlnow = 0; curl_off_t all_ulnow = 0; + curl_off_t xfers_added = 0; + curl_off_t xfers_running = 0; bool dlknown = TRUE; bool ulknown = TRUE; - curl_off_t all_running = 0; /* in progress */ curl_off_t speed = 0; unsigned int i; stamp = now; @@ -210,8 +210,6 @@ bool progress_meter(struct GlobalConfig *global, all_ultotal += per->ultotal; per->ultotal_added = TRUE; } - if(per->added) - all_running++; } if(dlknown && all_dltotal) msnprintf(dlpercen, sizeof(dlpercen), "%3" CURL_FORMAT_CURL_OFF_T, @@ -274,6 +272,8 @@ bool progress_meter(struct GlobalConfig *global, } time2str(time_spent, spent); + (void)curl_multi_get_offt(multi, CURLMINFO_XFERS_ADDED, &xfers_added); + (void)curl_multi_get_offt(multi, CURLMINFO_XFERS_RUNNING, &xfers_running); fprintf(tool_stderr, "\r" "%-3s " /* percent downloaded */ @@ -292,8 +292,8 @@ bool progress_meter(struct GlobalConfig *global, ulpercen, /* 3 letters */ max5data(all_dlnow, buffer[0]), max5data(all_ulnow, buffer[1]), - all_xfers, - all_running, + xfers_added, + xfers_running, time_total, time_spent, time_left, diff --git a/src/tool_progress.h b/src/tool_progress.h index 02c34d42ee..d9009504a1 100644 --- a/src/tool_progress.h +++ b/src/tool_progress.h @@ -32,10 +32,9 @@ int xferinfo_cb(void *clientp, curl_off_t ulnow); bool progress_meter(struct GlobalConfig *global, + CURLM *multi, struct curltime *start, bool final); void progress_finalize(struct per_transfer *per); -extern curl_off_t all_xfers; /* total number */ - #endif /* HEADER_CURL_TOOL_PROGRESS_H */ diff --git a/tests/data/test1135 b/tests/data/test1135 index 070d8dff4c..73668d9994 100644 --- a/tests/data/test1135 +++ b/tests/data/test1135 @@ -109,6 +109,7 @@ curl_multi_timeout curl_multi_setopt curl_multi_assign curl_multi_get_handles +curl_multi_get_offt curl_pushheader_bynum curl_pushheader_byname curl_multi_waitfds -- 2.47.2