From: Daniel Stenberg Date: Mon, 23 May 2022 15:59:56 +0000 (+0200) Subject: curl: add --rate to set max request rate per time unit X-Git-Tag: curl-7_84_0~146 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8f48b5d783bd3cc15fc2669d76c1d4931be1c501;p=thirdparty%2Fcurl.git curl: add --rate to set max request rate per time unit --rate "12/m" - for 12 per minute or --rate "5/h" - for 5 per hour Removed from TODO Closes #8671 --- diff --git a/docs/TODO b/docs/TODO index 5c5a77a835..97afcec3ac 100644 --- a/docs/TODO +++ b/docs/TODO @@ -149,7 +149,6 @@ 18.4 --proxycommand 18.5 UTF-8 filenames in Content-Disposition 18.6 Option to make -Z merge lined based outputs on stdout - 18.7 at least N milliseconds between requests 18.8 Consider convenience options for JSON and XML? 18.9 Choose the name of file in braces for complex URLs 18.10 improve how curl works in a windows console window @@ -999,17 +998,6 @@ https://github.com/curl/curl/issues/5175 -18.7 at least N milliseconds between requests - - Allow curl command lines issue a lot of request against services that limit - users to no more than N requests/second or similar. Could be implemented with - an option asking that at least a certain time has elapsed since the previous - request before the next one will be performed. Example: - - $ curl "https://example.com/api?input=[1-1000]" -d yadayada --after 500 - - See https://github.com/curl/curl/issues/3920 - 18.8 Consider convenience options for JSON and XML? Could we add `--xml` or `--json` to add headers needed to call rest API: diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc index 87819e087b..e486c77b6e 100644 --- a/docs/cmdline-opts/Makefile.inc +++ b/docs/cmdline-opts/Makefile.inc @@ -198,6 +198,7 @@ DPAGES = \ quote.d \ random-file.d \ range.d \ + rate.d \ raw.d \ referer.d \ remote-header-name.d \ diff --git a/docs/cmdline-opts/rate.d b/docs/cmdline-opts/rate.d new file mode 100644 index 0000000000..cf4a1709ed --- /dev/null +++ b/docs/cmdline-opts/rate.d @@ -0,0 +1,33 @@ +Long: rate +Arg: +Help: Request rate for serial transfers +Category: connection +Example: --rate 2/s $URL +Example: --rate 3/h $URL +Example: --rate 14/m $URL +Added: 7.84.0 +See-also: limit-rate retry-delay +--- +Specify the maximum transfer frequency you allow curl to use - in number of +transfer starts per time unit (sometimes called request rate). Without this +option, curl will start the next transfer as fast as possible. + +If given several URLs and a transfer completes faster than the allowed rate, +curl will wait until the next transfer is started to maintain the requested +rate. This option has no effect when --parallel is used. + +The request rate is provided as "N/U" where N is an integer number and U is a +time unit. Supported units are 's' (second), 'm' (minute), 'h' (hour) and 'd' +/(day, as in a 24 hour unit). The default time unit, if no "/U" is provided, +is number of transfers per hour. + +If curl is told to allow 10 requests per minute, it will not start the next +request until 6 seconds have elapsed since the previous transfer was started. + +This function uses millisecond resolution. If the allowed frequency is set +more than 1000 per second, it will instead run unrestricted. + +When retrying transfers, enabled with --retry, the separate retry delay logic +is used and not this setting. + +If this option is used several times, the last one will be used. diff --git a/docs/options-in-versions b/docs/options-in-versions index e6359ab928..a6c75cfd09 100644 --- a/docs/options-in-versions +++ b/docs/options-in-versions @@ -186,6 +186,7 @@ --quote (-Q) 5.3 --random-file 7.7 --range (-r) 4.0 +--rate 7.84.0 --raw 7.16.2 --referer (-e) 4.0 --remote-header-name (-J) 7.20.0 diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 058bbd41c5..20a653c87d 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -323,6 +323,8 @@ struct GlobalConfig { char *libcurl; /* Output libcurl code to this file name */ bool fail_early; /* exit on first transfer error */ bool styled_output; /* enable fancy output style detection */ + long ms_per_transfer; /* start next transfer after (at least) this + many milliseconds */ #ifdef CURLDEBUG bool test_event_based; #endif diff --git a/src/tool_getparam.c b/src/tool_getparam.c index 020567565e..bd459847b3 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -93,6 +93,7 @@ static const struct LongShort aliases[]= { {"*h", "trace-ascii", ARG_FILENAME}, {"*H", "alpn", ARG_BOOL}, {"*i", "limit-rate", ARG_STRING}, + {"*I", "rate", ARG_STRING}, {"*j", "compressed", ARG_BOOL}, {"*J", "tr-encoding", ARG_BOOL}, {"*k", "digest", ARG_BOOL}, @@ -738,6 +739,51 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ config->sendpersecond = value; } break; + case 'I': /* --rate (request rate) */ + { + /* support a few different suffixes, extract the suffix first, then + get the number and convert to per hour. + /s == per second + /m == per minute + /h == per hour (default) + /d == per day (24 hours) + */ + char *div = strchr(nextarg, '/'); + char number[26]; + long denominator; + long numerator = 60*60*1000; /* default per hour */ + size_t numlen = div ? (size_t)(div - nextarg) : strlen(nextarg); + if(numlen > sizeof(number)-1) + return PARAM_NUMBER_TOO_LARGE; + strncpy(number, nextarg, numlen); + number[numlen] = 0; + err = str2unum(&denominator, number); + if(err) + return err; + if(denominator < 1) + return PARAM_BAD_USE; + if(div) { + char unit = div[1]; + switch(unit) { + case 's': /* per second */ + numerator = 1000; + break; + case 'm': /* per minute */ + numerator = 60*1000; + break; + case 'h': /* per hour */ + break; + case 'd': /* per day */ + numerator = 24*60*60*1000; + break; + default: + errorf(global, "unsupported --rate unit\n"); + return PARAM_BAD_USE; + } + } + global->ms_per_transfer = numerator/denominator; + } + break; case 'j': /* --compressed */ if(toggle && diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c index 93b7b899ea..8c35789558 100644 --- a/src/tool_listhelp.c +++ b/src/tool_listhelp.c @@ -559,6 +559,9 @@ const struct helptxt helptext[] = { {"-r, --range ", "Retrieve only the bytes within RANGE", CURLHELP_HTTP | CURLHELP_FTP | CURLHELP_SFTP | CURLHELP_FILE}, + {" --rate ", + "Request rate for serial transfers", + CURLHELP_CONNECTION}, {" --raw", "Do HTTP \"raw\"; no transfer decoding", CURLHELP_HTTP}, diff --git a/src/tool_operate.c b/src/tool_operate.c index e977cce7ad..07766f3c7b 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -2359,8 +2359,9 @@ static CURLcode serial_transfers(struct GlobalConfig *global, } for(per = transfers; per;) { bool retry; - long delay; + long delay_ms; bool bailout = FALSE; + struct timeval start; result = pre_transfer(global, per); if(result) break; @@ -2372,6 +2373,7 @@ static CURLcode serial_transfers(struct GlobalConfig *global, break; } #endif + start = tvnow(); #ifdef CURLDEBUG if(global->test_event_based) result = curl_easy_perform_ev(per->curl); @@ -2379,9 +2381,9 @@ static CURLcode serial_transfers(struct GlobalConfig *global, #endif result = curl_easy_perform(per->curl); - returncode = post_per_transfer(global, per, result, &retry, &delay); + returncode = post_per_transfer(global, per, result, &retry, &delay_ms); if(retry) { - tool_go_sleep(delay); + tool_go_sleep(delay_ms); continue; } @@ -2399,6 +2401,18 @@ static CURLcode serial_transfers(struct GlobalConfig *global, if(bailout) break; + + if(per && global->ms_per_transfer) { + /* how long time did the most recent transfer take in number of + milliseconds */ + long milli = tvdiff(tvnow(), start); + if(milli < global->ms_per_transfer) { + notef(global, "Transfer took %ld ms, waits %ldms as set by --rate\n", + milli, global->ms_per_transfer - milli); + /* The transfer took less time than wanted. Wait a little. */ + tool_go_sleep(global->ms_per_transfer - milli); + } + } } if(returncode) /* returncode errors have priority */