18.4 --proxycommand
18.5 UTF-8 filenames in Content-Disposition
18.6 Option to make -Z merge lined based outputs on stdout
- 18.7 at least N milliseconds between requests
18.8 Consider convenience options for JSON and XML?
18.9 Choose the name of file in braces for complex URLs
18.10 improve how curl works in a windows console window
https://github.com/curl/curl/issues/5175
-18.7 at least N milliseconds between requests
-
- Allow curl command lines issue a lot of request against services that limit
- users to no more than N requests/second or similar. Could be implemented with
- an option asking that at least a certain time has elapsed since the previous
- request before the next one will be performed. Example:
-
- $ curl "https://example.com/api?input=[1-1000]" -d yadayada --after 500
-
- See https://github.com/curl/curl/issues/3920
-
18.8 Consider convenience options for JSON and XML?
Could we add `--xml` or `--json` to add headers needed to call rest API:
quote.d \
random-file.d \
range.d \
+ rate.d \
raw.d \
referer.d \
remote-header-name.d \
--- /dev/null
+Long: rate
+Arg: <max request rate>
+Help: Request rate for serial transfers
+Category: connection
+Example: --rate 2/s $URL
+Example: --rate 3/h $URL
+Example: --rate 14/m $URL
+Added: 7.84.0
+See-also: limit-rate retry-delay
+---
+Specify the maximum transfer frequency you allow curl to use - in number of
+transfer starts per time unit (sometimes called request rate). Without this
+option, curl will start the next transfer as fast as possible.
+
+If given several URLs and a transfer completes faster than the allowed rate,
+curl will wait until the next transfer is started to maintain the requested
+rate. This option has no effect when --parallel is used.
+
+The request rate is provided as "N/U" where N is an integer number and U is a
+time unit. Supported units are 's' (second), 'm' (minute), 'h' (hour) and 'd'
+/(day, as in a 24 hour unit). The default time unit, if no "/U" is provided,
+is number of transfers per hour.
+
+If curl is told to allow 10 requests per minute, it will not start the next
+request until 6 seconds have elapsed since the previous transfer was started.
+
+This function uses millisecond resolution. If the allowed frequency is set
+more than 1000 per second, it will instead run unrestricted.
+
+When retrying transfers, enabled with --retry, the separate retry delay logic
+is used and not this setting.
+
+If this option is used several times, the last one will be used.
--quote (-Q) 5.3
--random-file 7.7
--range (-r) 4.0
+--rate 7.84.0
--raw 7.16.2
--referer (-e) 4.0
--remote-header-name (-J) 7.20.0
char *libcurl; /* Output libcurl code to this file name */
bool fail_early; /* exit on first transfer error */
bool styled_output; /* enable fancy output style detection */
+ long ms_per_transfer; /* start next transfer after (at least) this
+ many milliseconds */
#ifdef CURLDEBUG
bool test_event_based;
#endif
{"*h", "trace-ascii", ARG_FILENAME},
{"*H", "alpn", ARG_BOOL},
{"*i", "limit-rate", ARG_STRING},
+ {"*I", "rate", ARG_STRING},
{"*j", "compressed", ARG_BOOL},
{"*J", "tr-encoding", ARG_BOOL},
{"*k", "digest", ARG_BOOL},
config->sendpersecond = value;
}
break;
+ case 'I': /* --rate (request rate) */
+ {
+ /* support a few different suffixes, extract the suffix first, then
+ get the number and convert to per hour.
+ /s == per second
+ /m == per minute
+ /h == per hour (default)
+ /d == per day (24 hours)
+ */
+ char *div = strchr(nextarg, '/');
+ char number[26];
+ long denominator;
+ long numerator = 60*60*1000; /* default per hour */
+ size_t numlen = div ? (size_t)(div - nextarg) : strlen(nextarg);
+ if(numlen > sizeof(number)-1)
+ return PARAM_NUMBER_TOO_LARGE;
+ strncpy(number, nextarg, numlen);
+ number[numlen] = 0;
+ err = str2unum(&denominator, number);
+ if(err)
+ return err;
+ if(denominator < 1)
+ return PARAM_BAD_USE;
+ if(div) {
+ char unit = div[1];
+ switch(unit) {
+ case 's': /* per second */
+ numerator = 1000;
+ break;
+ case 'm': /* per minute */
+ numerator = 60*1000;
+ break;
+ case 'h': /* per hour */
+ break;
+ case 'd': /* per day */
+ numerator = 24*60*60*1000;
+ break;
+ default:
+ errorf(global, "unsupported --rate unit\n");
+ return PARAM_BAD_USE;
+ }
+ }
+ global->ms_per_transfer = numerator/denominator;
+ }
+ break;
case 'j': /* --compressed */
if(toggle &&
{"-r, --range <range>",
"Retrieve only the bytes within RANGE",
CURLHELP_HTTP | CURLHELP_FTP | CURLHELP_SFTP | CURLHELP_FILE},
+ {" --rate <max request rate>",
+ "Request rate for serial transfers",
+ CURLHELP_CONNECTION},
{" --raw",
"Do HTTP \"raw\"; no transfer decoding",
CURLHELP_HTTP},
}
for(per = transfers; per;) {
bool retry;
- long delay;
+ long delay_ms;
bool bailout = FALSE;
+ struct timeval start;
result = pre_transfer(global, per);
if(result)
break;
break;
}
#endif
+ start = tvnow();
#ifdef CURLDEBUG
if(global->test_event_based)
result = curl_easy_perform_ev(per->curl);
#endif
result = curl_easy_perform(per->curl);
- returncode = post_per_transfer(global, per, result, &retry, &delay);
+ returncode = post_per_transfer(global, per, result, &retry, &delay_ms);
if(retry) {
- tool_go_sleep(delay);
+ tool_go_sleep(delay_ms);
continue;
}
if(bailout)
break;
+
+ if(per && global->ms_per_transfer) {
+ /* how long time did the most recent transfer take in number of
+ milliseconds */
+ long milli = tvdiff(tvnow(), start);
+ if(milli < global->ms_per_transfer) {
+ notef(global, "Transfer took %ld ms, waits %ldms as set by --rate\n",
+ milli, global->ms_per_transfer - milli);
+ /* The transfer took less time than wanted. Wait a little. */
+ tool_go_sleep(global->ms_per_transfer - milli);
+ }
+ }
}
if(returncode)
/* returncode errors have priority */
projects / thirdparty / curl.git / commitdiff
