From: Stefan Eissing Date: Thu, 1 Aug 2024 11:05:49 +0000 (+0200) Subject: progress: ratelimit/progress tweaks X-Git-Tag: curl-8_10_0~438 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ba44ac62e322b5667104e619e77aa7586bd3d9d6;p=thirdparty%2Fcurl.git progress: ratelimit/progress tweaks - multi.c: when ratelimiting a transfer stops (MSTATE_RATELIMITING -> MSTATE_PERFORMING), run the MSTATE_PERFORMING state right away - urldata.h: factor out upload and download progress counters into a struct, use that for passing these to progress update functions - progress.c/getinfo.c: change names of moved progress counters - progress.c: use new structs and a helper struct to factor repeated calculation into static helpers Closes #14335 --- diff --git a/lib/getinfo.c b/lib/getinfo.c index 9ca02889c6..03bf85ae65 100644 --- a/lib/getinfo.c +++ b/lib/getinfo.c @@ -384,24 +384,24 @@ static CURLcode getinfo_offt(struct Curl_easy *data, CURLINFO info, *param_offt = (curl_off_t)data->info.filetime; break; case CURLINFO_SIZE_UPLOAD_T: - *param_offt = data->progress.uploaded; + *param_offt = data->progress.ul.cur_size; break; case CURLINFO_SIZE_DOWNLOAD_T: - *param_offt = data->progress.downloaded; + *param_offt = data->progress.dl.cur_size; break; case CURLINFO_SPEED_DOWNLOAD_T: - *param_offt = data->progress.dlspeed; + *param_offt = data->progress.dl.speed; break; case CURLINFO_SPEED_UPLOAD_T: - *param_offt = data->progress.ulspeed; + *param_offt = data->progress.ul.speed; break; case CURLINFO_CONTENT_LENGTH_DOWNLOAD_T: *param_offt = (data->progress.flags & PGRS_DL_SIZE_KNOWN)? - data->progress.size_dl:-1; + data->progress.dl.total_size:-1; break; case CURLINFO_CONTENT_LENGTH_UPLOAD_T: *param_offt = (data->progress.flags & PGRS_UL_SIZE_KNOWN)? - data->progress.size_ul:-1; + data->progress.ul.total_size:-1; break; case CURLINFO_TOTAL_TIME_T: *param_offt = data->progress.timespent; @@ -488,24 +488,24 @@ static CURLcode getinfo_double(struct Curl_easy *data, CURLINFO info, *param_doublep = DOUBLE_SECS(data->progress.t_starttransfer); break; case CURLINFO_SIZE_UPLOAD: - *param_doublep = (double)data->progress.uploaded; + *param_doublep = (double)data->progress.ul.cur_size; break; case CURLINFO_SIZE_DOWNLOAD: - *param_doublep = (double)data->progress.downloaded; + *param_doublep = (double)data->progress.dl.cur_size; break; case CURLINFO_SPEED_DOWNLOAD: - *param_doublep = (double)data->progress.dlspeed; + *param_doublep = (double)data->progress.dl.speed; break; case CURLINFO_SPEED_UPLOAD: - *param_doublep = (double)data->progress.ulspeed; + *param_doublep = (double)data->progress.ul.speed; break; case CURLINFO_CONTENT_LENGTH_DOWNLOAD: *param_doublep = (data->progress.flags & PGRS_DL_SIZE_KNOWN)? - (double)data->progress.size_dl:-1; + (double)data->progress.dl.total_size:-1; break; case CURLINFO_CONTENT_LENGTH_UPLOAD: *param_doublep = (data->progress.flags & PGRS_UL_SIZE_KNOWN)? - (double)data->progress.size_ul:-1; + (double)data->progress.ul.total_size:-1; break; case CURLINFO_REDIRECT_TIME: *param_doublep = DOUBLE_SECS(data->progress.t_redirect); diff --git a/lib/multi.c b/lib/multi.c index 896f730490..4a787f0059 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -2384,24 +2384,22 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, send_timeout_ms = 0; if(data->set.max_send_speed) send_timeout_ms = - Curl_pgrsLimitWaitTime(data->progress.uploaded, - data->progress.ul_limit_size, + Curl_pgrsLimitWaitTime(&data->progress.ul, data->set.max_send_speed, - data->progress.ul_limit_start, *nowp); recv_timeout_ms = 0; if(data->set.max_recv_speed) recv_timeout_ms = - Curl_pgrsLimitWaitTime(data->progress.downloaded, - data->progress.dl_limit_size, + Curl_pgrsLimitWaitTime(&data->progress.dl, data->set.max_recv_speed, - data->progress.dl_limit_start, *nowp); if(!send_timeout_ms && !recv_timeout_ms) { multistate(data, MSTATE_PERFORMING); Curl_ratelimit(data, *nowp); + /* start performing again right away */ + rc = CURLM_CALL_MULTI_PERFORM; } else if(send_timeout_ms >= recv_timeout_ms) Curl_expire(data, send_timeout_ms, EXPIRE_TOOFAST); @@ -2417,19 +2415,15 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, /* check if over send speed */ send_timeout_ms = 0; if(data->set.max_send_speed) - send_timeout_ms = Curl_pgrsLimitWaitTime(data->progress.uploaded, - data->progress.ul_limit_size, + send_timeout_ms = Curl_pgrsLimitWaitTime(&data->progress.ul, data->set.max_send_speed, - data->progress.ul_limit_start, *nowp); /* check if over recv speed */ recv_timeout_ms = 0; if(data->set.max_recv_speed) - recv_timeout_ms = Curl_pgrsLimitWaitTime(data->progress.downloaded, - data->progress.dl_limit_size, + recv_timeout_ms = Curl_pgrsLimitWaitTime(&data->progress.dl, data->set.max_recv_speed, - data->progress.dl_limit_start, *nowp); if(send_timeout_ms || recv_timeout_ms) { diff --git a/lib/progress.c b/lib/progress.c index 34c29a2701..67f4ba9c30 100644 --- a/lib/progress.c +++ b/lib/progress.c @@ -252,12 +252,12 @@ void Curl_pgrsStartNow(struct Curl_easy *data) data->progress.speeder_c = 0; /* reset the progress meter display */ data->progress.start = Curl_now(); data->progress.is_t_startransfer_set = false; - data->progress.ul_limit_start = data->progress.start; - data->progress.dl_limit_start = data->progress.start; - data->progress.ul_limit_size = 0; - data->progress.dl_limit_size = 0; - data->progress.downloaded = 0; - data->progress.uploaded = 0; + data->progress.ul.limit.start = data->progress.start; + data->progress.dl.limit.start = data->progress.start; + data->progress.ul.limit.start_size = 0; + data->progress.dl.limit.start_size = 0; + data->progress.dl.cur_size = 0; + data->progress.ul.cur_size = 0; /* clear all bits except HIDE and HEADERS_OUT */ data->progress.flags &= PGRS_HIDE|PGRS_HEADERS_OUT; Curl_ratelimit(data, data->progress.start); @@ -281,17 +281,15 @@ void Curl_pgrsStartNow(struct Curl_easy *data) * starting point should be reset (to current); or the number of milliseconds * to wait to get back under the speed limit. */ -timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize, - curl_off_t startsize, - curl_off_t limit, - struct curltime start, +timediff_t Curl_pgrsLimitWaitTime(struct pgrs_dir *d, + curl_off_t speed_limit, struct curltime now) { - curl_off_t size = cursize - startsize; + curl_off_t size = d->cur_size - d->limit.start_size; timediff_t minimum; timediff_t actual; - if(!limit || !size) + if(!speed_limit || !size) return 0; /* @@ -299,9 +297,9 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize, * stay below 'limit'. */ if(size < CURL_OFF_T_MAX/1000) - minimum = (timediff_t) (CURL_OFF_T_C(1000) * size / limit); + minimum = (timediff_t) (CURL_OFF_T_C(1000) * size / speed_limit); else { - minimum = (timediff_t) (size / limit); + minimum = (timediff_t) (size / speed_limit); if(minimum < TIMEDIFF_T_MAX/1000) minimum *= 1000; else @@ -312,7 +310,7 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize, * 'actual' is the time in milliseconds it took to actually download the * last 'size' bytes. */ - actual = Curl_timediff_ceil(now, start); + actual = Curl_timediff_ceil(now, d->limit.start); if(actual < minimum) { /* if it downloaded the data faster than the limit, make it wait the difference */ @@ -327,7 +325,7 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize, */ CURLcode Curl_pgrsSetDownloadCounter(struct Curl_easy *data, curl_off_t size) { - data->progress.downloaded = size; + data->progress.dl.cur_size = size; return CURLE_OK; } @@ -338,17 +336,17 @@ void Curl_ratelimit(struct Curl_easy *data, struct curltime now) { /* do not set a new stamp unless the time since last update is long enough */ if(data->set.max_recv_speed) { - if(Curl_timediff(now, data->progress.dl_limit_start) >= + if(Curl_timediff(now, data->progress.dl.limit.start) >= MIN_RATE_LIMIT_PERIOD) { - data->progress.dl_limit_start = now; - data->progress.dl_limit_size = data->progress.downloaded; + data->progress.dl.limit.start = now; + data->progress.dl.limit.start_size = data->progress.dl.cur_size; } } if(data->set.max_send_speed) { - if(Curl_timediff(now, data->progress.ul_limit_start) >= + if(Curl_timediff(now, data->progress.ul.limit.start) >= MIN_RATE_LIMIT_PERIOD) { - data->progress.ul_limit_start = now; - data->progress.ul_limit_size = data->progress.uploaded; + data->progress.ul.limit.start = now; + data->progress.ul.limit.start_size = data->progress.ul.cur_size; } } } @@ -358,17 +356,17 @@ void Curl_ratelimit(struct Curl_easy *data, struct curltime now) */ void Curl_pgrsSetUploadCounter(struct Curl_easy *data, curl_off_t size) { - data->progress.uploaded = size; + data->progress.ul.cur_size = size; } void Curl_pgrsSetDownloadSize(struct Curl_easy *data, curl_off_t size) { if(size >= 0) { - data->progress.size_dl = size; + data->progress.dl.total_size = size; data->progress.flags |= PGRS_DL_SIZE_KNOWN; } else { - data->progress.size_dl = 0; + data->progress.dl.total_size = 0; data->progress.flags &= ~PGRS_DL_SIZE_KNOWN; } } @@ -376,11 +374,11 @@ void Curl_pgrsSetDownloadSize(struct Curl_easy *data, curl_off_t size) void Curl_pgrsSetUploadSize(struct Curl_easy *data, curl_off_t size) { if(size >= 0) { - data->progress.size_ul = size; + data->progress.ul.total_size = size; data->progress.flags |= PGRS_UL_SIZE_KNOWN; } else { - data->progress.size_ul = 0; + data->progress.ul.total_size = 0; data->progress.flags &= ~PGRS_UL_SIZE_KNOWN; } } @@ -407,8 +405,8 @@ static bool progress_calc(struct Curl_easy *data, struct curltime now) /* The time spent so far (from the start) in microseconds */ p->timespent = Curl_timediff_us(now, p->start); - p->dlspeed = trspeed(p->downloaded, p->timespent); - p->ulspeed = trspeed(p->uploaded, p->timespent); + p->dl.speed = trspeed(p->dl.cur_size, p->timespent); + p->ul.speed = trspeed(p->ul.cur_size, p->timespent); /* Calculations done at most once a second, unless end is reached */ if(p->lastshow != now.tv_sec) { @@ -419,7 +417,7 @@ static bool progress_calc(struct Curl_easy *data, struct curltime now) /* Let's do the "current speed" thing, with the dl + ul speeds combined. Store the speed at entry 'nowindex'. */ - p->speeder[ nowindex ] = p->downloaded + p->uploaded; + p->speeder[ nowindex ] = p->dl.cur_size + p->ul.cur_size; /* remember the exact time for this moment */ p->speeder_time [ nowindex ] = now; @@ -465,31 +463,55 @@ static bool progress_calc(struct Curl_easy *data, struct curltime now) } else /* the first second we use the average */ - p->current_speed = p->ulspeed + p->dlspeed; + p->current_speed = p->ul.speed + p->dl.speed; } /* Calculations end */ return timetoshow; } #ifndef CURL_DISABLE_PROGRESS_METER + +struct pgrs_estimate { + curl_off_t secs; + curl_off_t percent; +}; + +static curl_off_t pgrs_est_percent(curl_off_t total, curl_off_t cur) +{ + if(total > CURL_OFF_T_C(10000)) + return cur / (total/CURL_OFF_T_C(100)); + else if(total > CURL_OFF_T_C(0)) + return (cur*100) / total; + return 0; +} + +static void pgrs_estimates(struct pgrs_dir *d, + bool total_known, + struct pgrs_estimate *est) +{ + est->secs = 0; + est->percent = 0; + if(total_known && (d->speed > CURL_OFF_T_C(0))) { + est->secs = d->total_size / d->speed; + est->percent = pgrs_est_percent(d->total_size, d->cur_size); + } +} + static void progress_meter(struct Curl_easy *data) { + struct Progress *p = &data->progress; char max5[6][10]; - curl_off_t dlpercen = 0; - curl_off_t ulpercen = 0; - curl_off_t total_percen = 0; - curl_off_t total_transfer; - curl_off_t total_expected_transfer; + struct pgrs_estimate dl_estm; + struct pgrs_estimate ul_estm; + struct pgrs_estimate total_estm; + curl_off_t total_cur_size; + curl_off_t total_expected_size; char time_left[10]; char time_total[10]; char time_spent[10]; - curl_off_t ulestimate = 0; - curl_off_t dlestimate = 0; - curl_off_t total_estimate; - curl_off_t timespent = - (curl_off_t)data->progress.timespent/1000000; /* seconds */ + curl_off_t cur_secs = (curl_off_t)p->timespent/1000000; /* seconds */ - if(!(data->progress.flags & PGRS_HEADERS_OUT)) { + if(!(p->flags & PGRS_HEADERS_OUT)) { if(data->state.resume_from) { fprintf(data->set.err, "** Resuming transfer from byte position %" @@ -500,78 +522,48 @@ static void progress_meter(struct Curl_easy *data) "Time Time Time Current\n" " Dload Upload " "Total Spent Left Speed\n"); - data->progress.flags |= PGRS_HEADERS_OUT; /* headers are shown */ - } - - /* Figure out the estimated time of arrival for the upload */ - if((data->progress.flags & PGRS_UL_SIZE_KNOWN) && - (data->progress.ulspeed > CURL_OFF_T_C(0))) { - ulestimate = data->progress.size_ul / data->progress.ulspeed; - - if(data->progress.size_ul > CURL_OFF_T_C(10000)) - ulpercen = data->progress.uploaded / - (data->progress.size_ul/CURL_OFF_T_C(100)); - else if(data->progress.size_ul > CURL_OFF_T_C(0)) - ulpercen = (data->progress.uploaded*100) / - data->progress.size_ul; - } - - /* ... and the download */ - if((data->progress.flags & PGRS_DL_SIZE_KNOWN) && - (data->progress.dlspeed > CURL_OFF_T_C(0))) { - dlestimate = data->progress.size_dl / data->progress.dlspeed; - - if(data->progress.size_dl > CURL_OFF_T_C(10000)) - dlpercen = data->progress.downloaded / - (data->progress.size_dl/CURL_OFF_T_C(100)); - else if(data->progress.size_dl > CURL_OFF_T_C(0)) - dlpercen = (data->progress.downloaded*100) / - data->progress.size_dl; + p->flags |= PGRS_HEADERS_OUT; /* headers are shown */ } - /* Now figure out which of them is slower and use that one for the - total estimate! */ - total_estimate = ulestimate>dlestimate?ulestimate:dlestimate; + /* Figure out the estimated time of arrival for upload and download */ + pgrs_estimates(&p->ul, (p->flags & PGRS_UL_SIZE_KNOWN), &ul_estm); + pgrs_estimates(&p->dl, (p->flags & PGRS_DL_SIZE_KNOWN), &dl_estm); + /* Since both happen at the same time, total expected duration is max. */ + total_estm.secs = CURLMAX(ul_estm.secs, dl_estm.secs); /* create the three time strings */ - time2str(time_left, total_estimate > 0?(total_estimate - timespent):0); - time2str(time_total, total_estimate); - time2str(time_spent, timespent); + time2str(time_left, total_estm.secs > 0?(total_estm.secs - cur_secs):0); + time2str(time_total, total_estm.secs); + time2str(time_spent, cur_secs); /* Get the total amount of data expected to get transferred */ - total_expected_transfer = - ((data->progress.flags & PGRS_UL_SIZE_KNOWN)? - data->progress.size_ul:data->progress.uploaded)+ - ((data->progress.flags & PGRS_DL_SIZE_KNOWN)? - data->progress.size_dl:data->progress.downloaded); + total_expected_size = + ((p->flags & PGRS_UL_SIZE_KNOWN)? p->ul.total_size:p->ul.cur_size) + + ((p->flags & PGRS_DL_SIZE_KNOWN)? p->dl.total_size:p->dl.cur_size); /* We have transferred this much so far */ - total_transfer = data->progress.downloaded + data->progress.uploaded; + total_cur_size = p->dl.cur_size + p->ul.cur_size; /* Get the percentage of data transferred so far */ - if(total_expected_transfer > CURL_OFF_T_C(10000)) - total_percen = total_transfer / - (total_expected_transfer/CURL_OFF_T_C(100)); - else if(total_expected_transfer > CURL_OFF_T_C(0)) - total_percen = (total_transfer*100) / total_expected_transfer; + total_estm.percent = pgrs_est_percent(total_expected_size, total_cur_size); fprintf(data->set.err, "\r" "%3" CURL_FORMAT_CURL_OFF_T " %s " "%3" CURL_FORMAT_CURL_OFF_T " %s " "%3" CURL_FORMAT_CURL_OFF_T " %s %s %s %s %s %s %s", - total_percen, /* 3 letters */ /* total % */ - max5data(total_expected_transfer, max5[2]), /* total size */ - dlpercen, /* 3 letters */ /* rcvd % */ - max5data(data->progress.downloaded, max5[0]), /* rcvd size */ - ulpercen, /* 3 letters */ /* xfer % */ - max5data(data->progress.uploaded, max5[1]), /* xfer size */ - max5data(data->progress.dlspeed, max5[3]), /* avrg dl speed */ - max5data(data->progress.ulspeed, max5[4]), /* avrg ul speed */ + total_estm.percent, /* 3 letters */ /* total % */ + max5data(total_expected_size, max5[2]), /* total size */ + dl_estm.percent, /* 3 letters */ /* rcvd % */ + max5data(p->dl.cur_size, max5[0]), /* rcvd size */ + ul_estm.percent, /* 3 letters */ /* xfer % */ + max5data(p->ul.cur_size, max5[1]), /* xfer size */ + max5data(p->dl.speed, max5[3]), /* avrg dl speed */ + max5data(p->ul.speed, max5[4]), /* avrg ul speed */ time_total, /* 8 letters */ /* total time */ time_spent, /* 8 letters */ /* time spent */ time_left, /* 8 letters */ /* time left */ - max5data(data->progress.current_speed, max5[5]) + max5data(p->current_speed, max5[5]) ); /* we flush the output stream to make it appear as soon as possible */ @@ -595,10 +587,10 @@ static int pgrsupdate(struct Curl_easy *data, bool showprogress) /* There is a callback set, call that */ Curl_set_in_callback(data, true); result = data->set.fxferinfo(data->set.progress_client, - data->progress.size_dl, - data->progress.downloaded, - data->progress.size_ul, - data->progress.uploaded); + data->progress.dl.total_size, + data->progress.dl.cur_size, + data->progress.ul.total_size, + data->progress.ul.cur_size); Curl_set_in_callback(data, false); if(result != CURL_PROGRESSFUNC_CONTINUE) { if(result) @@ -611,10 +603,10 @@ static int pgrsupdate(struct Curl_easy *data, bool showprogress) /* The older deprecated callback is set, call that */ Curl_set_in_callback(data, true); result = data->set.fprogress(data->set.progress_client, - (double)data->progress.size_dl, - (double)data->progress.downloaded, - (double)data->progress.size_ul, - (double)data->progress.uploaded); + (double)data->progress.dl.total_size, + (double)data->progress.dl.cur_size, + (double)data->progress.ul.total_size, + (double)data->progress.ul.cur_size); Curl_set_in_callback(data, false); if(result != CURL_PROGRESSFUNC_CONTINUE) { if(result) diff --git a/lib/progress.h b/lib/progress.h index 860257425b..04a8f5bce9 100644 --- a/lib/progress.h +++ b/lib/progress.h @@ -58,10 +58,8 @@ void Curl_pgrsUpdate_nometer(struct Curl_easy *data); void Curl_pgrsResetTransferSizes(struct Curl_easy *data); struct curltime Curl_pgrsTime(struct Curl_easy *data, timerid timer); -timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize, - curl_off_t startsize, - curl_off_t limit, - struct curltime start, +timediff_t Curl_pgrsLimitWaitTime(struct pgrs_dir *d, + curl_off_t speed_limit, struct curltime now); /** * Update progress timer with the elapsed time from its start to `timestamp`. diff --git a/lib/urldata.h b/lib/urldata.h index f34aa591ca..aab5748b04 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1058,14 +1058,23 @@ struct PureInfo { BIT(used_proxy); /* the transfer used a proxy */ }; +struct pgrs_measure { + struct curltime start; /* when measure started */ + curl_off_t start_size; /* the 'cur_size' the measure started at */ +}; + +struct pgrs_dir { + curl_off_t total_size; /* total expected bytes */ + curl_off_t cur_size; /* transferred bytes so far */ + curl_off_t speed; /* bytes per second transferred */ + struct pgrs_measure limit; +}; struct Progress { time_t lastshow; /* time() of the last displayed progress meter or NULL to force redraw at next call */ - curl_off_t size_dl; /* total expected size */ - curl_off_t size_ul; /* total expected size */ - curl_off_t downloaded; /* transferred so far */ - curl_off_t uploaded; /* transferred so far */ + struct pgrs_dir ul; + struct pgrs_dir dl; curl_off_t current_speed; /* uses the currently fastest transfer */ @@ -1074,9 +1083,6 @@ struct Progress { timediff_t timespent; - curl_off_t dlspeed; - curl_off_t ulspeed; - timediff_t t_postqueue; timediff_t t_nslookup; timediff_t t_connect; @@ -1090,14 +1096,6 @@ struct Progress { struct curltime t_startop; struct curltime t_acceptdata; - - /* upload speed limit */ - struct curltime ul_limit_start; - curl_off_t ul_limit_size; - /* download speed limit */ - struct curltime dl_limit_start; - curl_off_t dl_limit_size; - #define CURR_TIME (5 + 1) /* 6 entries for 5 seconds */ curl_off_t speeder[ CURR_TIME ];