From: Daniel Stenberg Date: Sun, 4 Aug 2024 14:14:24 +0000 (+0200) Subject: curl: add --skip-existing X-Git-Tag: curl-8_10_0~395 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=732cb15b9740521d8715ac46096dfb2f5598b857;p=thirdparty%2Fcurl.git curl: add --skip-existing With this option, the entire download is skipped if the selected target filename already exists when the opertion is about to begin. Test 994, 995 and 996 verify. Ref: #11012 Closes #13993 --- diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc index 963da4eae1..a7f635d8d9 100644 --- a/docs/cmdline-opts/Makefile.inc +++ b/docs/cmdline-opts/Makefile.inc @@ -253,6 +253,7 @@ DPAGES = \ show-error.md \ show-headers.md \ silent.md \ + skip-existing.md \ socks4.md \ socks4a.md \ socks5-basic.md \ diff --git a/docs/cmdline-opts/skip-existing.md b/docs/cmdline-opts/skip-existing.md new file mode 100644 index 0000000000..cfb7c2f953 --- /dev/null +++ b/docs/cmdline-opts/skip-existing.md @@ -0,0 +1,22 @@ +--- +c: Copyright (C) Daniel Stenberg, , et al. +SPDX-License-Identifier: curl +Long: skip-existing +Help: Skip download if local file already exists +Category: curl output +Added: 8.10.0 +Multi: boolean +See-also: + - output + - remote-name + - no-clobber +Example: + - --skip-existing --output local/dir/file $URL +--- + +# `--skip-existing` + +If there is a local file present when a download is requested, the operation +is skipped. Note that curl cannot know if the local file was previously +downloaded fine, or if it is incomplete etc, it just knows if there is a +filename present in the file system or not and it skips the transfer if it is. diff --git a/docs/options-in-versions b/docs/options-in-versions index e53935ccb6..62b61d94a8 100644 --- a/docs/options-in-versions +++ b/docs/options-in-versions @@ -218,6 +218,7 @@ --show-error (-S) 5.9 --show-headers (-i) 4.8 --silent (-s) 4.0 +--skip-existing 8.10.0 --socks4 7.15.2 --socks4a 7.18.0 --socks5 7.18.0 diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index a887881da0..e76ab46b0e 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -302,6 +302,7 @@ struct OperationConfig { struct State state; /* for create_transfer() */ bool rm_partial; /* on error, remove partially written output files */ + bool skip_existing; #ifdef USE_ECH char *ech; /* Config set by --ech keywords */ char *ech_config; /* Config set by "--ech esl:" option */ diff --git a/src/tool_getparam.c b/src/tool_getparam.c index 40b2d54928..a55cac44ca 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -287,6 +287,7 @@ static const struct LongShort aliases[]= { {"show-error", ARG_BOOL, 'S', C_SHOW_ERROR}, {"show-headers", ARG_BOOL, 'i', C_SHOW_HEADERS}, {"silent", ARG_BOOL, 's', C_SILENT}, + {"skip-existing", ARG_BOOL, ' ', C_SKIP_EXISTING}, {"socks4", ARG_STRG, ' ', C_SOCKS4}, {"socks4a", ARG_STRG, ' ', C_SOCKS4A}, {"socks5", ARG_STRG, ' ', C_SOCKS5}, @@ -2435,6 +2436,9 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ case C_SILENT: /* --silent */ global->silent = toggle; break; + case C_SKIP_EXISTING: /* --skip-existing */ + config->skip_existing = toggle; + break; case C_SHOW_ERROR: /* --show-error */ global->showerror = toggle; break; diff --git a/src/tool_getparam.h b/src/tool_getparam.h index c36d1d66f2..9d6c72ef82 100644 --- a/src/tool_getparam.h +++ b/src/tool_getparam.h @@ -242,6 +242,7 @@ typedef enum { C_SHOW_ERROR, C_SHOW_HEADERS, C_SILENT, + C_SKIP_EXISTING, C_SOCKS4, C_SOCKS4A, C_SOCKS5, diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c index 4a33c7db7c..1c5b5e9ef7 100644 --- a/src/tool_listhelp.c +++ b/src/tool_listhelp.c @@ -662,6 +662,9 @@ const struct helptxt helptext[] = { {"-s, --silent", "Silent mode", CURLHELP_IMPORTANT | CURLHELP_VERBOSE}, + {" --skip-existing", + "Skip download if local file already exists", + CURLHELP_CURL | CURLHELP_OUTPUT}, {" --socks4 ", "SOCKS4 proxy on given host + port", CURLHELP_PROXY}, diff --git a/src/tool_operate.c b/src/tool_operate.c index 25dc19a638..a3fff2a51a 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -460,6 +460,9 @@ static CURLcode post_per_transfer(struct GlobalConfig *global, if(per->infdopen) close(per->infd); + if(per->skip) + goto skip; + #ifdef __VMS if(is_vms_shell()) { /* VMS DCL shell behavior */ @@ -731,7 +734,7 @@ noretry: curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime); setfiletime(filetime, outs->filename, global); } - +skip: /* Write the --write-out data before cleanup but after result is final */ if(config->writeout) ourWriteOut(config, per, result); @@ -1197,6 +1200,15 @@ static CURLcode single_transfer(struct GlobalConfig *global, break; } + if(per->outfile && config->skip_existing) { + struct_stat fileinfo; + if(!stat(per->outfile, &fileinfo)) { + /* file is present */ + notef(global, "skips transfer, \"%s\" exists locally", + per->outfile); + per->skip = TRUE; + } + } if((urlnode->flags & GETOUT_USEREMOTE) && config->content_disposition) { /* Our header callback MIGHT set the filename */ @@ -2611,25 +2623,29 @@ static CURLcode serial_transfers(struct GlobalConfig *global, long delay_ms; bool bailout = FALSE; struct timeval start; - result = pre_transfer(global, per); - if(result) - break; - if(global->libcurl) { - result = easysrc_perform(); + start = tvnow(); + if(!per->skip) { + result = pre_transfer(global, per); if(result) break; - } - start = tvnow(); + + if(global->libcurl) { + result = easysrc_perform(); + if(result) + break; + } + #ifdef DEBUGBUILD - if(getenv("CURL_FORBID_REUSE")) - (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L); + if(getenv("CURL_FORBID_REUSE")) + (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L); - if(global->test_event_based) - result = curl_easy_perform_ev(per->curl); - else + if(global->test_event_based) + result = curl_easy_perform_ev(per->curl); + else #endif - result = curl_easy_perform(per->curl); + result = curl_easy_perform(per->curl); + } returncode = post_per_transfer(global, per, result, &retry, &delay_ms); if(retry) { diff --git a/src/tool_operate.h b/src/tool_operate.h index 820ac1395d..a2bd83b10b 100644 --- a/src/tool_operate.h +++ b/src/tool_operate.h @@ -44,25 +44,16 @@ struct per_transfer { char *this_url; unsigned int urlnum; /* the index of the given URL */ char *outfile; - bool infdopen; /* TRUE if infd needs closing */ int infd; - bool noprogress; struct ProgressData progressbar; struct OutStruct outs; struct OutStruct heads; struct OutStruct etag_save; struct HdrCbData hdrcbdata; long num_headers; - bool was_last_header_empty; - - bool added; /* set TRUE when added to the multi handle */ time_t startat; /* when doing parallel transfers, this is a retry transfer that has been set to sleep until this time before it should get started (again) */ - bool abort; /* when doing parallel transfers and this is TRUE then a critical - error (eg --fail-early) has occurred in another transfer and - this transfer will be aborted in the progress callback */ - /* for parallel progress bar */ curl_off_t dltotal; curl_off_t dlnow; @@ -77,6 +68,15 @@ struct per_transfer { char *uploadfile; char *errorbuffer; /* allocated and assigned while this is used for a transfer */ + bool infdopen; /* TRUE if infd needs closing */ + bool noprogress; + bool was_last_header_empty; + + bool added; /* set TRUE when added to the multi handle */ + bool abort; /* when doing parallel transfers and this is TRUE then a critical + error (eg --fail-early) has occurred in another transfer and + this transfer will be aborted in the progress callback */ + bool skip; /* considered already done */ }; CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]); diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index 39041bec7f..96de5fdcc3 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -129,7 +129,7 @@ test952 test953 test954 test955 test956 test957 test958 test959 test960 \ test961 test962 test963 test964 test965 test966 test967 test968 test969 \ test970 test971 test972 test973 test974 test975 test976 test977 test978 \ test979 test980 test981 test982 test983 test984 test985 test986 test987 \ -test988 test989 test990 test991 test992 test993 \ +test988 test989 test990 test991 test992 test993 test994 test995 test996 \ \ test1000 test1001 test1002 test1003 test1004 test1005 test1006 test1007 \ test1008 test1009 test1010 test1011 test1012 test1013 test1014 test1015 \ diff --git a/tests/data/test994 b/tests/data/test994 new file mode 100644 index 0000000000..e19d8c0108 --- /dev/null +++ b/tests/data/test994 @@ -0,0 +1,42 @@ + + + +HTTP +HTTP GET + + + +# +# Server-side + + + +# +# Client-side + + +http + + +--skip-existing with globbing + + +-o "%LOGDIR/#1" "http://%HOSTIP:%HTTPPORT/%TESTNUMBER/{hey,ho}" --skip-existing + + +content + + +content + + + +# +# Verify data after the test has been "shot" + + +Note: skips transfer, "%LOGDIR/hey" exists locally +Note: skips transfer, "%LOGDIR/ho" exists locally + + + diff --git a/tests/data/test995 b/tests/data/test995 new file mode 100644 index 0000000000..f2ec85ea7a --- /dev/null +++ b/tests/data/test995 @@ -0,0 +1,56 @@ + + + +HTTP +HTTP GET + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Tue, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 6 +Connection: close +Content-Type: text/html +Funny-head: yesyes + +-foo- + + + +# +# Client-side + + +http + + +--skip-existing without file present + + +-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing + + + +# +# Verify data after the test has been "shot" + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + + + +-foo- + + + diff --git a/tests/data/test996 b/tests/data/test996 new file mode 100644 index 0000000000..7c5e639013 --- /dev/null +++ b/tests/data/test996 @@ -0,0 +1,41 @@ + + + +HTTP +HTTP GET + + + +# +# Server-side + + + +# +# Client-side + + +http + + +--skip-existing with file present + + +-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing + + +content + + + +# +# Verify data after the test has been "shot" + + +Note: skips transfer, "%LOGDIR/there" exists locally + + +content + + +