curl: add --skip-existing

With this option, the entire download is skipped if the selected target
filename already exists when the opertion is about to begin.

Test 994, 995 and 996 verify.

Ref: #11012
Closes #13993

diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc
index 963da4eae1a65da6a8b421e40d02a8f94018337c..a7f635d8d9a45e9152e32d001f44b29780b4e011 100644 (file)
--- a/docs/cmdline-opts/Makefile.inc
+++ b/docs/cmdline-opts/Makefile.inc
@@ -253,6 +253,7 @@ DPAGES = \
   show-error.md \
   show-headers.md \
   silent.md \
+  skip-existing.md \
   socks4.md \
   socks4a.md \
   socks5-basic.md \

diff --git a/docs/cmdline-opts/skip-existing.md b/docs/cmdline-opts/skip-existing.md
new file mode 100644 (file)
index 0000000..cfb7c2f
--- /dev/null
+++ b/docs/cmdline-opts/skip-existing.md
@@ -0,0 +1,22 @@
+---
+c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+SPDX-License-Identifier: curl
+Long: skip-existing
+Help: Skip download if local file already exists
+Category: curl output
+Added: 8.10.0
+Multi: boolean
+See-also:
+  - output
+  - remote-name
+  - no-clobber
+Example:
+  - --skip-existing --output local/dir/file $URL
+---
+
+# `--skip-existing`
+
+If there is a local file present when a download is requested, the operation
+is skipped. Note that curl cannot know if the local file was previously
+downloaded fine, or if it is incomplete etc, it just knows if there is a
+filename present in the file system or not and it skips the transfer if it is.

diff --git a/docs/options-in-versions b/docs/options-in-versions
index e53935ccb6a582635b7670d54ee76b6b3b712e2b..62b61d94a8ff8bbcb7b414dd6fc0103f3ed152db 100644 (file)
--- a/docs/options-in-versions
+++ b/docs/options-in-versions
@@ -218,6 +218,7 @@
 --show-error (-S)                    5.9
 --show-headers (-i)                  4.8
 --silent (-s)                        4.0
+--skip-existing                      8.10.0
 --socks4                             7.15.2
 --socks4a                            7.18.0
 --socks5                             7.18.0

diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index a887881da02f50cb56e90dc7948b0fd52884485b..e76ab46b0e42e9bdc0abeeb0631d8a9b120aa3b8 100644 (file)
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -302,6 +302,7 @@ struct OperationConfig {
   struct State state;             /* for create_transfer() */
   bool rm_partial;                /* on error, remove partially written output
                                      files */
+  bool skip_existing;
 #ifdef USE_ECH
   char *ech;                      /* Config set by --ech keywords */
   char *ech_config;               /* Config set by "--ech esl:" option */

diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 40b2d54928a9cb08ce6205ba88efa5c0eae8cbd2..a55cac44ca20fb0cb413aefade51ec2ed65123b8 100644 (file)
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -287,6 +287,7 @@ static const struct LongShort aliases[]= {
   {"show-error",                 ARG_BOOL, 'S', C_SHOW_ERROR},
   {"show-headers",               ARG_BOOL, 'i', C_SHOW_HEADERS},
   {"silent",                     ARG_BOOL, 's', C_SILENT},
+  {"skip-existing",              ARG_BOOL, ' ', C_SKIP_EXISTING},
   {"socks4",                     ARG_STRG, ' ', C_SOCKS4},
   {"socks4a",                    ARG_STRG, ' ', C_SOCKS4A},
   {"socks5",                     ARG_STRG, ' ', C_SOCKS5},
@@ -2435,6 +2436,9 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
     case C_SILENT: /* --silent */
       global->silent = toggle;
       break;
+    case C_SKIP_EXISTING: /* --skip-existing */
+      config->skip_existing = toggle;
+      break;
     case C_SHOW_ERROR: /* --show-error */
       global->showerror = toggle;
       break;

diff --git a/src/tool_getparam.h b/src/tool_getparam.h
index c36d1d66f2ec3d4dfa5f74a7daebb69e04641e9e..9d6c72ef825e8501dbb731af3fe6345fa4e63552 100644 (file)
--- a/src/tool_getparam.h
+++ b/src/tool_getparam.h
@@ -242,6 +242,7 @@ typedef enum {
   C_SHOW_ERROR,
   C_SHOW_HEADERS,
   C_SILENT,
+  C_SKIP_EXISTING,
   C_SOCKS4,
   C_SOCKS4A,
   C_SOCKS5,

diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c
index 4a33c7db7cba7bbc44df970a2a7b7bf786209219..1c5b5e9ef7fccf57ae63d2e5b02be95bf8add458 100644 (file)
--- a/src/tool_listhelp.c
+++ b/src/tool_listhelp.c
@@ -662,6 +662,9 @@ const struct helptxt helptext[] = {
   {"-s, --silent",
    "Silent mode",
    CURLHELP_IMPORTANT | CURLHELP_VERBOSE},
+  {"    --skip-existing",
+   "Skip download if local file already exists",
+   CURLHELP_CURL | CURLHELP_OUTPUT},
   {"    --socks4 <host[:port]>",
    "SOCKS4 proxy on given host + port",
    CURLHELP_PROXY},

diff --git a/src/tool_operate.c b/src/tool_operate.c
index 25dc19a63883225da536a68782e3521bfaa0a1ee..a3fff2a51aba50b0a8f4dcd0cda744038b918a2e 100644 (file)
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -460,6 +460,9 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
   if(per->infdopen)
     close(per->infd);
 
+  if(per->skip)
+    goto skip;
+
 #ifdef __VMS
   if(is_vms_shell()) {
     /* VMS DCL shell behavior */
@@ -731,7 +734,7 @@ noretry:
     curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime);
     setfiletime(filetime, outs->filename, global);
   }
-
+skip:
   /* Write the --write-out data before cleanup but after result is final */
   if(config->writeout)
     ourWriteOut(config, per, result);
@@ -1197,6 +1200,15 @@ static CURLcode single_transfer(struct GlobalConfig *global,
               break;
           }
 
+          if(per->outfile && config->skip_existing) {
+            struct_stat fileinfo;
+            if(!stat(per->outfile, &fileinfo)) {
+              /* file is present */
+              notef(global, "skips transfer, \"%s\" exists locally",
+                    per->outfile);
+              per->skip = TRUE;
+            }
+          }
           if((urlnode->flags & GETOUT_USEREMOTE)
              && config->content_disposition) {
             /* Our header callback MIGHT set the filename */
@@ -2611,25 +2623,29 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
     long delay_ms;
     bool bailout = FALSE;
     struct timeval start;
-    result = pre_transfer(global, per);
-    if(result)
-      break;
 
-    if(global->libcurl) {
-      result = easysrc_perform();
+    start = tvnow();
+    if(!per->skip) {
+      result = pre_transfer(global, per);
       if(result)
         break;
-    }
-    start = tvnow();
+
+      if(global->libcurl) {
+        result = easysrc_perform();
+        if(result)
+          break;
+      }
+
 #ifdef DEBUGBUILD
-    if(getenv("CURL_FORBID_REUSE"))
-      (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
+      if(getenv("CURL_FORBID_REUSE"))
+        (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
 
-    if(global->test_event_based)
-      result = curl_easy_perform_ev(per->curl);
-    else
+      if(global->test_event_based)
+        result = curl_easy_perform_ev(per->curl);
+      else
 #endif
-      result = curl_easy_perform(per->curl);
+        result = curl_easy_perform(per->curl);
+    }
 
     returncode = post_per_transfer(global, per, result, &retry, &delay_ms);
     if(retry) {

diff --git a/src/tool_operate.h b/src/tool_operate.h
index 820ac1395d91a3a529f6827dd6ee10cc8f4eabfe..a2bd83b10bb8c91b5372eb3b147462b4059b6844 100644 (file)
--- a/src/tool_operate.h
+++ b/src/tool_operate.h
@@ -44,25 +44,16 @@ struct per_transfer {
   char *this_url;
   unsigned int urlnum; /* the index of the given URL */
   char *outfile;
-  bool infdopen; /* TRUE if infd needs closing */
   int infd;
-  bool noprogress;
   struct ProgressData progressbar;
   struct OutStruct outs;
   struct OutStruct heads;
   struct OutStruct etag_save;
   struct HdrCbData hdrcbdata;
   long num_headers;
-  bool was_last_header_empty;
-
-  bool added; /* set TRUE when added to the multi handle */
   time_t startat; /* when doing parallel transfers, this is a retry transfer
                      that has been set to sleep until this time before it
                      should get started (again) */
-  bool abort; /* when doing parallel transfers and this is TRUE then a critical
-                 error (eg --fail-early) has occurred in another transfer and
-                 this transfer will be aborted in the progress callback */
-
   /* for parallel progress bar */
   curl_off_t dltotal;
   curl_off_t dlnow;
@@ -77,6 +68,15 @@ struct per_transfer {
   char *uploadfile;
   char *errorbuffer; /* allocated and assigned while this is used for a
                         transfer */
+  bool infdopen; /* TRUE if infd needs closing */
+  bool noprogress;
+  bool was_last_header_empty;
+
+  bool added; /* set TRUE when added to the multi handle */
+  bool abort; /* when doing parallel transfers and this is TRUE then a critical
+                 error (eg --fail-early) has occurred in another transfer and
+                 this transfer will be aborted in the progress callback */
+  bool skip;  /* considered already done */
 };
 
 CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]);

diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am
index 39041bec7f3d2047022223a6d61e19ed11e5977a..96de5fdcc3e2fcf67f69c7729951ce46e48d6b11 100644 (file)
--- a/tests/data/Makefile.am
+++ b/tests/data/Makefile.am
@@ -129,7 +129,7 @@ test952 test953 test954 test955 test956 test957 test958 test959 test960 \
 test961 test962 test963 test964 test965 test966 test967 test968 test969 \
 test970 test971 test972 test973 test974 test975 test976 test977 test978 \
 test979 test980 test981 test982 test983 test984 test985 test986 test987 \
-test988 test989 test990 test991 test992 test993 \
+test988 test989 test990 test991 test992 test993 test994 test995 test996 \
 \
 test1000 test1001 test1002 test1003 test1004 test1005 test1006 test1007 \
 test1008 test1009 test1010 test1011 test1012 test1013 test1014 test1015 \

diff --git a/tests/data/test994 b/tests/data/test994
new file mode 100644 (file)
index 0000000..e19d8c0
--- /dev/null
+++ b/tests/data/test994
@@ -0,0 +1,42 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--skip-existing with globbing
+</name>
+<command option="no-output">
+-o "%LOGDIR/#1" "http://%HOSTIP:%HTTPPORT/%TESTNUMBER/{hey,ho}" --skip-existing
+</command>
+<file name="%LOGDIR/hey">
+content
+</file>
+<file2 name="%LOGDIR/ho">
+content
+</file2>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stderr mode="text">
+Note: skips transfer, "%LOGDIR/hey" exists locally
+Note: skips transfer, "%LOGDIR/ho" exists locally
+</stderr>
+</verify>
+</testcase>

diff --git a/tests/data/test995 b/tests/data/test995
new file mode 100644 (file)
index 0000000..f2ec85e
--- /dev/null
+++ b/tests/data/test995
@@ -0,0 +1,56 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--skip-existing without file present
+</name>
+<command option="no-output,no-include">
+-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+<file name="%LOGDIR/there">
+-foo-
+</file>
+</verify>
+</testcase>

diff --git a/tests/data/test996 b/tests/data/test996
new file mode 100644 (file)
index 0000000..7c5e639
--- /dev/null
+++ b/tests/data/test996
@@ -0,0 +1,41 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--skip-existing with file present
+</name>
+<command option="no-output">
+-o %LOGDIR/there http://%HOSTIP:%HTTPPORT/%TESTNUMBER --skip-existing
+</command>
+<file name="%LOGDIR/there">
+content
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stderr mode="text">
+Note: skips transfer, "%LOGDIR/there" exists locally
+</stderr>
+<file name="%LOGDIR/there">
+content
+</file>
+</verify>
+</testcase>