From: HexTheDragon Date: Sun, 12 Sep 2021 04:36:21 +0000 (-0800) Subject: curl: add --no-clobber X-Git-Tag: curl-7_83_0~156 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=1831a6e7f179165c3c2055dd00f68bc5e9d27240;p=thirdparty%2Fcurl.git curl: add --no-clobber Does not overwrite output files if they already exist Closes #7708 Co-authored-by: Daniel Stenberg --- diff --git a/docs/TODO b/docs/TODO index c4ff53a9eb..616611d508 100644 --- a/docs/TODO +++ b/docs/TODO @@ -142,7 +142,6 @@ 18. Command line tool 18.1 sync 18.2 glob posts - 18.3 prevent file overwriting 18.4 --proxycommand 18.5 UTF-8 filenames in Content-Disposition 18.6 Option to make -Z merge lined based outputs on stdout @@ -940,14 +939,6 @@ Globbing support for -d and -F, as in 'curl -d "name=foo[0-9]" URL'. This is easily scripted though. -18.3 prevent file overwriting - - Add an option that prevents curl from overwriting existing local files. When - used, and there already is an existing file with the target file name - (either -O or -o), a number should be appended (and increased if already - existing). So that index.html becomes first index.html.1 and then - index.html.2 etc. - 18.4 --proxycommand Allow the user to make curl run a command and use its stdio to make requests diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc index 3f6d00831a..87819e087b 100644 --- a/docs/cmdline-opts/Makefile.inc +++ b/docs/cmdline-opts/Makefile.inc @@ -140,6 +140,7 @@ DPAGES = \ next.d \ no-alpn.d \ no-buffer.d \ + no-clobber.d \ no-keepalive.d \ no-npn.d \ no-progress-meter.d \ diff --git a/docs/cmdline-opts/no-clobber.d b/docs/cmdline-opts/no-clobber.d new file mode 100644 index 0000000000..382e6786a2 --- /dev/null +++ b/docs/cmdline-opts/no-clobber.d @@ -0,0 +1,16 @@ +Long: no-clobber +Help: Do not overwrite files that already exist +Category: curl output +Added: 7.83.0 +See-also: output remote-name +Example: --no-clobber --output local/dir/file $URL +--- +When used in conjunction with the --output, --remote-header-name, +--remote-name, or --remote-name-all options, curl avoids overwriting files +that already exist. Instead, a dot and a number gets appended to the name +of the file that would be created, up to filename.100 after which it will not +create any file. + +Note that this is the negated option name documented. You can thus use +--clobber to enforce the clobbering, even if --remote-header-name or -J is +specified. diff --git a/docs/options-in-versions b/docs/options-in-versions index 559a332760..e6359ab928 100644 --- a/docs/options-in-versions +++ b/docs/options-in-versions @@ -128,6 +128,7 @@ --next (-:) 7.36.0 --no-alpn 7.36.0 --no-buffer (-N) 6.5 +--no-clobber 7.83.0 --no-keepalive 7.18.0 --no-npn 7.36.0 --no-progress-meter 7.67.0 diff --git a/src/tool_cb_wrt.c b/src/tool_cb_wrt.c index d5e96aa0b9..8d59d989c0 100644 --- a/src/tool_cb_wrt.c +++ b/src/tool_cb_wrt.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2022, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -48,50 +48,86 @@ #define OPENMODE S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH #endif -/* create a local file for writing, return TRUE on success */ +/* create/open a local file for writing, return TRUE on success */ bool tool_create_output_file(struct OutStruct *outs, struct OperationConfig *config) { struct GlobalConfig *global; FILE *file = NULL; + char *fname = outs->filename; + char *aname = NULL; DEBUGASSERT(outs); DEBUGASSERT(config); global = config->global; - if(!outs->filename || !*outs->filename) { + if(!fname || !*fname) { warnf(global, "Remote filename has no length!\n"); return FALSE; } - if(outs->is_cd_filename) { - /* don't overwrite existing files */ + if(config->output_dir && outs->is_cd_filename) { + aname = aprintf("%s/%s", config->output_dir, fname); + if(!aname) { + errorf(global, "out of memory\n"); + return FALSE; + } + fname = aname; + } + + if(config->file_clobber_mode == CLOBBER_ALWAYS || + (config->file_clobber_mode == CLOBBER_DEFAULT && + !outs->is_cd_filename)) { + /* open file for writing */ + file = fopen(fname, "wb"); + } + else { int fd; - char *name = outs->filename; - char *aname = NULL; - if(config->output_dir) { - aname = aprintf("%s/%s", config->output_dir, name); - if(!aname) { + do { + fd = open(fname, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE); + /* Keep retrying in the hope that it isn't interrupted sometime */ + } while(fd == -1 && errno == EINTR); + if(config->file_clobber_mode == CLOBBER_NEVER && fd == -1) { + int next_num = 1; + size_t len = strlen(fname); + char *newname = malloc(len + 13); /* nul + 1-11 digits + dot */ + if(!newname) { errorf(global, "out of memory\n"); return FALSE; } - name = aname; + memcpy(newname, fname, len); + newname[len] = '.'; + while(fd == -1 && /* haven't sucessfully opened a file */ + (errno == EEXIST || errno == EISDIR) && + /* because we keep having files that already exist */ + next_num < 100 /* and we haven't reached the retry limit */ ) { + curlx_msnprintf(newname + len + 1, 12, "%d", next_num); + next_num++; + do { + fd = open(newname, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE); + /* Keep retrying in the hope that it isn't interrupted sometime */ + } while(fd == -1 && errno == EINTR); + } + outs->filename = newname; /* remember the new one */ + outs->alloc_filename = TRUE; } - fd = open(name, O_CREAT | O_WRONLY | O_EXCL | O_BINARY, OPENMODE); + /* An else statement to not overwrite existing files and not retry with + new numbered names (which would cover + config->file_clobber_mode == CLOBBER_DEFAULT && outs->is_cd_filename) + is not needed because we would have failed earlier, in the while loop + and `fd` would now be -1 */ if(fd != -1) { file = fdopen(fd, "wb"); if(!file) close(fd); } - free(aname); } - else - /* open file for writing */ - file = fopen(outs->filename, "wb"); if(!file) { - warnf(global, "Failed to create the file %s: %s\n", outs->filename, + warnf(global, "Failed to open the file %s: %s\n", fname, strerror(errno)); + free(aname); return FALSE; } + free(aname); outs->s_isreg = TRUE; outs->fopened = TRUE; outs->stream = file; diff --git a/src/tool_cfgable.c b/src/tool_cfgable.c index 34e17ce55b..ecc266a065 100644 --- a/src/tool_cfgable.c +++ b/src/tool_cfgable.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2022, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -45,6 +45,7 @@ void config_init(struct OperationConfig *config) config->happy_eyeballs_timeout_ms = CURL_HET_DEFAULT; config->http09_allowed = FALSE; config->ftp_skip_ip = TRUE; + config->file_clobber_mode = CLOBBER_DEFAULT; } static void free_config_fields(struct OperationConfig *config) diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 4a420db32d..e5c43583ea 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -290,6 +290,15 @@ struct OperationConfig { bool haproxy_protocol; /* whether to send HAProxy protocol v1 */ bool disallow_username_in_url; /* disallow usernames in URLs */ char *aws_sigv4; + enum { + CLOBBER_DEFAULT, /* Provides compatability with previous versions of curl, + by using the default behavior for -o, -O, and -J. + If those options would have overwritten files, like + -o and -O would, then overwrite them. In the case of + -J, this will not overwrite any files. */ + CLOBBER_NEVER, /* If the file exists, always fail */ + CLOBBER_ALWAYS /* If the file exists, always overwrite it */ + } file_clobber_mode; struct GlobalConfig *global; struct OperationConfig *prev; struct OperationConfig *next; /* Always last in the struct */ diff --git a/src/tool_getparam.c b/src/tool_getparam.c index b31583299c..7558f2003b 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -314,6 +314,7 @@ static const struct LongShort aliases[]= { {"O", "remote-name", ARG_NONE}, {"Oa", "remote-name-all", ARG_BOOL}, {"Ob", "output-dir", ARG_STRING}, + {"Oc", "clobber", ARG_BOOL}, {"p", "proxytunnel", ARG_BOOL}, {"P", "ftp-port", ARG_STRING}, {"q", "disable", ARG_BOOL}, @@ -1999,10 +2000,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ case 'N': /* disable the output I/O buffering. note that the option is called --buffer but is mostly used in the negative form: --no-buffer */ - if(longopt) - config->nobuffer = (!toggle)?TRUE:FALSE; - else - config->nobuffer = toggle; + config->nobuffer = longopt ? !toggle : TRUE; break; case 'O': /* --remote-name */ if(subletter == 'a') { /* --remote-name-all */ @@ -2013,6 +2011,10 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ GetStr(&config->output_dir, nextarg); break; } + else if(subletter == 'c') { /* --clobber / --no-clobber */ + config->file_clobber_mode = toggle ? CLOBBER_ALWAYS : CLOBBER_NEVER; + break; + } /* FALLTHROUGH */ case 'o': /* --output */ /* output file */ diff --git a/src/tool_help.c b/src/tool_help.c index d49cccd058..7602a6926b 100644 --- a/src/tool_help.c +++ b/src/tool_help.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2022, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c index 33ebda25fd..d67e8563bf 100644 --- a/src/tool_listhelp.c +++ b/src/tool_listhelp.c @@ -385,6 +385,9 @@ const struct helptxt helptext[] = { {"-N, --no-buffer", "Disable buffering of the output stream", CURLHELP_CURL}, + {" --no-clobber", + "Do not overwrite files that already exist", + CURLHELP_CURL | CURLHELP_OUTPUT}, {" --no-keepalive", "Disable TCP keepalive on the connection", CURLHELP_CONNECTION}, diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index 6a32dd6782..a7a42d2953 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -209,6 +209,8 @@ test1630 test1631 test1632 test1633 test1634 \ test1650 test1651 test1652 test1653 test1654 test1655 \ test1660 test1661 \ \ +test1680 test1681 test1682 test1683 \ +\ test1700 test1701 test1702 test1703 \ \ test1800 test1801 \ diff --git a/tests/data/test1680 b/tests/data/test1680 new file mode 100644 index 0000000000..7d8167c0e2 --- /dev/null +++ b/tests/data/test1680 @@ -0,0 +1,55 @@ + + + +HTTP +HTTP GET +--clobber + + + +# +# Server-side + + +HTTP/1.0 200 OK +Connection: close +Content-Type: text/plain +Content-Length: 4 + +foo + + + +# +# Client-side + + +HTTP GET with explicit clobber + + +http + + +http + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --clobber + + +to be overwritten + + + +# +# Verify data after the test has been "shot" + + +HTTP/1.0 200 OK +Connection: close +Content-Type: text/plain +Content-Length: 4 + +foo + + + diff --git a/tests/data/test1681 b/tests/data/test1681 new file mode 100644 index 0000000000..cfc8a5db81 --- /dev/null +++ b/tests/data/test1681 @@ -0,0 +1,61 @@ + + + +HTTP +HTTP GET +--no-clobber + + + +# +# Server-side + + +HTTP/1.0 200 OK +Connection: close +Content-Type: text/plain +Content-Length: 4 + +foo + + + +# +# Client-side + + +HTTP GET without clobber + + +http + + +http + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --no-clobber -w '%{filename_effective}\n' + + +to stay the same + + + +# +# Verify data after the test has been "shot" + + +to stay the same + + +HTTP/1.0 200 OK +Connection: close +Content-Type: text/plain +Content-Length: 4 + +foo + + +log/exist%TESTNUMBER.1 + + + diff --git a/tests/data/test1682 b/tests/data/test1682 new file mode 100644 index 0000000000..e981c20e95 --- /dev/null +++ b/tests/data/test1682 @@ -0,0 +1,58 @@ + + + +HTTP +HTTP GET +--no-clobber + + + +# +# Server-side + + +HTTP/1.0 200 OK +Connection: close +Content-Type: text/plain +Content-Length: 4 + +foo + + + +# +# Client-side + + +HTTP GET without clobber and --output-dir + + +http + + +http + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --output-dir log -o exist%TESTNUMBER --no-clobber + + +to stay the same + + + +# +# Verify data after the test has been "shot" + + +to stay the same + + +HTTP/1.0 200 OK +Connection: close +Content-Type: text/plain +Content-Length: 4 + +foo + + + diff --git a/tests/data/test1683 b/tests/data/test1683 new file mode 100644 index 0000000000..93d27d73c6 --- /dev/null +++ b/tests/data/test1683 @@ -0,0 +1,61 @@ + + + +HTTP +HTTP GET +--no-clobber + + + +# +# Server-side + + +HTTP/1.0 200 OK +Connection: close +Content-Type: text/plain +Content-Length: 4 + +foo + + + +# +# Client-side + + +HTTP GET without clobber when 100 files already exist + + +http + + +http + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER -o log/exist%TESTNUMBER --no-clobber + + +to stay the same + + +perl -e 'for my $i ((1..100)) { my $filename = "log/exist%TESTNUMBER.$i"; open(FH, ">", $filename) or die $!; print FH "to stay the same" ; close(FH) }' +# python3 -c 'for i in range(1, 101): open("log/exist%TESTNUMBER.{}".format(i), mode="w").write("to stay the same")' + + +perl -e 'for my $i ((1..100)) { my $filename = "log/exist%TESTNUMBER.$i"; open(FH, "<", $filename) or die $!; ( eq "to stay the same" and eq "") or die "incorrect $filename" ; close(FH) }' +# python3 -c 'for i in range(1, 101): assert open("log/exist%TESTNUMBER.{}".format(i), mode="r").read(17) == "to stay the same"' + + + +# +# Verify data after the test has been "shot" + + +23 + + +to stay the same + + + diff --git a/tests/manpage-scan.pl b/tests/manpage-scan.pl index 219c4a4632..986dbd556f 100755 --- a/tests/manpage-scan.pl +++ b/tests/manpage-scan.pl @@ -6,7 +6,7 @@ # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # -# Copyright (C) 2016 - 2021, Daniel Stenberg, , et al. +# Copyright (C) 2016 - 2022, Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms @@ -146,6 +146,7 @@ my %opts = ( '--no-sessionid' => 1, '--no-keepalive' => 1, '--no-progress-meter' => 1, + '--no-clobber' => 1, # pretend these options without -no exist in curl.1 and tool_listhelp.c '--alpn' => 6, @@ -156,8 +157,9 @@ my %opts = ( '-N, --buffer' => 6, '--sessionid' => 6, '--progress-meter' => 6, + '--clobber' => 6, - # deprecated options do not need to be in tool_listhelp.c nor curl.1 + # deprecated options do not need to be in tool_help.c nor curl.1 '--krb4' => 6, '--ftp-ssl' => 6, '--ftp-ssl-reqd' => 6,