From: Wayne Davison Date: Fri, 22 May 2020 15:23:26 +0000 (-0700) Subject: Checksum negotiation & more bits for compat_flags X-Git-Tag: v3.2.0pre1~136 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=4f6c8c6652c74b958c631ca9a16f450c6ce6a23c;p=thirdparty%2Frsync.git Checksum negotiation & more bits for compat_flags - Add checksum negotiation to the protocol so that we can easily add new checksum algorithms and each will be used when both sides support it. - Increase the size of the compat_flags value in the protocol from a byte to an int. --- diff --git a/checksum.c b/checksum.c index 8698543d..980d262b 100644 --- a/checksum.c +++ b/checksum.c @@ -21,6 +21,10 @@ #include "rsync.h" +extern int am_server; +extern int local_server; +extern int whole_file; +extern int read_batch; extern int checksum_seed; extern int protocol_version; extern int proper_seed_order; @@ -33,27 +37,21 @@ extern char *checksum_choice; #define CSUM_MD4 4 #define CSUM_MD5 5 +const char *default_checksum_list = + "md5 md4"; + +#define MAX_CHECKSUM_LIST 1024 + int xfersum_type = 0; /* used for the file transfer checksums */ int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */ +const char *negotiated_csum_name = NULL; -/* Returns 1 if --whole-file must be enabled. */ -int parse_checksum_choice(void) -{ - char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL; - if (cp) { - xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice); - checksum_type = parse_csum_name(cp+1, -1); - } else - xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1); - return xfersum_type == CSUM_NONE; -} - -int parse_csum_name(const char *name, int len) +static int parse_csum_name(const char *name, int len, int allow_auto) { if (len < 0 && name) len = strlen(name); - if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) { + if (!name || (allow_auto && len == 4 && strncasecmp(name, "auto", 4) == 0)) { if (protocol_version >= 30) return CSUM_MD5; if (protocol_version >= 27) @@ -69,7 +67,65 @@ int parse_csum_name(const char *name, int len) if (len == 4 && strncasecmp(name, "none", 4) == 0) return CSUM_NONE; - rprintf(FERROR, "unknown checksum name: %s\n", name); + if (allow_auto) { + rprintf(FERROR, "unknown checksum name: %s\n", name); + exit_cleanup(RERR_UNSUPPORTED); + } + + return -1; +} + +void parse_checksum_choice(void) +{ + if (!negotiated_csum_name) { + char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL; + if (cp) { + xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice, 1); + checksum_type = parse_csum_name(cp+1, -1, 1); + } else + xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1, 1); + } + if (xfersum_type == CSUM_NONE) + whole_file = 1; +} + +void negotiate_checksum(int f_in, int f_out, const char *csum_list) +{ + char *tok, sumbuf[MAX_CHECKSUM_LIST]; + int sum_type, len; + + if (!am_server || local_server) { + if (!csum_list || !*csum_list) + csum_list = default_checksum_list; + len = strlen(csum_list); + if (len >= (int)sizeof sumbuf) { + rprintf(FERROR, "The checksum list is too long.\n"); + exit_cleanup(RERR_UNSUPPORTED); + } + if (!local_server) + write_vstring(f_out, csum_list, len); + } + + if (local_server && !read_batch) + memcpy(sumbuf, csum_list, len+1); + else + len = read_vstring(f_in, sumbuf, sizeof sumbuf); + + if (len > 0) { + for (tok = strtok(sumbuf, " \t"); tok; tok = strtok(NULL, " \t")) { + len = strlen(tok); + sum_type = parse_csum_name(tok, len, 0); + if (sum_type >= CSUM_NONE) { + xfersum_type = checksum_type = sum_type; + if (am_server && !local_server) + write_vstring(f_out, tok, len); + negotiated_csum_name = strdup(tok); + return; + } + } + } + + rprintf(FERROR, "Failed to negotiate a common checksum\n"); exit_cleanup(RERR_UNSUPPORTED); } @@ -260,7 +316,7 @@ void sum_init(int csum_type, int seed) char s[4]; if (csum_type < 0) - csum_type = parse_csum_name(NULL, 0); + csum_type = parse_csum_name(NULL, 0, 1); cursum_type = csum_type; switch (csum_type) { diff --git a/compat.c b/compat.c index bd313fa9..b29b9637 100644 --- a/compat.c +++ b/compat.c @@ -41,6 +41,7 @@ extern int preallocate_files; extern int append_mode; extern int fuzzy_basis; extern int read_batch; +extern int write_batch; extern int delay_updates; extern int checksum_seed; extern int basis_dir_cnt; @@ -60,12 +61,14 @@ extern char *partial_dir; extern char *dest_option; extern char *files_from; extern char *filesfrom_host; +extern char *checksum_choice; extern filter_rule_list filter_list; extern int need_unsorted_flist; #ifdef ICONV_OPTION extern iconv_t ic_send, ic_recv; extern char *iconv_opt; #endif +extern const char *negotiated_csum_name; /* These index values are for the file-list's extra-attribute array. */ int pathname_ndx, depth_ndx, atimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx; @@ -141,6 +144,8 @@ void set_allow_inc_recurse(void) void setup_protocol(int f_out,int f_in) { + int csum_exchange = 0; + assert(file_extra_cnt == 0); assert(EXTRA64_CNT == 2 || EXTRA64_CNT == 1); @@ -289,11 +294,23 @@ void setup_protocol(int f_out,int f_in) compat_flags |= CF_CHKSUM_SEED_FIX; if (local_server || strchr(client_info, 'I') != NULL) compat_flags |= CF_INPLACE_PARTIAL_DIR; - if (local_server || strchr(client_info, 'V') != NULL) - compat_flags |= CF_VARINT_FLIST_FLAGS; - write_byte(f_out, compat_flags); - } else - compat_flags = read_byte(f_in); + if (local_server || strchr(client_info, 'v') != NULL) { + if (!write_batch || protocol_version >= 30) { + csum_exchange = 1; + compat_flags |= CF_VARINT_FLIST_FLAGS; + } + } + if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */ + if (!write_batch) + compat_flags |= CF_VARINT_FLIST_FLAGS; + write_byte(f_out, compat_flags); + } else + write_varint(f_out, compat_flags); + } else { /* read_varint() is compatible with the older write_byte() when the 0x80 bit isn't on. */ + compat_flags = read_varint(f_in); + if (compat_flags & CF_VARINT_FLIST_FLAGS) + csum_exchange = 1; + } /* The inc_recurse var MUST be set to 0 or 1. */ inc_recurse = compat_flags & CF_INC_RECURSE ? 1 : 0; want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM); @@ -358,5 +375,22 @@ void setup_protocol(int f_out,int f_in) checksum_seed = read_int(f_in); } + if (!checksum_choice) { + const char *rcl = getenv("RSYNC_CHECKSUM_LIST"); + if (csum_exchange) + negotiate_checksum(f_in, f_out, rcl); + else if (!am_server && rcl && *rcl && strstr(rcl, "FAIL")) { + rprintf(FERROR, "Remote rsync is too old for checksum negotation\n"); + exit_cleanup(RERR_UNSUPPORTED); + } + } + init_flist(); } + +void maybe_write_checksum(int batch_fd) +{ + assert(negotiated_csum_name != NULL); + if (compat_flags & CF_VARINT_FLIST_FLAGS) + write_vstring(batch_fd, negotiated_csum_name, strlen(negotiated_csum_name)); +} diff --git a/io.c b/io.c index c6d2023c..446a5f34 100644 --- a/io.c +++ b/io.c @@ -2368,8 +2368,9 @@ void start_write_batch(int fd) * is involved. */ write_int(batch_fd, protocol_version); if (protocol_version >= 30) - write_byte(batch_fd, compat_flags); + write_varint(batch_fd, compat_flags); write_int(batch_fd, checksum_seed); + maybe_write_checksum(batch_fd); if (am_sender) write_batch_monitor_out = fd; diff --git a/options.c b/options.c index ca3b97e1..c1e957b8 100644 --- a/options.c +++ b/options.c @@ -1928,12 +1928,11 @@ int parse_arguments(int *argc_p, const char ***argv_p) } } - if (checksum_choice && strcmp(checksum_choice, "auto") != 0 && strcmp(checksum_choice, "auto,auto") != 0) { + if (checksum_choice && strcasecmp(checksum_choice, "auto") != 0 && strcasecmp(checksum_choice, "auto,auto") != 0) { /* Call this early to verify the args and figure out if we need to force * --whole-file. Note that the parse function will get called again later, * just in case an "auto" choice needs to know the protocol_version. */ - if (parse_checksum_choice()) - whole_file = 1; + parse_checksum_choice(); } else checksum_choice = NULL; @@ -2642,7 +2641,8 @@ void server_options(char **args, int *argc_p) eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */ eFlags[x++] = 'C'; /* support checksum seed order fix */ eFlags[x++] = 'I'; /* support inplace_partial behavior */ - eFlags[x++] = 'V'; /* use varint for flist flags */ + eFlags[x++] = 'v'; /* use varint for flist & compat flags; negotiate checksum */ + /* NOTE: Avoid using 'V' -- it was the high bit of a write_byte() that became write_varint(). */ #undef eFlags } diff --git a/rsync.yo b/rsync.yo index ed774968..a1917cf6 100644 --- a/rsync.yo +++ b/rsync.yo @@ -657,8 +657,9 @@ checksum that is generated as the file is transferred, but that automatic after-the-transfer verification has nothing to do with this option's before-the-transfer "Does this file need to be updated?" check. -For protocol 30 and beyond (first supported in 3.0.0), the checksum used is -MD5. For older protocols, the checksum used is MD4. +The checksum used is auto-negotiated between the client and the server, but +can be overridden using either the bf(--checksum-choice) option or an +environment variable that is discussed in that option's section. dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick way of saying you want recursion and want to preserve almost @@ -1371,16 +1372,38 @@ batch-writing option is in effect. dit(bf(--checksum-choice=STR)) This option overrides the checksum algorithms. If one algorithm name is specified, it is used for both the transfer checksums -and (assuming bf(--checksum) is specified) the pre-transfer checksumming. If two +and (assuming bf(--checksum) is specified) the pre-transfer checksums. If two comma-separated names are supplied, the first name affects the transfer -checksums, and the second name affects the pre-transfer checksumming. - -The algorithm choices are "auto", "md4", "md5", and "none". If "none" is -specified for the first name, the bf(--whole-file) option is forced on and no -checksum verification is performed on the transferred data. If "none" is -specified for the second name, the bf(--checksum) option cannot be used. The -"auto" option is the default, where rsync bases its algorithm choice on the -protocol version (for backward compatibility with older rsync versions). +checksums, and the second name affects the pre-transfer checksums (bf(-c)). + +The algorithm choices are "auto", "MD5", "MD4", and "none". + +If "none" is specified for the first (or only) name, the bf(--whole-file) option +is forced on and no checksum verification is performed on the transferred data. +If "none" is specified for the second (or only) name, the bf(--checksum) option +cannot be used. + +The "auto" option is the default, where rsync bases its algorithm choice on a +negotation between the client and the server as follows: + +If both the client and the server are at least version 3.2.0, they will +exchange a list of checksum names and choose the first one in the list that +they have in common. +This typically means that they will choose MD5. +If one side of the transfer is not new enough to support this checksum +negotation, then a value is chosen based on the protocol version (which +chooses between MD5 and various flavors of MD4 based on protocol age). + +You can also override the checksum using the RSYNC_CHECKSUM_LIST environment +variable by setting it to a space-separated list of checksum names that you +consider acceptable. If no common checksum is found, the client exits with an +error. This method does not allow you to specify the transfer checksum +separately from the pre-transfer checksum, and it ignores "auto" and all +unknown checksum names. If the remote rsync is not new enough to handle a +checksum negotiation list, the list is silently ignored unless it contains the +string "FAIL" in it. + +The use of the bf(--checksum-choice) option overrides this environment list. dit(bf(-x, --one-file-system)) This tells rsync to avoid crossing a filesystem boundary when recursing. This does not limit the user's ability