Use a varint to send the file-list flags

If both sides support the "V" compatibility flag, we send the file-list
flags as a varint instead of a 1-or-2 byte value.  This upgrades the
number of reserved flag bits from 1 to 17 with very few extra bytes in
typical file-list data.

diff --git a/compat.c b/compat.c
index 13a3b9f9832c7838f0237dffb7ebe17115f6c58e..bd313fa9fe715d1960c20d116b32848f7fc352ba 100644 (file)
--- a/compat.c
+++ b/compat.c
@@ -52,6 +52,7 @@ extern int preserve_gid;
 extern int preserve_atimes;
 extern int preserve_acls;
 extern int preserve_xattrs;
+extern int xfer_flags_as_varint;
 extern int need_messages_from_generator;
 extern int delete_mode, delete_before, delete_during, delete_after;
 extern char *shell_cmd;
@@ -83,6 +84,7 @@ int filesfrom_convert = 0;
 #define CF_AVOID_XATTR_OPTIM (1<<4)
 #define CF_CHKSUM_SEED_FIX (1<<5)
 #define CF_INPLACE_PARTIAL_DIR (1<<6)
+#define CF_VARINT_FLIST_FLAGS (1<<7)
 
 static const char *client_info;
 
@@ -287,6 +289,8 @@ void setup_protocol(int f_out,int f_in)
                                compat_flags |= CF_CHKSUM_SEED_FIX;
                        if (local_server || strchr(client_info, 'I') != NULL)
                                compat_flags |= CF_INPLACE_PARTIAL_DIR;
+                       if (local_server || strchr(client_info, 'V') != NULL)
+                               compat_flags |= CF_VARINT_FLIST_FLAGS;
                        write_byte(f_out, compat_flags);
                } else
                        compat_flags = read_byte(f_in);
@@ -294,6 +298,7 @@ void setup_protocol(int f_out,int f_in)
                inc_recurse = compat_flags & CF_INC_RECURSE ? 1 : 0;
                want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM);
                proper_seed_order = compat_flags & CF_CHKSUM_SEED_FIX ? 1 : 0;
+               xfer_flags_as_varint = compat_flags & CF_VARINT_FLIST_FLAGS ? 1 : 0;
                if (am_sender) {
                        receiver_symlink_times = am_server
                            ? strchr(client_info, 'L') != NULL

diff --git a/flist.c b/flist.c
index d5f169ce52daf8abea60bc5d8bdd2e8298ed1fc2..f95889e906d1bda703b0701087d1e188532f5aec 100644 (file)
--- a/flist.c
+++ b/flist.c
@@ -101,6 +101,7 @@ int flist_cnt = 0; /* how many (non-tmp) file list objects exist */
 int file_total = 0; /* total of all active items over all file-lists */
 int file_old_total = 0; /* total of active items that will soon be gone */
 int flist_eof = 0; /* all the file-lists are now known */
+int xfer_flags_as_varint = 0;
 
 #define NORMAL_NAME 0
 #define SLASH_ENDING_NAME 1
@@ -528,11 +529,14 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
        if (l2 > 255)
                xflags |= XMIT_LONG_NAME;
 
-       /* We must make sure we don't send a zero flag byte or the
-        * other end will terminate the flist transfer.  Note that
-        * the use of XMIT_TOP_DIR on a non-dir has no meaning, so
-        * it's harmless way to add a bit to the first flag byte. */
-       if (protocol_version >= 28) {
+       /* We must avoid sending a flag value of 0 (or an initial byte of
+        * 0 for the older xflags protocol) or it will signal the end of
+        * the list.  Note that the use of XMIT_TOP_DIR on a non-dir has
+        * no meaning, so it's a harmless way to add a bit to the first
+        * flag byte. */
+       if (xfer_flags_as_varint)
+               write_varint(f, xflags ? xflags : XMIT_EXTENDED_FLAGS);
+       else if (protocol_version >= 28) {
                if (!xflags && !S_ISDIR(mode))
                        xflags |= XMIT_TOP_DIR;
                if ((xflags & 0xFF00) || !xflags) {
@@ -1969,6 +1973,18 @@ static void send1extra(int f, struct file_struct *file, struct file_list *flist)
        free(relname_list);
 }
 
+static void write_end_of_flist(int f, int send_io_error)
+{
+       if (xfer_flags_as_varint) {
+               write_varint(f, 0);
+               write_varint(f, send_io_error ? io_error : 0);
+       } else if (send_io_error) {
+               write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST);
+               write_varint(f, io_error);
+       } else
+               write_byte(f, 0);
+}
+
 void send_extra_file_list(int f, int at_least)
 {
        struct file_list *flist;
@@ -2020,14 +2036,13 @@ void send_extra_file_list(int f, int at_least)
                }
 
                if (io_error == save_io_error || ignore_errors)
-                       write_byte(f, 0);
+                       write_end_of_flist(f, 0);
                else if (use_safe_inc_flist) {
-                       write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST);
-                       write_varint(f, io_error);
+                       write_end_of_flist(f, 1);
                } else {
                        if (delete_during)
                                fatal_unsafe_io_error();
-                       write_byte(f, 0);
+                       write_end_of_flist(f, 0);
                }
 
                if (need_unsorted_flist) {
@@ -2356,14 +2371,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 
        /* Indicate end of file list */
        if (io_error == 0 || ignore_errors)
-               write_byte(f, 0);
-       else if (use_safe_inc_flist) {
-               write_shortint(f, XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST);
-               write_varint(f, io_error);
-       } else {
+               write_end_of_flist(f, 0);
+       else if (use_safe_inc_flist)
+               write_end_of_flist(f, 1);
+       else {
                if (delete_during && inc_recurse)
                        fatal_unsafe_io_error();
-               write_byte(f, 0);
+               write_end_of_flist(f, 0);
        }
 
 #ifdef SUPPORT_HARD_LINKS
@@ -2482,22 +2496,34 @@ struct file_list *recv_file_list(int f, int dir_ndx)
                dstart = 0;
        }
 
-       while ((flags = read_byte(f)) != 0) {
+       while (1) {
                struct file_struct *file;
 
-               if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
-                       flags |= read_byte(f) << 8;
+               if (xfer_flags_as_varint) {
+                       if ((flags = read_varint(f)) == 0) {
+                               int err = read_varint(f);
+                               if (!ignore_errors)
+                                       io_error |= err;
+                               break;
+                       }
+               } else {
+                       if ((flags = read_byte(f)) == 0)
+                               break;
+
+                       if (protocol_version >= 28 && (flags & XMIT_EXTENDED_FLAGS))
+                               flags |= read_byte(f) << 8;
 
-               if (flags == (XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST)) {
-                       int err;
-                       if (!use_safe_inc_flist) {
-                               rprintf(FERROR, "Invalid flist flag: %x\n", flags);
-                               exit_cleanup(RERR_PROTOCOL);
+                       if (flags == (XMIT_EXTENDED_FLAGS|XMIT_IO_ERROR_ENDLIST)) {
+                               int err;
+                               if (!use_safe_inc_flist) {
+                                       rprintf(FERROR, "Invalid flist flag: %x\n", flags);
+                                       exit_cleanup(RERR_PROTOCOL);
+                               }
+                               err = read_varint(f);
+                               if (!ignore_errors)
+                                       io_error |= err;
+                               break;
                        }
-                       err = read_varint(f);
-                       if (!ignore_errors)
-                               io_error |= err;
-                       break;
                }
 
                flist_expand(flist, 1);

diff --git a/io.c b/io.c
index 923d6ec5c975e5c79dff07240746508606125dcd..19ed388d6749e18b315109d3652f511f62740c56 100644 (file)
--- a/io.c
+++ b/io.c
@@ -2026,7 +2026,7 @@ void write_varint(int f, int32 x)
        } else if (cnt > 1)
                *b = b[cnt] | ~(bit*2-1);
        else
-               *b = b[cnt];
+               *b = b[1];
 
        write_buf(f, b, cnt);
 }

diff --git a/options.c b/options.c
index b2cbda53ba065b6114107afd0a26e2b32d8b92ff..66854ca0bc6c429a4515c7b043e6db9442213490 100644 (file)
--- a/options.c
+++ b/options.c
@@ -2578,6 +2578,7 @@ void server_options(char **args, int *argc_p)
                eFlags[x++] = 'x'; /* xattr hardlink optimization not desired */
                eFlags[x++] = 'C'; /* support checksum seed order fix */
                eFlags[x++] = 'I'; /* support inplace_partial behavior */
+               eFlags[x++] = 'V'; /* use varint for flist flags */
 #undef eFlags
        }
 

diff --git a/rsync.h b/rsync.h
index 2a558e43072c79ddb836e77905e586ba3c9077d2..29012d950bcbf16bad7590b0369252fcaa869479 100644 (file)
--- a/rsync.h
+++ b/rsync.h
@@ -52,6 +52,7 @@
 #define XMIT_SAME_NAME (1<<5)
 #define XMIT_LONG_NAME (1<<6)
 #define XMIT_SAME_TIME (1<<7)
+
 #define XMIT_SAME_RDEV_MAJOR (1<<8)    /* protocols 28 - now (devices only) */
 #define XMIT_NO_CONTENT_DIR (1<<8)     /* protocols 30 - now (dirs only) */
 #define XMIT_HLINKED (1<<9)            /* protocols 28 - now (non-dirs) */
@@ -62,7 +63,12 @@
 #define XMIT_HLINK_FIRST (1<<12)       /* protocols 30 - now (HLINKED files only) */
 #define XMIT_IO_ERROR_ENDLIST (1<<12)  /* protocols 31*- now (w/XMIT_EXTENDED_FLAGS) (also protocol 30 w/'f' compat flag) */
 #define XMIT_MOD_NSEC (1<<13)          /* protocols 31 - now */
-#define XMIT_SAME_ATIME (1<<14)        /* protocols ?? - now */
+#define XMIT_SAME_ATIME (1<<14)        /* any protocol - restricted by command-line option */
+#define XMIT_RESERVED_15 (1<<15)       /* reserved for future use */
+
+/* The following XMIT flags require an rsync that uses a varint for the flag values */
+
+#define XMIT_RESERVED_16 (1<<16)       /* reserved for future use */
 
 /* These flags are used in the live flist data. */