Add new-style compression that skips matching data.

Adding new-style compression that only compresses the literal data that
is sent over the wire and not also matching file data that was not sent.
This new-style compression is compatible with external zlib instances,
and will eventually become the default (once enough time has passed that
all servers support the --new-compress and --old-compress options).

NOTE: if you build rsync with an external zlib (i.e. if you specified
configure --with-included-zlib=no) you will ONLY get support for the
--new-compress option!  A client will treat -z as uncompressed (with a
warning) and a server will exit with an error (unless -zz was used).

diff --git a/configure.ac b/configure.ac
index dccb2aafcc8cd8c45bf83ea3bfbb5e82bc4c4a6f..c7b28c5293c76ce4a739c1d5b5704e84a94aa95d 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -792,6 +792,7 @@ if test x"$with_included_zlib" = x"yes"; then
     BUILD_ZLIB='$(zlib_OBJS)'
     CFLAGS="$CFLAGS -I$srcdir/zlib"
 else
+    AC_DEFINE(EXTERNAL_ZLIB, 1, [Define to 1 if using external zlib])
     AC_MSG_RESULT(no)
 fi
 

diff --git a/options.c b/options.c
index dc9e62a45c94e45769eb2d0c9026fe2c74b571d9..0aa64bf0f845a9a291d35da674eabc71d5de6091 100644 (file)
--- a/options.c
+++ b/options.c
@@ -32,6 +32,8 @@ extern unsigned int module_dirlen;
 extern filter_rule_list filter_list;
 extern filter_rule_list daemon_filter_list;
 
+#define NOT_SPECIFIED (-42)
+
 int make_backups = 0;
 
 /**
@@ -75,7 +77,7 @@ int protocol_version = PROTOCOL_VERSION;
 int sparse_files = 0;
 int preallocate_files = 0;
 int do_compression = 0;
-int def_compress_level = Z_DEFAULT_COMPRESSION;
+int def_compress_level = NOT_SPECIFIED;
 int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
 int am_server = 0;
 int am_sender = 0;
@@ -965,10 +967,12 @@ static struct poptOption long_options[] = {
   {"no-fuzzy",         0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
   {"no-y",             0,  POPT_ARG_VAL,    &fuzzy_basis, 0, 0, 0 },
   {"compress",        'z', POPT_ARG_NONE,   0, 'z', 0, 0 },
+  {"old-compress",     0,  POPT_ARG_VAL,    &do_compression, 1, 0, 0 },
+  {"new-compress",     0,  POPT_ARG_VAL,    &do_compression, 2, 0, 0 },
   {"no-compress",      0,  POPT_ARG_VAL,    &do_compression, 0, 0, 0 },
   {"no-z",             0,  POPT_ARG_VAL,    &do_compression, 0, 0, 0 },
   {"skip-compress",    0,  POPT_ARG_STRING, &skip_compress, 0, 0, 0 },
-  {"compress-level",   0,  POPT_ARG_INT,    &def_compress_level, 'z', 0, 0 },
+  {"compress-level",   0,  POPT_ARG_INT,    &def_compress_level, 0, 0, 0 },
   {0,                 'P', POPT_ARG_NONE,   0, 'P', 0, 0 },
   {"progress",         0,  POPT_ARG_VAL,    &do_progress, 1, 0, 0 },
   {"no-progress",      0,  POPT_ARG_VAL,    &do_progress, 0, 0, 0 },
@@ -1543,18 +1547,7 @@ int parse_arguments(int *argc_p, const char ***argv_p)
                        break;
 
                case 'z':
-                       if (def_compress_level < Z_DEFAULT_COMPRESSION
-                        || def_compress_level > Z_BEST_COMPRESSION) {
-                               snprintf(err_buf, sizeof err_buf,
-                                       "--compress-level value is invalid: %d\n",
-                                       def_compress_level);
-                               return 0;
-                       }
-                       do_compression = def_compress_level != Z_NO_COMPRESSION;
-                       if (do_compression && refused_compress) {
-                               create_refuse_error(refused_compress);
-                               return 0;
-                       }
+                       do_compression++;
                        break;
 
                case 'M':
@@ -1829,6 +1822,33 @@ int parse_arguments(int *argc_p, const char ***argv_p)
                exit_cleanup(0);
        }
 
+       if (do_compression || def_compress_level != NOT_SPECIFIED) {
+               if (def_compress_level == NOT_SPECIFIED)
+                       def_compress_level = Z_DEFAULT_COMPRESSION;
+               else if (def_compress_level < Z_DEFAULT_COMPRESSION || def_compress_level > Z_BEST_COMPRESSION) {
+                       snprintf(err_buf, sizeof err_buf, "--compress-level value is invalid: %d\n",
+                                def_compress_level);
+                       return 0;
+               } else if (def_compress_level == Z_NO_COMPRESSION)
+                       do_compression = 0;
+               else if (!do_compression)
+                       do_compression = 1;
+               if (do_compression && refused_compress) {
+                       create_refuse_error(refused_compress);
+                       return 0;
+               }
+#ifdef EXTERNAL_ZLIB
+               if (do_compression == 1) {
+                       snprintf(err_buf, sizeof err_buf,
+                               "This rsync lacks old-style --compress due to its external zlib.  Try -zz.\n");
+                       if (am_server)
+                               return 0;
+                       fprintf(stderr, "%s" "Continuing without compression.\n\n", err_buf);
+                       do_compression = 0;
+               }
+#endif
+       }
+
 #ifdef HAVE_SETVBUF
        if (outbuf_mode && !am_server) {
                int mode = *(uchar *)outbuf_mode;
@@ -2451,7 +2471,7 @@ void server_options(char **args, int *argc_p)
        }
        if (sparse_files)
                argstr[x++] = 'S';
-       if (do_compression)
+       if (do_compression == 1)
                argstr[x++] = 'z';
 
        set_allow_inc_recurse();
@@ -2747,6 +2767,9 @@ void server_options(char **args, int *argc_p)
                exit_cleanup(RERR_MALLOC);
        }
 
+       if (do_compression > 1)
+               args[ac++] = "--new-compress";
+
        if (remote_option_cnt) {
                int j;
                if (ac + remote_option_cnt > MAX_SERVER_ARGS) {

diff --git a/rsync.yo b/rsync.yo
index eafa385496ac406a2ff32aaf8807167215f218f0..c99d150c397459f9987b9c64611711e726c22c62 100644 (file)
--- a/rsync.yo
+++ b/rsync.yo
@@ -1867,7 +1867,20 @@ being transmitted -- something that is useful over a slow connection.
 Note that this option typically achieves better compression ratios than can
 be achieved by using a compressing remote shell or a compressing transport
 because it takes advantage of the implicit information in the matching data
-blocks that are not explicitly sent over the connection.
+blocks that are not explicitly sent over the connection.  This matching-data
+compression comes at a cost of CPU, though, and can be disabled by repeating
+the bf(-z) option, but only if both sides are at least version 3.1.1.
+
+Note that if your version of rsync was compiled with an external zlib (instead
+of the zlib that comes packaged with rsync) then it will not support the
+old-style compression, only the new-style (repeated-option) compression.  In
+the future this new-style compression will likely become the default.
+
+The client rsync requests new-style compression on the server via the
+bf(--new-compress) option, so if you see that option rejected it means that
+the server is not new enough to support bf(-zz).  Rsync also accepts the
+bf(--old-compress) option for a future time when new-style compression
+becomes the default.
 
 See the bf(--skip-compress) option for the default list of file suffixes
 that will not be compressed.

diff --git a/testsuite/daemon-gzip-download.test b/testsuite/daemon-gzip-download.test
index 89a112ff2bad44e48aecb56134a14a35cfc30658..57dd820b8c7a00b9bf83d1c826789182b946d492 100644 (file)
--- a/testsuite/daemon-gzip-download.test
+++ b/testsuite/daemon-gzip-download.test
@@ -31,7 +31,7 @@ hands_setup
 # Build chkdir with a normal rsync and an --exclude.
 $RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/"
 
-checkit "$RSYNC -avvvvz localhost::test-from/ '$todir/'" "$chkdir" "$todir"
+checkit "$RSYNC -avvvvzz localhost::test-from/ '$todir/'" "$chkdir" "$todir"
 
 # The script would have aborted on error, so getting here means we've won.
 exit 0

diff --git a/testsuite/daemon-gzip-upload.test b/testsuite/daemon-gzip-upload.test
index 11c52bad3d79e51fa78cf1e5f00d2e2e853dfe34..b2110ea673d3df76d6e63d6a714789188a56792b 100644 (file)
--- a/testsuite/daemon-gzip-upload.test
+++ b/testsuite/daemon-gzip-upload.test
@@ -25,7 +25,7 @@ hands_setup
 # Build chkdir with a normal rsync and an --exclude.
 $RSYNC -av --exclude=foobar.baz "$fromdir/" "$chkdir/"
 
-checkit "'$ignore23' $RSYNC -avvvvz '$fromdir/' localhost::test-to/" "$chkdir" "$todir"
+checkit "'$ignore23' $RSYNC -avvvvzz '$fromdir/' localhost::test-to/" "$chkdir" "$todir"
 
 # The script would have aborted on error, so getting here means we've won.
 exit 0

diff --git a/token.c b/token.c
index bd4d52feed2c2820cd7ec61cf108ffc2c9cac1c2..8cc55329cb8902af9955cbc18016a0bdd00b5927 100644 (file)
--- a/token.c
+++ b/token.c
@@ -23,10 +23,6 @@
 #include "itypes.h"
 #include <zlib.h>
 
-#ifndef Z_INSERT_ONLY
-#define Z_INSERT_ONLY Z_SYNC_FLUSH
-#endif
-
 extern int do_compression;
 extern int protocol_version;
 extern int module_id;
@@ -406,9 +402,10 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
        if (token == -1) {
                /* end of file - clean up */
                write_byte(f, END_FLAG);
-       } else if (token != -2) {
+       } else if (token != -2 && do_compression == 1) {
                /* Add the data in the current block to the compressor's
                 * history and hash table. */
+#ifndef EXTERNAL_ZLIB
                do {
                        /* Break up long sections in the same way that
                         * see_deflate_token() does. */
@@ -418,17 +415,20 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset,
                        tx_strm.avail_in = n1;
                        if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */
                                offset += n1;
-                       do {
-                               tx_strm.next_out = (Bytef *) obuf;
-                               tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
-                               r = deflate(&tx_strm, Z_INSERT_ONLY);
-                               if (r != Z_OK) {
-                                       rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
-                                               r, tx_strm.avail_in);
-                                       exit_cleanup(RERR_STREAMIO);
-                               }
-                       } while (tx_strm.avail_in != 0);
+                       tx_strm.next_out = (Bytef *) obuf;
+                       tx_strm.avail_out = AVAIL_OUT_SIZE(CHUNK_SIZE);
+                       r = deflate(&tx_strm, Z_INSERT_ONLY);
+                       if (r != Z_OK || tx_strm.avail_in != 0) {
+                               rprintf(FERROR, "deflate on token returned %d (%d bytes left)\n",
+                                       r, tx_strm.avail_in);
+                               exit_cleanup(RERR_STREAMIO);
+                       }
                } while (toklen > 0);
+#else
+               toklen++;
+               rprintf(FERROR, "Impossible error in external-zlib code (1).\n");
+               exit_cleanup(RERR_STREAMIO);
+#endif
        }
 }
 
@@ -579,6 +579,7 @@ static int32 recv_deflated_token(int f, char **data)
  */
 static void see_deflate_token(char *buf, int32 len)
 {
+#ifndef EXTERNAL_ZLIB
        int r;
        int32 blklen;
        unsigned char hdr[5];
@@ -616,6 +617,11 @@ static void see_deflate_token(char *buf, int32 len)
                        exit_cleanup(RERR_STREAMIO);
                }
        } while (len || rx_strm.avail_out == 0);
+#else
+       buf++; len++;
+       rprintf(FERROR, "Impossible error in external-zlib code (2).\n");
+       exit_cleanup(RERR_STREAMIO);
+#endif
 }
 
 /**
@@ -655,6 +661,6 @@ int32 recv_token(int f, char **data)
  */
 void see_token(char *data, int32 toklen)
 {
-       if (do_compression)
+       if (do_compression == 1)
                see_deflate_token(data, toklen);
 }