gzip/patches/gzip-1.3.13-rsync.patch

   1 diff -up gzip-1.3.13/deflate.c.rsync gzip-1.3.13/deflate.c
   2 --- gzip-1.3.13/deflate.c.rsync 2009-09-26 20:43:28.000000000 +0200
   3 +++ gzip-1.3.13/deflate.c       2009-12-01 16:14:24.656387546 +0100
   4 @@ -131,6 +131,14 @@
   5  #endif
   6  /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
   7
   8 +#ifndef RSYNC_WIN
   9 +#  define RSYNC_WIN 4096
  10 +#endif
  11 +/* Size of rsync window, must be < MAX_DIST */
  12 +
  13 +#define RSYNC_SUM_MATCH(sum) ((sum) % RSYNC_WIN == 0)
  14 +/* Whether window sum matches magic value */
  15 +
  16  /* ===========================================================================
  17   * Local data used by the "longest match" routines.
  18   */
  19 @@ -212,6 +220,8 @@ local int compr_level;
  20  unsigned good_match;
  21  /* Use a faster search when the previous match is longer than this */
  22
  23 +local ulg rsync_sum;  /* rolling sum of rsync window */
  24 +local ulg rsync_chunk_end; /* next rsync sequence point */
  25
  26  /* Values for max_lazy_match, good_match and max_chain_length, depending on
  27   * the desired pack level (0..9). The values given below have been tuned to
  28 @@ -310,6 +320,10 @@ void lm_init (pack_level, flags)
  29  #endif
  30      /* prev will be initialized on the fly */
  31
  32 +    /* rsync params */
  33 +    rsync_chunk_end = 0xFFFFFFFFUL;
  34 +    rsync_sum = 0;
  35 +
  36      /* Set the default configuration parameters:
  37       */
  38      max_lazy_match   = configuration_table[pack_level].max_lazy;
  39 @@ -546,6 +560,8 @@ local void fill_window()
  40          memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
  41          match_start -= WSIZE;
  42          strstart    -= WSIZE; /* we now have strstart >= MAX_DIST: */
  43 +        if (rsync_chunk_end != 0xFFFFFFFFUL)
  44 +            rsync_chunk_end -= WSIZE;
  45
  46          block_start -= (long) WSIZE;
  47
  48 @@ -573,13 +589,46 @@ local void fill_window()
  49      }
  50  }
  51
  52 +local void rsync_roll(start, num)
  53 +    unsigned start;
  54 +    unsigned num;
  55 +{
  56 +    unsigned i;
  57 +
  58 +    if (start < RSYNC_WIN) {
  59 +       /* before window fills. */
  60 +       for (i = start; i < RSYNC_WIN; i++) {
  61 +           if (i == start + num) return;
  62 +           rsync_sum += (ulg)window[i];
  63 +       }
  64 +       num -= (RSYNC_WIN - start);
  65 +       start = RSYNC_WIN;
  66 +    }
  67 +
  68 +    /* buffer after window full */
  69 +    for (i = start; i < start+num; i++) {
  70 +       /* New character in */
  71 +       rsync_sum += (ulg)window[i];
  72 +       /* Old character out */
  73 +       rsync_sum -= (ulg)window[i - RSYNC_WIN];
  74 +       if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
  75 +           rsync_chunk_end = i;
  76 +    }
  77 +}
  78 +
  79 +/* ===========================================================================
  80 + * Set rsync_chunk_end if window sum matches magic value.
  81 + */
  82 +#define RSYNC_ROLL(s, n) \
  83 +   do { if (rsync) rsync_roll((s), (n)); } while(0)
  84 +
  85  /* ===========================================================================
  86   * Flush the current block, with given end-of-file flag.
  87   * IN assertion: strstart is set to the end of the current match.
  88   */
  89  #define FLUSH_BLOCK(eof) \
  90     flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
  91 -                (char*)NULL, (long)strstart - block_start, (eof))
  92 +                (char*)NULL, (long)strstart - block_start, flush-1, (eof))
  93
  94  /* ===========================================================================
  95   * Processes a new input file and return its compressed length. This
  96 @@ -590,7 +639,7 @@ local void fill_window()
  97  local off_t deflate_fast()
  98  {
  99      IPos hash_head; /* head of the hash chain */
 100 -    int flush;      /* set if current block must be flushed */
 101 +    int flush;      /* set if current block must be flushed, 2=>and padded  */
 102      unsigned match_length = 0;  /* length of best match */
 103
 104      prev_length = MIN_MATCH-1;
 105 @@ -674,7 +674,8 @@
 106              flush = ct_tally(strstart-match_start, match_length - MIN_MATCH);
 107
 108              lookahead -= match_length;
 109 -
 110 +
 111 +            RSYNC_ROLL(strstart, match_length);
 112              /* Insert new strings in the hash table only if the match length
 113               * is not too large. This saves time but degrades compression.
 114               */
 115 @@ -703,9 +704,14 @@
 116              /* No match, output a literal byte */
 117              Tracevv((stderr,"%c",window[strstart]));
 118              flush = ct_tally (0, window[strstart]);
 119 +            RSYNC_ROLL(strstart, 1);
 120              lookahead--;
 121              strstart++;
 122          }
 123 +        if (rsync && strstart > rsync_chunk_end) {
 124 +            rsync_chunk_end = 0xFFFFFFFFUL;
 125 +            flush = 2;
 126 +       }
 127          if (flush) FLUSH_BLOCK(0), block_start = strstart;
 128
 129          /* Make sure that we always have enough lookahead, except
 130 @@ -724,6 +779,7 @@ off_t deflate()
 131               */
 132              lookahead -= prev_length-1;
 133              prev_length -= 2;
 134 +            RSYNC_ROLL(strstart, prev_length+1);
 135              do {
 136                  strstart++;
 137                  INSERT_STRING(strstart, hash_head);
 138 @@ -736,24 +792,40 @@ off_t deflate()
 139              match_available = 0;
 140              match_length = MIN_MATCH-1;
 141              strstart++;
 142 -            if (flush) FLUSH_BLOCK(0), block_start = strstart;
 143
 144 +            if (rsync && strstart > rsync_chunk_end) {
 145 +                rsync_chunk_end = 0xFFFFFFFFUL;
 146 +                flush = 2;
 147 +            }
 148 +            if (flush) FLUSH_BLOCK(0), block_start = strstart;
 149          } else if (match_available) {
 150              /* If there was no match at the previous position, output a
 151               * single literal. If there was a match but the current match
 152               * is longer, truncate the previous match to a single literal.
 153               */
 154              Tracevv((stderr,"%c",window[strstart-1]));
 155 -            if (ct_tally (0, window[strstart-1])) {
 156 -                FLUSH_BLOCK(0), block_start = strstart;
 157 +            flush = ct_tally (0, window[strstart-1]);
 158 +            if (rsync && strstart > rsync_chunk_end) {
 159 +                rsync_chunk_end = 0xFFFFFFFFUL;
 160 +                flush = 2;
 161              }
 162 +            if (flush) FLUSH_BLOCK(0), block_start = strstart;
 163 +            RSYNC_ROLL(strstart, 1);
 164              strstart++;
 165              lookahead--;
 166          } else {
 167              /* There is no previous match to compare with, wait for
 168               * the next step to decide.
 169               */
 170 +            if (rsync && strstart > rsync_chunk_end) {
 171 +                /* Reset huffman tree */
 172 +                rsync_chunk_end = 0xFFFFFFFFUL;
 173 +                flush = 2;
 174 +                FLUSH_BLOCK(0), block_start = strstart;
 175 +            }
 176 +
 177              match_available = 1;
 178 +            RSYNC_ROLL(strstart, 1);
 179              strstart++;
 180              lookahead--;
 181          }
 182 diff -up gzip-1.3.13/doc/gzip.texi.rsync gzip-1.3.13/doc/gzip.texi
 183 --- gzip-1.3.13/doc/gzip.texi.rsync     2009-09-28 11:08:16.000000000 +0200
 184 +++ gzip-1.3.13/doc/gzip.texi   2009-12-01 16:14:24.664394713 +0100
 185 @@ -353,6 +353,14 @@ specified on the command line are direct
 186  into the directory and compress all the files it finds there (or
 187  decompress them in the case of @command{gunzip}).
 188
 189 +@item --rsyncable
 190 +While compressing, synchronize the output occasionally based on the
 191 +input.  This reduces compression by about 1 percent most cases, but
 192 +means that the @code{rsync} program can take advantage of similarities
 193 +in the uncompressed input when syncronizing two files compressed with
 194 +this flag.  @code{gunzip} cannot tell the difference between a
 195 +compressed file created with this option, and one created without it.
 196 +
 197  @item --suffix @var{suf}
 198  @itemx -S @var{suf}
 199  Use suffix @var{suf} instead of @samp{.gz}.  Any suffix can be
 200 diff -up gzip-1.3.13/gzip.c.rsync gzip-1.3.13/gzip.c
 201 --- gzip-1.3.13/gzip.c.rsync    2009-09-26 20:56:02.000000000 +0200
 202 +++ gzip-1.3.13/gzip.c  2009-12-01 16:18:17.121387126 +0100
 203 @@ -229,6 +229,7 @@ int  ofd;                  /* output fil
 204  unsigned insize;           /* valid bytes in inbuf */
 205  unsigned inptr;            /* index of next byte to be processed in inbuf */
 206  unsigned outcnt;           /* bytes in output buffer */
 207 +int rsync = 0;             /* make ryncable chunks */
 208
 209  static int handled_sig[] =
 210    {
 211 @@ -271,7 +271,7 @@ static const struct option longopts[] =
 212      {"best",       0, 0, '9'}, /* compress better */
 213      {"lzw",        0, 0, 'Z'}, /* make output compatible with old compress */
 214      {"bits",       1, 0, 'b'}, /* max number of bits per code (implies -Z) */
 215 -
 216 +    {"rsyncable",  0, 0, 'R'}, /* make rsync-friendly archive */
 217      { 0, 0, 0, 0 }
 218  };
 219
 220
 221 @@ -363,6 +365,7 @@ local void help()
 222   "  -Z, --lzw         produce output compatible with old compress",
 223   "  -b, --bits=BITS   max number of bits per code (implies -Z)",
 224  #endif
 225 + "    --rsyncable   Make rsync-friendly archive",
 226   "",
 227   "With no FILE, or when FILE is -, read standard input.",
 228   "",
 229 @@ -484,6 +484,9 @@ int main (int argc, char **argv)
 230              recursive = 1;
 231  #endif
 232              break;
 233 +
 234 +        case 'R':
 235 +            rsync = 1; break;
 236          case 'S':
 237  #ifdef NO_MULTIPLE_DOTS
 238              if (*optarg == '.') optarg++;
 239
 240 diff -up gzip-1.3.13/gzip.h.rsync gzip-1.3.13/gzip.h
 241 --- gzip-1.3.13/gzip.h.rsync    2009-09-26 20:43:28.000000000 +0200
 242 +++ gzip-1.3.13/gzip.h  2009-12-01 16:14:24.664394713 +0100
 243 @@ -158,6 +158,7 @@ EXTERN(uch, window);         /* Sliding
 244  extern unsigned insize; /* valid bytes in inbuf */
 245  extern unsigned inptr;  /* index of next byte to be processed in inbuf */
 246  extern unsigned outcnt; /* bytes in output buffer */
 247 +extern int rsync;  /* deflate into rsyncable chunks */
 248
 249  extern off_t bytes_in;   /* number of input bytes */
 250  extern off_t bytes_out;  /* number of output bytes */
 251 @@ -288,7 +288,7 @@ extern off_t deflate (void);
 252          /* in trees.c */
 253  extern void ct_init     (ush *attr, int *method);
 254  extern int  ct_tally    (int dist, int lc);
 255 -extern off_t flush_block (char *buf, ulg stored_len, int eof);
 256 +extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof);
 257
 258          /* in bits.c */
 259  extern void     bi_init    (file_t zipfile);
 260
 261 diff -up gzip-1.3.13/trees.c.rsync gzip-1.3.13/trees.c
 262 --- gzip-1.3.13/trees.c.rsync   2009-09-26 20:43:28.000000000 +0200
 263 +++ gzip-1.3.13/trees.c 2009-12-01 16:14:24.655388257 +0100
 264 @@ -856,9 +856,10 @@ local void send_all_trees(lcodes, dcodes
 265   * trees or store, and output the encoded block to the zip file. This function
 266   * returns the total compressed length for the file so far.
 267   */
 268 -off_t flush_block(buf, stored_len, eof)
 269 +off_t flush_block(buf, stored_len, pad, eof)
 270      char *buf;        /* input block, or NULL if too old */
 271      ulg stored_len;   /* length of input block */
 272 +    int pad;          /* pad output to byte boundary */
 273      int eof;          /* true if this is the last block for a file */
 274  {
 275      ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
 276 @@ -951,6 +952,10 @@ off_t flush_block(buf, stored_len, eof)
 277          Assert (input_len == bytes_in, "bad input size");
 278          bi_windup();
 279          compressed_len += 7;  /* align on byte boundary */
 280 +    } else if (pad && (compressed_len % 8) != 0) {
 281 +        send_bits((STORED_BLOCK<<1)+eof, 3);  /* send block type */
 282 +        compressed_len = (compressed_len + 3 + 7) & ~7L;
 283 +        copy_block(buf, 0, 1); /* with header */
 284      }
 285
 286      return compressed_len >> 3;