NEWS for rsync version 2.5.7
-Protocol: 26 (unchanged)
+Protocol: 27 (changed)
Changes since version 2.5.6:
ENHANCEMENTS:
Note that --from0 affects the line-ending character for all the
--*-from options. (Wayne Davison)
+ - Length of csum2 is now per-file starting with protocol verison
+ 27. (J.W. Schultz)
+
+ - Per-file dynamic block size is now sqrt(file length).
+
+ The per-file checksum size is determined according
+ to an algorythm provided by Donovan Baarda which
+ reduces the probability of rsync algorithm
+ corrupting data and falling back using the whole md4
+ checksums. (J.W. Schultz, Donovan Baarda)
+
BUG FIXES:
- - Fixed a crash bug when interacting with older rsync versions and
+ - for protocol version >= 27, mdfour_tail() is called when the
+ block size (including checksum_seed) is a multiple of 64.
+ Previously it was not called, giving the wrong MD4 checksum.
+ (Craig Barratt)
+
+ - for protocol version >= 27, a 64 bit bit counter is used in
+ mdfour.c as required by the RFC. Previously only a 32 bit bit
+ counter was used, causing incorrect MD4 file checksums for
+ file sizes >= 512MB - 4. (Craig Barratt)
+
+
+ - Fixed a crash bug when interacting with Craiolder rsync versioss and
multiple files of the same name are destined for the same dir.
(Wayne Davison)
extern int preserve_hard_links;
extern int update_only;
extern int opt_ignore_existing;
-extern int block_size;
extern int csum_length;
extern int ignore_times;
extern int size_only;
}
-/* use a larger block size for really big files */
-static int adapt_block_size(struct file_struct *file, int bsize)
-{
- int ret;
-
- if (bsize != BLOCK_SIZE) return bsize;
-
- ret = file->length / (10000); /* rough heuristic */
- ret = ret & ~15; /* multiple of 16 */
- if (ret < bsize) ret = bsize;
- if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
- return ret;
-}
-
-
/*
* NULL sum_struct means we have no checksums
- */
+ */
void write_sum_head(int f, struct sum_struct *sum)
{
write_int(f, sum->remainder);
}
+/*
+ * set (initialize) the size entries in the per-file sum_struct
+ * calulating dynamic block ans checksum sizes.
+ *
+ * This is only called from generate_and_send_sums() but is a seperate
+ * function to encapsulate the logic.
+ *
+ * The block size is a rounded square root of file length.
+ *
+ * The checksum size is determined according to:
+ * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len)
+ * provided by Donovan Baarda which gives a probability of rsync
+ * algorithm corrupting data and falling back using the whole md4
+ * checksums.
+ *
+ * This might be made one of several selectable heuristics.
+ */
+static void sum_sizes_sqroot_baarda(struct sum_struct *sum, uint64 len)
+{
+ extern int block_size;
+ int blength, s2length, b;
+ uint32 c;
+ uint64 l;
+
+ if (block_size) {
+ blength = block_size;
+ } else if (len <= BLOCK_SIZE * BLOCK_SIZE) {
+ blength = BLOCK_SIZE;
+ } else {
+ l = len;
+ c = 1;
+ while (l >>= 2) {
+ c <<= 1;
+ }
+ blength = 0;
+ do {
+ blength |= c;
+ if (len < (uint64)(blength * blength))
+ blength &= ~c;
+ c >>= 1;
+ } while (c >= 8); /* round to multiple of 8 */
+ blength = MAX(blength, BLOCK_SIZE);
+ }
+
+ if (remote_version < 27) {
+ s2length = csum_length;
+ } else if (csum_length == SUM_LENGTH) {
+ s2length = SUM_LENGTH;
+ } else {
+ b = BLOCKSUM_BIAS;
+ l = len;
+ while (l >>= 1) {
+ b += 2;
+ }
+ c = blength;
+ while (c >>= 1 && b) {
+ b--;
+ }
+ s2length = (b + 1 - 32 + 7) / 8; /* add a bit,
+ * subtract rollsum,
+ * round up
+ * --optimize in compiler--
+ */
+ s2length = MAX(s2length, csum_length);
+ s2length = MIN(s2length, SUM_LENGTH);
+ }
+
+ sum->flength = len;
+ sum->blength = blength;
+ sum->s2length = s2length;
+ sum->count = (len + (blength - 1)) / blength;
+ sum->remainder = (len % blength);
+
+ if (sum->count && verbose > 2) {
+ rprintf(FINFO, "count=%ld rem=%ld blength=%ld s2length=%ld flength=%.0f\n",
+ (long) sum->count, (long) sum->remainder,
+ (long) sum->blength, (long) sum->s2length,
+ (double) sum->flength);
+ }
+}
/**
* Perhaps we want to just send an empty checksum set for this file,
*
* Generate approximately one checksum every block_len bytes.
*/
-static void generate_and_send_sums(struct map_struct *buf, OFF_T len,
- int block_len, int f_out)
+static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out)
{
size_t i;
struct sum_struct sum;
OFF_T offset = 0;
- sum.count = (len + (block_len - 1)) / block_len;
- sum.remainder = (len % block_len);
- sum.blength = block_len;
- sum.flength = len;
- sum.s2length = csum_length;
- /* not needed here sum.sums = NULL; */
-
- if (sum.count && verbose > 3) {
- rprintf(FINFO, "count=%ld rem=%ld n=%ld flength=%.0f\n",
- (long) sum.count, (long) sum.remainder,
- (long) sum.blength, (double) sum.flength);
- }
+ sum_sizes_sqroot_baarda(&sum, len);
write_sum_head(f_out, &sum);
for (i = 0; i < sum.count; i++) {
- int n1 = MIN(len, block_len);
+ int n1 = MIN(len, sum.blength);
char *map = map_ptr(buf, offset, n1);
uint32 sum1 = get_checksum1(map, n1);
char sum2[SUM_LENGTH];
rprintf(FINFO, "generating and sending sums for %d\n", i);
write_int(f_out,i);
- generate_and_send_sums(buf, st.st_size,
- adapt_block_size(file, block_size), f_out);
+ generate_and_send_sums(buf, st.st_size, f_out);
close(fd);
if (buf) unmap_file(buf);
projects / thirdparty / rsync.git / commitdiff
