]>
Commit | Line | Data |
---|---|---|
0d7638ea | 1 | /* |
0f78b815 WD |
2 | * Routines to support checksumming of bytes. |
3 | * | |
4 | * Copyright (C) 1996 Andrew Tridgell | |
5 | * Copyright (C) 1996 Paul Mackerras | |
d821e4cb | 6 | * Copyright (C) 2004-2022 Wayne Davison |
0f78b815 WD |
7 | * |
8 | * This program is free software; you can redistribute it and/or modify | |
8e41b68e WD |
9 | * it under the terms of the GNU General Public License as published by |
10 | * the Free Software Foundation; either version 3 of the License, or | |
11 | * (at your option) any later version. | |
0f78b815 | 12 | * |
15c1162b WD |
13 | * In addition, as a special exception, the copyright holders give |
14 | * permission to dynamically link rsync with the OpenSSL and xxhash | |
15 | * libraries when those libraries are being distributed in compliance | |
16 | * with their license terms, and to distribute a dynamically linked | |
17 | * combination of rsync and these libraries. This is also considered | |
18 | * to be covered under the GPL's System Libraries exception. | |
19 | * | |
0f78b815 WD |
20 | * This program is distributed in the hope that it will be useful, |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
23 | * GNU General Public License for more details. | |
24 | * | |
e7c67065 | 25 | * You should have received a copy of the GNU General Public License along |
4fd842f9 | 26 | * with this program; if not, visit the http://fsf.org website. |
0f78b815 | 27 | */ |
c627d613 AT |
28 | |
29 | #include "rsync.h" | |
1bdf68b9 | 30 | |
a28bc3eb | 31 | #ifdef SUPPORT_XXHASH |
5eda68f1 | 32 | #include <xxhash.h> |
1bdf68b9 WD |
33 | # if XXH_VERSION_NUMBER >= 800 |
34 | # define SUPPORT_XXH3 1 | |
35 | # endif | |
a28bc3eb | 36 | #endif |
c627d613 | 37 | |
4f6c8c66 | 38 | extern int am_server; |
4f6c8c66 | 39 | extern int whole_file; |
8ee6eb71 | 40 | extern int checksum_seed; |
d04e9c51 | 41 | extern int protocol_version; |
eac85808 | 42 | extern int proper_seed_order; |
87f2984d | 43 | extern const char *checksum_choice; |
a5a7d3a2 | 44 | |
7e2711bb WD |
45 | #define NNI_BUILTIN (1<<0) |
46 | #define NNI_EVP (1<<1) | |
47 | #define NNI_EVP_OK (1<<2) | |
48 | ||
3258534e | 49 | struct name_num_item valid_checksums_items[] = { |
1bdf68b9 | 50 | #ifdef SUPPORT_XXH3 |
7e2711bb WD |
51 | { CSUM_XXH3_128, 0, "xxh128", NULL }, |
52 | { CSUM_XXH3_64, 0, "xxh3", NULL }, | |
1bdf68b9 | 53 | #endif |
a28bc3eb | 54 | #ifdef SUPPORT_XXHASH |
7e2711bb WD |
55 | { CSUM_XXH64, 0, "xxh64", NULL }, |
56 | { CSUM_XXH64, 0, "xxhash", NULL }, | |
a28bc3eb | 57 | #endif |
7e2711bb WD |
58 | { CSUM_MD5, NNI_BUILTIN|NNI_EVP, "md5", NULL }, |
59 | { CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL }, | |
ae16850d WD |
60 | #ifdef SHA_DIGEST_LENGTH |
61 | { CSUM_SHA1, NNI_EVP, "sha1", NULL }, | |
62 | #endif | |
63 | #ifdef SHA256_DIGEST_LENGTH | |
64 | { CSUM_SHA256, NNI_EVP, "sha256", NULL }, | |
65 | #endif | |
66 | #ifdef SHA512_DIGEST_LENGTH | |
67 | { CSUM_SHA512, NNI_EVP, "sha512", NULL }, | |
68 | #endif | |
7e2711bb WD |
69 | { CSUM_NONE, 0, "none", NULL }, |
70 | { 0, 0, NULL, NULL } | |
3258534e WD |
71 | }; |
72 | ||
73 | struct name_num_obj valid_checksums = { | |
7e2711bb WD |
74 | "checksum", NULL, 0, 0, valid_checksums_items |
75 | }; | |
76 | ||
77 | struct name_num_item valid_auth_checksums_items[] = { | |
ae16850d WD |
78 | #ifdef SHA512_DIGEST_LENGTH |
79 | { CSUM_SHA512, NNI_EVP, "sha512", NULL }, | |
80 | #endif | |
81 | #ifdef SHA256_DIGEST_LENGTH | |
82 | { CSUM_SHA256, NNI_EVP, "sha256", NULL }, | |
83 | #endif | |
84 | #ifdef SHA_DIGEST_LENGTH | |
85 | { CSUM_SHA1, NNI_EVP, "sha1", NULL }, | |
86 | #endif | |
7e2711bb WD |
87 | { CSUM_MD5, NNI_BUILTIN|NNI_EVP, "md5", NULL }, |
88 | { CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL }, | |
89 | { 0, 0, NULL, NULL } | |
55bb4dab | 90 | }; |
4f6c8c66 | 91 | |
7e2711bb WD |
92 | struct name_num_obj valid_auth_checksums = { |
93 | "daemon auth checksum", NULL, 0, 0, valid_auth_checksums_items | |
94 | }; | |
a5a7d3a2 | 95 | |
7e2711bb WD |
96 | /* These cannot make use of openssl, so they're marked just as built-in */ |
97 | struct name_num_item implied_checksum_md4 = | |
98 | { CSUM_MD4, NNI_BUILTIN, "md4", NULL }; | |
99 | struct name_num_item implied_checksum_md5 = | |
100 | { CSUM_MD5, NNI_BUILTIN, "md5", NULL }; | |
101 | ||
102 | struct name_num_item *xfer_sum_nni; /* used for the transfer checksum2 computations */ | |
103 | const EVP_MD *xfer_sum_evp_md; | |
104 | int xfer_sum_len; | |
105 | struct name_num_item *file_sum_nni; /* used for the pre-transfer --checksum computations */ | |
106 | const EVP_MD *file_sum_evp_md; | |
107 | int file_sum_len; | |
108 | ||
109 | #ifdef USE_OPENSSL | |
110 | EVP_MD_CTX *ctx_evp = NULL; | |
111 | #endif | |
b7ea3fcd WD |
112 | static int initialized_choices = 0; |
113 | ||
7e2711bb | 114 | struct name_num_item *parse_csum_name(const char *name, int len) |
a5a7d3a2 | 115 | { |
2f84a6bd | 116 | struct name_num_item *nni; |
55bb4dab | 117 | |
a5a7d3a2 WD |
118 | if (len < 0 && name) |
119 | len = strlen(name); | |
120 | ||
7e2711bb WD |
121 | init_checksum_choices(); |
122 | ||
2f84a6bd | 123 | if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) { |
7e2711bb WD |
124 | if (protocol_version >= 30) { |
125 | if (!proper_seed_order) | |
126 | return &implied_checksum_md5; | |
127 | name = "md5"; | |
128 | len = 3; | |
129 | } else { | |
130 | if (protocol_version >= 27) | |
131 | implied_checksum_md4.num = CSUM_MD4_OLD; | |
132 | else if (protocol_version >= 21) | |
133 | implied_checksum_md4.num = CSUM_MD4_BUSTED; | |
134 | else | |
135 | implied_checksum_md4.num = CSUM_MD4_ARCHAIC; | |
136 | return &implied_checksum_md4; | |
137 | } | |
a5a7d3a2 | 138 | } |
55bb4dab | 139 | |
2f84a6bd | 140 | nni = get_nni_by_name(&valid_checksums, name, len); |
a5a7d3a2 | 141 | |
2f84a6bd | 142 | if (!nni) { |
4f6c8c66 WD |
143 | rprintf(FERROR, "unknown checksum name: %s\n", name); |
144 | exit_cleanup(RERR_UNSUPPORTED); | |
145 | } | |
146 | ||
7e2711bb | 147 | return nni; |
4f6c8c66 WD |
148 | } |
149 | ||
7e2711bb | 150 | static const EVP_MD *csum_evp_md(struct name_num_item *nni) |
55bb4dab | 151 | { |
7e2711bb WD |
152 | #ifdef USE_OPENSSL |
153 | const EVP_MD *emd; | |
154 | if (!(nni->flags & NNI_EVP)) | |
155 | return NULL; | |
156 | ||
157 | #ifdef USE_MD5_ASM | |
158 | if (nni->num == CSUM_MD5) | |
159 | emd = NULL; | |
160 | else | |
161 | #endif | |
162 | emd = EVP_get_digestbyname(nni->name); | |
163 | if (emd && !(nni->flags & NNI_EVP_OK)) { /* Make sure it works before we advertise it */ | |
164 | if (!ctx_evp && !(ctx_evp = EVP_MD_CTX_create())) | |
165 | out_of_memory("csum_evp_md"); | |
166 | /* Some routines are marked as legacy and are not enabled in the openssl.cnf file. | |
167 | * If we can't init the emd, we'll fall back to our built-in code. */ | |
168 | if (EVP_DigestInit_ex(ctx_evp, emd, NULL) == 0) | |
169 | emd = NULL; | |
170 | else | |
171 | nni->flags = (nni->flags & ~NNI_BUILTIN) | NNI_EVP_OK; | |
172 | } | |
173 | if (!emd) | |
174 | nni->flags &= ~NNI_EVP; | |
175 | return emd; | |
176 | #else | |
177 | return NULL; | |
178 | #endif | |
55bb4dab WD |
179 | } |
180 | ||
181 | void parse_checksum_choice(int final_call) | |
4f6c8c66 | 182 | { |
7e2711bb WD |
183 | if (valid_checksums.negotiated_nni) |
184 | xfer_sum_nni = file_sum_nni = valid_checksums.negotiated_nni; | |
97e8c55e | 185 | else { |
4f6c8c66 WD |
186 | char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL; |
187 | if (cp) { | |
7e2711bb WD |
188 | xfer_sum_nni = parse_csum_name(checksum_choice, cp - checksum_choice); |
189 | file_sum_nni = parse_csum_name(cp+1, -1); | |
4f6c8c66 | 190 | } else |
7e2711bb | 191 | xfer_sum_nni = file_sum_nni = parse_csum_name(checksum_choice, -1); |
28f9c960 | 192 | if (am_server && checksum_choice) |
7e2711bb | 193 | validate_choice_vs_env(NSTR_CHECKSUM, xfer_sum_nni->num, file_sum_nni->num); |
4f6c8c66 | 194 | } |
7e2711bb WD |
195 | xfer_sum_len = csum_len_for_type(xfer_sum_nni->num, 0); |
196 | file_sum_len = csum_len_for_type(file_sum_nni->num, 0); | |
197 | xfer_sum_evp_md = csum_evp_md(xfer_sum_nni); | |
198 | file_sum_evp_md = csum_evp_md(file_sum_nni); | |
55bb4dab | 199 | |
7e2711bb | 200 | if (xfer_sum_nni->num == CSUM_NONE) |
4f6c8c66 | 201 | whole_file = 1; |
55bb4dab | 202 | |
87f2984d | 203 | /* Snag the checksum name for both write_batch's option output & the following debug output. */ |
7e2711bb WD |
204 | if (valid_checksums.negotiated_nni) |
205 | checksum_choice = valid_checksums.negotiated_nni->name; | |
87f2984d | 206 | else if (checksum_choice == NULL) |
7e2711bb | 207 | checksum_choice = xfer_sum_nni->name; |
87f2984d | 208 | |
d999efe6 | 209 | if (final_call && DEBUG_GTE(NSTR, am_server ? 3 : 1)) { |
87f2984d WD |
210 | rprintf(FINFO, "%s%s checksum: %s\n", |
211 | am_server ? "Server" : "Client", | |
7e2711bb | 212 | valid_checksums.negotiated_nni ? " negotiated" : "", |
87f2984d | 213 | checksum_choice); |
55bb4dab | 214 | } |
4f6c8c66 WD |
215 | } |
216 | ||
f5e8a17e | 217 | int csum_len_for_type(int cst, BOOL flist_csum) |
a5a7d3a2 WD |
218 | { |
219 | switch (cst) { | |
220 | case CSUM_NONE: | |
221 | return 1; | |
7b8a4ecd | 222 | case CSUM_MD4_ARCHAIC: |
416e719b WD |
223 | /* The oldest checksum code is rather weird: the file-list code only sent |
224 | * 2-byte checksums, but all other checksums were full MD4 length. */ | |
225 | return flist_csum ? 2 : MD4_DIGEST_LEN; | |
a5a7d3a2 WD |
226 | case CSUM_MD4: |
227 | case CSUM_MD4_OLD: | |
228 | case CSUM_MD4_BUSTED: | |
229 | return MD4_DIGEST_LEN; | |
230 | case CSUM_MD5: | |
231 | return MD5_DIGEST_LEN; | |
ae16850d WD |
232 | #ifdef SHA_DIGEST_LENGTH |
233 | case CSUM_SHA1: | |
234 | return SHA_DIGEST_LENGTH; | |
235 | #endif | |
236 | #ifdef SHA256_DIGEST_LENGTH | |
237 | case CSUM_SHA256: | |
238 | return SHA256_DIGEST_LENGTH; | |
239 | #endif | |
240 | #ifdef SHA512_DIGEST_LENGTH | |
241 | case CSUM_SHA512: | |
242 | return SHA512_DIGEST_LENGTH; | |
243 | #endif | |
4f92fd8d | 244 | case CSUM_XXH64: |
1bdf68b9 | 245 | case CSUM_XXH3_64: |
4f92fd8d | 246 | return 64/8; |
1bdf68b9 WD |
247 | case CSUM_XXH3_128: |
248 | return 128/8; | |
bc112b0e WD |
249 | default: /* paranoia to prevent missing case values */ |
250 | exit_cleanup(RERR_UNSUPPORTED); | |
a5a7d3a2 WD |
251 | } |
252 | return 0; | |
253 | } | |
254 | ||
d7521f54 WD |
255 | /* Returns 0 if the checksum is not canonical (i.e. it includes a seed value). |
256 | * Returns 1 if the public sum order matches our internal sum order. | |
257 | * Returns -1 if the public sum order is the reverse of our internal sum order. | |
258 | */ | |
a5a7d3a2 WD |
259 | int canonical_checksum(int csum_type) |
260 | { | |
d7521f54 WD |
261 | switch (csum_type) { |
262 | case CSUM_NONE: | |
263 | case CSUM_MD4_ARCHAIC: | |
264 | case CSUM_MD4_OLD: | |
265 | case CSUM_MD4_BUSTED: | |
266 | break; | |
267 | case CSUM_MD4: | |
268 | case CSUM_MD5: | |
ae16850d WD |
269 | case CSUM_SHA1: |
270 | case CSUM_SHA256: | |
271 | case CSUM_SHA512: | |
d7521f54 | 272 | return -1; |
d7521f54 | 273 | case CSUM_XXH64: |
1bdf68b9 WD |
274 | case CSUM_XXH3_64: |
275 | case CSUM_XXH3_128: | |
d7521f54 | 276 | return 1; |
d7521f54 WD |
277 | default: /* paranoia to prevent missing case values */ |
278 | exit_cleanup(RERR_UNSUPPORTED); | |
279 | } | |
280 | return 0; | |
a5a7d3a2 | 281 | } |
c627d613 | 282 | |
b81a5095 | 283 | #ifndef USE_ROLL_SIMD /* See simd-checksum-*.cpp. */ |
c627d613 | 284 | /* |
d2970213 | 285 | a simple 32 bit checksum that can be updated from either end |
c627d613 AT |
286 | (inspired by Mark Adler's Adler-32 checksum) |
287 | */ | |
deb5bf1d | 288 | uint32 get_checksum1(char *buf1, int32 len) |
c627d613 | 289 | { |
8475e0e4 WD |
290 | int32 i; |
291 | uint32 s1, s2; | |
292 | schar *buf = (schar *)buf1; | |
293 | ||
294 | s1 = s2 = 0; | |
295 | for (i = 0; i < (len-4); i+=4) { | |
296 | s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + 10*CHAR_OFFSET; | |
297 | s1 += (buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET); | |
298 | } | |
299 | for (; i < len; i++) { | |
300 | s1 += (buf[i]+CHAR_OFFSET); s2 += s1; | |
301 | } | |
302 | return (s1 & 0xffff) + (s2 << 16); | |
c627d613 | 303 | } |
70c6b408 | 304 | #endif |
c627d613 | 305 | |
deb5bf1d | 306 | void get_checksum2(char *buf, int32 len, char *sum) |
c627d613 | 307 | { |
7e2711bb WD |
308 | #ifdef USE_OPENSSL |
309 | if (xfer_sum_evp_md) { | |
310 | static EVP_MD_CTX *evp = NULL; | |
311 | uchar seedbuf[4]; | |
312 | if (!evp && !(evp = EVP_MD_CTX_create())) | |
313 | out_of_memory("get_checksum2"); | |
314 | EVP_DigestInit_ex(evp, xfer_sum_evp_md, NULL); | |
315 | if (checksum_seed) { | |
316 | SIVALu(seedbuf, 0, checksum_seed); | |
317 | EVP_DigestUpdate(evp, seedbuf, 4); | |
318 | } | |
319 | EVP_DigestUpdate(evp, (uchar *)buf, len); | |
320 | EVP_DigestFinal_ex(evp, (uchar *)sum, NULL); | |
321 | } else | |
322 | #endif | |
323 | switch (xfer_sum_nni->num) { | |
60e71c1b WD |
324 | #ifdef SUPPORT_XXHASH |
325 | case CSUM_XXH64: | |
326 | SIVAL64(sum, 0, XXH64(buf, len, checksum_seed)); | |
327 | break; | |
1bdf68b9 WD |
328 | #endif |
329 | #ifdef SUPPORT_XXH3 | |
330 | case CSUM_XXH3_64: | |
331 | SIVAL64(sum, 0, XXH3_64bits_withSeed(buf, len, checksum_seed)); | |
332 | break; | |
333 | case CSUM_XXH3_128: { | |
334 | XXH128_hash_t digest = XXH3_128bits_withSeed(buf, len, checksum_seed); | |
335 | SIVAL64(sum, 0, digest.low64); | |
336 | SIVAL64(sum, 8, digest.high64); | |
337 | break; | |
338 | } | |
60e71c1b | 339 | #endif |
a5a7d3a2 | 340 | case CSUM_MD5: { |
7e2711bb | 341 | md_context m5; |
a0456b9c | 342 | uchar seedbuf[4]; |
b81a5095 | 343 | md5_begin(&m5); |
eac85808 WD |
344 | if (proper_seed_order) { |
345 | if (checksum_seed) { | |
346 | SIVALu(seedbuf, 0, checksum_seed); | |
b81a5095 | 347 | md5_update(&m5, seedbuf, 4); |
eac85808 | 348 | } |
b81a5095 | 349 | md5_update(&m5, (uchar *)buf, len); |
eac85808 | 350 | } else { |
b81a5095 | 351 | md5_update(&m5, (uchar *)buf, len); |
eac85808 WD |
352 | if (checksum_seed) { |
353 | SIVALu(seedbuf, 0, checksum_seed); | |
b81a5095 | 354 | md5_update(&m5, seedbuf, 4); |
eac85808 | 355 | } |
a0456b9c | 356 | } |
b81a5095 | 357 | md5_result(&m5, (uchar *)sum); |
a5a7d3a2 WD |
358 | break; |
359 | } | |
360 | case CSUM_MD4: | |
361 | case CSUM_MD4_OLD: | |
7b8a4ecd WD |
362 | case CSUM_MD4_BUSTED: |
363 | case CSUM_MD4_ARCHAIC: { | |
1c9bb168 | 364 | md_context m; |
a0456b9c WD |
365 | int32 i; |
366 | static char *buf1; | |
367 | static int32 len1; | |
368 | ||
369 | mdfour_begin(&m); | |
370 | ||
371 | if (len > len1) { | |
372 | if (buf1) | |
373 | free(buf1); | |
374 | buf1 = new_array(char, len+4); | |
375 | len1 = len; | |
a0456b9c WD |
376 | } |
377 | ||
378 | memcpy(buf1, buf, len); | |
379 | if (checksum_seed) { | |
380 | SIVAL(buf1,len,checksum_seed); | |
381 | len += 4; | |
382 | } | |
383 | ||
384 | for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) | |
385 | mdfour_update(&m, (uchar *)(buf1+i), CSUM_CHUNK); | |
386 | ||
387 | /* | |
388 | * Prior to version 27 an incorrect MD4 checksum was computed | |
389 | * by failing to call mdfour_tail() for block sizes that | |
390 | * are multiples of 64. This is fixed by calling mdfour_update() | |
391 | * even when there are no more bytes. | |
392 | */ | |
7e2711bb | 393 | if (len - i > 0 || xfer_sum_nni->num > CSUM_MD4_BUSTED) |
a0456b9c WD |
394 | mdfour_update(&m, (uchar *)(buf1+i), len-i); |
395 | ||
396 | mdfour_result(&m, (uchar *)sum); | |
a5a7d3a2 WD |
397 | break; |
398 | } | |
bc112b0e WD |
399 | default: /* paranoia to prevent missing case values */ |
400 | exit_cleanup(RERR_UNSUPPORTED); | |
8de330a3 | 401 | } |
c627d613 AT |
402 | } |
403 | ||
43d6d0c5 | 404 | void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) |
c627d613 | 405 | { |
8de330a3 | 406 | struct map_struct *buf; |
43d6d0c5 | 407 | OFF_T i, len = st_p->st_size; |
26404276 WD |
408 | int32 remainder; |
409 | int fd; | |
0d7638ea | 410 | |
8c9fd200 | 411 | fd = do_open(fname, O_RDONLY, 0); |
7e2711bb WD |
412 | if (fd == -1) { |
413 | memset(sum, 0, file_sum_len); | |
efa95a18 | 414 | return; |
7e2711bb | 415 | } |
0d7638ea | 416 | |
d474f298 | 417 | buf = map_file(fd, len, MAX_MAP_SIZE, CHUNK_SIZE); |
0d7638ea | 418 | |
7e2711bb WD |
419 | #ifdef USE_OPENSSL |
420 | if (file_sum_evp_md) { | |
421 | static EVP_MD_CTX *evp = NULL; | |
422 | if (!evp && !(evp = EVP_MD_CTX_create())) | |
423 | out_of_memory("file_checksum"); | |
424 | ||
425 | EVP_DigestInit_ex(evp, file_sum_evp_md, NULL); | |
426 | ||
427 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
428 | EVP_DigestUpdate(evp, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
429 | ||
430 | remainder = (int32)(len - i); | |
431 | if (remainder > 0) | |
432 | EVP_DigestUpdate(evp, (uchar *)map_ptr(buf, i, remainder), remainder); | |
433 | ||
434 | EVP_DigestFinal_ex(evp, (uchar *)sum, NULL); | |
435 | } else | |
436 | #endif | |
437 | switch (file_sum_nni->num) { | |
60e71c1b WD |
438 | #ifdef SUPPORT_XXHASH |
439 | case CSUM_XXH64: { | |
440 | static XXH64_state_t* state = NULL; | |
441 | if (!state && !(state = XXH64_createState())) | |
442 | out_of_memory("file_checksum"); | |
443 | ||
444 | XXH64_reset(state, 0); | |
445 | ||
446 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
447 | XXH64_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
448 | ||
449 | remainder = (int32)(len - i); | |
450 | if (remainder > 0) | |
451 | XXH64_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); | |
452 | ||
453 | SIVAL64(sum, 0, XXH64_digest(state)); | |
454 | break; | |
455 | } | |
1bdf68b9 WD |
456 | #endif |
457 | #ifdef SUPPORT_XXH3 | |
458 | case CSUM_XXH3_64: { | |
459 | static XXH3_state_t* state = NULL; | |
460 | if (!state && !(state = XXH3_createState())) | |
461 | out_of_memory("file_checksum"); | |
462 | ||
463 | XXH3_64bits_reset(state); | |
464 | ||
465 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
466 | XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
467 | ||
468 | remainder = (int32)(len - i); | |
469 | if (remainder > 0) | |
470 | XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); | |
471 | ||
472 | SIVAL64(sum, 0, XXH3_64bits_digest(state)); | |
473 | break; | |
474 | } | |
475 | case CSUM_XXH3_128: { | |
476 | XXH128_hash_t digest; | |
477 | static XXH3_state_t* state = NULL; | |
478 | if (!state && !(state = XXH3_createState())) | |
479 | out_of_memory("file_checksum"); | |
480 | ||
481 | XXH3_128bits_reset(state); | |
482 | ||
483 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
484 | XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
485 | ||
486 | remainder = (int32)(len - i); | |
487 | if (remainder > 0) | |
488 | XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); | |
489 | ||
490 | digest = XXH3_128bits_digest(state); | |
491 | SIVAL64(sum, 0, digest.low64); | |
492 | SIVAL64(sum, 8, digest.high64); | |
493 | break; | |
494 | } | |
60e71c1b | 495 | #endif |
1c9bb168 | 496 | case CSUM_MD5: { |
7e2711bb | 497 | md_context m5; |
1c9bb168 | 498 | |
b81a5095 | 499 | md5_begin(&m5); |
8de330a3 | 500 | |
d474f298 | 501 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) |
b81a5095 | 502 | md5_update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); |
8de330a3 | 503 | |
a0456b9c WD |
504 | remainder = (int32)(len - i); |
505 | if (remainder > 0) | |
b81a5095 | 506 | md5_update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder); |
8de330a3 | 507 | |
b81a5095 | 508 | md5_result(&m5, (uchar *)sum); |
a5a7d3a2 | 509 | break; |
1c9bb168 | 510 | } |
a5a7d3a2 WD |
511 | case CSUM_MD4: |
512 | case CSUM_MD4_OLD: | |
513 | case CSUM_MD4_BUSTED: | |
1c9bb168 WD |
514 | case CSUM_MD4_ARCHAIC: { |
515 | md_context m; | |
516 | ||
a0456b9c WD |
517 | mdfour_begin(&m); |
518 | ||
6ac2c7b6 WD |
519 | for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) |
520 | mdfour_update(&m, (uchar *)map_ptr(buf, i, CSUM_CHUNK), CSUM_CHUNK); | |
a0456b9c WD |
521 | |
522 | /* Prior to version 27 an incorrect MD4 checksum was computed | |
523 | * by failing to call mdfour_tail() for block sizes that | |
524 | * are multiples of 64. This is fixed by calling mdfour_update() | |
525 | * even when there are no more bytes. */ | |
526 | remainder = (int32)(len - i); | |
7e2711bb | 527 | if (remainder > 0 || file_sum_nni->num > CSUM_MD4_BUSTED) |
a0456b9c WD |
528 | mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder); |
529 | ||
530 | mdfour_result(&m, (uchar *)sum); | |
a5a7d3a2 | 531 | break; |
1c9bb168 | 532 | } |
a5a7d3a2 | 533 | default: |
2f84a6bd | 534 | rprintf(FERROR, "Invalid checksum-choice for --checksum: %s (%d)\n", |
7e2711bb | 535 | file_sum_nni->name, file_sum_nni->num); |
a5a7d3a2 | 536 | exit_cleanup(RERR_UNSUPPORTED); |
a0456b9c | 537 | } |
8de330a3 AT |
538 | |
539 | close(fd); | |
540 | unmap_file(buf); | |
c627d613 | 541 | } |
34ccb63e | 542 | |
deb5bf1d | 543 | static int32 sumresidue; |
7e2711bb | 544 | static md_context ctx_md; |
a28bc3eb | 545 | #ifdef SUPPORT_XXHASH |
4f92fd8d | 546 | static XXH64_state_t* xxh64_state; |
a28bc3eb | 547 | #endif |
1bdf68b9 WD |
548 | #ifdef SUPPORT_XXH3 |
549 | static XXH3_state_t* xxh3_state; | |
550 | #endif | |
7e2711bb WD |
551 | static struct name_num_item *cur_sum_nni; |
552 | static const EVP_MD *cur_sum_evp_md; | |
553 | int cur_sum_len; | |
9e31c482 | 554 | |
7e2711bb | 555 | int sum_init(struct name_num_item *nni, int seed) |
9e31c482 | 556 | { |
8de330a3 | 557 | char s[4]; |
a0456b9c | 558 | |
7e2711bb WD |
559 | if (!nni) |
560 | nni = parse_csum_name(NULL, 0); | |
561 | cur_sum_nni = nni; | |
562 | cur_sum_len = csum_len_for_type(nni->num, 0); | |
563 | cur_sum_evp_md = csum_evp_md(nni); | |
a5a7d3a2 | 564 | |
7e2711bb WD |
565 | #ifdef USE_OPENSSL |
566 | if (cur_sum_evp_md) { | |
567 | if (!ctx_evp && !(ctx_evp = EVP_MD_CTX_create())) | |
568 | out_of_memory("file_checksum"); | |
569 | EVP_DigestInit_ex(ctx_evp, cur_sum_evp_md, NULL); | |
570 | } else | |
571 | #endif | |
572 | switch (cur_sum_nni->num) { | |
60e71c1b WD |
573 | #ifdef SUPPORT_XXHASH |
574 | case CSUM_XXH64: | |
575 | if (!xxh64_state && !(xxh64_state = XXH64_createState())) | |
576 | out_of_memory("sum_init"); | |
577 | XXH64_reset(xxh64_state, 0); | |
578 | break; | |
1bdf68b9 WD |
579 | #endif |
580 | #ifdef SUPPORT_XXH3 | |
581 | case CSUM_XXH3_64: | |
582 | if (!xxh3_state && !(xxh3_state = XXH3_createState())) | |
583 | out_of_memory("sum_init"); | |
584 | XXH3_64bits_reset(xxh3_state); | |
585 | break; | |
586 | case CSUM_XXH3_128: | |
587 | if (!xxh3_state && !(xxh3_state = XXH3_createState())) | |
588 | out_of_memory("sum_init"); | |
589 | XXH3_128bits_reset(xxh3_state); | |
590 | break; | |
60e71c1b | 591 | #endif |
a5a7d3a2 | 592 | case CSUM_MD5: |
7e2711bb | 593 | md5_begin(&ctx_md); |
a5a7d3a2 WD |
594 | break; |
595 | case CSUM_MD4: | |
7e2711bb | 596 | mdfour_begin(&ctx_md); |
a5a7d3a2 WD |
597 | sumresidue = 0; |
598 | break; | |
599 | case CSUM_MD4_OLD: | |
600 | case CSUM_MD4_BUSTED: | |
7b8a4ecd | 601 | case CSUM_MD4_ARCHAIC: |
7e2711bb | 602 | mdfour_begin(&ctx_md); |
a0456b9c | 603 | sumresidue = 0; |
9889a34b WD |
604 | SIVAL(s, 0, seed); |
605 | sum_update(s, 4); | |
a5a7d3a2 WD |
606 | break; |
607 | case CSUM_NONE: | |
608 | break; | |
bc112b0e WD |
609 | default: /* paranoia to prevent missing case values */ |
610 | exit_cleanup(RERR_UNSUPPORTED); | |
a0456b9c | 611 | } |
7e2711bb WD |
612 | |
613 | return cur_sum_len; | |
9e31c482 AT |
614 | } |
615 | ||
d37d8d7b MP |
616 | /** |
617 | * Feed data into an MD4 accumulator, md. The results may be | |
618 | * retrieved using sum_end(). md is used for different purposes at | |
619 | * different points during execution. | |
620 | * | |
621 | * @todo Perhaps get rid of md and just pass in the address each time. | |
622 | * Very slightly clearer and slower. | |
623 | **/ | |
4a19c3b2 | 624 | void sum_update(const char *p, int32 len) |
9e31c482 | 625 | { |
7e2711bb WD |
626 | #ifdef USE_OPENSSL |
627 | if (cur_sum_evp_md) { | |
628 | EVP_DigestUpdate(ctx_evp, (uchar *)p, len); | |
629 | } else | |
630 | #endif | |
631 | switch (cur_sum_nni->num) { | |
60e71c1b WD |
632 | #ifdef SUPPORT_XXHASH |
633 | case CSUM_XXH64: | |
634 | XXH64_update(xxh64_state, p, len); | |
635 | break; | |
1bdf68b9 WD |
636 | #endif |
637 | #ifdef SUPPORT_XXH3 | |
638 | case CSUM_XXH3_64: | |
639 | XXH3_64bits_update(xxh3_state, p, len); | |
640 | break; | |
641 | case CSUM_XXH3_128: | |
642 | XXH3_128bits_update(xxh3_state, p, len); | |
643 | break; | |
60e71c1b | 644 | #endif |
a5a7d3a2 | 645 | case CSUM_MD5: |
7e2711bb | 646 | md5_update(&ctx_md, (uchar *)p, len); |
a5a7d3a2 WD |
647 | break; |
648 | case CSUM_MD4: | |
649 | case CSUM_MD4_OLD: | |
650 | case CSUM_MD4_BUSTED: | |
c252546c | 651 | case CSUM_MD4_ARCHAIC: |
4fc78878 | 652 | if (len + sumresidue < CSUM_CHUNK) { |
7e2711bb | 653 | memcpy(ctx_md.buffer + sumresidue, p, len); |
4fc78878 | 654 | sumresidue += len; |
a5a7d3a2 | 655 | break; |
4fc78878 | 656 | } |
a0456b9c | 657 | |
4fc78878 WD |
658 | if (sumresidue) { |
659 | int32 i = CSUM_CHUNK - sumresidue; | |
7e2711bb WD |
660 | memcpy(ctx_md.buffer + sumresidue, p, i); |
661 | mdfour_update(&ctx_md, (uchar *)ctx_md.buffer, CSUM_CHUNK); | |
4fc78878 WD |
662 | len -= i; |
663 | p += i; | |
664 | } | |
8de330a3 | 665 | |
4fc78878 | 666 | while (len >= CSUM_CHUNK) { |
7e2711bb | 667 | mdfour_update(&ctx_md, (uchar *)p, CSUM_CHUNK); |
4fc78878 WD |
668 | len -= CSUM_CHUNK; |
669 | p += CSUM_CHUNK; | |
670 | } | |
8de330a3 | 671 | |
4fc78878 WD |
672 | sumresidue = len; |
673 | if (sumresidue) | |
7e2711bb | 674 | memcpy(ctx_md.buffer, p, sumresidue); |
a5a7d3a2 WD |
675 | break; |
676 | case CSUM_NONE: | |
677 | break; | |
bc112b0e WD |
678 | default: /* paranoia to prevent missing case values */ |
679 | exit_cleanup(RERR_UNSUPPORTED); | |
8de330a3 | 680 | } |
9e31c482 AT |
681 | } |
682 | ||
7e2711bb WD |
683 | /* The sum buffer only needs to be as long as the current checksum's digest |
684 | * len, not MAX_DIGEST_LEN. Note that for CSUM_MD4_ARCHAIC that is the full | |
685 | * MD4_DIGEST_LEN even if the file-list code is going to ignore all but the | |
686 | * first 2 bytes of it. */ | |
687 | void sum_end(char *sum) | |
9e31c482 | 688 | { |
7e2711bb WD |
689 | #ifdef USE_OPENSSL |
690 | if (cur_sum_evp_md) { | |
691 | EVP_DigestFinal_ex(ctx_evp, (uchar *)sum, NULL); | |
692 | } else | |
693 | #endif | |
694 | switch (cur_sum_nni->num) { | |
60e71c1b WD |
695 | #ifdef SUPPORT_XXHASH |
696 | case CSUM_XXH64: | |
697 | SIVAL64(sum, 0, XXH64_digest(xxh64_state)); | |
698 | break; | |
1bdf68b9 WD |
699 | #endif |
700 | #ifdef SUPPORT_XXH3 | |
701 | case CSUM_XXH3_64: | |
702 | SIVAL64(sum, 0, XXH3_64bits_digest(xxh3_state)); | |
703 | break; | |
704 | case CSUM_XXH3_128: { | |
705 | XXH128_hash_t digest = XXH3_128bits_digest(xxh3_state); | |
706 | SIVAL64(sum, 0, digest.low64); | |
707 | SIVAL64(sum, 8, digest.high64); | |
708 | break; | |
709 | } | |
60e71c1b | 710 | #endif |
a5a7d3a2 | 711 | case CSUM_MD5: |
7e2711bb | 712 | md5_result(&ctx_md, (uchar *)sum); |
a5a7d3a2 WD |
713 | break; |
714 | case CSUM_MD4: | |
715 | case CSUM_MD4_OLD: | |
7e2711bb WD |
716 | mdfour_update(&ctx_md, (uchar *)ctx_md.buffer, sumresidue); |
717 | mdfour_result(&ctx_md, (uchar *)sum); | |
a5a7d3a2 WD |
718 | break; |
719 | case CSUM_MD4_BUSTED: | |
7b8a4ecd | 720 | case CSUM_MD4_ARCHAIC: |
a5a7d3a2 | 721 | if (sumresidue) |
7e2711bb WD |
722 | mdfour_update(&ctx_md, (uchar *)ctx_md.buffer, sumresidue); |
723 | mdfour_result(&ctx_md, (uchar *)sum); | |
a5a7d3a2 WD |
724 | break; |
725 | case CSUM_NONE: | |
726 | *sum = '\0'; | |
727 | break; | |
bc112b0e WD |
728 | default: /* paranoia to prevent missing case values */ |
729 | exit_cleanup(RERR_UNSUPPORTED); | |
4fc78878 | 730 | } |
9e31c482 | 731 | } |
b7ea3fcd | 732 | |
7e2711bb WD |
733 | #if defined SUPPORT_XXH3 || defined USE_OPENSSL |
734 | static void verify_digest(struct name_num_item *nni, BOOL check_auth_list) | |
b7ea3fcd WD |
735 | { |
736 | #ifdef SUPPORT_XXH3 | |
7e2711bb WD |
737 | static int xxh3_result = 0; |
738 | #endif | |
739 | #ifdef USE_OPENSSL | |
740 | static int prior_num = 0, prior_flags = 0, prior_result = 0; | |
741 | #endif | |
742 | ||
743 | #ifdef SUPPORT_XXH3 | |
744 | if (nni->num == CSUM_XXH3_64 || nni->num == CSUM_XXH3_128) { | |
745 | if (!xxh3_result) { | |
746 | char buf[32816]; | |
747 | int j; | |
748 | for (j = 0; j < (int)sizeof buf; j++) | |
749 | buf[j] = ' ' + (j % 96); | |
750 | sum_init(nni, 0); | |
751 | sum_update(buf, 32816); | |
752 | sum_update(buf, 31152); | |
753 | sum_update(buf, 32474); | |
754 | sum_update(buf, 9322); | |
755 | xxh3_result = XXH3_64bits_digest(xxh3_state) != 0xadbcf16d4678d1de ? -1 : 1; | |
756 | } | |
757 | if (xxh3_result < 0) | |
758 | nni->num = CSUM_gone; | |
759 | return; | |
b7ea3fcd | 760 | } |
7e2711bb WD |
761 | #endif |
762 | ||
763 | #ifdef USE_OPENSSL | |
764 | if (BITS_SETnUNSET(nni->flags, NNI_EVP, NNI_BUILTIN|NNI_EVP_OK)) { | |
765 | if (nni->num == prior_num && nni->flags == prior_flags) { | |
766 | nni->flags = prior_result; | |
767 | if (!(nni->flags & NNI_EVP)) | |
768 | nni->num = CSUM_gone; | |
769 | } else { | |
770 | prior_num = nni->num; | |
771 | prior_flags = nni->flags; | |
772 | if (!csum_evp_md(nni)) | |
773 | nni->num = CSUM_gone; | |
774 | prior_result = nni->flags; | |
775 | if (check_auth_list && (nni = get_nni_by_num(&valid_auth_checksums, prior_num)) != NULL) | |
776 | verify_digest(nni, False); | |
b7ea3fcd | 777 | } |
b7ea3fcd WD |
778 | } |
779 | #endif | |
7e2711bb WD |
780 | } |
781 | #endif | |
782 | ||
783 | void init_checksum_choices() | |
784 | { | |
785 | struct name_num_item *nni; | |
786 | ||
787 | if (initialized_choices) | |
788 | return; | |
789 | ||
790 | #if defined SUPPORT_XXH3 || defined USE_OPENSSL | |
791 | for (nni = valid_checksums.list; nni->name; nni++) | |
792 | verify_digest(nni, True); | |
793 | ||
794 | for (nni = valid_auth_checksums.list; nni->name; nni++) | |
795 | verify_digest(nni, False); | |
796 | #endif | |
797 | ||
b7ea3fcd WD |
798 | initialized_choices = 1; |
799 | } |