]>
Commit | Line | Data |
---|---|---|
0d7638ea | 1 | /* |
0f78b815 WD |
2 | * Routines to support checksumming of bytes. |
3 | * | |
4 | * Copyright (C) 1996 Andrew Tridgell | |
5 | * Copyright (C) 1996 Paul Mackerras | |
9a06b2ed | 6 | * Copyright (C) 2004-2023 Wayne Davison |
0f78b815 WD |
7 | * |
8 | * This program is free software; you can redistribute it and/or modify | |
8e41b68e WD |
9 | * it under the terms of the GNU General Public License as published by |
10 | * the Free Software Foundation; either version 3 of the License, or | |
11 | * (at your option) any later version. | |
0f78b815 | 12 | * |
15c1162b WD |
13 | * In addition, as a special exception, the copyright holders give |
14 | * permission to dynamically link rsync with the OpenSSL and xxhash | |
15 | * libraries when those libraries are being distributed in compliance | |
16 | * with their license terms, and to distribute a dynamically linked | |
17 | * combination of rsync and these libraries. This is also considered | |
18 | * to be covered under the GPL's System Libraries exception. | |
19 | * | |
0f78b815 WD |
20 | * This program is distributed in the hope that it will be useful, |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
23 | * GNU General Public License for more details. | |
24 | * | |
e7c67065 | 25 | * You should have received a copy of the GNU General Public License along |
4fd842f9 | 26 | * with this program; if not, visit the http://fsf.org website. |
0f78b815 | 27 | */ |
c627d613 AT |
28 | |
29 | #include "rsync.h" | |
1bdf68b9 | 30 | |
a28bc3eb | 31 | #ifdef SUPPORT_XXHASH |
5eda68f1 | 32 | #include <xxhash.h> |
1bdf68b9 WD |
33 | # if XXH_VERSION_NUMBER >= 800 |
34 | # define SUPPORT_XXH3 1 | |
35 | # endif | |
a28bc3eb | 36 | #endif |
c627d613 | 37 | |
4f6c8c66 | 38 | extern int am_server; |
4f6c8c66 | 39 | extern int whole_file; |
8ee6eb71 | 40 | extern int checksum_seed; |
d04e9c51 | 41 | extern int protocol_version; |
eac85808 | 42 | extern int proper_seed_order; |
87f2984d | 43 | extern const char *checksum_choice; |
a5a7d3a2 | 44 | |
7e2711bb WD |
45 | #define NNI_BUILTIN (1<<0) |
46 | #define NNI_EVP (1<<1) | |
47 | #define NNI_EVP_OK (1<<2) | |
48 | ||
3258534e | 49 | struct name_num_item valid_checksums_items[] = { |
1bdf68b9 | 50 | #ifdef SUPPORT_XXH3 |
7e2711bb WD |
51 | { CSUM_XXH3_128, 0, "xxh128", NULL }, |
52 | { CSUM_XXH3_64, 0, "xxh3", NULL }, | |
1bdf68b9 | 53 | #endif |
a28bc3eb | 54 | #ifdef SUPPORT_XXHASH |
7e2711bb WD |
55 | { CSUM_XXH64, 0, "xxh64", NULL }, |
56 | { CSUM_XXH64, 0, "xxhash", NULL }, | |
a28bc3eb | 57 | #endif |
7e2711bb WD |
58 | { CSUM_MD5, NNI_BUILTIN|NNI_EVP, "md5", NULL }, |
59 | { CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL }, | |
ae16850d WD |
60 | #ifdef SHA_DIGEST_LENGTH |
61 | { CSUM_SHA1, NNI_EVP, "sha1", NULL }, | |
ae16850d | 62 | #endif |
7e2711bb WD |
63 | { CSUM_NONE, 0, "none", NULL }, |
64 | { 0, 0, NULL, NULL } | |
3258534e WD |
65 | }; |
66 | ||
67 | struct name_num_obj valid_checksums = { | |
7e2711bb WD |
68 | "checksum", NULL, 0, 0, valid_checksums_items |
69 | }; | |
70 | ||
71 | struct name_num_item valid_auth_checksums_items[] = { | |
ae16850d WD |
72 | #ifdef SHA512_DIGEST_LENGTH |
73 | { CSUM_SHA512, NNI_EVP, "sha512", NULL }, | |
74 | #endif | |
75 | #ifdef SHA256_DIGEST_LENGTH | |
76 | { CSUM_SHA256, NNI_EVP, "sha256", NULL }, | |
77 | #endif | |
78 | #ifdef SHA_DIGEST_LENGTH | |
79 | { CSUM_SHA1, NNI_EVP, "sha1", NULL }, | |
80 | #endif | |
7e2711bb WD |
81 | { CSUM_MD5, NNI_BUILTIN|NNI_EVP, "md5", NULL }, |
82 | { CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL }, | |
83 | { 0, 0, NULL, NULL } | |
55bb4dab | 84 | }; |
4f6c8c66 | 85 | |
7e2711bb WD |
86 | struct name_num_obj valid_auth_checksums = { |
87 | "daemon auth checksum", NULL, 0, 0, valid_auth_checksums_items | |
88 | }; | |
a5a7d3a2 | 89 | |
7e2711bb WD |
90 | /* These cannot make use of openssl, so they're marked just as built-in */ |
91 | struct name_num_item implied_checksum_md4 = | |
92 | { CSUM_MD4, NNI_BUILTIN, "md4", NULL }; | |
93 | struct name_num_item implied_checksum_md5 = | |
94 | { CSUM_MD5, NNI_BUILTIN, "md5", NULL }; | |
95 | ||
96 | struct name_num_item *xfer_sum_nni; /* used for the transfer checksum2 computations */ | |
7e2711bb WD |
97 | int xfer_sum_len; |
98 | struct name_num_item *file_sum_nni; /* used for the pre-transfer --checksum computations */ | |
5b27d2e6 | 99 | int file_sum_len, file_sum_extra_cnt; |
7e2711bb WD |
100 | |
101 | #ifdef USE_OPENSSL | |
591de7ce WD |
102 | const EVP_MD *xfer_sum_evp_md; |
103 | const EVP_MD *file_sum_evp_md; | |
7e2711bb WD |
104 | EVP_MD_CTX *ctx_evp = NULL; |
105 | #endif | |
591de7ce | 106 | |
b7ea3fcd WD |
107 | static int initialized_choices = 0; |
108 | ||
7e2711bb | 109 | struct name_num_item *parse_csum_name(const char *name, int len) |
a5a7d3a2 | 110 | { |
2f84a6bd | 111 | struct name_num_item *nni; |
55bb4dab | 112 | |
a5a7d3a2 WD |
113 | if (len < 0 && name) |
114 | len = strlen(name); | |
115 | ||
7e2711bb WD |
116 | init_checksum_choices(); |
117 | ||
2f84a6bd | 118 | if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) { |
7e2711bb WD |
119 | if (protocol_version >= 30) { |
120 | if (!proper_seed_order) | |
121 | return &implied_checksum_md5; | |
122 | name = "md5"; | |
123 | len = 3; | |
124 | } else { | |
125 | if (protocol_version >= 27) | |
126 | implied_checksum_md4.num = CSUM_MD4_OLD; | |
127 | else if (protocol_version >= 21) | |
128 | implied_checksum_md4.num = CSUM_MD4_BUSTED; | |
129 | else | |
130 | implied_checksum_md4.num = CSUM_MD4_ARCHAIC; | |
131 | return &implied_checksum_md4; | |
132 | } | |
a5a7d3a2 | 133 | } |
55bb4dab | 134 | |
2f84a6bd | 135 | nni = get_nni_by_name(&valid_checksums, name, len); |
a5a7d3a2 | 136 | |
2f84a6bd | 137 | if (!nni) { |
4f6c8c66 WD |
138 | rprintf(FERROR, "unknown checksum name: %s\n", name); |
139 | exit_cleanup(RERR_UNSUPPORTED); | |
140 | } | |
141 | ||
7e2711bb | 142 | return nni; |
4f6c8c66 WD |
143 | } |
144 | ||
591de7ce | 145 | #ifdef USE_OPENSSL |
7e2711bb | 146 | static const EVP_MD *csum_evp_md(struct name_num_item *nni) |
55bb4dab | 147 | { |
7e2711bb WD |
148 | const EVP_MD *emd; |
149 | if (!(nni->flags & NNI_EVP)) | |
150 | return NULL; | |
151 | ||
152 | #ifdef USE_MD5_ASM | |
153 | if (nni->num == CSUM_MD5) | |
154 | emd = NULL; | |
155 | else | |
156 | #endif | |
27feda04 | 157 | emd = EVP_get_digestbyname(nni->name); |
7e2711bb WD |
158 | if (emd && !(nni->flags & NNI_EVP_OK)) { /* Make sure it works before we advertise it */ |
159 | if (!ctx_evp && !(ctx_evp = EVP_MD_CTX_create())) | |
160 | out_of_memory("csum_evp_md"); | |
161 | /* Some routines are marked as legacy and are not enabled in the openssl.cnf file. | |
162 | * If we can't init the emd, we'll fall back to our built-in code. */ | |
163 | if (EVP_DigestInit_ex(ctx_evp, emd, NULL) == 0) | |
164 | emd = NULL; | |
165 | else | |
166 | nni->flags = (nni->flags & ~NNI_BUILTIN) | NNI_EVP_OK; | |
167 | } | |
168 | if (!emd) | |
169 | nni->flags &= ~NNI_EVP; | |
170 | return emd; | |
55bb4dab | 171 | } |
591de7ce | 172 | #endif |
55bb4dab WD |
173 | |
174 | void parse_checksum_choice(int final_call) | |
4f6c8c66 | 175 | { |
7e2711bb WD |
176 | if (valid_checksums.negotiated_nni) |
177 | xfer_sum_nni = file_sum_nni = valid_checksums.negotiated_nni; | |
97e8c55e | 178 | else { |
4f6c8c66 WD |
179 | char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL; |
180 | if (cp) { | |
7e2711bb WD |
181 | xfer_sum_nni = parse_csum_name(checksum_choice, cp - checksum_choice); |
182 | file_sum_nni = parse_csum_name(cp+1, -1); | |
4f6c8c66 | 183 | } else |
7e2711bb | 184 | xfer_sum_nni = file_sum_nni = parse_csum_name(checksum_choice, -1); |
28f9c960 | 185 | if (am_server && checksum_choice) |
7e2711bb | 186 | validate_choice_vs_env(NSTR_CHECKSUM, xfer_sum_nni->num, file_sum_nni->num); |
4f6c8c66 | 187 | } |
7e2711bb WD |
188 | xfer_sum_len = csum_len_for_type(xfer_sum_nni->num, 0); |
189 | file_sum_len = csum_len_for_type(file_sum_nni->num, 0); | |
591de7ce | 190 | #ifdef USE_OPENSSL |
7e2711bb WD |
191 | xfer_sum_evp_md = csum_evp_md(xfer_sum_nni); |
192 | file_sum_evp_md = csum_evp_md(file_sum_nni); | |
591de7ce | 193 | #endif |
55bb4dab | 194 | |
5b27d2e6 WD |
195 | file_sum_extra_cnt = (file_sum_len + EXTRA_LEN - 1) / EXTRA_LEN; |
196 | ||
7e2711bb | 197 | if (xfer_sum_nni->num == CSUM_NONE) |
4f6c8c66 | 198 | whole_file = 1; |
55bb4dab | 199 | |
87f2984d | 200 | /* Snag the checksum name for both write_batch's option output & the following debug output. */ |
7e2711bb WD |
201 | if (valid_checksums.negotiated_nni) |
202 | checksum_choice = valid_checksums.negotiated_nni->name; | |
87f2984d | 203 | else if (checksum_choice == NULL) |
7e2711bb | 204 | checksum_choice = xfer_sum_nni->name; |
87f2984d | 205 | |
d999efe6 | 206 | if (final_call && DEBUG_GTE(NSTR, am_server ? 3 : 1)) { |
87f2984d WD |
207 | rprintf(FINFO, "%s%s checksum: %s\n", |
208 | am_server ? "Server" : "Client", | |
7e2711bb | 209 | valid_checksums.negotiated_nni ? " negotiated" : "", |
87f2984d | 210 | checksum_choice); |
55bb4dab | 211 | } |
4f6c8c66 WD |
212 | } |
213 | ||
f5e8a17e | 214 | int csum_len_for_type(int cst, BOOL flist_csum) |
a5a7d3a2 WD |
215 | { |
216 | switch (cst) { | |
217 | case CSUM_NONE: | |
218 | return 1; | |
7b8a4ecd | 219 | case CSUM_MD4_ARCHAIC: |
416e719b WD |
220 | /* The oldest checksum code is rather weird: the file-list code only sent |
221 | * 2-byte checksums, but all other checksums were full MD4 length. */ | |
222 | return flist_csum ? 2 : MD4_DIGEST_LEN; | |
a5a7d3a2 WD |
223 | case CSUM_MD4: |
224 | case CSUM_MD4_OLD: | |
225 | case CSUM_MD4_BUSTED: | |
226 | return MD4_DIGEST_LEN; | |
227 | case CSUM_MD5: | |
228 | return MD5_DIGEST_LEN; | |
ae16850d WD |
229 | #ifdef SHA_DIGEST_LENGTH |
230 | case CSUM_SHA1: | |
231 | return SHA_DIGEST_LENGTH; | |
232 | #endif | |
233 | #ifdef SHA256_DIGEST_LENGTH | |
234 | case CSUM_SHA256: | |
235 | return SHA256_DIGEST_LENGTH; | |
236 | #endif | |
237 | #ifdef SHA512_DIGEST_LENGTH | |
238 | case CSUM_SHA512: | |
239 | return SHA512_DIGEST_LENGTH; | |
240 | #endif | |
4f92fd8d | 241 | case CSUM_XXH64: |
1bdf68b9 | 242 | case CSUM_XXH3_64: |
4f92fd8d | 243 | return 64/8; |
1bdf68b9 WD |
244 | case CSUM_XXH3_128: |
245 | return 128/8; | |
bc112b0e WD |
246 | default: /* paranoia to prevent missing case values */ |
247 | exit_cleanup(RERR_UNSUPPORTED); | |
a5a7d3a2 WD |
248 | } |
249 | return 0; | |
250 | } | |
251 | ||
d7521f54 WD |
252 | /* Returns 0 if the checksum is not canonical (i.e. it includes a seed value). |
253 | * Returns 1 if the public sum order matches our internal sum order. | |
254 | * Returns -1 if the public sum order is the reverse of our internal sum order. | |
255 | */ | |
a5a7d3a2 WD |
256 | int canonical_checksum(int csum_type) |
257 | { | |
d7521f54 WD |
258 | switch (csum_type) { |
259 | case CSUM_NONE: | |
260 | case CSUM_MD4_ARCHAIC: | |
261 | case CSUM_MD4_OLD: | |
262 | case CSUM_MD4_BUSTED: | |
263 | break; | |
264 | case CSUM_MD4: | |
265 | case CSUM_MD5: | |
ae16850d WD |
266 | case CSUM_SHA1: |
267 | case CSUM_SHA256: | |
268 | case CSUM_SHA512: | |
d7521f54 | 269 | return -1; |
d7521f54 | 270 | case CSUM_XXH64: |
1bdf68b9 WD |
271 | case CSUM_XXH3_64: |
272 | case CSUM_XXH3_128: | |
d7521f54 | 273 | return 1; |
d7521f54 WD |
274 | default: /* paranoia to prevent missing case values */ |
275 | exit_cleanup(RERR_UNSUPPORTED); | |
276 | } | |
277 | return 0; | |
a5a7d3a2 | 278 | } |
c627d613 | 279 | |
b81a5095 | 280 | #ifndef USE_ROLL_SIMD /* See simd-checksum-*.cpp. */ |
c627d613 | 281 | /* |
d2970213 | 282 | a simple 32 bit checksum that can be updated from either end |
c627d613 AT |
283 | (inspired by Mark Adler's Adler-32 checksum) |
284 | */ | |
deb5bf1d | 285 | uint32 get_checksum1(char *buf1, int32 len) |
c627d613 | 286 | { |
8475e0e4 WD |
287 | int32 i; |
288 | uint32 s1, s2; | |
289 | schar *buf = (schar *)buf1; | |
290 | ||
291 | s1 = s2 = 0; | |
292 | for (i = 0; i < (len-4); i+=4) { | |
293 | s2 += 4*(s1 + buf[i]) + 3*buf[i+1] + 2*buf[i+2] + buf[i+3] + 10*CHAR_OFFSET; | |
294 | s1 += (buf[i+0] + buf[i+1] + buf[i+2] + buf[i+3] + 4*CHAR_OFFSET); | |
295 | } | |
296 | for (; i < len; i++) { | |
297 | s1 += (buf[i]+CHAR_OFFSET); s2 += s1; | |
298 | } | |
299 | return (s1 & 0xffff) + (s2 << 16); | |
c627d613 | 300 | } |
70c6b408 | 301 | #endif |
c627d613 | 302 | |
0f599d36 | 303 | /* The "sum" buffer must be at least MAX_DIGEST_LEN bytes! */ |
deb5bf1d | 304 | void get_checksum2(char *buf, int32 len, char *sum) |
c627d613 | 305 | { |
7e2711bb WD |
306 | #ifdef USE_OPENSSL |
307 | if (xfer_sum_evp_md) { | |
308 | static EVP_MD_CTX *evp = NULL; | |
309 | uchar seedbuf[4]; | |
310 | if (!evp && !(evp = EVP_MD_CTX_create())) | |
311 | out_of_memory("get_checksum2"); | |
312 | EVP_DigestInit_ex(evp, xfer_sum_evp_md, NULL); | |
313 | if (checksum_seed) { | |
314 | SIVALu(seedbuf, 0, checksum_seed); | |
315 | EVP_DigestUpdate(evp, seedbuf, 4); | |
316 | } | |
317 | EVP_DigestUpdate(evp, (uchar *)buf, len); | |
318 | EVP_DigestFinal_ex(evp, (uchar *)sum, NULL); | |
319 | } else | |
320 | #endif | |
321 | switch (xfer_sum_nni->num) { | |
60e71c1b WD |
322 | #ifdef SUPPORT_XXHASH |
323 | case CSUM_XXH64: | |
324 | SIVAL64(sum, 0, XXH64(buf, len, checksum_seed)); | |
325 | break; | |
1bdf68b9 WD |
326 | #endif |
327 | #ifdef SUPPORT_XXH3 | |
328 | case CSUM_XXH3_64: | |
329 | SIVAL64(sum, 0, XXH3_64bits_withSeed(buf, len, checksum_seed)); | |
330 | break; | |
331 | case CSUM_XXH3_128: { | |
332 | XXH128_hash_t digest = XXH3_128bits_withSeed(buf, len, checksum_seed); | |
333 | SIVAL64(sum, 0, digest.low64); | |
334 | SIVAL64(sum, 8, digest.high64); | |
335 | break; | |
336 | } | |
60e71c1b | 337 | #endif |
a5a7d3a2 | 338 | case CSUM_MD5: { |
7e2711bb | 339 | md_context m5; |
a0456b9c | 340 | uchar seedbuf[4]; |
b81a5095 | 341 | md5_begin(&m5); |
eac85808 WD |
342 | if (proper_seed_order) { |
343 | if (checksum_seed) { | |
344 | SIVALu(seedbuf, 0, checksum_seed); | |
b81a5095 | 345 | md5_update(&m5, seedbuf, 4); |
eac85808 | 346 | } |
b81a5095 | 347 | md5_update(&m5, (uchar *)buf, len); |
eac85808 | 348 | } else { |
b81a5095 | 349 | md5_update(&m5, (uchar *)buf, len); |
eac85808 WD |
350 | if (checksum_seed) { |
351 | SIVALu(seedbuf, 0, checksum_seed); | |
b81a5095 | 352 | md5_update(&m5, seedbuf, 4); |
eac85808 | 353 | } |
a0456b9c | 354 | } |
b81a5095 | 355 | md5_result(&m5, (uchar *)sum); |
a5a7d3a2 WD |
356 | break; |
357 | } | |
358 | case CSUM_MD4: | |
359 | case CSUM_MD4_OLD: | |
7b8a4ecd WD |
360 | case CSUM_MD4_BUSTED: |
361 | case CSUM_MD4_ARCHAIC: { | |
1c9bb168 | 362 | md_context m; |
a0456b9c WD |
363 | int32 i; |
364 | static char *buf1; | |
365 | static int32 len1; | |
366 | ||
367 | mdfour_begin(&m); | |
368 | ||
369 | if (len > len1) { | |
370 | if (buf1) | |
371 | free(buf1); | |
372 | buf1 = new_array(char, len+4); | |
373 | len1 = len; | |
a0456b9c WD |
374 | } |
375 | ||
376 | memcpy(buf1, buf, len); | |
377 | if (checksum_seed) { | |
378 | SIVAL(buf1,len,checksum_seed); | |
379 | len += 4; | |
380 | } | |
381 | ||
382 | for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) | |
383 | mdfour_update(&m, (uchar *)(buf1+i), CSUM_CHUNK); | |
384 | ||
385 | /* | |
386 | * Prior to version 27 an incorrect MD4 checksum was computed | |
387 | * by failing to call mdfour_tail() for block sizes that | |
388 | * are multiples of 64. This is fixed by calling mdfour_update() | |
389 | * even when there are no more bytes. | |
390 | */ | |
7e2711bb | 391 | if (len - i > 0 || xfer_sum_nni->num > CSUM_MD4_BUSTED) |
a0456b9c WD |
392 | mdfour_update(&m, (uchar *)(buf1+i), len-i); |
393 | ||
394 | mdfour_result(&m, (uchar *)sum); | |
a5a7d3a2 WD |
395 | break; |
396 | } | |
bc112b0e WD |
397 | default: /* paranoia to prevent missing case values */ |
398 | exit_cleanup(RERR_UNSUPPORTED); | |
8de330a3 | 399 | } |
c627d613 AT |
400 | } |
401 | ||
43d6d0c5 | 402 | void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) |
c627d613 | 403 | { |
8de330a3 | 404 | struct map_struct *buf; |
43d6d0c5 | 405 | OFF_T i, len = st_p->st_size; |
26404276 WD |
406 | int32 remainder; |
407 | int fd; | |
0d7638ea | 408 | |
8c9fd200 | 409 | fd = do_open(fname, O_RDONLY, 0); |
7e2711bb WD |
410 | if (fd == -1) { |
411 | memset(sum, 0, file_sum_len); | |
efa95a18 | 412 | return; |
7e2711bb | 413 | } |
0d7638ea | 414 | |
d474f298 | 415 | buf = map_file(fd, len, MAX_MAP_SIZE, CHUNK_SIZE); |
0d7638ea | 416 | |
7e2711bb WD |
417 | #ifdef USE_OPENSSL |
418 | if (file_sum_evp_md) { | |
419 | static EVP_MD_CTX *evp = NULL; | |
420 | if (!evp && !(evp = EVP_MD_CTX_create())) | |
421 | out_of_memory("file_checksum"); | |
422 | ||
423 | EVP_DigestInit_ex(evp, file_sum_evp_md, NULL); | |
424 | ||
425 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
426 | EVP_DigestUpdate(evp, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
427 | ||
428 | remainder = (int32)(len - i); | |
429 | if (remainder > 0) | |
430 | EVP_DigestUpdate(evp, (uchar *)map_ptr(buf, i, remainder), remainder); | |
431 | ||
432 | EVP_DigestFinal_ex(evp, (uchar *)sum, NULL); | |
433 | } else | |
434 | #endif | |
435 | switch (file_sum_nni->num) { | |
60e71c1b WD |
436 | #ifdef SUPPORT_XXHASH |
437 | case CSUM_XXH64: { | |
438 | static XXH64_state_t* state = NULL; | |
439 | if (!state && !(state = XXH64_createState())) | |
440 | out_of_memory("file_checksum"); | |
441 | ||
442 | XXH64_reset(state, 0); | |
443 | ||
444 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
445 | XXH64_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
446 | ||
447 | remainder = (int32)(len - i); | |
448 | if (remainder > 0) | |
449 | XXH64_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); | |
450 | ||
451 | SIVAL64(sum, 0, XXH64_digest(state)); | |
452 | break; | |
453 | } | |
1bdf68b9 WD |
454 | #endif |
455 | #ifdef SUPPORT_XXH3 | |
456 | case CSUM_XXH3_64: { | |
457 | static XXH3_state_t* state = NULL; | |
458 | if (!state && !(state = XXH3_createState())) | |
459 | out_of_memory("file_checksum"); | |
460 | ||
461 | XXH3_64bits_reset(state); | |
462 | ||
463 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
464 | XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
465 | ||
466 | remainder = (int32)(len - i); | |
467 | if (remainder > 0) | |
468 | XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); | |
469 | ||
470 | SIVAL64(sum, 0, XXH3_64bits_digest(state)); | |
471 | break; | |
472 | } | |
473 | case CSUM_XXH3_128: { | |
474 | XXH128_hash_t digest; | |
475 | static XXH3_state_t* state = NULL; | |
476 | if (!state && !(state = XXH3_createState())) | |
477 | out_of_memory("file_checksum"); | |
478 | ||
479 | XXH3_128bits_reset(state); | |
480 | ||
481 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) | |
482 | XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); | |
483 | ||
484 | remainder = (int32)(len - i); | |
485 | if (remainder > 0) | |
486 | XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); | |
487 | ||
488 | digest = XXH3_128bits_digest(state); | |
489 | SIVAL64(sum, 0, digest.low64); | |
490 | SIVAL64(sum, 8, digest.high64); | |
491 | break; | |
492 | } | |
60e71c1b | 493 | #endif |
1c9bb168 | 494 | case CSUM_MD5: { |
7e2711bb | 495 | md_context m5; |
1c9bb168 | 496 | |
b81a5095 | 497 | md5_begin(&m5); |
8de330a3 | 498 | |
d474f298 | 499 | for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) |
b81a5095 | 500 | md5_update(&m5, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); |
8de330a3 | 501 | |
a0456b9c WD |
502 | remainder = (int32)(len - i); |
503 | if (remainder > 0) | |
b81a5095 | 504 | md5_update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder); |
8de330a3 | 505 | |
b81a5095 | 506 | md5_result(&m5, (uchar *)sum); |
a5a7d3a2 | 507 | break; |
1c9bb168 | 508 | } |
a5a7d3a2 WD |
509 | case CSUM_MD4: |
510 | case CSUM_MD4_OLD: | |
511 | case CSUM_MD4_BUSTED: | |
1c9bb168 WD |
512 | case CSUM_MD4_ARCHAIC: { |
513 | md_context m; | |
514 | ||
a0456b9c WD |
515 | mdfour_begin(&m); |
516 | ||
6ac2c7b6 WD |
517 | for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) |
518 | mdfour_update(&m, (uchar *)map_ptr(buf, i, CSUM_CHUNK), CSUM_CHUNK); | |
a0456b9c WD |
519 | |
520 | /* Prior to version 27 an incorrect MD4 checksum was computed | |
521 | * by failing to call mdfour_tail() for block sizes that | |
522 | * are multiples of 64. This is fixed by calling mdfour_update() | |
523 | * even when there are no more bytes. */ | |
524 | remainder = (int32)(len - i); | |
7e2711bb | 525 | if (remainder > 0 || file_sum_nni->num > CSUM_MD4_BUSTED) |
a0456b9c WD |
526 | mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder); |
527 | ||
528 | mdfour_result(&m, (uchar *)sum); | |
a5a7d3a2 | 529 | break; |
1c9bb168 | 530 | } |
a5a7d3a2 | 531 | default: |
2f84a6bd | 532 | rprintf(FERROR, "Invalid checksum-choice for --checksum: %s (%d)\n", |
7e2711bb | 533 | file_sum_nni->name, file_sum_nni->num); |
a5a7d3a2 | 534 | exit_cleanup(RERR_UNSUPPORTED); |
a0456b9c | 535 | } |
8de330a3 AT |
536 | |
537 | close(fd); | |
538 | unmap_file(buf); | |
c627d613 | 539 | } |
34ccb63e | 540 | |
deb5bf1d | 541 | static int32 sumresidue; |
7e2711bb | 542 | static md_context ctx_md; |
a28bc3eb | 543 | #ifdef SUPPORT_XXHASH |
4f92fd8d | 544 | static XXH64_state_t* xxh64_state; |
a28bc3eb | 545 | #endif |
1bdf68b9 WD |
546 | #ifdef SUPPORT_XXH3 |
547 | static XXH3_state_t* xxh3_state; | |
548 | #endif | |
7e2711bb | 549 | static struct name_num_item *cur_sum_nni; |
7e2711bb | 550 | int cur_sum_len; |
9e31c482 | 551 | |
591de7ce WD |
552 | #ifdef USE_OPENSSL |
553 | static const EVP_MD *cur_sum_evp_md; | |
554 | #endif | |
555 | ||
4d44bf12 WD |
556 | /* Initialize a hash digest accumulator. Data is supplied via |
557 | * sum_update() and the resulting binary digest is retrieved via | |
558 | * sum_end(). This only supports one active sum at a time. */ | |
7e2711bb | 559 | int sum_init(struct name_num_item *nni, int seed) |
9e31c482 | 560 | { |
8de330a3 | 561 | char s[4]; |
a0456b9c | 562 | |
7e2711bb WD |
563 | if (!nni) |
564 | nni = parse_csum_name(NULL, 0); | |
565 | cur_sum_nni = nni; | |
566 | cur_sum_len = csum_len_for_type(nni->num, 0); | |
591de7ce | 567 | #ifdef USE_OPENSSL |
7e2711bb | 568 | cur_sum_evp_md = csum_evp_md(nni); |
591de7ce | 569 | #endif |
a5a7d3a2 | 570 | |
7e2711bb WD |
571 | #ifdef USE_OPENSSL |
572 | if (cur_sum_evp_md) { | |
573 | if (!ctx_evp && !(ctx_evp = EVP_MD_CTX_create())) | |
574 | out_of_memory("file_checksum"); | |
575 | EVP_DigestInit_ex(ctx_evp, cur_sum_evp_md, NULL); | |
576 | } else | |
577 | #endif | |
578 | switch (cur_sum_nni->num) { | |
60e71c1b WD |
579 | #ifdef SUPPORT_XXHASH |
580 | case CSUM_XXH64: | |
581 | if (!xxh64_state && !(xxh64_state = XXH64_createState())) | |
582 | out_of_memory("sum_init"); | |
583 | XXH64_reset(xxh64_state, 0); | |
584 | break; | |
1bdf68b9 WD |
585 | #endif |
586 | #ifdef SUPPORT_XXH3 | |
587 | case CSUM_XXH3_64: | |
588 | if (!xxh3_state && !(xxh3_state = XXH3_createState())) | |
589 | out_of_memory("sum_init"); | |
590 | XXH3_64bits_reset(xxh3_state); | |
591 | break; | |
592 | case CSUM_XXH3_128: | |
593 | if (!xxh3_state && !(xxh3_state = XXH3_createState())) | |
594 | out_of_memory("sum_init"); | |
595 | XXH3_128bits_reset(xxh3_state); | |
596 | break; | |
60e71c1b | 597 | #endif |
a5a7d3a2 | 598 | case CSUM_MD5: |
7e2711bb | 599 | md5_begin(&ctx_md); |
a5a7d3a2 WD |
600 | break; |
601 | case CSUM_MD4: | |
7e2711bb | 602 | mdfour_begin(&ctx_md); |
a5a7d3a2 WD |
603 | sumresidue = 0; |
604 | break; | |
605 | case CSUM_MD4_OLD: | |
606 | case CSUM_MD4_BUSTED: | |
7b8a4ecd | 607 | case CSUM_MD4_ARCHAIC: |
7e2711bb | 608 | mdfour_begin(&ctx_md); |
a0456b9c | 609 | sumresidue = 0; |
9889a34b WD |
610 | SIVAL(s, 0, seed); |
611 | sum_update(s, 4); | |
a5a7d3a2 WD |
612 | break; |
613 | case CSUM_NONE: | |
614 | break; | |
bc112b0e WD |
615 | default: /* paranoia to prevent missing case values */ |
616 | exit_cleanup(RERR_UNSUPPORTED); | |
a0456b9c | 617 | } |
7e2711bb WD |
618 | |
619 | return cur_sum_len; | |
9e31c482 AT |
620 | } |
621 | ||
4d44bf12 | 622 | /* Feed data into a hash digest accumulator. */ |
4a19c3b2 | 623 | void sum_update(const char *p, int32 len) |
9e31c482 | 624 | { |
7e2711bb WD |
625 | #ifdef USE_OPENSSL |
626 | if (cur_sum_evp_md) { | |
627 | EVP_DigestUpdate(ctx_evp, (uchar *)p, len); | |
628 | } else | |
629 | #endif | |
630 | switch (cur_sum_nni->num) { | |
60e71c1b WD |
631 | #ifdef SUPPORT_XXHASH |
632 | case CSUM_XXH64: | |
633 | XXH64_update(xxh64_state, p, len); | |
634 | break; | |
1bdf68b9 WD |
635 | #endif |
636 | #ifdef SUPPORT_XXH3 | |
637 | case CSUM_XXH3_64: | |
638 | XXH3_64bits_update(xxh3_state, p, len); | |
639 | break; | |
640 | case CSUM_XXH3_128: | |
641 | XXH3_128bits_update(xxh3_state, p, len); | |
642 | break; | |
60e71c1b | 643 | #endif |
a5a7d3a2 | 644 | case CSUM_MD5: |
7e2711bb | 645 | md5_update(&ctx_md, (uchar *)p, len); |
a5a7d3a2 WD |
646 | break; |
647 | case CSUM_MD4: | |
648 | case CSUM_MD4_OLD: | |
649 | case CSUM_MD4_BUSTED: | |
c252546c | 650 | case CSUM_MD4_ARCHAIC: |
4fc78878 | 651 | if (len + sumresidue < CSUM_CHUNK) { |
7e2711bb | 652 | memcpy(ctx_md.buffer + sumresidue, p, len); |
4fc78878 | 653 | sumresidue += len; |
a5a7d3a2 | 654 | break; |
4fc78878 | 655 | } |
a0456b9c | 656 | |
4fc78878 WD |
657 | if (sumresidue) { |
658 | int32 i = CSUM_CHUNK - sumresidue; | |
7e2711bb WD |
659 | memcpy(ctx_md.buffer + sumresidue, p, i); |
660 | mdfour_update(&ctx_md, (uchar *)ctx_md.buffer, CSUM_CHUNK); | |
4fc78878 WD |
661 | len -= i; |
662 | p += i; | |
663 | } | |
8de330a3 | 664 | |
4fc78878 | 665 | while (len >= CSUM_CHUNK) { |
7e2711bb | 666 | mdfour_update(&ctx_md, (uchar *)p, CSUM_CHUNK); |
4fc78878 WD |
667 | len -= CSUM_CHUNK; |
668 | p += CSUM_CHUNK; | |
669 | } | |
8de330a3 | 670 | |
4fc78878 WD |
671 | sumresidue = len; |
672 | if (sumresidue) | |
7e2711bb | 673 | memcpy(ctx_md.buffer, p, sumresidue); |
a5a7d3a2 WD |
674 | break; |
675 | case CSUM_NONE: | |
676 | break; | |
bc112b0e WD |
677 | default: /* paranoia to prevent missing case values */ |
678 | exit_cleanup(RERR_UNSUPPORTED); | |
8de330a3 | 679 | } |
9e31c482 AT |
680 | } |
681 | ||
7e2711bb WD |
682 | /* The sum buffer only needs to be as long as the current checksum's digest |
683 | * len, not MAX_DIGEST_LEN. Note that for CSUM_MD4_ARCHAIC that is the full | |
684 | * MD4_DIGEST_LEN even if the file-list code is going to ignore all but the | |
685 | * first 2 bytes of it. */ | |
686 | void sum_end(char *sum) | |
9e31c482 | 687 | { |
7e2711bb WD |
688 | #ifdef USE_OPENSSL |
689 | if (cur_sum_evp_md) { | |
690 | EVP_DigestFinal_ex(ctx_evp, (uchar *)sum, NULL); | |
691 | } else | |
692 | #endif | |
693 | switch (cur_sum_nni->num) { | |
60e71c1b WD |
694 | #ifdef SUPPORT_XXHASH |
695 | case CSUM_XXH64: | |
696 | SIVAL64(sum, 0, XXH64_digest(xxh64_state)); | |
697 | break; | |
1bdf68b9 WD |
698 | #endif |
699 | #ifdef SUPPORT_XXH3 | |
700 | case CSUM_XXH3_64: | |
701 | SIVAL64(sum, 0, XXH3_64bits_digest(xxh3_state)); | |
702 | break; | |
703 | case CSUM_XXH3_128: { | |
704 | XXH128_hash_t digest = XXH3_128bits_digest(xxh3_state); | |
705 | SIVAL64(sum, 0, digest.low64); | |
706 | SIVAL64(sum, 8, digest.high64); | |
707 | break; | |
708 | } | |
60e71c1b | 709 | #endif |
a5a7d3a2 | 710 | case CSUM_MD5: |
7e2711bb | 711 | md5_result(&ctx_md, (uchar *)sum); |
a5a7d3a2 WD |
712 | break; |
713 | case CSUM_MD4: | |
714 | case CSUM_MD4_OLD: | |
7e2711bb WD |
715 | mdfour_update(&ctx_md, (uchar *)ctx_md.buffer, sumresidue); |
716 | mdfour_result(&ctx_md, (uchar *)sum); | |
a5a7d3a2 WD |
717 | break; |
718 | case CSUM_MD4_BUSTED: | |
7b8a4ecd | 719 | case CSUM_MD4_ARCHAIC: |
a5a7d3a2 | 720 | if (sumresidue) |
7e2711bb WD |
721 | mdfour_update(&ctx_md, (uchar *)ctx_md.buffer, sumresidue); |
722 | mdfour_result(&ctx_md, (uchar *)sum); | |
a5a7d3a2 WD |
723 | break; |
724 | case CSUM_NONE: | |
725 | *sum = '\0'; | |
726 | break; | |
bc112b0e WD |
727 | default: /* paranoia to prevent missing case values */ |
728 | exit_cleanup(RERR_UNSUPPORTED); | |
4fc78878 | 729 | } |
9e31c482 | 730 | } |
b7ea3fcd | 731 | |
7e2711bb WD |
732 | #if defined SUPPORT_XXH3 || defined USE_OPENSSL |
733 | static void verify_digest(struct name_num_item *nni, BOOL check_auth_list) | |
b7ea3fcd WD |
734 | { |
735 | #ifdef SUPPORT_XXH3 | |
7e2711bb WD |
736 | static int xxh3_result = 0; |
737 | #endif | |
738 | #ifdef USE_OPENSSL | |
739 | static int prior_num = 0, prior_flags = 0, prior_result = 0; | |
740 | #endif | |
741 | ||
742 | #ifdef SUPPORT_XXH3 | |
743 | if (nni->num == CSUM_XXH3_64 || nni->num == CSUM_XXH3_128) { | |
744 | if (!xxh3_result) { | |
745 | char buf[32816]; | |
746 | int j; | |
747 | for (j = 0; j < (int)sizeof buf; j++) | |
748 | buf[j] = ' ' + (j % 96); | |
749 | sum_init(nni, 0); | |
750 | sum_update(buf, 32816); | |
751 | sum_update(buf, 31152); | |
752 | sum_update(buf, 32474); | |
753 | sum_update(buf, 9322); | |
754 | xxh3_result = XXH3_64bits_digest(xxh3_state) != 0xadbcf16d4678d1de ? -1 : 1; | |
755 | } | |
756 | if (xxh3_result < 0) | |
757 | nni->num = CSUM_gone; | |
758 | return; | |
b7ea3fcd | 759 | } |
7e2711bb WD |
760 | #endif |
761 | ||
762 | #ifdef USE_OPENSSL | |
763 | if (BITS_SETnUNSET(nni->flags, NNI_EVP, NNI_BUILTIN|NNI_EVP_OK)) { | |
764 | if (nni->num == prior_num && nni->flags == prior_flags) { | |
765 | nni->flags = prior_result; | |
766 | if (!(nni->flags & NNI_EVP)) | |
767 | nni->num = CSUM_gone; | |
768 | } else { | |
769 | prior_num = nni->num; | |
770 | prior_flags = nni->flags; | |
771 | if (!csum_evp_md(nni)) | |
772 | nni->num = CSUM_gone; | |
773 | prior_result = nni->flags; | |
774 | if (check_auth_list && (nni = get_nni_by_num(&valid_auth_checksums, prior_num)) != NULL) | |
775 | verify_digest(nni, False); | |
b7ea3fcd | 776 | } |
b7ea3fcd WD |
777 | } |
778 | #endif | |
7e2711bb WD |
779 | } |
780 | #endif | |
781 | ||
782 | void init_checksum_choices() | |
783 | { | |
591de7ce | 784 | #if defined SUPPORT_XXH3 || defined USE_OPENSSL |
7e2711bb | 785 | struct name_num_item *nni; |
591de7ce | 786 | #endif |
7e2711bb WD |
787 | |
788 | if (initialized_choices) | |
789 | return; | |
790 | ||
27feda04 WD |
791 | #if defined USE_OPENSSL && OPENSSL_VERSION_NUMBER < 0x10100000L |
792 | OpenSSL_add_all_algorithms(); | |
793 | #endif | |
794 | ||
7e2711bb WD |
795 | #if defined SUPPORT_XXH3 || defined USE_OPENSSL |
796 | for (nni = valid_checksums.list; nni->name; nni++) | |
797 | verify_digest(nni, True); | |
798 | ||
799 | for (nni = valid_auth_checksums.list; nni->name; nni++) | |
800 | verify_digest(nni, False); | |
801 | #endif | |
802 | ||
b7ea3fcd WD |
803 | initialized_choices = 1; |
804 | } |