]>
Commit | Line | Data |
---|---|---|
5e3f94df | 1 | #include "git-compat-util.h" |
c4d25228 | 2 | #include "config.h" |
396f2570 | 3 | #include "dir.h" |
41771fa4 | 4 | #include "hex.h" |
396f2570 | 5 | #include "packfile.h" |
87bed179 | 6 | #include "object-file.h" |
bc626927 | 7 | #include "hash-lookup.h" |
a3407730 | 8 | #include "midx.h" |
144d7033 | 9 | #include "progress.h" |
d829223a | 10 | #include "trace2.h" |
63a8f0e9 | 11 | #include "chunk-format.h" |
c528e179 | 12 | #include "pack-bitmap.h" |
b1e33330 | 13 | #include "pack-revindex.h" |
a3407730 | 14 | |
748b88a0 TB |
15 | int midx_checksum_valid(struct multi_pack_index *m); |
16 | void clear_midx_files_ext(const char *object_dir, const char *ext, | |
17 | unsigned char *keep_hash); | |
18 | int cmp_idx_or_pack_name(const char *idx_or_pack_name, | |
19 | const char *idx_name); | |
19575c7c | 20 | |
0f533c72 | 21 | const unsigned char *get_midx_checksum(struct multi_pack_index *m) |
f894081d TB |
22 | { |
23 | return m->data + m->data_len - the_hash_algo->rawsz; | |
24 | } | |
25 | ||
60980aed | 26 | void get_midx_filename(struct strbuf *out, const char *object_dir) |
fc59e748 | 27 | { |
60980aed | 28 | strbuf_addf(out, "%s/pack/multi-pack-index", object_dir); |
fc59e748 DS |
29 | } |
30 | ||
60980aed | 31 | void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m) |
f894081d | 32 | { |
60980aed TB |
33 | get_midx_filename(out, m->object_dir); |
34 | strbuf_addf(out, "-%s.rev", hash_to_hex(get_midx_checksum(m))); | |
f894081d TB |
35 | } |
36 | ||
6ab3b8b8 DS |
37 | static int midx_read_oid_fanout(const unsigned char *chunk_start, |
38 | size_t chunk_size, void *data) | |
39 | { | |
9d78fb0e | 40 | int i; |
6ab3b8b8 DS |
41 | struct multi_pack_index *m = data; |
42 | m->chunk_oid_fanout = (uint32_t *)chunk_start; | |
43 | ||
44 | if (chunk_size != 4 * 256) { | |
45 | error(_("multi-pack-index OID fanout is of the wrong size")); | |
46 | return 1; | |
47 | } | |
9d78fb0e JK |
48 | for (i = 0; i < 255; i++) { |
49 | uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]); | |
50 | uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i+1]); | |
51 | ||
52 | if (oid_fanout1 > oid_fanout2) { | |
53 | error(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"), | |
54 | i, oid_fanout1, oid_fanout2, i + 1); | |
55 | return 1; | |
56 | } | |
57 | } | |
fc926567 JK |
58 | m->num_objects = ntohl(m->chunk_oid_fanout[255]); |
59 | return 0; | |
60 | } | |
61 | ||
62 | static int midx_read_oid_lookup(const unsigned char *chunk_start, | |
63 | size_t chunk_size, void *data) | |
64 | { | |
65 | struct multi_pack_index *m = data; | |
66 | m->chunk_oid_lookup = chunk_start; | |
67 | ||
68 | if (chunk_size != st_mult(m->hash_len, m->num_objects)) { | |
69 | error(_("multi-pack-index OID lookup chunk is the wrong size")); | |
70 | return 1; | |
71 | } | |
6ab3b8b8 DS |
72 | return 0; |
73 | } | |
74 | ||
0924869b JK |
75 | static int midx_read_object_offsets(const unsigned char *chunk_start, |
76 | size_t chunk_size, void *data) | |
77 | { | |
78 | struct multi_pack_index *m = data; | |
79 | m->chunk_object_offsets = chunk_start; | |
80 | ||
81 | if (chunk_size != st_mult(m->num_objects, MIDX_CHUNK_OFFSET_WIDTH)) { | |
82 | error(_("multi-pack-index object offset chunk is the wrong size")); | |
83 | return 1; | |
84 | } | |
85 | return 0; | |
86 | } | |
87 | ||
748b88a0 TB |
88 | #define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz) |
89 | ||
2cf489a3 | 90 | struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local) |
4d80560c DS |
91 | { |
92 | struct multi_pack_index *m = NULL; | |
93 | int fd; | |
94 | struct stat st; | |
95 | size_t midx_size; | |
96 | void *midx_map = NULL; | |
97 | uint32_t hash_version; | |
60980aed | 98 | struct strbuf midx_name = STRBUF_INIT; |
32f3c541 | 99 | uint32_t i; |
3227565c | 100 | const char *cur_pack_name; |
6ab3b8b8 | 101 | struct chunkfile *cf = NULL; |
4d80560c | 102 | |
60980aed TB |
103 | get_midx_filename(&midx_name, object_dir); |
104 | ||
105 | fd = git_open(midx_name.buf); | |
4d80560c DS |
106 | |
107 | if (fd < 0) | |
108 | goto cleanup_fail; | |
109 | if (fstat(fd, &st)) { | |
60980aed | 110 | error_errno(_("failed to read %s"), midx_name.buf); |
4d80560c DS |
111 | goto cleanup_fail; |
112 | } | |
113 | ||
114 | midx_size = xsize_t(st.st_size); | |
115 | ||
116 | if (midx_size < MIDX_MIN_SIZE) { | |
60980aed | 117 | error(_("multi-pack-index file %s is too small"), midx_name.buf); |
4d80560c DS |
118 | goto cleanup_fail; |
119 | } | |
120 | ||
60980aed | 121 | strbuf_release(&midx_name); |
4d80560c DS |
122 | |
123 | midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); | |
6c7ff7cf | 124 | close(fd); |
4d80560c | 125 | |
577314ca | 126 | FLEX_ALLOC_STR(m, object_dir, object_dir); |
4d80560c DS |
127 | m->data = midx_map; |
128 | m->data_len = midx_size; | |
2cf489a3 | 129 | m->local = local; |
4d80560c DS |
130 | |
131 | m->signature = get_be32(m->data); | |
53ad0407 DS |
132 | if (m->signature != MIDX_SIGNATURE) |
133 | die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"), | |
4d80560c | 134 | m->signature, MIDX_SIGNATURE); |
4d80560c DS |
135 | |
136 | m->version = m->data[MIDX_BYTE_FILE_VERSION]; | |
53ad0407 DS |
137 | if (m->version != MIDX_VERSION) |
138 | die(_("multi-pack-index version %d not recognized"), | |
4d80560c | 139 | m->version); |
4d80560c DS |
140 | |
141 | hash_version = m->data[MIDX_BYTE_HASH_VERSION]; | |
d9fef9d9 | 142 | if (hash_version != oid_version(the_hash_algo)) { |
d9607542 | 143 | error(_("multi-pack-index hash version %u does not match version %u"), |
d9fef9d9 | 144 | hash_version, oid_version(the_hash_algo)); |
d9607542 DS |
145 | goto cleanup_fail; |
146 | } | |
aaa95dfa | 147 | m->hash_len = the_hash_algo->rawsz; |
4d80560c DS |
148 | |
149 | m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS]; | |
150 | ||
151 | m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS); | |
152 | ||
b1e33330 TB |
153 | m->preferred_pack_idx = -1; |
154 | ||
6ab3b8b8 | 155 | cf = init_chunkfile(NULL); |
32f3c541 | 156 | |
6ab3b8b8 | 157 | if (read_table_of_contents(cf, m->data, midx_size, |
c9b9fefc JK |
158 | MIDX_HEADER_SIZE, m->num_chunks, |
159 | MIDX_CHUNK_ALIGNMENT)) | |
6ab3b8b8 | 160 | goto cleanup_fail; |
32f3c541 | 161 | |
72a9a082 | 162 | if (pair_chunk(cf, MIDX_CHUNKID_PACKNAMES, &m->chunk_pack_names, &m->chunk_pack_names_len)) |
e3c96003 JK |
163 | die(_("multi-pack-index required pack-name chunk missing or corrupted")); |
164 | if (read_chunk(cf, MIDX_CHUNKID_OIDFANOUT, midx_read_oid_fanout, m)) | |
165 | die(_("multi-pack-index required OID fanout chunk missing or corrupted")); | |
fc926567 | 166 | if (read_chunk(cf, MIDX_CHUNKID_OIDLOOKUP, midx_read_oid_lookup, m)) |
e3c96003 | 167 | die(_("multi-pack-index required OID lookup chunk missing or corrupted")); |
0924869b | 168 | if (read_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS, midx_read_object_offsets, m)) |
e3c96003 | 169 | die(_("multi-pack-index required object offsets chunk missing or corrupted")); |
32f3c541 | 170 | |
2abd56e9 JK |
171 | pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets, |
172 | &m->chunk_large_offsets_len); | |
5f5ccd95 TB |
173 | pair_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS, |
174 | (const unsigned char **)&m->chunk_bitmapped_packs, | |
175 | &m->chunk_bitmapped_packs_len); | |
6ab3b8b8 | 176 | |
7f514b7a | 177 | if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1)) |
c0fe9b2d JK |
178 | pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex, |
179 | &m->chunk_revindex_len); | |
7f514b7a | 180 | |
ca56dadb RS |
181 | CALLOC_ARRAY(m->pack_names, m->num_packs); |
182 | CALLOC_ARRAY(m->packs, m->num_packs); | |
3227565c DS |
183 | |
184 | cur_pack_name = (const char *)m->chunk_pack_names; | |
185 | for (i = 0; i < m->num_packs; i++) { | |
72a9a082 JK |
186 | const char *end; |
187 | size_t avail = m->chunk_pack_names_len - | |
188 | (cur_pack_name - (const char *)m->chunk_pack_names); | |
189 | ||
3227565c DS |
190 | m->pack_names[i] = cur_pack_name; |
191 | ||
72a9a082 JK |
192 | end = memchr(cur_pack_name, '\0', avail); |
193 | if (!end) | |
194 | die(_("multi-pack-index pack-name chunk is too short")); | |
195 | cur_pack_name = end + 1; | |
3227565c | 196 | |
8e72a3c3 DS |
197 | if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0) |
198 | die(_("multi-pack-index pack names out of order: '%s' before '%s'"), | |
3227565c DS |
199 | m->pack_names[i - 1], |
200 | m->pack_names[i]); | |
3227565c DS |
201 | } |
202 | ||
d829223a JH |
203 | trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs); |
204 | trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects); | |
205 | ||
692305ec | 206 | free_chunkfile(cf); |
4d80560c DS |
207 | return m; |
208 | ||
209 | cleanup_fail: | |
210 | free(m); | |
60980aed | 211 | strbuf_release(&midx_name); |
692305ec | 212 | free_chunkfile(cf); |
4d80560c DS |
213 | if (midx_map) |
214 | munmap(midx_map, midx_size); | |
215 | if (0 <= fd) | |
216 | close(fd); | |
217 | return NULL; | |
218 | } | |
219 | ||
1dcd9f20 | 220 | void close_midx(struct multi_pack_index *m) |
a40498a1 DS |
221 | { |
222 | uint32_t i; | |
1dcd9f20 DS |
223 | |
224 | if (!m) | |
225 | return; | |
226 | ||
9bb6c2e5 TB |
227 | close_midx(m->next); |
228 | ||
a40498a1 | 229 | munmap((unsigned char *)m->data, m->data_len); |
a40498a1 DS |
230 | |
231 | for (i = 0; i < m->num_packs; i++) { | |
af96fe33 DS |
232 | if (m->packs[i]) |
233 | m->packs[i]->multi_pack_index = 0; | |
a40498a1 DS |
234 | } |
235 | FREE_AND_NULL(m->packs); | |
236 | FREE_AND_NULL(m->pack_names); | |
9bb6c2e5 | 237 | free(m); |
a40498a1 DS |
238 | } |
239 | ||
64404a24 | 240 | int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id) |
3715a633 DS |
241 | { |
242 | struct strbuf pack_name = STRBUF_INIT; | |
af96fe33 | 243 | struct packed_git *p; |
3715a633 DS |
244 | |
245 | if (pack_int_id >= m->num_packs) | |
d355e46a | 246 | die(_("bad pack-int-id: %u (%u total packs)"), |
cc6af73c | 247 | pack_int_id, m->num_packs); |
3715a633 DS |
248 | |
249 | if (m->packs[pack_int_id]) | |
250 | return 0; | |
251 | ||
252 | strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir, | |
253 | m->pack_names[pack_int_id]); | |
254 | ||
af96fe33 | 255 | p = add_packed_git(pack_name.buf, pack_name.len, m->local); |
3715a633 | 256 | strbuf_release(&pack_name); |
af96fe33 DS |
257 | |
258 | if (!p) | |
259 | return 1; | |
260 | ||
261 | p->multi_pack_index = 1; | |
262 | m->packs[pack_int_id] = p; | |
263 | install_packed_git(r, p); | |
264 | list_add_tail(&p->mru, &r->objects->packed_git_mru); | |
265 | ||
266 | return 0; | |
3715a633 DS |
267 | } |
268 | ||
748b88a0 TB |
269 | #define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t)) |
270 | ||
5f5ccd95 TB |
271 | int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, |
272 | struct bitmapped_pack *bp, uint32_t pack_int_id) | |
273 | { | |
274 | if (!m->chunk_bitmapped_packs) | |
275 | return error(_("MIDX does not contain the BTMP chunk")); | |
276 | ||
277 | if (prepare_midx_pack(r, m, pack_int_id)) | |
278 | return error(_("could not load bitmapped pack %"PRIu32), pack_int_id); | |
279 | ||
280 | bp->p = m->packs[pack_int_id]; | |
281 | bp->bitmap_pos = get_be32((char *)m->chunk_bitmapped_packs + | |
282 | MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id); | |
283 | bp->bitmap_nr = get_be32((char *)m->chunk_bitmapped_packs + | |
284 | MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id + | |
285 | sizeof(uint32_t)); | |
286 | bp->pack_int_id = pack_int_id; | |
287 | ||
288 | return 0; | |
289 | } | |
290 | ||
3715a633 DS |
291 | int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result) |
292 | { | |
293 | return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup, | |
aaa95dfa | 294 | the_hash_algo->rawsz, result); |
3715a633 DS |
295 | } |
296 | ||
8aac67a1 DS |
297 | struct object_id *nth_midxed_object_oid(struct object_id *oid, |
298 | struct multi_pack_index *m, | |
299 | uint32_t n) | |
300 | { | |
301 | if (n >= m->num_objects) | |
302 | return NULL; | |
303 | ||
c2b24ede | 304 | oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n)); |
8aac67a1 DS |
305 | return oid; |
306 | } | |
307 | ||
62f2c1b5 | 308 | off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos) |
3715a633 DS |
309 | { |
310 | const unsigned char *offset_data; | |
311 | uint32_t offset32; | |
312 | ||
329fac3a | 313 | offset_data = m->chunk_object_offsets + (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH; |
3715a633 DS |
314 | offset32 = get_be32(offset_data + sizeof(uint32_t)); |
315 | ||
316 | if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) { | |
d8ac9ee1 | 317 | if (sizeof(off_t) < sizeof(uint64_t)) |
3715a633 DS |
318 | die(_("multi-pack-index stores a 64-bit offset, but off_t is too small")); |
319 | ||
320 | offset32 ^= MIDX_LARGE_OFFSET_NEEDED; | |
2abd56e9 JK |
321 | if (offset32 >= m->chunk_large_offsets_len / sizeof(uint64_t)) |
322 | die(_("multi-pack-index large offset out of bounds")); | |
323 | return get_be64(m->chunk_large_offsets + sizeof(uint64_t) * offset32); | |
3715a633 DS |
324 | } |
325 | ||
326 | return offset32; | |
327 | } | |
328 | ||
62f2c1b5 | 329 | uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos) |
3715a633 | 330 | { |
329fac3a DS |
331 | return get_be32(m->chunk_object_offsets + |
332 | (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH); | |
3715a633 DS |
333 | } |
334 | ||
a8437f3c | 335 | int fill_midx_entry(struct repository *r, |
893b5635 RS |
336 | const struct object_id *oid, |
337 | struct pack_entry *e, | |
338 | struct multi_pack_index *m) | |
3715a633 | 339 | { |
893b5635 | 340 | uint32_t pos; |
3715a633 DS |
341 | uint32_t pack_int_id; |
342 | struct packed_git *p; | |
343 | ||
893b5635 RS |
344 | if (!bsearch_midx(oid, m, &pos)) |
345 | return 0; | |
346 | ||
3715a633 DS |
347 | if (pos >= m->num_objects) |
348 | return 0; | |
349 | ||
350 | pack_int_id = nth_midxed_pack_int_id(m, pos); | |
351 | ||
64404a24 | 352 | if (prepare_midx_pack(r, m, pack_int_id)) |
506ec2fb | 353 | return 0; |
3715a633 DS |
354 | p = m->packs[pack_int_id]; |
355 | ||
356 | /* | |
357 | * We are about to tell the caller where they can locate the | |
358 | * requested object. We better make sure the packfile is | |
359 | * still here and can be accessed before supplying that | |
360 | * answer, as it may have been deleted since the MIDX was | |
361 | * loaded! | |
362 | */ | |
363 | if (!is_pack_valid(p)) | |
364 | return 0; | |
365 | ||
09ef6617 RS |
366 | if (oidset_size(&p->bad_objects) && |
367 | oidset_contains(&p->bad_objects, oid)) | |
368 | return 0; | |
c39b02ae | 369 | |
3715a633 DS |
370 | e->offset = nth_midxed_offset(m, pos); |
371 | e->p = p; | |
372 | ||
373 | return 1; | |
374 | } | |
375 | ||
013fd7ad | 376 | /* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */ |
748b88a0 TB |
377 | int cmp_idx_or_pack_name(const char *idx_or_pack_name, |
378 | const char *idx_name) | |
013fd7ad JK |
379 | { |
380 | /* Skip past any initial matching prefix. */ | |
381 | while (*idx_name && *idx_name == *idx_or_pack_name) { | |
382 | idx_name++; | |
383 | idx_or_pack_name++; | |
384 | } | |
385 | ||
386 | /* | |
387 | * If we didn't match completely, we may have matched "pack-1234." and | |
388 | * be left with "idx" and "pack" respectively, which is also OK. We do | |
389 | * not have to check for "idx" and "idx", because that would have been | |
390 | * a complete match (and in that case these strcmps will be false, but | |
391 | * we'll correctly return 0 from the final strcmp() below. | |
392 | * | |
393 | * Technically this matches "fooidx" and "foopack", but we'd never have | |
394 | * such names in the first place. | |
395 | */ | |
396 | if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack")) | |
397 | return 0; | |
398 | ||
399 | /* | |
400 | * This not only checks for a complete match, but also orders based on | |
401 | * the first non-identical character, which means our ordering will | |
402 | * match a raw strcmp(). That makes it OK to use this to binary search | |
403 | * a naively-sorted list. | |
404 | */ | |
405 | return strcmp(idx_or_pack_name, idx_name); | |
406 | } | |
407 | ||
307d75bb TB |
408 | int midx_locate_pack(struct multi_pack_index *m, const char *idx_or_pack_name, |
409 | uint32_t *pos) | |
a40498a1 DS |
410 | { |
411 | uint32_t first = 0, last = m->num_packs; | |
412 | ||
413 | while (first < last) { | |
414 | uint32_t mid = first + (last - first) / 2; | |
415 | const char *current; | |
416 | int cmp; | |
417 | ||
418 | current = m->pack_names[mid]; | |
013fd7ad | 419 | cmp = cmp_idx_or_pack_name(idx_or_pack_name, current); |
307d75bb TB |
420 | if (!cmp) { |
421 | if (pos) | |
422 | *pos = mid; | |
a40498a1 | 423 | return 1; |
307d75bb | 424 | } |
a40498a1 DS |
425 | if (cmp > 0) { |
426 | first = mid + 1; | |
427 | continue; | |
428 | } | |
429 | last = mid; | |
430 | } | |
431 | ||
432 | return 0; | |
433 | } | |
434 | ||
307d75bb TB |
435 | int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) |
436 | { | |
437 | return midx_locate_pack(m, idx_or_pack_name, NULL); | |
438 | } | |
439 | ||
b1e33330 TB |
440 | int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) |
441 | { | |
442 | if (m->preferred_pack_idx == -1) { | |
443 | if (load_midx_revindex(m) < 0) { | |
444 | m->preferred_pack_idx = -2; | |
445 | return -1; | |
446 | } | |
447 | ||
448 | m->preferred_pack_idx = | |
449 | nth_midxed_pack_int_id(m, pack_pos_to_midx(m, 0)); | |
450 | } else if (m->preferred_pack_idx == -2) | |
451 | return -1; /* no revindex */ | |
452 | ||
453 | *pack_int_id = m->preferred_pack_idx; | |
454 | return 0; | |
455 | } | |
456 | ||
2cf489a3 | 457 | int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local) |
c4d25228 | 458 | { |
29e2016b | 459 | struct multi_pack_index *m; |
c4d25228 | 460 | struct multi_pack_index *m_search; |
c4d25228 | 461 | |
18e449f8 DS |
462 | prepare_repo_settings(r); |
463 | if (!r->settings.core_multi_pack_index) | |
c4d25228 DS |
464 | return 0; |
465 | ||
29e2016b | 466 | for (m_search = r->objects->multi_pack_index; m_search; m_search = m_search->next) |
c4d25228 DS |
467 | if (!strcmp(object_dir, m_search->object_dir)) |
468 | return 1; | |
469 | ||
29e2016b | 470 | m = load_multi_pack_index(object_dir, local); |
c4d25228 | 471 | |
29e2016b | 472 | if (m) { |
59552fb3 TB |
473 | struct multi_pack_index *mp = r->objects->multi_pack_index; |
474 | if (mp) { | |
475 | m->next = mp->next; | |
476 | mp->next = m; | |
477 | } else | |
478 | r->objects->multi_pack_index = m; | |
c4d25228 DS |
479 | return 1; |
480 | } | |
481 | ||
482 | return 0; | |
483 | } | |
484 | ||
748b88a0 | 485 | int midx_checksum_valid(struct multi_pack_index *m) |
9218c6a4 | 486 | { |
748b88a0 | 487 | return hashfile_checksum_valid(m->data, m->data_len); |
9218c6a4 TB |
488 | } |
489 | ||
748b88a0 TB |
490 | struct clear_midx_data { |
491 | char *keep; | |
492 | const char *ext; | |
396f2570 DS |
493 | }; |
494 | ||
748b88a0 TB |
495 | static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUSED, |
496 | const char *file_name, void *_data) | |
396f2570 | 497 | { |
748b88a0 | 498 | struct clear_midx_data *data = _data; |
396f2570 | 499 | |
748b88a0 TB |
500 | if (!(starts_with(file_name, "multi-pack-index-") && |
501 | ends_with(file_name, data->ext))) | |
502 | return; | |
503 | if (data->keep && !strcmp(data->keep, file_name)) | |
504 | return; | |
fe1ed56f | 505 | |
748b88a0 TB |
506 | if (unlink(full_path)) |
507 | die_errno(_("failed to remove %s"), full_path); | |
396f2570 DS |
508 | } |
509 | ||
748b88a0 TB |
510 | void clear_midx_files_ext(const char *object_dir, const char *ext, |
511 | unsigned char *keep_hash) | |
fe1ed56f | 512 | { |
748b88a0 TB |
513 | struct clear_midx_data data; |
514 | memset(&data, 0, sizeof(struct clear_midx_data)); | |
fe1ed56f | 515 | |
748b88a0 TB |
516 | if (keep_hash) |
517 | data.keep = xstrfmt("multi-pack-index-%s%s", | |
518 | hash_to_hex(keep_hash), ext); | |
519 | data.ext = ext; | |
9218c6a4 | 520 | |
748b88a0 TB |
521 | for_each_file_in_pack_dir(object_dir, |
522 | clear_midx_file_ext, | |
523 | &data); | |
fe1ed56f | 524 | |
748b88a0 | 525 | free(data.keep); |
fe1ed56f DS |
526 | } |
527 | ||
748b88a0 | 528 | void clear_midx_file(struct repository *r) |
a40498a1 | 529 | { |
748b88a0 | 530 | struct strbuf midx = STRBUF_INIT; |
a40498a1 | 531 | |
748b88a0 | 532 | get_midx_filename(&midx, r->objects->odb->path); |
a40498a1 | 533 | |
748b88a0 TB |
534 | if (r->objects && r->objects->multi_pack_index) { |
535 | close_midx(r->objects->multi_pack_index); | |
536 | r->objects->multi_pack_index = NULL; | |
537 | } | |
a40498a1 | 538 | |
748b88a0 TB |
539 | if (remove_path(midx.buf)) |
540 | die(_("failed to clear multi-pack-index at %s"), midx.buf); | |
fe1ed56f | 541 | |
748b88a0 TB |
542 | clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL); |
543 | clear_midx_files_ext(r->objects->odb->path, ".rev", NULL); | |
fe1ed56f | 544 | |
748b88a0 | 545 | strbuf_release(&midx); |
fe1ed56f DS |
546 | } |
547 | ||
748b88a0 | 548 | static int verify_midx_error; |
989d9cbd | 549 | |
748b88a0 TB |
550 | __attribute__((format (printf, 1, 2))) |
551 | static void midx_report(const char *fmt, ...) | |
989d9cbd | 552 | { |
748b88a0 TB |
553 | va_list ap; |
554 | verify_midx_error = 1; | |
555 | va_start(ap, fmt); | |
556 | vfprintf(stderr, fmt, ap); | |
557 | fprintf(stderr, "\n"); | |
558 | va_end(ap); | |
989d9cbd TB |
559 | } |
560 | ||
748b88a0 | 561 | struct pair_pos_vs_id |
989d9cbd | 562 | { |
748b88a0 TB |
563 | uint32_t pos; |
564 | uint32_t pack_int_id; | |
565 | }; | |
989d9cbd | 566 | |
748b88a0 | 567 | static int compare_pair_pos_vs_id(const void *_a, const void *_b) |
852c5301 | 568 | { |
748b88a0 TB |
569 | struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a; |
570 | struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b; | |
852c5301 | 571 | |
748b88a0 | 572 | return b->pack_int_id - a->pack_int_id; |
1d6f4c64 TB |
573 | } |
574 | ||
fe1ed56f | 575 | /* |
748b88a0 TB |
576 | * Limit calls to display_progress() for performance reasons. |
577 | * The interval here was arbitrarily chosen. | |
fe1ed56f | 578 | */ |
748b88a0 TB |
579 | #define SPARSE_PROGRESS_INTERVAL (1 << 12) |
580 | #define midx_display_sparse_progress(progress, n) \ | |
581 | do { \ | |
582 | uint64_t _n = (n); \ | |
583 | if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \ | |
584 | display_progress(progress, _n); \ | |
585 | } while (0) | |
a40498a1 | 586 | |
748b88a0 TB |
587 | int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags) |
588 | { | |
589 | struct pair_pos_vs_id *pairs = NULL; | |
590 | uint32_t i; | |
591 | struct progress *progress = NULL; | |
592 | struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); | |
593 | verify_midx_error = 0; | |
fe1ed56f | 594 | |
748b88a0 TB |
595 | if (!m) { |
596 | int result = 0; | |
597 | struct stat sb; | |
598 | struct strbuf filename = STRBUF_INIT; | |
cdf517be | 599 | |
748b88a0 | 600 | get_midx_filename(&filename, object_dir); |
fe1ed56f | 601 | |
748b88a0 TB |
602 | if (!stat(filename.buf, &sb)) { |
603 | error(_("multi-pack-index file exists, but failed to parse")); | |
604 | result = 1; | |
fe1ed56f | 605 | } |
748b88a0 TB |
606 | strbuf_release(&filename); |
607 | return result; | |
fe1ed56f DS |
608 | } |
609 | ||
748b88a0 TB |
610 | if (!midx_checksum_valid(m)) |
611 | midx_report(_("incorrect checksum")); | |
32f3c541 | 612 | |
748b88a0 TB |
613 | if (flags & MIDX_PROGRESS) |
614 | progress = start_delayed_progress(_("Looking for referenced packfiles"), | |
615 | m->num_packs); | |
616 | for (i = 0; i < m->num_packs; i++) { | |
617 | if (prepare_midx_pack(r, m, i)) | |
618 | midx_report("failed to load pack in position %d", i); | |
32f3c541 | 619 | |
748b88a0 | 620 | display_progress(progress, i + 1); |
32f3c541 | 621 | } |
748b88a0 | 622 | stop_progress(&progress); |
32f3c541 | 623 | |
748b88a0 TB |
624 | if (m->num_objects == 0) { |
625 | midx_report(_("the midx contains no oid")); | |
626 | /* | |
627 | * Remaining tests assume that we have objects, so we can | |
628 | * return here. | |
629 | */ | |
630 | goto cleanup; | |
32f3c541 DS |
631 | } |
632 | ||
748b88a0 TB |
633 | if (flags & MIDX_PROGRESS) |
634 | progress = start_sparse_progress(_("Verifying OID order in multi-pack-index"), | |
635 | m->num_objects - 1); | |
636 | for (i = 0; i < m->num_objects - 1; i++) { | |
637 | struct object_id oid1, oid2; | |
5f5ccd95 | 638 | |
748b88a0 TB |
639 | nth_midxed_object_oid(&oid1, m, i); |
640 | nth_midxed_object_oid(&oid2, m, i + 1); | |
5f5ccd95 | 641 | |
748b88a0 TB |
642 | if (oidcmp(&oid1, &oid2) >= 0) |
643 | midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"), | |
644 | i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1); | |
5f5ccd95 | 645 | |
748b88a0 | 646 | midx_display_sparse_progress(progress, i + 1); |
5f5ccd95 | 647 | } |
748b88a0 | 648 | stop_progress(&progress); |
d7cacf29 DS |
649 | |
650 | /* | |
748b88a0 TB |
651 | * Create an array mapping each object to its packfile id. Sort it |
652 | * to group the objects by packfile. Use this permutation to visit | |
653 | * each of the objects and only require 1 packfile to be open at a | |
654 | * time. | |
655 | */ | |
656 | ALLOC_ARRAY(pairs, m->num_objects); | |
657 | for (i = 0; i < m->num_objects; i++) { | |
658 | pairs[i].pos = i; | |
659 | pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i); | |
660 | } | |
d7cacf29 | 661 | |
748b88a0 TB |
662 | if (flags & MIDX_PROGRESS) |
663 | progress = start_sparse_progress(_("Sorting objects by packfile"), | |
664 | m->num_objects); | |
665 | display_progress(progress, 0); /* TODO: Measure QSORT() progress */ | |
666 | QSORT(pairs, m->num_objects, compare_pair_pos_vs_id); | |
667 | stop_progress(&progress); | |
d7cacf29 | 668 | |
748b88a0 TB |
669 | if (flags & MIDX_PROGRESS) |
670 | progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects); | |
671 | for (i = 0; i < m->num_objects; i++) { | |
672 | struct object_id oid; | |
673 | struct pack_entry e; | |
674 | off_t m_offset, p_offset; | |
d7cacf29 | 675 | |
748b88a0 TB |
676 | if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id && |
677 | m->packs[pairs[i-1].pack_int_id]) | |
678 | { | |
679 | close_pack_fd(m->packs[pairs[i-1].pack_int_id]); | |
680 | close_pack_index(m->packs[pairs[i-1].pack_int_id]); | |
681 | } | |
d7cacf29 | 682 | |
748b88a0 | 683 | nth_midxed_object_oid(&oid, m, pairs[i].pos); |
0d5b3a5e | 684 | |
748b88a0 TB |
685 | if (!fill_midx_entry(r, &oid, &e, m)) { |
686 | midx_report(_("failed to load pack entry for oid[%d] = %s"), | |
687 | pairs[i].pos, oid_to_hex(&oid)); | |
688 | continue; | |
689 | } | |
0d5b3a5e | 690 | |
748b88a0 TB |
691 | if (open_pack_index(e.p)) { |
692 | midx_report(_("failed to load pack-index for packfile %s"), | |
693 | e.p->pack_name); | |
694 | break; | |
0d5b3a5e DS |
695 | } |
696 | ||
748b88a0 TB |
697 | m_offset = e.offset; |
698 | p_offset = find_pack_entry_one(oid.hash, e.p); | |
0d5b3a5e | 699 | |
748b88a0 TB |
700 | if (m_offset != p_offset) |
701 | midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64), | |
702 | pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset); | |
0d5b3a5e | 703 | |
748b88a0 | 704 | midx_display_sparse_progress(progress, i + 1); |
662148c4 | 705 | } |
748b88a0 | 706 | stop_progress(&progress); |
662148c4 | 707 | |
748b88a0 TB |
708 | cleanup: |
709 | free(pairs); | |
710 | close_midx(m); | |
5ae18df9 | 711 | |
56ee7ff1 DS |
712 | return verify_midx_error; |
713 | } |