]>
Commit | Line | Data |
---|---|---|
5e3f94df | 1 | #include "git-compat-util.h" |
c4d25228 | 2 | #include "config.h" |
396f2570 | 3 | #include "dir.h" |
41771fa4 | 4 | #include "hex.h" |
396f2570 | 5 | #include "packfile.h" |
87bed179 | 6 | #include "object-file.h" |
bc626927 | 7 | #include "hash-lookup.h" |
a3407730 | 8 | #include "midx.h" |
144d7033 | 9 | #include "progress.h" |
d829223a | 10 | #include "trace2.h" |
63a8f0e9 | 11 | #include "chunk-format.h" |
c528e179 | 12 | #include "pack-bitmap.h" |
b1e33330 | 13 | #include "pack-revindex.h" |
a3407730 | 14 | |
748b88a0 TB |
15 | int midx_checksum_valid(struct multi_pack_index *m); |
16 | void clear_midx_files_ext(const char *object_dir, const char *ext, | |
17 | unsigned char *keep_hash); | |
18 | int cmp_idx_or_pack_name(const char *idx_or_pack_name, | |
19 | const char *idx_name); | |
19575c7c | 20 | |
0f533c72 | 21 | const unsigned char *get_midx_checksum(struct multi_pack_index *m) |
f894081d TB |
22 | { |
23 | return m->data + m->data_len - the_hash_algo->rawsz; | |
24 | } | |
25 | ||
60980aed | 26 | void get_midx_filename(struct strbuf *out, const char *object_dir) |
fc59e748 | 27 | { |
60980aed | 28 | strbuf_addf(out, "%s/pack/multi-pack-index", object_dir); |
fc59e748 DS |
29 | } |
30 | ||
60980aed | 31 | void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m) |
f894081d | 32 | { |
60980aed TB |
33 | get_midx_filename(out, m->object_dir); |
34 | strbuf_addf(out, "-%s.rev", hash_to_hex(get_midx_checksum(m))); | |
f894081d TB |
35 | } |
36 | ||
6ab3b8b8 DS |
37 | static int midx_read_oid_fanout(const unsigned char *chunk_start, |
38 | size_t chunk_size, void *data) | |
39 | { | |
9d78fb0e | 40 | int i; |
6ab3b8b8 DS |
41 | struct multi_pack_index *m = data; |
42 | m->chunk_oid_fanout = (uint32_t *)chunk_start; | |
43 | ||
44 | if (chunk_size != 4 * 256) { | |
45 | error(_("multi-pack-index OID fanout is of the wrong size")); | |
46 | return 1; | |
47 | } | |
9d78fb0e JK |
48 | for (i = 0; i < 255; i++) { |
49 | uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]); | |
50 | uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i+1]); | |
51 | ||
52 | if (oid_fanout1 > oid_fanout2) { | |
53 | error(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"), | |
54 | i, oid_fanout1, oid_fanout2, i + 1); | |
55 | return 1; | |
56 | } | |
57 | } | |
fc926567 JK |
58 | m->num_objects = ntohl(m->chunk_oid_fanout[255]); |
59 | return 0; | |
60 | } | |
61 | ||
62 | static int midx_read_oid_lookup(const unsigned char *chunk_start, | |
63 | size_t chunk_size, void *data) | |
64 | { | |
65 | struct multi_pack_index *m = data; | |
66 | m->chunk_oid_lookup = chunk_start; | |
67 | ||
68 | if (chunk_size != st_mult(m->hash_len, m->num_objects)) { | |
69 | error(_("multi-pack-index OID lookup chunk is the wrong size")); | |
70 | return 1; | |
71 | } | |
6ab3b8b8 DS |
72 | return 0; |
73 | } | |
74 | ||
0924869b JK |
75 | static int midx_read_object_offsets(const unsigned char *chunk_start, |
76 | size_t chunk_size, void *data) | |
77 | { | |
78 | struct multi_pack_index *m = data; | |
79 | m->chunk_object_offsets = chunk_start; | |
80 | ||
81 | if (chunk_size != st_mult(m->num_objects, MIDX_CHUNK_OFFSET_WIDTH)) { | |
82 | error(_("multi-pack-index object offset chunk is the wrong size")); | |
83 | return 1; | |
84 | } | |
85 | return 0; | |
86 | } | |
87 | ||
748b88a0 TB |
88 | #define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz) |
89 | ||
2cf489a3 | 90 | struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local) |
4d80560c DS |
91 | { |
92 | struct multi_pack_index *m = NULL; | |
93 | int fd; | |
94 | struct stat st; | |
95 | size_t midx_size; | |
96 | void *midx_map = NULL; | |
97 | uint32_t hash_version; | |
60980aed | 98 | struct strbuf midx_name = STRBUF_INIT; |
32f3c541 | 99 | uint32_t i; |
3227565c | 100 | const char *cur_pack_name; |
6ab3b8b8 | 101 | struct chunkfile *cf = NULL; |
4d80560c | 102 | |
60980aed TB |
103 | get_midx_filename(&midx_name, object_dir); |
104 | ||
105 | fd = git_open(midx_name.buf); | |
4d80560c DS |
106 | |
107 | if (fd < 0) | |
108 | goto cleanup_fail; | |
109 | if (fstat(fd, &st)) { | |
60980aed | 110 | error_errno(_("failed to read %s"), midx_name.buf); |
4d80560c DS |
111 | goto cleanup_fail; |
112 | } | |
113 | ||
114 | midx_size = xsize_t(st.st_size); | |
115 | ||
116 | if (midx_size < MIDX_MIN_SIZE) { | |
60980aed | 117 | error(_("multi-pack-index file %s is too small"), midx_name.buf); |
4d80560c DS |
118 | goto cleanup_fail; |
119 | } | |
120 | ||
60980aed | 121 | strbuf_release(&midx_name); |
4d80560c DS |
122 | |
123 | midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); | |
6c7ff7cf | 124 | close(fd); |
4d80560c | 125 | |
577314ca | 126 | FLEX_ALLOC_STR(m, object_dir, object_dir); |
4d80560c DS |
127 | m->data = midx_map; |
128 | m->data_len = midx_size; | |
2cf489a3 | 129 | m->local = local; |
4d80560c DS |
130 | |
131 | m->signature = get_be32(m->data); | |
53ad0407 DS |
132 | if (m->signature != MIDX_SIGNATURE) |
133 | die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"), | |
4d80560c | 134 | m->signature, MIDX_SIGNATURE); |
4d80560c DS |
135 | |
136 | m->version = m->data[MIDX_BYTE_FILE_VERSION]; | |
53ad0407 DS |
137 | if (m->version != MIDX_VERSION) |
138 | die(_("multi-pack-index version %d not recognized"), | |
4d80560c | 139 | m->version); |
4d80560c DS |
140 | |
141 | hash_version = m->data[MIDX_BYTE_HASH_VERSION]; | |
d9fef9d9 | 142 | if (hash_version != oid_version(the_hash_algo)) { |
d9607542 | 143 | error(_("multi-pack-index hash version %u does not match version %u"), |
d9fef9d9 | 144 | hash_version, oid_version(the_hash_algo)); |
d9607542 DS |
145 | goto cleanup_fail; |
146 | } | |
aaa95dfa | 147 | m->hash_len = the_hash_algo->rawsz; |
4d80560c DS |
148 | |
149 | m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS]; | |
150 | ||
151 | m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS); | |
152 | ||
b1e33330 TB |
153 | m->preferred_pack_idx = -1; |
154 | ||
6ab3b8b8 | 155 | cf = init_chunkfile(NULL); |
32f3c541 | 156 | |
6ab3b8b8 | 157 | if (read_table_of_contents(cf, m->data, midx_size, |
c9b9fefc JK |
158 | MIDX_HEADER_SIZE, m->num_chunks, |
159 | MIDX_CHUNK_ALIGNMENT)) | |
6ab3b8b8 | 160 | goto cleanup_fail; |
32f3c541 | 161 | |
72a9a082 | 162 | if (pair_chunk(cf, MIDX_CHUNKID_PACKNAMES, &m->chunk_pack_names, &m->chunk_pack_names_len)) |
e3c96003 JK |
163 | die(_("multi-pack-index required pack-name chunk missing or corrupted")); |
164 | if (read_chunk(cf, MIDX_CHUNKID_OIDFANOUT, midx_read_oid_fanout, m)) | |
165 | die(_("multi-pack-index required OID fanout chunk missing or corrupted")); | |
fc926567 | 166 | if (read_chunk(cf, MIDX_CHUNKID_OIDLOOKUP, midx_read_oid_lookup, m)) |
e3c96003 | 167 | die(_("multi-pack-index required OID lookup chunk missing or corrupted")); |
0924869b | 168 | if (read_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS, midx_read_object_offsets, m)) |
e3c96003 | 169 | die(_("multi-pack-index required object offsets chunk missing or corrupted")); |
32f3c541 | 170 | |
2abd56e9 JK |
171 | pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets, |
172 | &m->chunk_large_offsets_len); | |
795006ff PS |
173 | if (git_env_bool("GIT_TEST_MIDX_READ_BTMP", 1)) |
174 | pair_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS, | |
175 | (const unsigned char **)&m->chunk_bitmapped_packs, | |
176 | &m->chunk_bitmapped_packs_len); | |
6ab3b8b8 | 177 | |
7f514b7a | 178 | if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1)) |
c0fe9b2d JK |
179 | pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex, |
180 | &m->chunk_revindex_len); | |
7f514b7a | 181 | |
ca56dadb RS |
182 | CALLOC_ARRAY(m->pack_names, m->num_packs); |
183 | CALLOC_ARRAY(m->packs, m->num_packs); | |
3227565c DS |
184 | |
185 | cur_pack_name = (const char *)m->chunk_pack_names; | |
186 | for (i = 0; i < m->num_packs; i++) { | |
72a9a082 JK |
187 | const char *end; |
188 | size_t avail = m->chunk_pack_names_len - | |
189 | (cur_pack_name - (const char *)m->chunk_pack_names); | |
190 | ||
3227565c DS |
191 | m->pack_names[i] = cur_pack_name; |
192 | ||
72a9a082 JK |
193 | end = memchr(cur_pack_name, '\0', avail); |
194 | if (!end) | |
195 | die(_("multi-pack-index pack-name chunk is too short")); | |
196 | cur_pack_name = end + 1; | |
3227565c | 197 | |
8e72a3c3 DS |
198 | if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0) |
199 | die(_("multi-pack-index pack names out of order: '%s' before '%s'"), | |
3227565c DS |
200 | m->pack_names[i - 1], |
201 | m->pack_names[i]); | |
3227565c DS |
202 | } |
203 | ||
d829223a JH |
204 | trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs); |
205 | trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects); | |
206 | ||
692305ec | 207 | free_chunkfile(cf); |
4d80560c DS |
208 | return m; |
209 | ||
210 | cleanup_fail: | |
211 | free(m); | |
60980aed | 212 | strbuf_release(&midx_name); |
692305ec | 213 | free_chunkfile(cf); |
4d80560c DS |
214 | if (midx_map) |
215 | munmap(midx_map, midx_size); | |
216 | if (0 <= fd) | |
217 | close(fd); | |
218 | return NULL; | |
219 | } | |
220 | ||
1dcd9f20 | 221 | void close_midx(struct multi_pack_index *m) |
a40498a1 DS |
222 | { |
223 | uint32_t i; | |
1dcd9f20 DS |
224 | |
225 | if (!m) | |
226 | return; | |
227 | ||
9bb6c2e5 TB |
228 | close_midx(m->next); |
229 | ||
a40498a1 | 230 | munmap((unsigned char *)m->data, m->data_len); |
a40498a1 DS |
231 | |
232 | for (i = 0; i < m->num_packs; i++) { | |
af96fe33 DS |
233 | if (m->packs[i]) |
234 | m->packs[i]->multi_pack_index = 0; | |
a40498a1 DS |
235 | } |
236 | FREE_AND_NULL(m->packs); | |
237 | FREE_AND_NULL(m->pack_names); | |
9bb6c2e5 | 238 | free(m); |
a40498a1 DS |
239 | } |
240 | ||
64404a24 | 241 | int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id) |
3715a633 DS |
242 | { |
243 | struct strbuf pack_name = STRBUF_INIT; | |
af96fe33 | 244 | struct packed_git *p; |
3715a633 DS |
245 | |
246 | if (pack_int_id >= m->num_packs) | |
d355e46a | 247 | die(_("bad pack-int-id: %u (%u total packs)"), |
cc6af73c | 248 | pack_int_id, m->num_packs); |
3715a633 DS |
249 | |
250 | if (m->packs[pack_int_id]) | |
251 | return 0; | |
252 | ||
253 | strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir, | |
254 | m->pack_names[pack_int_id]); | |
255 | ||
af96fe33 | 256 | p = add_packed_git(pack_name.buf, pack_name.len, m->local); |
3715a633 | 257 | strbuf_release(&pack_name); |
af96fe33 DS |
258 | |
259 | if (!p) | |
260 | return 1; | |
261 | ||
262 | p->multi_pack_index = 1; | |
263 | m->packs[pack_int_id] = p; | |
264 | install_packed_git(r, p); | |
265 | list_add_tail(&p->mru, &r->objects->packed_git_mru); | |
266 | ||
267 | return 0; | |
3715a633 DS |
268 | } |
269 | ||
748b88a0 TB |
270 | #define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t)) |
271 | ||
5f5ccd95 TB |
272 | int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, |
273 | struct bitmapped_pack *bp, uint32_t pack_int_id) | |
274 | { | |
275 | if (!m->chunk_bitmapped_packs) | |
276 | return error(_("MIDX does not contain the BTMP chunk")); | |
277 | ||
278 | if (prepare_midx_pack(r, m, pack_int_id)) | |
279 | return error(_("could not load bitmapped pack %"PRIu32), pack_int_id); | |
280 | ||
281 | bp->p = m->packs[pack_int_id]; | |
282 | bp->bitmap_pos = get_be32((char *)m->chunk_bitmapped_packs + | |
283 | MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id); | |
284 | bp->bitmap_nr = get_be32((char *)m->chunk_bitmapped_packs + | |
285 | MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id + | |
286 | sizeof(uint32_t)); | |
287 | bp->pack_int_id = pack_int_id; | |
288 | ||
289 | return 0; | |
290 | } | |
291 | ||
3715a633 DS |
292 | int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result) |
293 | { | |
294 | return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup, | |
aaa95dfa | 295 | the_hash_algo->rawsz, result); |
3715a633 DS |
296 | } |
297 | ||
8aac67a1 DS |
298 | struct object_id *nth_midxed_object_oid(struct object_id *oid, |
299 | struct multi_pack_index *m, | |
300 | uint32_t n) | |
301 | { | |
302 | if (n >= m->num_objects) | |
303 | return NULL; | |
304 | ||
c2b24ede | 305 | oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n)); |
8aac67a1 DS |
306 | return oid; |
307 | } | |
308 | ||
62f2c1b5 | 309 | off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos) |
3715a633 DS |
310 | { |
311 | const unsigned char *offset_data; | |
312 | uint32_t offset32; | |
313 | ||
329fac3a | 314 | offset_data = m->chunk_object_offsets + (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH; |
3715a633 DS |
315 | offset32 = get_be32(offset_data + sizeof(uint32_t)); |
316 | ||
317 | if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) { | |
d8ac9ee1 | 318 | if (sizeof(off_t) < sizeof(uint64_t)) |
3715a633 DS |
319 | die(_("multi-pack-index stores a 64-bit offset, but off_t is too small")); |
320 | ||
321 | offset32 ^= MIDX_LARGE_OFFSET_NEEDED; | |
2abd56e9 JK |
322 | if (offset32 >= m->chunk_large_offsets_len / sizeof(uint64_t)) |
323 | die(_("multi-pack-index large offset out of bounds")); | |
324 | return get_be64(m->chunk_large_offsets + sizeof(uint64_t) * offset32); | |
3715a633 DS |
325 | } |
326 | ||
327 | return offset32; | |
328 | } | |
329 | ||
62f2c1b5 | 330 | uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos) |
3715a633 | 331 | { |
329fac3a DS |
332 | return get_be32(m->chunk_object_offsets + |
333 | (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH); | |
3715a633 DS |
334 | } |
335 | ||
a8437f3c | 336 | int fill_midx_entry(struct repository *r, |
893b5635 RS |
337 | const struct object_id *oid, |
338 | struct pack_entry *e, | |
339 | struct multi_pack_index *m) | |
3715a633 | 340 | { |
893b5635 | 341 | uint32_t pos; |
3715a633 DS |
342 | uint32_t pack_int_id; |
343 | struct packed_git *p; | |
344 | ||
893b5635 RS |
345 | if (!bsearch_midx(oid, m, &pos)) |
346 | return 0; | |
347 | ||
3715a633 DS |
348 | if (pos >= m->num_objects) |
349 | return 0; | |
350 | ||
351 | pack_int_id = nth_midxed_pack_int_id(m, pos); | |
352 | ||
64404a24 | 353 | if (prepare_midx_pack(r, m, pack_int_id)) |
506ec2fb | 354 | return 0; |
3715a633 DS |
355 | p = m->packs[pack_int_id]; |
356 | ||
357 | /* | |
358 | * We are about to tell the caller where they can locate the | |
359 | * requested object. We better make sure the packfile is | |
360 | * still here and can be accessed before supplying that | |
361 | * answer, as it may have been deleted since the MIDX was | |
362 | * loaded! | |
363 | */ | |
364 | if (!is_pack_valid(p)) | |
365 | return 0; | |
366 | ||
09ef6617 RS |
367 | if (oidset_size(&p->bad_objects) && |
368 | oidset_contains(&p->bad_objects, oid)) | |
369 | return 0; | |
c39b02ae | 370 | |
3715a633 DS |
371 | e->offset = nth_midxed_offset(m, pos); |
372 | e->p = p; | |
373 | ||
374 | return 1; | |
375 | } | |
376 | ||
013fd7ad | 377 | /* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */ |
748b88a0 TB |
378 | int cmp_idx_or_pack_name(const char *idx_or_pack_name, |
379 | const char *idx_name) | |
013fd7ad JK |
380 | { |
381 | /* Skip past any initial matching prefix. */ | |
382 | while (*idx_name && *idx_name == *idx_or_pack_name) { | |
383 | idx_name++; | |
384 | idx_or_pack_name++; | |
385 | } | |
386 | ||
387 | /* | |
388 | * If we didn't match completely, we may have matched "pack-1234." and | |
389 | * be left with "idx" and "pack" respectively, which is also OK. We do | |
390 | * not have to check for "idx" and "idx", because that would have been | |
391 | * a complete match (and in that case these strcmps will be false, but | |
392 | * we'll correctly return 0 from the final strcmp() below. | |
393 | * | |
394 | * Technically this matches "fooidx" and "foopack", but we'd never have | |
395 | * such names in the first place. | |
396 | */ | |
397 | if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack")) | |
398 | return 0; | |
399 | ||
400 | /* | |
401 | * This not only checks for a complete match, but also orders based on | |
402 | * the first non-identical character, which means our ordering will | |
403 | * match a raw strcmp(). That makes it OK to use this to binary search | |
404 | * a naively-sorted list. | |
405 | */ | |
406 | return strcmp(idx_or_pack_name, idx_name); | |
407 | } | |
408 | ||
307d75bb TB |
409 | int midx_locate_pack(struct multi_pack_index *m, const char *idx_or_pack_name, |
410 | uint32_t *pos) | |
a40498a1 DS |
411 | { |
412 | uint32_t first = 0, last = m->num_packs; | |
413 | ||
414 | while (first < last) { | |
415 | uint32_t mid = first + (last - first) / 2; | |
416 | const char *current; | |
417 | int cmp; | |
418 | ||
419 | current = m->pack_names[mid]; | |
013fd7ad | 420 | cmp = cmp_idx_or_pack_name(idx_or_pack_name, current); |
307d75bb TB |
421 | if (!cmp) { |
422 | if (pos) | |
423 | *pos = mid; | |
a40498a1 | 424 | return 1; |
307d75bb | 425 | } |
a40498a1 DS |
426 | if (cmp > 0) { |
427 | first = mid + 1; | |
428 | continue; | |
429 | } | |
430 | last = mid; | |
431 | } | |
432 | ||
433 | return 0; | |
434 | } | |
435 | ||
307d75bb TB |
436 | int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) |
437 | { | |
438 | return midx_locate_pack(m, idx_or_pack_name, NULL); | |
439 | } | |
440 | ||
b1e33330 TB |
441 | int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) |
442 | { | |
443 | if (m->preferred_pack_idx == -1) { | |
444 | if (load_midx_revindex(m) < 0) { | |
445 | m->preferred_pack_idx = -2; | |
446 | return -1; | |
447 | } | |
448 | ||
449 | m->preferred_pack_idx = | |
450 | nth_midxed_pack_int_id(m, pack_pos_to_midx(m, 0)); | |
451 | } else if (m->preferred_pack_idx == -2) | |
452 | return -1; /* no revindex */ | |
453 | ||
454 | *pack_int_id = m->preferred_pack_idx; | |
455 | return 0; | |
456 | } | |
457 | ||
2cf489a3 | 458 | int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local) |
c4d25228 | 459 | { |
29e2016b | 460 | struct multi_pack_index *m; |
c4d25228 | 461 | struct multi_pack_index *m_search; |
c4d25228 | 462 | |
18e449f8 DS |
463 | prepare_repo_settings(r); |
464 | if (!r->settings.core_multi_pack_index) | |
c4d25228 DS |
465 | return 0; |
466 | ||
29e2016b | 467 | for (m_search = r->objects->multi_pack_index; m_search; m_search = m_search->next) |
c4d25228 DS |
468 | if (!strcmp(object_dir, m_search->object_dir)) |
469 | return 1; | |
470 | ||
29e2016b | 471 | m = load_multi_pack_index(object_dir, local); |
c4d25228 | 472 | |
29e2016b | 473 | if (m) { |
59552fb3 TB |
474 | struct multi_pack_index *mp = r->objects->multi_pack_index; |
475 | if (mp) { | |
476 | m->next = mp->next; | |
477 | mp->next = m; | |
478 | } else | |
479 | r->objects->multi_pack_index = m; | |
c4d25228 DS |
480 | return 1; |
481 | } | |
482 | ||
483 | return 0; | |
484 | } | |
485 | ||
748b88a0 | 486 | int midx_checksum_valid(struct multi_pack_index *m) |
9218c6a4 | 487 | { |
748b88a0 | 488 | return hashfile_checksum_valid(m->data, m->data_len); |
9218c6a4 TB |
489 | } |
490 | ||
748b88a0 TB |
491 | struct clear_midx_data { |
492 | char *keep; | |
493 | const char *ext; | |
396f2570 DS |
494 | }; |
495 | ||
748b88a0 TB |
496 | static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUSED, |
497 | const char *file_name, void *_data) | |
396f2570 | 498 | { |
748b88a0 | 499 | struct clear_midx_data *data = _data; |
396f2570 | 500 | |
748b88a0 TB |
501 | if (!(starts_with(file_name, "multi-pack-index-") && |
502 | ends_with(file_name, data->ext))) | |
503 | return; | |
504 | if (data->keep && !strcmp(data->keep, file_name)) | |
505 | return; | |
fe1ed56f | 506 | |
748b88a0 TB |
507 | if (unlink(full_path)) |
508 | die_errno(_("failed to remove %s"), full_path); | |
396f2570 DS |
509 | } |
510 | ||
748b88a0 TB |
511 | void clear_midx_files_ext(const char *object_dir, const char *ext, |
512 | unsigned char *keep_hash) | |
fe1ed56f | 513 | { |
748b88a0 TB |
514 | struct clear_midx_data data; |
515 | memset(&data, 0, sizeof(struct clear_midx_data)); | |
fe1ed56f | 516 | |
748b88a0 TB |
517 | if (keep_hash) |
518 | data.keep = xstrfmt("multi-pack-index-%s%s", | |
519 | hash_to_hex(keep_hash), ext); | |
520 | data.ext = ext; | |
9218c6a4 | 521 | |
748b88a0 TB |
522 | for_each_file_in_pack_dir(object_dir, |
523 | clear_midx_file_ext, | |
524 | &data); | |
fe1ed56f | 525 | |
748b88a0 | 526 | free(data.keep); |
fe1ed56f DS |
527 | } |
528 | ||
748b88a0 | 529 | void clear_midx_file(struct repository *r) |
a40498a1 | 530 | { |
748b88a0 | 531 | struct strbuf midx = STRBUF_INIT; |
a40498a1 | 532 | |
748b88a0 | 533 | get_midx_filename(&midx, r->objects->odb->path); |
a40498a1 | 534 | |
748b88a0 TB |
535 | if (r->objects && r->objects->multi_pack_index) { |
536 | close_midx(r->objects->multi_pack_index); | |
537 | r->objects->multi_pack_index = NULL; | |
538 | } | |
a40498a1 | 539 | |
748b88a0 TB |
540 | if (remove_path(midx.buf)) |
541 | die(_("failed to clear multi-pack-index at %s"), midx.buf); | |
fe1ed56f | 542 | |
748b88a0 TB |
543 | clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL); |
544 | clear_midx_files_ext(r->objects->odb->path, ".rev", NULL); | |
fe1ed56f | 545 | |
748b88a0 | 546 | strbuf_release(&midx); |
fe1ed56f DS |
547 | } |
548 | ||
748b88a0 | 549 | static int verify_midx_error; |
989d9cbd | 550 | |
748b88a0 TB |
551 | __attribute__((format (printf, 1, 2))) |
552 | static void midx_report(const char *fmt, ...) | |
989d9cbd | 553 | { |
748b88a0 TB |
554 | va_list ap; |
555 | verify_midx_error = 1; | |
556 | va_start(ap, fmt); | |
557 | vfprintf(stderr, fmt, ap); | |
558 | fprintf(stderr, "\n"); | |
559 | va_end(ap); | |
989d9cbd TB |
560 | } |
561 | ||
748b88a0 | 562 | struct pair_pos_vs_id |
989d9cbd | 563 | { |
748b88a0 TB |
564 | uint32_t pos; |
565 | uint32_t pack_int_id; | |
566 | }; | |
989d9cbd | 567 | |
748b88a0 | 568 | static int compare_pair_pos_vs_id(const void *_a, const void *_b) |
852c5301 | 569 | { |
748b88a0 TB |
570 | struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a; |
571 | struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b; | |
852c5301 | 572 | |
748b88a0 | 573 | return b->pack_int_id - a->pack_int_id; |
1d6f4c64 TB |
574 | } |
575 | ||
fe1ed56f | 576 | /* |
748b88a0 TB |
577 | * Limit calls to display_progress() for performance reasons. |
578 | * The interval here was arbitrarily chosen. | |
fe1ed56f | 579 | */ |
748b88a0 TB |
580 | #define SPARSE_PROGRESS_INTERVAL (1 << 12) |
581 | #define midx_display_sparse_progress(progress, n) \ | |
582 | do { \ | |
583 | uint64_t _n = (n); \ | |
584 | if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \ | |
585 | display_progress(progress, _n); \ | |
586 | } while (0) | |
a40498a1 | 587 | |
748b88a0 TB |
588 | int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags) |
589 | { | |
590 | struct pair_pos_vs_id *pairs = NULL; | |
591 | uint32_t i; | |
592 | struct progress *progress = NULL; | |
593 | struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); | |
594 | verify_midx_error = 0; | |
fe1ed56f | 595 | |
748b88a0 TB |
596 | if (!m) { |
597 | int result = 0; | |
598 | struct stat sb; | |
599 | struct strbuf filename = STRBUF_INIT; | |
cdf517be | 600 | |
748b88a0 | 601 | get_midx_filename(&filename, object_dir); |
fe1ed56f | 602 | |
748b88a0 TB |
603 | if (!stat(filename.buf, &sb)) { |
604 | error(_("multi-pack-index file exists, but failed to parse")); | |
605 | result = 1; | |
fe1ed56f | 606 | } |
748b88a0 TB |
607 | strbuf_release(&filename); |
608 | return result; | |
fe1ed56f DS |
609 | } |
610 | ||
748b88a0 TB |
611 | if (!midx_checksum_valid(m)) |
612 | midx_report(_("incorrect checksum")); | |
32f3c541 | 613 | |
748b88a0 TB |
614 | if (flags & MIDX_PROGRESS) |
615 | progress = start_delayed_progress(_("Looking for referenced packfiles"), | |
616 | m->num_packs); | |
617 | for (i = 0; i < m->num_packs; i++) { | |
618 | if (prepare_midx_pack(r, m, i)) | |
619 | midx_report("failed to load pack in position %d", i); | |
32f3c541 | 620 | |
748b88a0 | 621 | display_progress(progress, i + 1); |
32f3c541 | 622 | } |
748b88a0 | 623 | stop_progress(&progress); |
32f3c541 | 624 | |
748b88a0 TB |
625 | if (m->num_objects == 0) { |
626 | midx_report(_("the midx contains no oid")); | |
627 | /* | |
628 | * Remaining tests assume that we have objects, so we can | |
629 | * return here. | |
630 | */ | |
631 | goto cleanup; | |
32f3c541 DS |
632 | } |
633 | ||
748b88a0 TB |
634 | if (flags & MIDX_PROGRESS) |
635 | progress = start_sparse_progress(_("Verifying OID order in multi-pack-index"), | |
636 | m->num_objects - 1); | |
637 | for (i = 0; i < m->num_objects - 1; i++) { | |
638 | struct object_id oid1, oid2; | |
5f5ccd95 | 639 | |
748b88a0 TB |
640 | nth_midxed_object_oid(&oid1, m, i); |
641 | nth_midxed_object_oid(&oid2, m, i + 1); | |
5f5ccd95 | 642 | |
748b88a0 TB |
643 | if (oidcmp(&oid1, &oid2) >= 0) |
644 | midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"), | |
645 | i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1); | |
5f5ccd95 | 646 | |
748b88a0 | 647 | midx_display_sparse_progress(progress, i + 1); |
5f5ccd95 | 648 | } |
748b88a0 | 649 | stop_progress(&progress); |
d7cacf29 DS |
650 | |
651 | /* | |
748b88a0 TB |
652 | * Create an array mapping each object to its packfile id. Sort it |
653 | * to group the objects by packfile. Use this permutation to visit | |
654 | * each of the objects and only require 1 packfile to be open at a | |
655 | * time. | |
656 | */ | |
657 | ALLOC_ARRAY(pairs, m->num_objects); | |
658 | for (i = 0; i < m->num_objects; i++) { | |
659 | pairs[i].pos = i; | |
660 | pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i); | |
661 | } | |
d7cacf29 | 662 | |
748b88a0 TB |
663 | if (flags & MIDX_PROGRESS) |
664 | progress = start_sparse_progress(_("Sorting objects by packfile"), | |
665 | m->num_objects); | |
666 | display_progress(progress, 0); /* TODO: Measure QSORT() progress */ | |
667 | QSORT(pairs, m->num_objects, compare_pair_pos_vs_id); | |
668 | stop_progress(&progress); | |
d7cacf29 | 669 | |
748b88a0 TB |
670 | if (flags & MIDX_PROGRESS) |
671 | progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects); | |
672 | for (i = 0; i < m->num_objects; i++) { | |
673 | struct object_id oid; | |
674 | struct pack_entry e; | |
675 | off_t m_offset, p_offset; | |
d7cacf29 | 676 | |
748b88a0 TB |
677 | if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id && |
678 | m->packs[pairs[i-1].pack_int_id]) | |
679 | { | |
680 | close_pack_fd(m->packs[pairs[i-1].pack_int_id]); | |
681 | close_pack_index(m->packs[pairs[i-1].pack_int_id]); | |
682 | } | |
d7cacf29 | 683 | |
748b88a0 | 684 | nth_midxed_object_oid(&oid, m, pairs[i].pos); |
0d5b3a5e | 685 | |
748b88a0 TB |
686 | if (!fill_midx_entry(r, &oid, &e, m)) { |
687 | midx_report(_("failed to load pack entry for oid[%d] = %s"), | |
688 | pairs[i].pos, oid_to_hex(&oid)); | |
689 | continue; | |
690 | } | |
0d5b3a5e | 691 | |
748b88a0 TB |
692 | if (open_pack_index(e.p)) { |
693 | midx_report(_("failed to load pack-index for packfile %s"), | |
694 | e.p->pack_name); | |
695 | break; | |
0d5b3a5e DS |
696 | } |
697 | ||
748b88a0 TB |
698 | m_offset = e.offset; |
699 | p_offset = find_pack_entry_one(oid.hash, e.p); | |
0d5b3a5e | 700 | |
748b88a0 TB |
701 | if (m_offset != p_offset) |
702 | midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64), | |
703 | pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset); | |
0d5b3a5e | 704 | |
748b88a0 | 705 | midx_display_sparse_progress(progress, i + 1); |
662148c4 | 706 | } |
748b88a0 | 707 | stop_progress(&progress); |
662148c4 | 708 | |
748b88a0 TB |
709 | cleanup: |
710 | free(pairs); | |
711 | close_midx(m); | |
5ae18df9 | 712 | |
56ee7ff1 DS |
713 | return verify_midx_error; |
714 | } |