]>
Commit | Line | Data |
---|---|---|
7cc8f971 | 1 | #include "cache.h" |
cbd53a21 | 2 | #include "object-store.h" |
7cc8f971 VM |
3 | #include "commit.h" |
4 | #include "tag.h" | |
5 | #include "diff.h" | |
6 | #include "revision.h" | |
7 | #include "list-objects.h" | |
8 | #include "progress.h" | |
9 | #include "pack-revindex.h" | |
10 | #include "pack.h" | |
11 | #include "pack-bitmap.h" | |
bc626927 | 12 | #include "hash-lookup.h" |
7cc8f971 | 13 | #include "pack-objects.h" |
64043556 | 14 | #include "commit-reach.h" |
6dc5ef75 | 15 | #include "prio-queue.h" |
7cc8f971 VM |
16 | |
17 | struct bitmapped_commit { | |
18 | struct commit *commit; | |
19 | struct ewah_bitmap *bitmap; | |
20 | struct ewah_bitmap *write_as; | |
21 | int flags; | |
22 | int xor_offset; | |
23 | uint32_t commit_pos; | |
24 | }; | |
25 | ||
26 | struct bitmap_writer { | |
27 | struct ewah_bitmap *commits; | |
28 | struct ewah_bitmap *trees; | |
29 | struct ewah_bitmap *blobs; | |
30 | struct ewah_bitmap *tags; | |
31 | ||
d2bc62b1 | 32 | kh_oid_map_t *bitmaps; |
7cc8f971 VM |
33 | struct packing_data *to_pack; |
34 | ||
35 | struct bitmapped_commit *selected; | |
36 | unsigned int selected_nr, selected_alloc; | |
37 | ||
38 | struct progress *progress; | |
39 | int show_progress; | |
825544a3 | 40 | unsigned char pack_checksum[GIT_MAX_RAWSZ]; |
7cc8f971 VM |
41 | }; |
42 | ||
43 | static struct bitmap_writer writer; | |
44 | ||
45 | void bitmap_writer_show_progress(int show) | |
46 | { | |
47 | writer.show_progress = show; | |
48 | } | |
49 | ||
50 | /** | |
51 | * Build the initial type index for the packfile | |
52 | */ | |
06af3bba NTND |
53 | void bitmap_writer_build_type_index(struct packing_data *to_pack, |
54 | struct pack_idx_entry **index, | |
7cc8f971 VM |
55 | uint32_t index_nr) |
56 | { | |
57 | uint32_t i; | |
58 | ||
59 | writer.commits = ewah_new(); | |
60 | writer.trees = ewah_new(); | |
61 | writer.blobs = ewah_new(); | |
62 | writer.tags = ewah_new(); | |
06af3bba | 63 | ALLOC_ARRAY(to_pack->in_pack_pos, to_pack->nr_objects); |
7cc8f971 VM |
64 | |
65 | for (i = 0; i < index_nr; ++i) { | |
66 | struct object_entry *entry = (struct object_entry *)index[i]; | |
67 | enum object_type real_type; | |
68 | ||
06af3bba | 69 | oe_set_in_pack_pos(to_pack, entry, i); |
7cc8f971 | 70 | |
fd9b1bae | 71 | switch (oe_type(entry)) { |
7cc8f971 VM |
72 | case OBJ_COMMIT: |
73 | case OBJ_TREE: | |
74 | case OBJ_BLOB: | |
75 | case OBJ_TAG: | |
fd9b1bae | 76 | real_type = oe_type(entry); |
7cc8f971 VM |
77 | break; |
78 | ||
79 | default: | |
7c141127 | 80 | real_type = oid_object_info(to_pack->repo, |
0df8e965 | 81 | &entry->idx.oid, NULL); |
7cc8f971 VM |
82 | break; |
83 | } | |
84 | ||
85 | switch (real_type) { | |
86 | case OBJ_COMMIT: | |
87 | ewah_set(writer.commits, i); | |
88 | break; | |
89 | ||
90 | case OBJ_TREE: | |
91 | ewah_set(writer.trees, i); | |
92 | break; | |
93 | ||
94 | case OBJ_BLOB: | |
95 | ewah_set(writer.blobs, i); | |
96 | break; | |
97 | ||
98 | case OBJ_TAG: | |
99 | ewah_set(writer.tags, i); | |
100 | break; | |
101 | ||
102 | default: | |
103 | die("Missing type information for %s (%d/%d)", | |
e6a492b7 | 104 | oid_to_hex(&entry->idx.oid), real_type, |
fd9b1bae | 105 | oe_type(entry)); |
7cc8f971 VM |
106 | } |
107 | } | |
108 | } | |
109 | ||
110 | /** | |
111 | * Compute the actual bitmaps | |
112 | */ | |
7cc8f971 | 113 | |
449fa5ee | 114 | static inline void push_bitmapped_commit(struct commit *commit) |
7cc8f971 VM |
115 | { |
116 | if (writer.selected_nr >= writer.selected_alloc) { | |
117 | writer.selected_alloc = (writer.selected_alloc + 32) * 2; | |
2756ca43 | 118 | REALLOC_ARRAY(writer.selected, writer.selected_alloc); |
7cc8f971 VM |
119 | } |
120 | ||
121 | writer.selected[writer.selected_nr].commit = commit; | |
449fa5ee | 122 | writer.selected[writer.selected_nr].bitmap = NULL; |
7cc8f971 VM |
123 | writer.selected[writer.selected_nr].flags = 0; |
124 | ||
125 | writer.selected_nr++; | |
126 | } | |
127 | ||
05805d74 | 128 | static uint32_t find_object_pos(const struct object_id *oid) |
7cc8f971 | 129 | { |
3a37876b | 130 | struct object_entry *entry = packlist_find(writer.to_pack, oid); |
7cc8f971 VM |
131 | |
132 | if (!entry) { | |
133 | die("Failed to write bitmap index. Packfile doesn't have full closure " | |
05805d74 | 134 | "(object %s is missing)", oid_to_hex(oid)); |
7cc8f971 VM |
135 | } |
136 | ||
06af3bba | 137 | return oe_in_pack_pos(writer.to_pack, entry); |
7cc8f971 VM |
138 | } |
139 | ||
7cc8f971 VM |
140 | static void compute_xor_offsets(void) |
141 | { | |
142 | static const int MAX_XOR_OFFSET_SEARCH = 10; | |
143 | ||
144 | int i, next = 0; | |
145 | ||
146 | while (next < writer.selected_nr) { | |
147 | struct bitmapped_commit *stored = &writer.selected[next]; | |
148 | ||
149 | int best_offset = 0; | |
150 | struct ewah_bitmap *best_bitmap = stored->bitmap; | |
151 | struct ewah_bitmap *test_xor; | |
152 | ||
153 | for (i = 1; i <= MAX_XOR_OFFSET_SEARCH; ++i) { | |
154 | int curr = next - i; | |
155 | ||
156 | if (curr < 0) | |
157 | break; | |
158 | ||
159 | test_xor = ewah_pool_new(); | |
160 | ewah_xor(writer.selected[curr].bitmap, stored->bitmap, test_xor); | |
161 | ||
162 | if (test_xor->buffer_size < best_bitmap->buffer_size) { | |
163 | if (best_bitmap != stored->bitmap) | |
164 | ewah_pool_free(best_bitmap); | |
165 | ||
166 | best_bitmap = test_xor; | |
167 | best_offset = i; | |
168 | } else { | |
169 | ewah_pool_free(test_xor); | |
170 | } | |
171 | } | |
172 | ||
173 | stored->xor_offset = best_offset; | |
174 | stored->write_as = best_bitmap; | |
175 | ||
176 | next++; | |
177 | } | |
178 | } | |
179 | ||
4a9c5817 | 180 | struct bb_commit { |
928e3f42 | 181 | struct commit_list *reverse_edges; |
089f7513 | 182 | struct bitmap *commit_mask; |
4a9c5817 | 183 | struct bitmap *bitmap; |
089f7513 DS |
184 | unsigned selected:1, |
185 | maximal:1; | |
4a9c5817 JK |
186 | unsigned idx; /* within selected array */ |
187 | }; | |
7cc8f971 | 188 | |
4a9c5817 | 189 | define_commit_slab(bb_data, struct bb_commit); |
7cc8f971 | 190 | |
4a9c5817 JK |
191 | struct bitmap_builder { |
192 | struct bb_data data; | |
193 | struct commit **commits; | |
194 | size_t commits_nr, commits_alloc; | |
195 | }; | |
7cc8f971 | 196 | |
4a9c5817 | 197 | static void bitmap_builder_init(struct bitmap_builder *bb, |
f077b0a9 DS |
198 | struct bitmap_writer *writer, |
199 | struct bitmap_index *old_bitmap) | |
4a9c5817 JK |
200 | { |
201 | struct rev_info revs; | |
202 | struct commit *commit; | |
f077b0a9 DS |
203 | struct commit_list *reusable = NULL; |
204 | struct commit_list *r; | |
45f4eeb2 | 205 | unsigned int i, num_maximal = 0; |
7cc8f971 | 206 | |
4a9c5817 JK |
207 | memset(bb, 0, sizeof(*bb)); |
208 | init_bb_data(&bb->data); | |
7cc8f971 | 209 | |
7cc8f971 | 210 | reset_revision_walk(); |
4a9c5817 JK |
211 | repo_init_revisions(writer->to_pack->repo, &revs, NULL); |
212 | revs.topo_order = 1; | |
45f4eeb2 | 213 | revs.first_parent_only = 1; |
4a9c5817 JK |
214 | |
215 | for (i = 0; i < writer->selected_nr; i++) { | |
216 | struct commit *c = writer->selected[i].commit; | |
217 | struct bb_commit *ent = bb_data_at(&bb->data, c); | |
089f7513 | 218 | |
4a9c5817 | 219 | ent->selected = 1; |
089f7513 | 220 | ent->maximal = 1; |
4a9c5817 | 221 | ent->idx = i; |
089f7513 DS |
222 | |
223 | ent->commit_mask = bitmap_new(); | |
224 | bitmap_set(ent->commit_mask, i); | |
225 | ||
4a9c5817 JK |
226 | add_pending_object(&revs, &c->object, ""); |
227 | } | |
7cc8f971 | 228 | |
4a9c5817 JK |
229 | if (prepare_revision_walk(&revs)) |
230 | die("revision walk setup failed"); | |
7cc8f971 | 231 | |
4a9c5817 | 232 | while ((commit = get_revision(&revs))) { |
45f4eeb2 | 233 | struct commit_list *p = commit->parents; |
089f7513 | 234 | struct bb_commit *c_ent; |
7cc8f971 | 235 | |
4a9c5817 | 236 | parse_commit_or_die(commit); |
7cc8f971 | 237 | |
089f7513 DS |
238 | c_ent = bb_data_at(&bb->data, commit); |
239 | ||
f077b0a9 DS |
240 | /* |
241 | * If there is no commit_mask, there is no reason to iterate | |
242 | * over this commit; it is not selected (if it were, it would | |
243 | * not have a blank commit mask) and all its children have | |
244 | * existing bitmaps (see the comment starting with "This commit | |
245 | * has an existing bitmap" below), so it does not contribute | |
246 | * anything to the final bitmap file or its descendants. | |
247 | */ | |
248 | if (!c_ent->commit_mask) | |
249 | continue; | |
250 | ||
251 | if (old_bitmap && bitmap_for_commit(old_bitmap, commit)) { | |
252 | /* | |
253 | * This commit has an existing bitmap, so we can | |
254 | * get its bits immediately without an object | |
255 | * walk. That is, it is reusable as-is and there is no | |
256 | * need to continue walking beyond it. | |
257 | * | |
258 | * Mark it as such and add it to bb->commits separately | |
259 | * to avoid allocating a position in the commit mask. | |
260 | */ | |
261 | commit_list_insert(commit, &reusable); | |
262 | goto next; | |
263 | } | |
264 | ||
089f7513 | 265 | if (c_ent->maximal) { |
45f4eeb2 | 266 | num_maximal++; |
089f7513 DS |
267 | ALLOC_GROW(bb->commits, bb->commits_nr + 1, bb->commits_alloc); |
268 | bb->commits[bb->commits_nr++] = commit; | |
269 | } | |
7cc8f971 | 270 | |
45f4eeb2 | 271 | if (p) { |
089f7513 DS |
272 | struct bb_commit *p_ent = bb_data_at(&bb->data, p->item); |
273 | int c_not_p, p_not_c; | |
274 | ||
275 | if (!p_ent->commit_mask) { | |
276 | p_ent->commit_mask = bitmap_new(); | |
277 | c_not_p = 1; | |
278 | p_not_c = 0; | |
279 | } else { | |
280 | c_not_p = bitmap_is_subset(c_ent->commit_mask, p_ent->commit_mask); | |
281 | p_not_c = bitmap_is_subset(p_ent->commit_mask, c_ent->commit_mask); | |
282 | } | |
283 | ||
284 | if (!c_not_p) | |
285 | continue; | |
286 | ||
287 | bitmap_or(p_ent->commit_mask, c_ent->commit_mask); | |
288 | ||
289 | if (p_not_c) | |
290 | p_ent->maximal = 1; | |
291 | else { | |
292 | p_ent->maximal = 0; | |
293 | free_commit_list(p_ent->reverse_edges); | |
294 | p_ent->reverse_edges = NULL; | |
295 | } | |
296 | ||
297 | if (c_ent->maximal) { | |
298 | commit_list_insert(commit, &p_ent->reverse_edges); | |
299 | } else { | |
300 | struct commit_list *cc = c_ent->reverse_edges; | |
301 | ||
302 | for (; cc; cc = cc->next) { | |
303 | if (!commit_list_contains(cc->item, p_ent->reverse_edges)) | |
304 | commit_list_insert(cc->item, &p_ent->reverse_edges); | |
305 | } | |
306 | } | |
4a9c5817 | 307 | } |
089f7513 | 308 | |
f077b0a9 | 309 | next: |
089f7513 DS |
310 | bitmap_free(c_ent->commit_mask); |
311 | c_ent->commit_mask = NULL; | |
4a9c5817 | 312 | } |
089f7513 | 313 | |
f077b0a9 DS |
314 | for (r = reusable; r; r = r->next) { |
315 | ALLOC_GROW(bb->commits, bb->commits_nr + 1, bb->commits_alloc); | |
316 | bb->commits[bb->commits_nr++] = r->item; | |
317 | } | |
318 | ||
089f7513 DS |
319 | trace2_data_intmax("pack-bitmap-write", the_repository, |
320 | "num_selected_commits", writer->selected_nr); | |
321 | trace2_data_intmax("pack-bitmap-write", the_repository, | |
322 | "num_maximal_commits", num_maximal); | |
f077b0a9 DS |
323 | |
324 | free_commit_list(reusable); | |
4a9c5817 JK |
325 | } |
326 | ||
327 | static void bitmap_builder_clear(struct bitmap_builder *bb) | |
328 | { | |
329 | clear_bb_data(&bb->data); | |
330 | free(bb->commits); | |
331 | bb->commits_nr = bb->commits_alloc = 0; | |
332 | } | |
333 | ||
334 | static void fill_bitmap_tree(struct bitmap *bitmap, | |
335 | struct tree *tree) | |
336 | { | |
337 | uint32_t pos; | |
338 | struct tree_desc desc; | |
339 | struct name_entry entry; | |
340 | ||
341 | /* | |
342 | * If our bit is already set, then there is nothing to do. Both this | |
343 | * tree and all of its children will be set. | |
344 | */ | |
345 | pos = find_object_pos(&tree->object.oid); | |
346 | if (bitmap_get(bitmap, pos)) | |
347 | return; | |
348 | bitmap_set(bitmap, pos); | |
349 | ||
350 | if (parse_tree(tree) < 0) | |
351 | die("unable to load tree object %s", | |
352 | oid_to_hex(&tree->object.oid)); | |
353 | init_tree_desc(&desc, tree->buffer, tree->size); | |
354 | ||
355 | while (tree_entry(&desc, &entry)) { | |
356 | switch (object_type(entry.mode)) { | |
357 | case OBJ_TREE: | |
358 | fill_bitmap_tree(bitmap, | |
359 | lookup_tree(the_repository, &entry.oid)); | |
360 | break; | |
361 | case OBJ_BLOB: | |
362 | bitmap_set(bitmap, find_object_pos(&entry.oid)); | |
363 | break; | |
364 | default: | |
365 | /* Gitlink, etc; not reachable */ | |
366 | break; | |
367 | } | |
368 | } | |
7cc8f971 | 369 | |
4a9c5817 JK |
370 | free_tree_buffer(tree); |
371 | } | |
7cc8f971 | 372 | |
4a9c5817 | 373 | static void fill_bitmap_commit(struct bb_commit *ent, |
6dc5ef75 | 374 | struct commit *commit, |
341fa348 DS |
375 | struct prio_queue *queue, |
376 | struct prio_queue *tree_queue, | |
377 | struct bitmap_index *old_bitmap, | |
378 | const uint32_t *mapping) | |
4a9c5817 JK |
379 | { |
380 | if (!ent->bitmap) | |
381 | ent->bitmap = bitmap_new(); | |
382 | ||
6dc5ef75 DS |
383 | prio_queue_put(queue, commit); |
384 | ||
385 | while (queue->nr) { | |
386 | struct commit_list *p; | |
387 | struct commit *c = prio_queue_get(queue); | |
388 | ||
341fa348 DS |
389 | if (old_bitmap && mapping) { |
390 | struct ewah_bitmap *old = bitmap_for_commit(old_bitmap, c); | |
391 | /* | |
392 | * If this commit has an old bitmap, then translate that | |
393 | * bitmap and add its bits to this one. No need to walk | |
394 | * parents or the tree for this commit. | |
395 | */ | |
396 | if (old && !rebuild_bitmap(mapping, old, ent->bitmap)) | |
397 | continue; | |
398 | } | |
399 | ||
400 | /* | |
401 | * Mark ourselves and queue our tree. The commit | |
402 | * walk ensures we cover all parents. | |
403 | */ | |
6dc5ef75 | 404 | bitmap_set(ent->bitmap, find_object_pos(&c->object.oid)); |
341fa348 | 405 | prio_queue_put(tree_queue, get_commit_tree(c)); |
6dc5ef75 DS |
406 | |
407 | for (p = c->parents; p; p = p->next) { | |
408 | int pos = find_object_pos(&p->item->object.oid); | |
409 | if (!bitmap_get(ent->bitmap, pos)) { | |
410 | bitmap_set(ent->bitmap, pos); | |
411 | prio_queue_put(queue, p->item); | |
412 | } | |
413 | } | |
414 | } | |
341fa348 DS |
415 | |
416 | while (tree_queue->nr) | |
417 | fill_bitmap_tree(ent->bitmap, prio_queue_get(tree_queue)); | |
4a9c5817 | 418 | } |
7cc8f971 | 419 | |
4a9c5817 JK |
420 | static void store_selected(struct bb_commit *ent, struct commit *commit) |
421 | { | |
422 | struct bitmapped_commit *stored = &writer.selected[ent->idx]; | |
423 | khiter_t hash_pos; | |
424 | int hash_ret; | |
425 | ||
4a9c5817 JK |
426 | stored->bitmap = bitmap_to_ewah(ent->bitmap); |
427 | ||
428 | hash_pos = kh_put_oid_map(writer.bitmaps, commit->object.oid, &hash_ret); | |
429 | if (hash_ret == 0) | |
430 | die("Duplicate entry when writing index: %s", | |
431 | oid_to_hex(&commit->object.oid)); | |
432 | kh_value(writer.bitmaps, hash_pos) = stored; | |
433 | } | |
7cc8f971 | 434 | |
4a9c5817 JK |
435 | void bitmap_writer_build(struct packing_data *to_pack) |
436 | { | |
437 | struct bitmap_builder bb; | |
438 | size_t i; | |
439 | int nr_stored = 0; /* for progress */ | |
6dc5ef75 | 440 | struct prio_queue queue = { compare_commits_by_gen_then_commit_date }; |
341fa348 DS |
441 | struct prio_queue tree_queue = { NULL }; |
442 | struct bitmap_index *old_bitmap; | |
443 | uint32_t *mapping; | |
7cc8f971 | 444 | |
4a9c5817 JK |
445 | writer.bitmaps = kh_init_oid_map(); |
446 | writer.to_pack = to_pack; | |
447 | ||
448 | if (writer.show_progress) | |
449 | writer.progress = start_progress("Building bitmaps", writer.selected_nr); | |
450 | trace2_region_enter("pack-bitmap-write", "building_bitmaps_total", | |
451 | the_repository); | |
7cc8f971 | 452 | |
341fa348 DS |
453 | old_bitmap = prepare_bitmap_git(to_pack->repo); |
454 | if (old_bitmap) | |
455 | mapping = create_bitmap_mapping(old_bitmap, to_pack); | |
456 | else | |
457 | mapping = NULL; | |
458 | ||
f077b0a9 | 459 | bitmap_builder_init(&bb, &writer, old_bitmap); |
4a9c5817 JK |
460 | for (i = bb.commits_nr; i > 0; i--) { |
461 | struct commit *commit = bb.commits[i-1]; | |
462 | struct bb_commit *ent = bb_data_at(&bb.data, commit); | |
463 | struct commit *child; | |
010e5eac | 464 | int reused = 0; |
7cc8f971 | 465 | |
341fa348 DS |
466 | fill_bitmap_commit(ent, commit, &queue, &tree_queue, |
467 | old_bitmap, mapping); | |
7cc8f971 | 468 | |
4a9c5817 JK |
469 | if (ent->selected) { |
470 | store_selected(ent, commit); | |
471 | nr_stored++; | |
472 | display_progress(writer.progress, nr_stored); | |
473 | } | |
474 | ||
928e3f42 | 475 | while ((child = pop_commit(&ent->reverse_edges))) { |
4a9c5817 JK |
476 | struct bb_commit *child_ent = |
477 | bb_data_at(&bb.data, child); | |
478 | ||
479 | if (child_ent->bitmap) | |
480 | bitmap_or(child_ent->bitmap, ent->bitmap); | |
010e5eac | 481 | else if (reused) |
4a9c5817 | 482 | child_ent->bitmap = bitmap_dup(ent->bitmap); |
010e5eac JK |
483 | else { |
484 | child_ent->bitmap = ent->bitmap; | |
485 | reused = 1; | |
486 | } | |
4a9c5817 | 487 | } |
010e5eac JK |
488 | if (!reused) |
489 | bitmap_free(ent->bitmap); | |
4a9c5817 | 490 | ent->bitmap = NULL; |
7cc8f971 | 491 | } |
6dc5ef75 | 492 | clear_prio_queue(&queue); |
341fa348 | 493 | clear_prio_queue(&tree_queue); |
4a9c5817 | 494 | bitmap_builder_clear(&bb); |
341fa348 | 495 | free(mapping); |
4a9c5817 JK |
496 | |
497 | trace2_region_leave("pack-bitmap-write", "building_bitmaps_total", | |
498 | the_repository); | |
7cc8f971 | 499 | |
7cc8f971 VM |
500 | stop_progress(&writer.progress); |
501 | ||
502 | compute_xor_offsets(); | |
503 | } | |
504 | ||
505 | /** | |
506 | * Select the commits that will be bitmapped | |
507 | */ | |
508 | static inline unsigned int next_commit_index(unsigned int idx) | |
509 | { | |
510 | static const unsigned int MIN_COMMITS = 100; | |
511 | static const unsigned int MAX_COMMITS = 5000; | |
512 | ||
513 | static const unsigned int MUST_REGION = 100; | |
514 | static const unsigned int MIN_REGION = 20000; | |
515 | ||
516 | unsigned int offset, next; | |
517 | ||
518 | if (idx <= MUST_REGION) | |
519 | return 0; | |
520 | ||
521 | if (idx <= MIN_REGION) { | |
522 | offset = idx - MUST_REGION; | |
523 | return (offset < MIN_COMMITS) ? offset : MIN_COMMITS; | |
524 | } | |
525 | ||
526 | offset = idx - MIN_REGION; | |
527 | next = (offset < MAX_COMMITS) ? offset : MAX_COMMITS; | |
528 | ||
529 | return (next > MIN_COMMITS) ? next : MIN_COMMITS; | |
530 | } | |
531 | ||
532 | static int date_compare(const void *_a, const void *_b) | |
533 | { | |
534 | struct commit *a = *(struct commit **)_a; | |
535 | struct commit *b = *(struct commit **)_b; | |
536 | return (long)b->date - (long)a->date; | |
537 | } | |
538 | ||
7cc8f971 VM |
539 | void bitmap_writer_select_commits(struct commit **indexed_commits, |
540 | unsigned int indexed_commits_nr, | |
541 | int max_bitmaps) | |
542 | { | |
543 | unsigned int i = 0, j, next; | |
544 | ||
9ed0d8d6 | 545 | QSORT(indexed_commits, indexed_commits_nr, date_compare); |
7cc8f971 VM |
546 | |
547 | if (writer.show_progress) | |
548 | writer.progress = start_progress("Selecting bitmap commits", 0); | |
549 | ||
550 | if (indexed_commits_nr < 100) { | |
551 | for (i = 0; i < indexed_commits_nr; ++i) | |
449fa5ee | 552 | push_bitmapped_commit(indexed_commits[i]); |
7cc8f971 VM |
553 | return; |
554 | } | |
555 | ||
556 | for (;;) { | |
7cc8f971 VM |
557 | struct commit *chosen = NULL; |
558 | ||
559 | next = next_commit_index(i); | |
560 | ||
561 | if (i + next >= indexed_commits_nr) | |
562 | break; | |
563 | ||
564 | if (max_bitmaps > 0 && writer.selected_nr >= max_bitmaps) { | |
565 | writer.selected_nr = max_bitmaps; | |
566 | break; | |
567 | } | |
568 | ||
569 | if (next == 0) { | |
570 | chosen = indexed_commits[i]; | |
7cc8f971 VM |
571 | } else { |
572 | chosen = indexed_commits[i + next]; | |
573 | ||
574 | for (j = 0; j <= next; ++j) { | |
575 | struct commit *cm = indexed_commits[i + j]; | |
576 | ||
449fa5ee | 577 | if ((cm->object.flags & NEEDS_BITMAP) != 0) { |
7cc8f971 VM |
578 | chosen = cm; |
579 | break; | |
580 | } | |
581 | ||
582 | if (cm->parents && cm->parents->next) | |
583 | chosen = cm; | |
584 | } | |
585 | } | |
586 | ||
449fa5ee | 587 | push_bitmapped_commit(chosen); |
7cc8f971 VM |
588 | |
589 | i += next + 1; | |
590 | display_progress(writer.progress, i); | |
591 | } | |
592 | ||
593 | stop_progress(&writer.progress); | |
594 | } | |
595 | ||
596 | ||
98a3beab | 597 | static int hashwrite_ewah_helper(void *f, const void *buf, size_t len) |
7cc8f971 | 598 | { |
98a3beab | 599 | /* hashwrite will die on error */ |
600 | hashwrite(f, buf, len); | |
7cc8f971 VM |
601 | return len; |
602 | } | |
603 | ||
604 | /** | |
605 | * Write the bitmap index to disk | |
606 | */ | |
98a3beab | 607 | static inline void dump_bitmap(struct hashfile *f, struct ewah_bitmap *bitmap) |
7cc8f971 | 608 | { |
98a3beab | 609 | if (ewah_serialize_to(bitmap, hashwrite_ewah_helper, f) < 0) |
7cc8f971 VM |
610 | die("Failed to write bitmap index"); |
611 | } | |
612 | ||
613 | static const unsigned char *sha1_access(size_t pos, void *table) | |
614 | { | |
615 | struct pack_idx_entry **index = table; | |
e6a492b7 | 616 | return index[pos]->oid.hash; |
7cc8f971 VM |
617 | } |
618 | ||
98a3beab | 619 | static void write_selected_commits_v1(struct hashfile *f, |
7cc8f971 VM |
620 | struct pack_idx_entry **index, |
621 | uint32_t index_nr) | |
622 | { | |
623 | int i; | |
624 | ||
625 | for (i = 0; i < writer.selected_nr; ++i) { | |
626 | struct bitmapped_commit *stored = &writer.selected[i]; | |
7cc8f971 VM |
627 | |
628 | int commit_pos = | |
7a7d992d | 629 | hash_pos(stored->commit->object.oid.hash, index, index_nr, sha1_access); |
7cc8f971 VM |
630 | |
631 | if (commit_pos < 0) | |
033abf97 | 632 | BUG("trying to write commit not in index"); |
7cc8f971 | 633 | |
98a3beab | 634 | hashwrite_be32(f, commit_pos); |
635 | hashwrite_u8(f, stored->xor_offset); | |
636 | hashwrite_u8(f, stored->flags); | |
7cc8f971 | 637 | |
7cc8f971 VM |
638 | dump_bitmap(f, stored->write_as); |
639 | } | |
640 | } | |
641 | ||
98a3beab | 642 | static void write_hash_cache(struct hashfile *f, |
ae4f07fb VM |
643 | struct pack_idx_entry **index, |
644 | uint32_t index_nr) | |
645 | { | |
646 | uint32_t i; | |
647 | ||
648 | for (i = 0; i < index_nr; ++i) { | |
649 | struct object_entry *entry = (struct object_entry *)index[i]; | |
7744a5d6 | 650 | hashwrite_be32(f, entry->hash); |
ae4f07fb VM |
651 | } |
652 | } | |
653 | ||
7cc8f971 VM |
654 | void bitmap_writer_set_checksum(unsigned char *sha1) |
655 | { | |
656 | hashcpy(writer.pack_checksum, sha1); | |
657 | } | |
658 | ||
659 | void bitmap_writer_finish(struct pack_idx_entry **index, | |
660 | uint32_t index_nr, | |
ae4f07fb VM |
661 | const char *filename, |
662 | uint16_t options) | |
7cc8f971 | 663 | { |
7cc8f971 VM |
664 | static uint16_t default_version = 1; |
665 | static uint16_t flags = BITMAP_OPT_FULL_DAG; | |
594fa999 | 666 | struct strbuf tmp_file = STRBUF_INIT; |
98a3beab | 667 | struct hashfile *f; |
7cc8f971 VM |
668 | |
669 | struct bitmap_disk_header header; | |
670 | ||
594fa999 | 671 | int fd = odb_mkstemp(&tmp_file, "pack/tmp_bitmap_XXXXXX"); |
7cc8f971 | 672 | |
98a3beab | 673 | f = hashfd(fd, tmp_file.buf); |
7cc8f971 VM |
674 | |
675 | memcpy(header.magic, BITMAP_IDX_SIGNATURE, sizeof(BITMAP_IDX_SIGNATURE)); | |
676 | header.version = htons(default_version); | |
ae4f07fb | 677 | header.options = htons(flags | options); |
7cc8f971 | 678 | header.entry_count = htonl(writer.selected_nr); |
50546b15 | 679 | hashcpy(header.checksum, writer.pack_checksum); |
7cc8f971 | 680 | |
0f4d6cad | 681 | hashwrite(f, &header, sizeof(header) - GIT_MAX_RAWSZ + the_hash_algo->rawsz); |
7cc8f971 VM |
682 | dump_bitmap(f, writer.commits); |
683 | dump_bitmap(f, writer.trees); | |
684 | dump_bitmap(f, writer.blobs); | |
685 | dump_bitmap(f, writer.tags); | |
686 | write_selected_commits_v1(f, index, index_nr); | |
687 | ||
ae4f07fb VM |
688 | if (options & BITMAP_OPT_HASH_CACHE) |
689 | write_hash_cache(f, index, index_nr); | |
690 | ||
cfe83216 | 691 | finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); |
7cc8f971 | 692 | |
594fa999 | 693 | if (adjust_shared_perm(tmp_file.buf)) |
7cc8f971 VM |
694 | die_errno("unable to make temporary bitmap file readable"); |
695 | ||
594fa999 | 696 | if (rename(tmp_file.buf, filename)) |
7cc8f971 | 697 | die_errno("unable to rename temporary bitmap file to '%s'", filename); |
594fa999 JK |
698 | |
699 | strbuf_release(&tmp_file); | |
7cc8f971 | 700 | } |