]> git.ipfire.org Git - thirdparty/git.git/blob - builtin/pack-objects.c
Merge branch 'jk/clone-allow-bare-and-o-together'
[thirdparty/git.git] / builtin / pack-objects.c
1 #include "builtin.h"
2 #include "cache.h"
3 #include "repository.h"
4 #include "config.h"
5 #include "attr.h"
6 #include "object.h"
7 #include "blob.h"
8 #include "commit.h"
9 #include "tag.h"
10 #include "tree.h"
11 #include "delta.h"
12 #include "pack.h"
13 #include "pack-revindex.h"
14 #include "csum-file.h"
15 #include "tree-walk.h"
16 #include "diff.h"
17 #include "revision.h"
18 #include "list-objects.h"
19 #include "list-objects-filter.h"
20 #include "list-objects-filter-options.h"
21 #include "pack-objects.h"
22 #include "progress.h"
23 #include "refs.h"
24 #include "streaming.h"
25 #include "thread-utils.h"
26 #include "pack-bitmap.h"
27 #include "delta-islands.h"
28 #include "reachable.h"
29 #include "oid-array.h"
30 #include "strvec.h"
31 #include "list.h"
32 #include "packfile.h"
33 #include "object-store.h"
34 #include "dir.h"
35 #include "midx.h"
36 #include "trace2.h"
37 #include "shallow.h"
38 #include "promisor-remote.h"
39 #include "pack-mtimes.h"
40
41 /*
42 * Objects we are going to pack are collected in the `to_pack` structure.
43 * It contains an array (dynamically expanded) of the object data, and a map
44 * that can resolve SHA1s to their position in the array.
45 */
46 static struct packing_data to_pack;
47
48 static inline struct object_entry *oe_delta(
49 const struct packing_data *pack,
50 const struct object_entry *e)
51 {
52 if (!e->delta_idx)
53 return NULL;
54 if (e->ext_base)
55 return &pack->ext_bases[e->delta_idx - 1];
56 else
57 return &pack->objects[e->delta_idx - 1];
58 }
59
60 static inline unsigned long oe_delta_size(struct packing_data *pack,
61 const struct object_entry *e)
62 {
63 if (e->delta_size_valid)
64 return e->delta_size_;
65
66 /*
67 * pack->delta_size[] can't be NULL because oe_set_delta_size()
68 * must have been called when a new delta is saved with
69 * oe_set_delta().
70 * If oe_delta() returns NULL (i.e. default state, which means
71 * delta_size_valid is also false), then the caller must never
72 * call oe_delta_size().
73 */
74 return pack->delta_size[e - pack->objects];
75 }
76
77 unsigned long oe_get_size_slow(struct packing_data *pack,
78 const struct object_entry *e);
79
80 static inline unsigned long oe_size(struct packing_data *pack,
81 const struct object_entry *e)
82 {
83 if (e->size_valid)
84 return e->size_;
85
86 return oe_get_size_slow(pack, e);
87 }
88
89 static inline void oe_set_delta(struct packing_data *pack,
90 struct object_entry *e,
91 struct object_entry *delta)
92 {
93 if (delta)
94 e->delta_idx = (delta - pack->objects) + 1;
95 else
96 e->delta_idx = 0;
97 }
98
99 static inline struct object_entry *oe_delta_sibling(
100 const struct packing_data *pack,
101 const struct object_entry *e)
102 {
103 if (e->delta_sibling_idx)
104 return &pack->objects[e->delta_sibling_idx - 1];
105 return NULL;
106 }
107
108 static inline struct object_entry *oe_delta_child(
109 const struct packing_data *pack,
110 const struct object_entry *e)
111 {
112 if (e->delta_child_idx)
113 return &pack->objects[e->delta_child_idx - 1];
114 return NULL;
115 }
116
117 static inline void oe_set_delta_child(struct packing_data *pack,
118 struct object_entry *e,
119 struct object_entry *delta)
120 {
121 if (delta)
122 e->delta_child_idx = (delta - pack->objects) + 1;
123 else
124 e->delta_child_idx = 0;
125 }
126
127 static inline void oe_set_delta_sibling(struct packing_data *pack,
128 struct object_entry *e,
129 struct object_entry *delta)
130 {
131 if (delta)
132 e->delta_sibling_idx = (delta - pack->objects) + 1;
133 else
134 e->delta_sibling_idx = 0;
135 }
136
137 static inline void oe_set_size(struct packing_data *pack,
138 struct object_entry *e,
139 unsigned long size)
140 {
141 if (size < pack->oe_size_limit) {
142 e->size_ = size;
143 e->size_valid = 1;
144 } else {
145 e->size_valid = 0;
146 if (oe_get_size_slow(pack, e) != size)
147 BUG("'size' is supposed to be the object size!");
148 }
149 }
150
151 static inline void oe_set_delta_size(struct packing_data *pack,
152 struct object_entry *e,
153 unsigned long size)
154 {
155 if (size < pack->oe_delta_size_limit) {
156 e->delta_size_ = size;
157 e->delta_size_valid = 1;
158 } else {
159 packing_data_lock(pack);
160 if (!pack->delta_size)
161 ALLOC_ARRAY(pack->delta_size, pack->nr_alloc);
162 packing_data_unlock(pack);
163
164 pack->delta_size[e - pack->objects] = size;
165 e->delta_size_valid = 0;
166 }
167 }
168
169 #define IN_PACK(obj) oe_in_pack(&to_pack, obj)
170 #define SIZE(obj) oe_size(&to_pack, obj)
171 #define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size)
172 #define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj)
173 #define DELTA(obj) oe_delta(&to_pack, obj)
174 #define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
175 #define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
176 #define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
177 #define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid)
178 #define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val)
179 #define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
180 #define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
181
182 static const char *pack_usage[] = {
183 N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"),
184 N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"),
185 NULL
186 };
187
188 static struct pack_idx_entry **written_list;
189 static uint32_t nr_result, nr_written, nr_seen;
190 static struct bitmap_index *bitmap_git;
191 static uint32_t write_layer;
192
193 static int non_empty;
194 static int reuse_delta = 1, reuse_object = 1;
195 static int keep_unreachable, unpack_unreachable, include_tag;
196 static timestamp_t unpack_unreachable_expiration;
197 static int pack_loose_unreachable;
198 static int cruft;
199 static timestamp_t cruft_expiration;
200 static int local;
201 static int have_non_local_packs;
202 static int incremental;
203 static int ignore_packed_keep_on_disk;
204 static int ignore_packed_keep_in_core;
205 static int allow_ofs_delta;
206 static struct pack_idx_option pack_idx_opts;
207 static const char *base_name;
208 static int progress = 1;
209 static int window = 10;
210 static unsigned long pack_size_limit;
211 static int depth = 50;
212 static int delta_search_threads;
213 static int pack_to_stdout;
214 static int sparse;
215 static int thin;
216 static int num_preferred_base;
217 static struct progress *progress_state;
218
219 static struct packed_git *reuse_packfile;
220 static uint32_t reuse_packfile_objects;
221 static struct bitmap *reuse_packfile_bitmap;
222
223 static int use_bitmap_index_default = 1;
224 static int use_bitmap_index = -1;
225 static int allow_pack_reuse = 1;
226 static enum {
227 WRITE_BITMAP_FALSE = 0,
228 WRITE_BITMAP_QUIET,
229 WRITE_BITMAP_TRUE,
230 } write_bitmap_index;
231 static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE;
232
233 static int exclude_promisor_objects;
234
235 static int use_delta_islands;
236
237 static unsigned long delta_cache_size = 0;
238 static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
239 static unsigned long cache_max_small_delta_size = 1000;
240
241 static unsigned long window_memory_limit = 0;
242
243 static struct string_list uri_protocols = STRING_LIST_INIT_NODUP;
244
245 enum missing_action {
246 MA_ERROR = 0, /* fail if any missing objects are encountered */
247 MA_ALLOW_ANY, /* silently allow ALL missing objects */
248 MA_ALLOW_PROMISOR, /* silently allow all missing PROMISOR objects */
249 };
250 static enum missing_action arg_missing_action;
251 static show_object_fn fn_show_object;
252
253 struct configured_exclusion {
254 struct oidmap_entry e;
255 char *pack_hash_hex;
256 char *uri;
257 };
258 static struct oidmap configured_exclusions;
259
260 static struct oidset excluded_by_config;
261
262 /*
263 * stats
264 */
265 static uint32_t written, written_delta;
266 static uint32_t reused, reused_delta;
267
268 /*
269 * Indexed commits
270 */
271 static struct commit **indexed_commits;
272 static unsigned int indexed_commits_nr;
273 static unsigned int indexed_commits_alloc;
274
275 static void index_commit_for_bitmap(struct commit *commit)
276 {
277 if (indexed_commits_nr >= indexed_commits_alloc) {
278 indexed_commits_alloc = (indexed_commits_alloc + 32) * 2;
279 REALLOC_ARRAY(indexed_commits, indexed_commits_alloc);
280 }
281
282 indexed_commits[indexed_commits_nr++] = commit;
283 }
284
285 static void *get_delta(struct object_entry *entry)
286 {
287 unsigned long size, base_size, delta_size;
288 void *buf, *base_buf, *delta_buf;
289 enum object_type type;
290
291 buf = read_object_file(&entry->idx.oid, &type, &size);
292 if (!buf)
293 die(_("unable to read %s"), oid_to_hex(&entry->idx.oid));
294 base_buf = read_object_file(&DELTA(entry)->idx.oid, &type,
295 &base_size);
296 if (!base_buf)
297 die("unable to read %s",
298 oid_to_hex(&DELTA(entry)->idx.oid));
299 delta_buf = diff_delta(base_buf, base_size,
300 buf, size, &delta_size, 0);
301 /*
302 * We successfully computed this delta once but dropped it for
303 * memory reasons. Something is very wrong if this time we
304 * recompute and create a different delta.
305 */
306 if (!delta_buf || delta_size != DELTA_SIZE(entry))
307 BUG("delta size changed");
308 free(buf);
309 free(base_buf);
310 return delta_buf;
311 }
312
313 static unsigned long do_compress(void **pptr, unsigned long size)
314 {
315 git_zstream stream;
316 void *in, *out;
317 unsigned long maxsize;
318
319 git_deflate_init(&stream, pack_compression_level);
320 maxsize = git_deflate_bound(&stream, size);
321
322 in = *pptr;
323 out = xmalloc(maxsize);
324 *pptr = out;
325
326 stream.next_in = in;
327 stream.avail_in = size;
328 stream.next_out = out;
329 stream.avail_out = maxsize;
330 while (git_deflate(&stream, Z_FINISH) == Z_OK)
331 ; /* nothing */
332 git_deflate_end(&stream);
333
334 free(in);
335 return stream.total_out;
336 }
337
338 static unsigned long write_large_blob_data(struct git_istream *st, struct hashfile *f,
339 const struct object_id *oid)
340 {
341 git_zstream stream;
342 unsigned char ibuf[1024 * 16];
343 unsigned char obuf[1024 * 16];
344 unsigned long olen = 0;
345
346 git_deflate_init(&stream, pack_compression_level);
347
348 for (;;) {
349 ssize_t readlen;
350 int zret = Z_OK;
351 readlen = read_istream(st, ibuf, sizeof(ibuf));
352 if (readlen == -1)
353 die(_("unable to read %s"), oid_to_hex(oid));
354
355 stream.next_in = ibuf;
356 stream.avail_in = readlen;
357 while ((stream.avail_in || readlen == 0) &&
358 (zret == Z_OK || zret == Z_BUF_ERROR)) {
359 stream.next_out = obuf;
360 stream.avail_out = sizeof(obuf);
361 zret = git_deflate(&stream, readlen ? 0 : Z_FINISH);
362 hashwrite(f, obuf, stream.next_out - obuf);
363 olen += stream.next_out - obuf;
364 }
365 if (stream.avail_in)
366 die(_("deflate error (%d)"), zret);
367 if (readlen == 0) {
368 if (zret != Z_STREAM_END)
369 die(_("deflate error (%d)"), zret);
370 break;
371 }
372 }
373 git_deflate_end(&stream);
374 return olen;
375 }
376
377 /*
378 * we are going to reuse the existing object data as is. make
379 * sure it is not corrupt.
380 */
381 static int check_pack_inflate(struct packed_git *p,
382 struct pack_window **w_curs,
383 off_t offset,
384 off_t len,
385 unsigned long expect)
386 {
387 git_zstream stream;
388 unsigned char fakebuf[4096], *in;
389 int st;
390
391 memset(&stream, 0, sizeof(stream));
392 git_inflate_init(&stream);
393 do {
394 in = use_pack(p, w_curs, offset, &stream.avail_in);
395 stream.next_in = in;
396 stream.next_out = fakebuf;
397 stream.avail_out = sizeof(fakebuf);
398 st = git_inflate(&stream, Z_FINISH);
399 offset += stream.next_in - in;
400 } while (st == Z_OK || st == Z_BUF_ERROR);
401 git_inflate_end(&stream);
402 return (st == Z_STREAM_END &&
403 stream.total_out == expect &&
404 stream.total_in == len) ? 0 : -1;
405 }
406
407 static void copy_pack_data(struct hashfile *f,
408 struct packed_git *p,
409 struct pack_window **w_curs,
410 off_t offset,
411 off_t len)
412 {
413 unsigned char *in;
414 unsigned long avail;
415
416 while (len) {
417 in = use_pack(p, w_curs, offset, &avail);
418 if (avail > len)
419 avail = (unsigned long)len;
420 hashwrite(f, in, avail);
421 offset += avail;
422 len -= avail;
423 }
424 }
425
426 static inline int oe_size_greater_than(struct packing_data *pack,
427 const struct object_entry *lhs,
428 unsigned long rhs)
429 {
430 if (lhs->size_valid)
431 return lhs->size_ > rhs;
432 if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */
433 return 1;
434 return oe_get_size_slow(pack, lhs) > rhs;
435 }
436
437 /* Return 0 if we will bust the pack-size limit */
438 static unsigned long write_no_reuse_object(struct hashfile *f, struct object_entry *entry,
439 unsigned long limit, int usable_delta)
440 {
441 unsigned long size, datalen;
442 unsigned char header[MAX_PACK_OBJECT_HEADER],
443 dheader[MAX_PACK_OBJECT_HEADER];
444 unsigned hdrlen;
445 enum object_type type;
446 void *buf;
447 struct git_istream *st = NULL;
448 const unsigned hashsz = the_hash_algo->rawsz;
449
450 if (!usable_delta) {
451 if (oe_type(entry) == OBJ_BLOB &&
452 oe_size_greater_than(&to_pack, entry, big_file_threshold) &&
453 (st = open_istream(the_repository, &entry->idx.oid, &type,
454 &size, NULL)) != NULL)
455 buf = NULL;
456 else {
457 buf = read_object_file(&entry->idx.oid, &type, &size);
458 if (!buf)
459 die(_("unable to read %s"),
460 oid_to_hex(&entry->idx.oid));
461 }
462 /*
463 * make sure no cached delta data remains from a
464 * previous attempt before a pack split occurred.
465 */
466 FREE_AND_NULL(entry->delta_data);
467 entry->z_delta_size = 0;
468 } else if (entry->delta_data) {
469 size = DELTA_SIZE(entry);
470 buf = entry->delta_data;
471 entry->delta_data = NULL;
472 type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
473 OBJ_OFS_DELTA : OBJ_REF_DELTA;
474 } else {
475 buf = get_delta(entry);
476 size = DELTA_SIZE(entry);
477 type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
478 OBJ_OFS_DELTA : OBJ_REF_DELTA;
479 }
480
481 if (st) /* large blob case, just assume we don't compress well */
482 datalen = size;
483 else if (entry->z_delta_size)
484 datalen = entry->z_delta_size;
485 else
486 datalen = do_compress(&buf, size);
487
488 /*
489 * The object header is a byte of 'type' followed by zero or
490 * more bytes of length.
491 */
492 hdrlen = encode_in_pack_object_header(header, sizeof(header),
493 type, size);
494
495 if (type == OBJ_OFS_DELTA) {
496 /*
497 * Deltas with relative base contain an additional
498 * encoding of the relative offset for the delta
499 * base from this object's position in the pack.
500 */
501 off_t ofs = entry->idx.offset - DELTA(entry)->idx.offset;
502 unsigned pos = sizeof(dheader) - 1;
503 dheader[pos] = ofs & 127;
504 while (ofs >>= 7)
505 dheader[--pos] = 128 | (--ofs & 127);
506 if (limit && hdrlen + sizeof(dheader) - pos + datalen + hashsz >= limit) {
507 if (st)
508 close_istream(st);
509 free(buf);
510 return 0;
511 }
512 hashwrite(f, header, hdrlen);
513 hashwrite(f, dheader + pos, sizeof(dheader) - pos);
514 hdrlen += sizeof(dheader) - pos;
515 } else if (type == OBJ_REF_DELTA) {
516 /*
517 * Deltas with a base reference contain
518 * additional bytes for the base object ID.
519 */
520 if (limit && hdrlen + hashsz + datalen + hashsz >= limit) {
521 if (st)
522 close_istream(st);
523 free(buf);
524 return 0;
525 }
526 hashwrite(f, header, hdrlen);
527 hashwrite(f, DELTA(entry)->idx.oid.hash, hashsz);
528 hdrlen += hashsz;
529 } else {
530 if (limit && hdrlen + datalen + hashsz >= limit) {
531 if (st)
532 close_istream(st);
533 free(buf);
534 return 0;
535 }
536 hashwrite(f, header, hdrlen);
537 }
538 if (st) {
539 datalen = write_large_blob_data(st, f, &entry->idx.oid);
540 close_istream(st);
541 } else {
542 hashwrite(f, buf, datalen);
543 free(buf);
544 }
545
546 return hdrlen + datalen;
547 }
548
549 /* Return 0 if we will bust the pack-size limit */
550 static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
551 unsigned long limit, int usable_delta)
552 {
553 struct packed_git *p = IN_PACK(entry);
554 struct pack_window *w_curs = NULL;
555 uint32_t pos;
556 off_t offset;
557 enum object_type type = oe_type(entry);
558 off_t datalen;
559 unsigned char header[MAX_PACK_OBJECT_HEADER],
560 dheader[MAX_PACK_OBJECT_HEADER];
561 unsigned hdrlen;
562 const unsigned hashsz = the_hash_algo->rawsz;
563 unsigned long entry_size = SIZE(entry);
564
565 if (DELTA(entry))
566 type = (allow_ofs_delta && DELTA(entry)->idx.offset) ?
567 OBJ_OFS_DELTA : OBJ_REF_DELTA;
568 hdrlen = encode_in_pack_object_header(header, sizeof(header),
569 type, entry_size);
570
571 offset = entry->in_pack_offset;
572 if (offset_to_pack_pos(p, offset, &pos) < 0)
573 die(_("write_reuse_object: could not locate %s, expected at "
574 "offset %"PRIuMAX" in pack %s"),
575 oid_to_hex(&entry->idx.oid), (uintmax_t)offset,
576 p->pack_name);
577 datalen = pack_pos_to_offset(p, pos + 1) - offset;
578 if (!pack_to_stdout && p->index_version > 1 &&
579 check_pack_crc(p, &w_curs, offset, datalen,
580 pack_pos_to_index(p, pos))) {
581 error(_("bad packed object CRC for %s"),
582 oid_to_hex(&entry->idx.oid));
583 unuse_pack(&w_curs);
584 return write_no_reuse_object(f, entry, limit, usable_delta);
585 }
586
587 offset += entry->in_pack_header_size;
588 datalen -= entry->in_pack_header_size;
589
590 if (!pack_to_stdout && p->index_version == 1 &&
591 check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) {
592 error(_("corrupt packed object for %s"),
593 oid_to_hex(&entry->idx.oid));
594 unuse_pack(&w_curs);
595 return write_no_reuse_object(f, entry, limit, usable_delta);
596 }
597
598 if (type == OBJ_OFS_DELTA) {
599 off_t ofs = entry->idx.offset - DELTA(entry)->idx.offset;
600 unsigned pos = sizeof(dheader) - 1;
601 dheader[pos] = ofs & 127;
602 while (ofs >>= 7)
603 dheader[--pos] = 128 | (--ofs & 127);
604 if (limit && hdrlen + sizeof(dheader) - pos + datalen + hashsz >= limit) {
605 unuse_pack(&w_curs);
606 return 0;
607 }
608 hashwrite(f, header, hdrlen);
609 hashwrite(f, dheader + pos, sizeof(dheader) - pos);
610 hdrlen += sizeof(dheader) - pos;
611 reused_delta++;
612 } else if (type == OBJ_REF_DELTA) {
613 if (limit && hdrlen + hashsz + datalen + hashsz >= limit) {
614 unuse_pack(&w_curs);
615 return 0;
616 }
617 hashwrite(f, header, hdrlen);
618 hashwrite(f, DELTA(entry)->idx.oid.hash, hashsz);
619 hdrlen += hashsz;
620 reused_delta++;
621 } else {
622 if (limit && hdrlen + datalen + hashsz >= limit) {
623 unuse_pack(&w_curs);
624 return 0;
625 }
626 hashwrite(f, header, hdrlen);
627 }
628 copy_pack_data(f, p, &w_curs, offset, datalen);
629 unuse_pack(&w_curs);
630 reused++;
631 return hdrlen + datalen;
632 }
633
634 /* Return 0 if we will bust the pack-size limit */
635 static off_t write_object(struct hashfile *f,
636 struct object_entry *entry,
637 off_t write_offset)
638 {
639 unsigned long limit;
640 off_t len;
641 int usable_delta, to_reuse;
642
643 if (!pack_to_stdout)
644 crc32_begin(f);
645
646 /* apply size limit if limited packsize and not first object */
647 if (!pack_size_limit || !nr_written)
648 limit = 0;
649 else if (pack_size_limit <= write_offset)
650 /*
651 * the earlier object did not fit the limit; avoid
652 * mistaking this with unlimited (i.e. limit = 0).
653 */
654 limit = 1;
655 else
656 limit = pack_size_limit - write_offset;
657
658 if (!DELTA(entry))
659 usable_delta = 0; /* no delta */
660 else if (!pack_size_limit)
661 usable_delta = 1; /* unlimited packfile */
662 else if (DELTA(entry)->idx.offset == (off_t)-1)
663 usable_delta = 0; /* base was written to another pack */
664 else if (DELTA(entry)->idx.offset)
665 usable_delta = 1; /* base already exists in this pack */
666 else
667 usable_delta = 0; /* base could end up in another pack */
668
669 if (!reuse_object)
670 to_reuse = 0; /* explicit */
671 else if (!IN_PACK(entry))
672 to_reuse = 0; /* can't reuse what we don't have */
673 else if (oe_type(entry) == OBJ_REF_DELTA ||
674 oe_type(entry) == OBJ_OFS_DELTA)
675 /* check_object() decided it for us ... */
676 to_reuse = usable_delta;
677 /* ... but pack split may override that */
678 else if (oe_type(entry) != entry->in_pack_type)
679 to_reuse = 0; /* pack has delta which is unusable */
680 else if (DELTA(entry))
681 to_reuse = 0; /* we want to pack afresh */
682 else
683 to_reuse = 1; /* we have it in-pack undeltified,
684 * and we do not need to deltify it.
685 */
686
687 if (!to_reuse)
688 len = write_no_reuse_object(f, entry, limit, usable_delta);
689 else
690 len = write_reuse_object(f, entry, limit, usable_delta);
691 if (!len)
692 return 0;
693
694 if (usable_delta)
695 written_delta++;
696 written++;
697 if (!pack_to_stdout)
698 entry->idx.crc32 = crc32_end(f);
699 return len;
700 }
701
702 enum write_one_status {
703 WRITE_ONE_SKIP = -1, /* already written */
704 WRITE_ONE_BREAK = 0, /* writing this will bust the limit; not written */
705 WRITE_ONE_WRITTEN = 1, /* normal */
706 WRITE_ONE_RECURSIVE = 2 /* already scheduled to be written */
707 };
708
709 static enum write_one_status write_one(struct hashfile *f,
710 struct object_entry *e,
711 off_t *offset)
712 {
713 off_t size;
714 int recursing;
715
716 /*
717 * we set offset to 1 (which is an impossible value) to mark
718 * the fact that this object is involved in "write its base
719 * first before writing a deltified object" recursion.
720 */
721 recursing = (e->idx.offset == 1);
722 if (recursing) {
723 warning(_("recursive delta detected for object %s"),
724 oid_to_hex(&e->idx.oid));
725 return WRITE_ONE_RECURSIVE;
726 } else if (e->idx.offset || e->preferred_base) {
727 /* offset is non zero if object is written already. */
728 return WRITE_ONE_SKIP;
729 }
730
731 /* if we are deltified, write out base object first. */
732 if (DELTA(e)) {
733 e->idx.offset = 1; /* now recurse */
734 switch (write_one(f, DELTA(e), offset)) {
735 case WRITE_ONE_RECURSIVE:
736 /* we cannot depend on this one */
737 SET_DELTA(e, NULL);
738 break;
739 default:
740 break;
741 case WRITE_ONE_BREAK:
742 e->idx.offset = recursing;
743 return WRITE_ONE_BREAK;
744 }
745 }
746
747 e->idx.offset = *offset;
748 size = write_object(f, e, *offset);
749 if (!size) {
750 e->idx.offset = recursing;
751 return WRITE_ONE_BREAK;
752 }
753 written_list[nr_written++] = &e->idx;
754
755 /* make sure off_t is sufficiently large not to wrap */
756 if (signed_add_overflows(*offset, size))
757 die(_("pack too large for current definition of off_t"));
758 *offset += size;
759 return WRITE_ONE_WRITTEN;
760 }
761
762 static int mark_tagged(const char *path UNUSED, const struct object_id *oid,
763 int flag UNUSED, void *cb_data UNUSED)
764 {
765 struct object_id peeled;
766 struct object_entry *entry = packlist_find(&to_pack, oid);
767
768 if (entry)
769 entry->tagged = 1;
770 if (!peel_iterated_oid(oid, &peeled)) {
771 entry = packlist_find(&to_pack, &peeled);
772 if (entry)
773 entry->tagged = 1;
774 }
775 return 0;
776 }
777
778 static inline unsigned char oe_layer(struct packing_data *pack,
779 struct object_entry *e)
780 {
781 if (!pack->layer)
782 return 0;
783 return pack->layer[e - pack->objects];
784 }
785
786 static inline void add_to_write_order(struct object_entry **wo,
787 unsigned int *endp,
788 struct object_entry *e)
789 {
790 if (e->filled || oe_layer(&to_pack, e) != write_layer)
791 return;
792 wo[(*endp)++] = e;
793 e->filled = 1;
794 }
795
796 static void add_descendants_to_write_order(struct object_entry **wo,
797 unsigned int *endp,
798 struct object_entry *e)
799 {
800 int add_to_order = 1;
801 while (e) {
802 if (add_to_order) {
803 struct object_entry *s;
804 /* add this node... */
805 add_to_write_order(wo, endp, e);
806 /* all its siblings... */
807 for (s = DELTA_SIBLING(e); s; s = DELTA_SIBLING(s)) {
808 add_to_write_order(wo, endp, s);
809 }
810 }
811 /* drop down a level to add left subtree nodes if possible */
812 if (DELTA_CHILD(e)) {
813 add_to_order = 1;
814 e = DELTA_CHILD(e);
815 } else {
816 add_to_order = 0;
817 /* our sibling might have some children, it is next */
818 if (DELTA_SIBLING(e)) {
819 e = DELTA_SIBLING(e);
820 continue;
821 }
822 /* go back to our parent node */
823 e = DELTA(e);
824 while (e && !DELTA_SIBLING(e)) {
825 /* we're on the right side of a subtree, keep
826 * going up until we can go right again */
827 e = DELTA(e);
828 }
829 if (!e) {
830 /* done- we hit our original root node */
831 return;
832 }
833 /* pass it off to sibling at this level */
834 e = DELTA_SIBLING(e);
835 }
836 };
837 }
838
839 static void add_family_to_write_order(struct object_entry **wo,
840 unsigned int *endp,
841 struct object_entry *e)
842 {
843 struct object_entry *root;
844
845 for (root = e; DELTA(root); root = DELTA(root))
846 ; /* nothing */
847 add_descendants_to_write_order(wo, endp, root);
848 }
849
850 static void compute_layer_order(struct object_entry **wo, unsigned int *wo_end)
851 {
852 unsigned int i, last_untagged;
853 struct object_entry *objects = to_pack.objects;
854
855 for (i = 0; i < to_pack.nr_objects; i++) {
856 if (objects[i].tagged)
857 break;
858 add_to_write_order(wo, wo_end, &objects[i]);
859 }
860 last_untagged = i;
861
862 /*
863 * Then fill all the tagged tips.
864 */
865 for (; i < to_pack.nr_objects; i++) {
866 if (objects[i].tagged)
867 add_to_write_order(wo, wo_end, &objects[i]);
868 }
869
870 /*
871 * And then all remaining commits and tags.
872 */
873 for (i = last_untagged; i < to_pack.nr_objects; i++) {
874 if (oe_type(&objects[i]) != OBJ_COMMIT &&
875 oe_type(&objects[i]) != OBJ_TAG)
876 continue;
877 add_to_write_order(wo, wo_end, &objects[i]);
878 }
879
880 /*
881 * And then all the trees.
882 */
883 for (i = last_untagged; i < to_pack.nr_objects; i++) {
884 if (oe_type(&objects[i]) != OBJ_TREE)
885 continue;
886 add_to_write_order(wo, wo_end, &objects[i]);
887 }
888
889 /*
890 * Finally all the rest in really tight order
891 */
892 for (i = last_untagged; i < to_pack.nr_objects; i++) {
893 if (!objects[i].filled && oe_layer(&to_pack, &objects[i]) == write_layer)
894 add_family_to_write_order(wo, wo_end, &objects[i]);
895 }
896 }
897
898 static struct object_entry **compute_write_order(void)
899 {
900 uint32_t max_layers = 1;
901 unsigned int i, wo_end;
902
903 struct object_entry **wo;
904 struct object_entry *objects = to_pack.objects;
905
906 for (i = 0; i < to_pack.nr_objects; i++) {
907 objects[i].tagged = 0;
908 objects[i].filled = 0;
909 SET_DELTA_CHILD(&objects[i], NULL);
910 SET_DELTA_SIBLING(&objects[i], NULL);
911 }
912
913 /*
914 * Fully connect delta_child/delta_sibling network.
915 * Make sure delta_sibling is sorted in the original
916 * recency order.
917 */
918 for (i = to_pack.nr_objects; i > 0;) {
919 struct object_entry *e = &objects[--i];
920 if (!DELTA(e))
921 continue;
922 /* Mark me as the first child */
923 e->delta_sibling_idx = DELTA(e)->delta_child_idx;
924 SET_DELTA_CHILD(DELTA(e), e);
925 }
926
927 /*
928 * Mark objects that are at the tip of tags.
929 */
930 for_each_tag_ref(mark_tagged, NULL);
931
932 if (use_delta_islands)
933 max_layers = compute_pack_layers(&to_pack);
934
935 ALLOC_ARRAY(wo, to_pack.nr_objects);
936 wo_end = 0;
937
938 for (; write_layer < max_layers; ++write_layer)
939 compute_layer_order(wo, &wo_end);
940
941 if (wo_end != to_pack.nr_objects)
942 die(_("ordered %u objects, expected %"PRIu32),
943 wo_end, to_pack.nr_objects);
944
945 return wo;
946 }
947
948
949 /*
950 * A reused set of objects. All objects in a chunk have the same
951 * relative position in the original packfile and the generated
952 * packfile.
953 */
954
955 static struct reused_chunk {
956 /* The offset of the first object of this chunk in the original
957 * packfile. */
958 off_t original;
959 /* The difference for "original" minus the offset of the first object of
960 * this chunk in the generated packfile. */
961 off_t difference;
962 } *reused_chunks;
963 static int reused_chunks_nr;
964 static int reused_chunks_alloc;
965
966 static void record_reused_object(off_t where, off_t offset)
967 {
968 if (reused_chunks_nr && reused_chunks[reused_chunks_nr-1].difference == offset)
969 return;
970
971 ALLOC_GROW(reused_chunks, reused_chunks_nr + 1,
972 reused_chunks_alloc);
973 reused_chunks[reused_chunks_nr].original = where;
974 reused_chunks[reused_chunks_nr].difference = offset;
975 reused_chunks_nr++;
976 }
977
978 /*
979 * Binary search to find the chunk that "where" is in. Note
980 * that we're not looking for an exact match, just the first
981 * chunk that contains it (which implicitly ends at the start
982 * of the next chunk.
983 */
984 static off_t find_reused_offset(off_t where)
985 {
986 int lo = 0, hi = reused_chunks_nr;
987 while (lo < hi) {
988 int mi = lo + ((hi - lo) / 2);
989 if (where == reused_chunks[mi].original)
990 return reused_chunks[mi].difference;
991 if (where < reused_chunks[mi].original)
992 hi = mi;
993 else
994 lo = mi + 1;
995 }
996
997 /*
998 * The first chunk starts at zero, so we can't have gone below
999 * there.
1000 */
1001 assert(lo);
1002 return reused_chunks[lo-1].difference;
1003 }
1004
1005 static void write_reused_pack_one(size_t pos, struct hashfile *out,
1006 struct pack_window **w_curs)
1007 {
1008 off_t offset, next, cur;
1009 enum object_type type;
1010 unsigned long size;
1011
1012 offset = pack_pos_to_offset(reuse_packfile, pos);
1013 next = pack_pos_to_offset(reuse_packfile, pos + 1);
1014
1015 record_reused_object(offset, offset - hashfile_total(out));
1016
1017 cur = offset;
1018 type = unpack_object_header(reuse_packfile, w_curs, &cur, &size);
1019 assert(type >= 0);
1020
1021 if (type == OBJ_OFS_DELTA) {
1022 off_t base_offset;
1023 off_t fixup;
1024
1025 unsigned char header[MAX_PACK_OBJECT_HEADER];
1026 unsigned len;
1027
1028 base_offset = get_delta_base(reuse_packfile, w_curs, &cur, type, offset);
1029 assert(base_offset != 0);
1030
1031 /* Convert to REF_DELTA if we must... */
1032 if (!allow_ofs_delta) {
1033 uint32_t base_pos;
1034 struct object_id base_oid;
1035
1036 if (offset_to_pack_pos(reuse_packfile, base_offset, &base_pos) < 0)
1037 die(_("expected object at offset %"PRIuMAX" "
1038 "in pack %s"),
1039 (uintmax_t)base_offset,
1040 reuse_packfile->pack_name);
1041
1042 nth_packed_object_id(&base_oid, reuse_packfile,
1043 pack_pos_to_index(reuse_packfile, base_pos));
1044
1045 len = encode_in_pack_object_header(header, sizeof(header),
1046 OBJ_REF_DELTA, size);
1047 hashwrite(out, header, len);
1048 hashwrite(out, base_oid.hash, the_hash_algo->rawsz);
1049 copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
1050 return;
1051 }
1052
1053 /* Otherwise see if we need to rewrite the offset... */
1054 fixup = find_reused_offset(offset) -
1055 find_reused_offset(base_offset);
1056 if (fixup) {
1057 unsigned char ofs_header[10];
1058 unsigned i, ofs_len;
1059 off_t ofs = offset - base_offset - fixup;
1060
1061 len = encode_in_pack_object_header(header, sizeof(header),
1062 OBJ_OFS_DELTA, size);
1063
1064 i = sizeof(ofs_header) - 1;
1065 ofs_header[i] = ofs & 127;
1066 while (ofs >>= 7)
1067 ofs_header[--i] = 128 | (--ofs & 127);
1068
1069 ofs_len = sizeof(ofs_header) - i;
1070
1071 hashwrite(out, header, len);
1072 hashwrite(out, ofs_header + sizeof(ofs_header) - ofs_len, ofs_len);
1073 copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
1074 return;
1075 }
1076
1077 /* ...otherwise we have no fixup, and can write it verbatim */
1078 }
1079
1080 copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset);
1081 }
1082
1083 static size_t write_reused_pack_verbatim(struct hashfile *out,
1084 struct pack_window **w_curs)
1085 {
1086 size_t pos = 0;
1087
1088 while (pos < reuse_packfile_bitmap->word_alloc &&
1089 reuse_packfile_bitmap->words[pos] == (eword_t)~0)
1090 pos++;
1091
1092 if (pos) {
1093 off_t to_write;
1094
1095 written = (pos * BITS_IN_EWORD);
1096 to_write = pack_pos_to_offset(reuse_packfile, written)
1097 - sizeof(struct pack_header);
1098
1099 /* We're recording one chunk, not one object. */
1100 record_reused_object(sizeof(struct pack_header), 0);
1101 hashflush(out);
1102 copy_pack_data(out, reuse_packfile, w_curs,
1103 sizeof(struct pack_header), to_write);
1104
1105 display_progress(progress_state, written);
1106 }
1107 return pos;
1108 }
1109
1110 static void write_reused_pack(struct hashfile *f)
1111 {
1112 size_t i = 0;
1113 uint32_t offset;
1114 struct pack_window *w_curs = NULL;
1115
1116 if (allow_ofs_delta)
1117 i = write_reused_pack_verbatim(f, &w_curs);
1118
1119 for (; i < reuse_packfile_bitmap->word_alloc; ++i) {
1120 eword_t word = reuse_packfile_bitmap->words[i];
1121 size_t pos = (i * BITS_IN_EWORD);
1122
1123 for (offset = 0; offset < BITS_IN_EWORD; ++offset) {
1124 if ((word >> offset) == 0)
1125 break;
1126
1127 offset += ewah_bit_ctz64(word >> offset);
1128 /*
1129 * Can use bit positions directly, even for MIDX
1130 * bitmaps. See comment in try_partial_reuse()
1131 * for why.
1132 */
1133 write_reused_pack_one(pos + offset, f, &w_curs);
1134 display_progress(progress_state, ++written);
1135 }
1136 }
1137
1138 unuse_pack(&w_curs);
1139 }
1140
1141 static void write_excluded_by_configs(void)
1142 {
1143 struct oidset_iter iter;
1144 const struct object_id *oid;
1145
1146 oidset_iter_init(&excluded_by_config, &iter);
1147 while ((oid = oidset_iter_next(&iter))) {
1148 struct configured_exclusion *ex =
1149 oidmap_get(&configured_exclusions, oid);
1150
1151 if (!ex)
1152 BUG("configured exclusion wasn't configured");
1153 write_in_full(1, ex->pack_hash_hex, strlen(ex->pack_hash_hex));
1154 write_in_full(1, " ", 1);
1155 write_in_full(1, ex->uri, strlen(ex->uri));
1156 write_in_full(1, "\n", 1);
1157 }
1158 }
1159
1160 static const char no_split_warning[] = N_(
1161 "disabling bitmap writing, packs are split due to pack.packSizeLimit"
1162 );
1163
1164 static void write_pack_file(void)
1165 {
1166 uint32_t i = 0, j;
1167 struct hashfile *f;
1168 off_t offset;
1169 uint32_t nr_remaining = nr_result;
1170 time_t last_mtime = 0;
1171 struct object_entry **write_order;
1172
1173 if (progress > pack_to_stdout)
1174 progress_state = start_progress(_("Writing objects"), nr_result);
1175 ALLOC_ARRAY(written_list, to_pack.nr_objects);
1176 write_order = compute_write_order();
1177
1178 do {
1179 unsigned char hash[GIT_MAX_RAWSZ];
1180 char *pack_tmp_name = NULL;
1181
1182 if (pack_to_stdout)
1183 f = hashfd_throughput(1, "<stdout>", progress_state);
1184 else
1185 f = create_tmp_packfile(&pack_tmp_name);
1186
1187 offset = write_pack_header(f, nr_remaining);
1188
1189 if (reuse_packfile) {
1190 assert(pack_to_stdout);
1191 write_reused_pack(f);
1192 offset = hashfile_total(f);
1193 }
1194
1195 nr_written = 0;
1196 for (; i < to_pack.nr_objects; i++) {
1197 struct object_entry *e = write_order[i];
1198 if (write_one(f, e, &offset) == WRITE_ONE_BREAK)
1199 break;
1200 display_progress(progress_state, written);
1201 }
1202
1203 if (pack_to_stdout) {
1204 /*
1205 * We never fsync when writing to stdout since we may
1206 * not be writing to an actual pack file. For instance,
1207 * the upload-pack code passes a pipe here. Calling
1208 * fsync on a pipe results in unnecessary
1209 * synchronization with the reader on some platforms.
1210 */
1211 finalize_hashfile(f, hash, FSYNC_COMPONENT_NONE,
1212 CSUM_HASH_IN_STREAM | CSUM_CLOSE);
1213 } else if (nr_written == nr_remaining) {
1214 finalize_hashfile(f, hash, FSYNC_COMPONENT_PACK,
1215 CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
1216 } else {
1217 /*
1218 * If we wrote the wrong number of entries in the
1219 * header, rewrite it like in fast-import.
1220 */
1221
1222 int fd = finalize_hashfile(f, hash, FSYNC_COMPONENT_PACK, 0);
1223 fixup_pack_header_footer(fd, hash, pack_tmp_name,
1224 nr_written, hash, offset);
1225 close(fd);
1226 if (write_bitmap_index) {
1227 if (write_bitmap_index != WRITE_BITMAP_QUIET)
1228 warning(_(no_split_warning));
1229 write_bitmap_index = 0;
1230 }
1231 }
1232
1233 if (!pack_to_stdout) {
1234 struct stat st;
1235 struct strbuf tmpname = STRBUF_INIT;
1236 char *idx_tmp_name = NULL;
1237
1238 /*
1239 * Packs are runtime accessed in their mtime
1240 * order since newer packs are more likely to contain
1241 * younger objects. So if we are creating multiple
1242 * packs then we should modify the mtime of later ones
1243 * to preserve this property.
1244 */
1245 if (stat(pack_tmp_name, &st) < 0) {
1246 warning_errno(_("failed to stat %s"), pack_tmp_name);
1247 } else if (!last_mtime) {
1248 last_mtime = st.st_mtime;
1249 } else {
1250 struct utimbuf utb;
1251 utb.actime = st.st_atime;
1252 utb.modtime = --last_mtime;
1253 if (utime(pack_tmp_name, &utb) < 0)
1254 warning_errno(_("failed utime() on %s"), pack_tmp_name);
1255 }
1256
1257 strbuf_addf(&tmpname, "%s-%s.", base_name,
1258 hash_to_hex(hash));
1259
1260 if (write_bitmap_index) {
1261 bitmap_writer_set_checksum(hash);
1262 bitmap_writer_build_type_index(
1263 &to_pack, written_list, nr_written);
1264 }
1265
1266 if (cruft)
1267 pack_idx_opts.flags |= WRITE_MTIMES;
1268
1269 stage_tmp_packfiles(&tmpname, pack_tmp_name,
1270 written_list, nr_written,
1271 &to_pack, &pack_idx_opts, hash,
1272 &idx_tmp_name);
1273
1274 if (write_bitmap_index) {
1275 size_t tmpname_len = tmpname.len;
1276
1277 strbuf_addstr(&tmpname, "bitmap");
1278 stop_progress(&progress_state);
1279
1280 bitmap_writer_show_progress(progress);
1281 bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1);
1282 if (bitmap_writer_build(&to_pack) < 0)
1283 die(_("failed to write bitmap index"));
1284 bitmap_writer_finish(written_list, nr_written,
1285 tmpname.buf, write_bitmap_options);
1286 write_bitmap_index = 0;
1287 strbuf_setlen(&tmpname, tmpname_len);
1288 }
1289
1290 rename_tmp_packfile_idx(&tmpname, &idx_tmp_name);
1291
1292 free(idx_tmp_name);
1293 strbuf_release(&tmpname);
1294 free(pack_tmp_name);
1295 puts(hash_to_hex(hash));
1296 }
1297
1298 /* mark written objects as written to previous pack */
1299 for (j = 0; j < nr_written; j++) {
1300 written_list[j]->offset = (off_t)-1;
1301 }
1302 nr_remaining -= nr_written;
1303 } while (nr_remaining && i < to_pack.nr_objects);
1304
1305 free(written_list);
1306 free(write_order);
1307 stop_progress(&progress_state);
1308 if (written != nr_result)
1309 die(_("wrote %"PRIu32" objects while expecting %"PRIu32),
1310 written, nr_result);
1311 trace2_data_intmax("pack-objects", the_repository,
1312 "write_pack_file/wrote", nr_result);
1313 }
1314
1315 static int no_try_delta(const char *path)
1316 {
1317 static struct attr_check *check;
1318
1319 if (!check)
1320 check = attr_check_initl("delta", NULL);
1321 git_check_attr(the_repository->index, path, check);
1322 if (ATTR_FALSE(check->items[0].value))
1323 return 1;
1324 return 0;
1325 }
1326
1327 /*
1328 * When adding an object, check whether we have already added it
1329 * to our packing list. If so, we can skip. However, if we are
1330 * being asked to excludei t, but the previous mention was to include
1331 * it, make sure to adjust its flags and tweak our numbers accordingly.
1332 *
1333 * As an optimization, we pass out the index position where we would have
1334 * found the item, since that saves us from having to look it up again a
1335 * few lines later when we want to add the new entry.
1336 */
1337 static int have_duplicate_entry(const struct object_id *oid,
1338 int exclude)
1339 {
1340 struct object_entry *entry;
1341
1342 if (reuse_packfile_bitmap &&
1343 bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid))
1344 return 1;
1345
1346 entry = packlist_find(&to_pack, oid);
1347 if (!entry)
1348 return 0;
1349
1350 if (exclude) {
1351 if (!entry->preferred_base)
1352 nr_result--;
1353 entry->preferred_base = 1;
1354 }
1355
1356 return 1;
1357 }
1358
1359 static int want_found_object(const struct object_id *oid, int exclude,
1360 struct packed_git *p)
1361 {
1362 if (exclude)
1363 return 1;
1364 if (incremental)
1365 return 0;
1366
1367 if (!is_pack_valid(p))
1368 return -1;
1369
1370 /*
1371 * When asked to do --local (do not include an object that appears in a
1372 * pack we borrow from elsewhere) or --honor-pack-keep (do not include
1373 * an object that appears in a pack marked with .keep), finding a pack
1374 * that matches the criteria is sufficient for us to decide to omit it.
1375 * However, even if this pack does not satisfy the criteria, we need to
1376 * make sure no copy of this object appears in _any_ pack that makes us
1377 * to omit the object, so we need to check all the packs.
1378 *
1379 * We can however first check whether these options can possibly matter;
1380 * if they do not matter we know we want the object in generated pack.
1381 * Otherwise, we signal "-1" at the end to tell the caller that we do
1382 * not know either way, and it needs to check more packs.
1383 */
1384
1385 /*
1386 * Objects in packs borrowed from elsewhere are discarded regardless of
1387 * if they appear in other packs that weren't borrowed.
1388 */
1389 if (local && !p->pack_local)
1390 return 0;
1391
1392 /*
1393 * Then handle .keep first, as we have a fast(er) path there.
1394 */
1395 if (ignore_packed_keep_on_disk || ignore_packed_keep_in_core) {
1396 /*
1397 * Set the flags for the kept-pack cache to be the ones we want
1398 * to ignore.
1399 *
1400 * That is, if we are ignoring objects in on-disk keep packs,
1401 * then we want to search through the on-disk keep and ignore
1402 * the in-core ones.
1403 */
1404 unsigned flags = 0;
1405 if (ignore_packed_keep_on_disk)
1406 flags |= ON_DISK_KEEP_PACKS;
1407 if (ignore_packed_keep_in_core)
1408 flags |= IN_CORE_KEEP_PACKS;
1409
1410 if (ignore_packed_keep_on_disk && p->pack_keep)
1411 return 0;
1412 if (ignore_packed_keep_in_core && p->pack_keep_in_core)
1413 return 0;
1414 if (has_object_kept_pack(oid, flags))
1415 return 0;
1416 }
1417
1418 /*
1419 * At this point we know definitively that either we don't care about
1420 * keep-packs, or the object is not in one. Keep checking other
1421 * conditions...
1422 */
1423 if (!local || !have_non_local_packs)
1424 return 1;
1425
1426 /* we don't know yet; keep looking for more packs */
1427 return -1;
1428 }
1429
1430 static int want_object_in_pack_one(struct packed_git *p,
1431 const struct object_id *oid,
1432 int exclude,
1433 struct packed_git **found_pack,
1434 off_t *found_offset)
1435 {
1436 off_t offset;
1437
1438 if (p == *found_pack)
1439 offset = *found_offset;
1440 else
1441 offset = find_pack_entry_one(oid->hash, p);
1442
1443 if (offset) {
1444 if (!*found_pack) {
1445 if (!is_pack_valid(p))
1446 return -1;
1447 *found_offset = offset;
1448 *found_pack = p;
1449 }
1450 return want_found_object(oid, exclude, p);
1451 }
1452 return -1;
1453 }
1454
1455 /*
1456 * Check whether we want the object in the pack (e.g., we do not want
1457 * objects found in non-local stores if the "--local" option was used).
1458 *
1459 * If the caller already knows an existing pack it wants to take the object
1460 * from, that is passed in *found_pack and *found_offset; otherwise this
1461 * function finds if there is any pack that has the object and returns the pack
1462 * and its offset in these variables.
1463 */
1464 static int want_object_in_pack(const struct object_id *oid,
1465 int exclude,
1466 struct packed_git **found_pack,
1467 off_t *found_offset)
1468 {
1469 int want;
1470 struct list_head *pos;
1471 struct multi_pack_index *m;
1472
1473 if (!exclude && local && has_loose_object_nonlocal(oid))
1474 return 0;
1475
1476 /*
1477 * If we already know the pack object lives in, start checks from that
1478 * pack - in the usual case when neither --local was given nor .keep files
1479 * are present we will determine the answer right now.
1480 */
1481 if (*found_pack) {
1482 want = want_found_object(oid, exclude, *found_pack);
1483 if (want != -1)
1484 return want;
1485
1486 *found_pack = NULL;
1487 *found_offset = 0;
1488 }
1489
1490 for (m = get_multi_pack_index(the_repository); m; m = m->next) {
1491 struct pack_entry e;
1492 if (fill_midx_entry(the_repository, oid, &e, m)) {
1493 want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset);
1494 if (want != -1)
1495 return want;
1496 }
1497 }
1498
1499 list_for_each(pos, get_packed_git_mru(the_repository)) {
1500 struct packed_git *p = list_entry(pos, struct packed_git, mru);
1501 want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset);
1502 if (!exclude && want > 0)
1503 list_move(&p->mru,
1504 get_packed_git_mru(the_repository));
1505 if (want != -1)
1506 return want;
1507 }
1508
1509 if (uri_protocols.nr) {
1510 struct configured_exclusion *ex =
1511 oidmap_get(&configured_exclusions, oid);
1512 int i;
1513 const char *p;
1514
1515 if (ex) {
1516 for (i = 0; i < uri_protocols.nr; i++) {
1517 if (skip_prefix(ex->uri,
1518 uri_protocols.items[i].string,
1519 &p) &&
1520 *p == ':') {
1521 oidset_insert(&excluded_by_config, oid);
1522 return 0;
1523 }
1524 }
1525 }
1526 }
1527
1528 return 1;
1529 }
1530
1531 static struct object_entry *create_object_entry(const struct object_id *oid,
1532 enum object_type type,
1533 uint32_t hash,
1534 int exclude,
1535 int no_try_delta,
1536 struct packed_git *found_pack,
1537 off_t found_offset)
1538 {
1539 struct object_entry *entry;
1540
1541 entry = packlist_alloc(&to_pack, oid);
1542 entry->hash = hash;
1543 oe_set_type(entry, type);
1544 if (exclude)
1545 entry->preferred_base = 1;
1546 else
1547 nr_result++;
1548 if (found_pack) {
1549 oe_set_in_pack(&to_pack, entry, found_pack);
1550 entry->in_pack_offset = found_offset;
1551 }
1552
1553 entry->no_try_delta = no_try_delta;
1554
1555 return entry;
1556 }
1557
1558 static const char no_closure_warning[] = N_(
1559 "disabling bitmap writing, as some objects are not being packed"
1560 );
1561
1562 static int add_object_entry(const struct object_id *oid, enum object_type type,
1563 const char *name, int exclude)
1564 {
1565 struct packed_git *found_pack = NULL;
1566 off_t found_offset = 0;
1567
1568 display_progress(progress_state, ++nr_seen);
1569
1570 if (have_duplicate_entry(oid, exclude))
1571 return 0;
1572
1573 if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {
1574 /* The pack is missing an object, so it will not have closure */
1575 if (write_bitmap_index) {
1576 if (write_bitmap_index != WRITE_BITMAP_QUIET)
1577 warning(_(no_closure_warning));
1578 write_bitmap_index = 0;
1579 }
1580 return 0;
1581 }
1582
1583 create_object_entry(oid, type, pack_name_hash(name),
1584 exclude, name && no_try_delta(name),
1585 found_pack, found_offset);
1586 return 1;
1587 }
1588
1589 static int add_object_entry_from_bitmap(const struct object_id *oid,
1590 enum object_type type,
1591 int flags, uint32_t name_hash,
1592 struct packed_git *pack, off_t offset)
1593 {
1594 display_progress(progress_state, ++nr_seen);
1595
1596 if (have_duplicate_entry(oid, 0))
1597 return 0;
1598
1599 if (!want_object_in_pack(oid, 0, &pack, &offset))
1600 return 0;
1601
1602 create_object_entry(oid, type, name_hash, 0, 0, pack, offset);
1603 return 1;
1604 }
1605
1606 struct pbase_tree_cache {
1607 struct object_id oid;
1608 int ref;
1609 int temporary;
1610 void *tree_data;
1611 unsigned long tree_size;
1612 };
1613
1614 static struct pbase_tree_cache *(pbase_tree_cache[256]);
1615 static int pbase_tree_cache_ix(const struct object_id *oid)
1616 {
1617 return oid->hash[0] % ARRAY_SIZE(pbase_tree_cache);
1618 }
1619 static int pbase_tree_cache_ix_incr(int ix)
1620 {
1621 return (ix+1) % ARRAY_SIZE(pbase_tree_cache);
1622 }
1623
1624 static struct pbase_tree {
1625 struct pbase_tree *next;
1626 /* This is a phony "cache" entry; we are not
1627 * going to evict it or find it through _get()
1628 * mechanism -- this is for the toplevel node that
1629 * would almost always change with any commit.
1630 */
1631 struct pbase_tree_cache pcache;
1632 } *pbase_tree;
1633
1634 static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)
1635 {
1636 struct pbase_tree_cache *ent, *nent;
1637 void *data;
1638 unsigned long size;
1639 enum object_type type;
1640 int neigh;
1641 int my_ix = pbase_tree_cache_ix(oid);
1642 int available_ix = -1;
1643
1644 /* pbase-tree-cache acts as a limited hashtable.
1645 * your object will be found at your index or within a few
1646 * slots after that slot if it is cached.
1647 */
1648 for (neigh = 0; neigh < 8; neigh++) {
1649 ent = pbase_tree_cache[my_ix];
1650 if (ent && oideq(&ent->oid, oid)) {
1651 ent->ref++;
1652 return ent;
1653 }
1654 else if (((available_ix < 0) && (!ent || !ent->ref)) ||
1655 ((0 <= available_ix) &&
1656 (!ent && pbase_tree_cache[available_ix])))
1657 available_ix = my_ix;
1658 if (!ent)
1659 break;
1660 my_ix = pbase_tree_cache_ix_incr(my_ix);
1661 }
1662
1663 /* Did not find one. Either we got a bogus request or
1664 * we need to read and perhaps cache.
1665 */
1666 data = read_object_file(oid, &type, &size);
1667 if (!data)
1668 return NULL;
1669 if (type != OBJ_TREE) {
1670 free(data);
1671 return NULL;
1672 }
1673
1674 /* We need to either cache or return a throwaway copy */
1675
1676 if (available_ix < 0)
1677 ent = NULL;
1678 else {
1679 ent = pbase_tree_cache[available_ix];
1680 my_ix = available_ix;
1681 }
1682
1683 if (!ent) {
1684 nent = xmalloc(sizeof(*nent));
1685 nent->temporary = (available_ix < 0);
1686 }
1687 else {
1688 /* evict and reuse */
1689 free(ent->tree_data);
1690 nent = ent;
1691 }
1692 oidcpy(&nent->oid, oid);
1693 nent->tree_data = data;
1694 nent->tree_size = size;
1695 nent->ref = 1;
1696 if (!nent->temporary)
1697 pbase_tree_cache[my_ix] = nent;
1698 return nent;
1699 }
1700
1701 static void pbase_tree_put(struct pbase_tree_cache *cache)
1702 {
1703 if (!cache->temporary) {
1704 cache->ref--;
1705 return;
1706 }
1707 free(cache->tree_data);
1708 free(cache);
1709 }
1710
1711 static int name_cmp_len(const char *name)
1712 {
1713 int i;
1714 for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)
1715 ;
1716 return i;
1717 }
1718
1719 static void add_pbase_object(struct tree_desc *tree,
1720 const char *name,
1721 int cmplen,
1722 const char *fullname)
1723 {
1724 struct name_entry entry;
1725 int cmp;
1726
1727 while (tree_entry(tree,&entry)) {
1728 if (S_ISGITLINK(entry.mode))
1729 continue;
1730 cmp = tree_entry_len(&entry) != cmplen ? 1 :
1731 memcmp(name, entry.path, cmplen);
1732 if (cmp > 0)
1733 continue;
1734 if (cmp < 0)
1735 return;
1736 if (name[cmplen] != '/') {
1737 add_object_entry(&entry.oid,
1738 object_type(entry.mode),
1739 fullname, 1);
1740 return;
1741 }
1742 if (S_ISDIR(entry.mode)) {
1743 struct tree_desc sub;
1744 struct pbase_tree_cache *tree;
1745 const char *down = name+cmplen+1;
1746 int downlen = name_cmp_len(down);
1747
1748 tree = pbase_tree_get(&entry.oid);
1749 if (!tree)
1750 return;
1751 init_tree_desc(&sub, tree->tree_data, tree->tree_size);
1752
1753 add_pbase_object(&sub, down, downlen, fullname);
1754 pbase_tree_put(tree);
1755 }
1756 }
1757 }
1758
1759 static unsigned *done_pbase_paths;
1760 static int done_pbase_paths_num;
1761 static int done_pbase_paths_alloc;
1762 static int done_pbase_path_pos(unsigned hash)
1763 {
1764 int lo = 0;
1765 int hi = done_pbase_paths_num;
1766 while (lo < hi) {
1767 int mi = lo + (hi - lo) / 2;
1768 if (done_pbase_paths[mi] == hash)
1769 return mi;
1770 if (done_pbase_paths[mi] < hash)
1771 hi = mi;
1772 else
1773 lo = mi + 1;
1774 }
1775 return -lo-1;
1776 }
1777
1778 static int check_pbase_path(unsigned hash)
1779 {
1780 int pos = done_pbase_path_pos(hash);
1781 if (0 <= pos)
1782 return 1;
1783 pos = -pos - 1;
1784 ALLOC_GROW(done_pbase_paths,
1785 done_pbase_paths_num + 1,
1786 done_pbase_paths_alloc);
1787 done_pbase_paths_num++;
1788 if (pos < done_pbase_paths_num)
1789 MOVE_ARRAY(done_pbase_paths + pos + 1, done_pbase_paths + pos,
1790 done_pbase_paths_num - pos - 1);
1791 done_pbase_paths[pos] = hash;
1792 return 0;
1793 }
1794
1795 static void add_preferred_base_object(const char *name)
1796 {
1797 struct pbase_tree *it;
1798 int cmplen;
1799 unsigned hash = pack_name_hash(name);
1800
1801 if (!num_preferred_base || check_pbase_path(hash))
1802 return;
1803
1804 cmplen = name_cmp_len(name);
1805 for (it = pbase_tree; it; it = it->next) {
1806 if (cmplen == 0) {
1807 add_object_entry(&it->pcache.oid, OBJ_TREE, NULL, 1);
1808 }
1809 else {
1810 struct tree_desc tree;
1811 init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);
1812 add_pbase_object(&tree, name, cmplen, name);
1813 }
1814 }
1815 }
1816
1817 static void add_preferred_base(struct object_id *oid)
1818 {
1819 struct pbase_tree *it;
1820 void *data;
1821 unsigned long size;
1822 struct object_id tree_oid;
1823
1824 if (window <= num_preferred_base++)
1825 return;
1826
1827 data = read_object_with_reference(the_repository, oid,
1828 OBJ_TREE, &size, &tree_oid);
1829 if (!data)
1830 return;
1831
1832 for (it = pbase_tree; it; it = it->next) {
1833 if (oideq(&it->pcache.oid, &tree_oid)) {
1834 free(data);
1835 return;
1836 }
1837 }
1838
1839 CALLOC_ARRAY(it, 1);
1840 it->next = pbase_tree;
1841 pbase_tree = it;
1842
1843 oidcpy(&it->pcache.oid, &tree_oid);
1844 it->pcache.tree_data = data;
1845 it->pcache.tree_size = size;
1846 }
1847
1848 static void cleanup_preferred_base(void)
1849 {
1850 struct pbase_tree *it;
1851 unsigned i;
1852
1853 it = pbase_tree;
1854 pbase_tree = NULL;
1855 while (it) {
1856 struct pbase_tree *tmp = it;
1857 it = tmp->next;
1858 free(tmp->pcache.tree_data);
1859 free(tmp);
1860 }
1861
1862 for (i = 0; i < ARRAY_SIZE(pbase_tree_cache); i++) {
1863 if (!pbase_tree_cache[i])
1864 continue;
1865 free(pbase_tree_cache[i]->tree_data);
1866 FREE_AND_NULL(pbase_tree_cache[i]);
1867 }
1868
1869 FREE_AND_NULL(done_pbase_paths);
1870 done_pbase_paths_num = done_pbase_paths_alloc = 0;
1871 }
1872
1873 /*
1874 * Return 1 iff the object specified by "delta" can be sent
1875 * literally as a delta against the base in "base_sha1". If
1876 * so, then *base_out will point to the entry in our packing
1877 * list, or NULL if we must use the external-base list.
1878 *
1879 * Depth value does not matter - find_deltas() will
1880 * never consider reused delta as the base object to
1881 * deltify other objects against, in order to avoid
1882 * circular deltas.
1883 */
1884 static int can_reuse_delta(const struct object_id *base_oid,
1885 struct object_entry *delta,
1886 struct object_entry **base_out)
1887 {
1888 struct object_entry *base;
1889
1890 /*
1891 * First see if we're already sending the base (or it's explicitly in
1892 * our "excluded" list).
1893 */
1894 base = packlist_find(&to_pack, base_oid);
1895 if (base) {
1896 if (!in_same_island(&delta->idx.oid, &base->idx.oid))
1897 return 0;
1898 *base_out = base;
1899 return 1;
1900 }
1901
1902 /*
1903 * Otherwise, reachability bitmaps may tell us if the receiver has it,
1904 * even if it was buried too deep in history to make it into the
1905 * packing list.
1906 */
1907 if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) {
1908 if (use_delta_islands) {
1909 if (!in_same_island(&delta->idx.oid, base_oid))
1910 return 0;
1911 }
1912 *base_out = NULL;
1913 return 1;
1914 }
1915
1916 return 0;
1917 }
1918
1919 static void prefetch_to_pack(uint32_t object_index_start) {
1920 struct oid_array to_fetch = OID_ARRAY_INIT;
1921 uint32_t i;
1922
1923 for (i = object_index_start; i < to_pack.nr_objects; i++) {
1924 struct object_entry *entry = to_pack.objects + i;
1925
1926 if (!oid_object_info_extended(the_repository,
1927 &entry->idx.oid,
1928 NULL,
1929 OBJECT_INFO_FOR_PREFETCH))
1930 continue;
1931 oid_array_append(&to_fetch, &entry->idx.oid);
1932 }
1933 promisor_remote_get_direct(the_repository,
1934 to_fetch.oid, to_fetch.nr);
1935 oid_array_clear(&to_fetch);
1936 }
1937
1938 static void check_object(struct object_entry *entry, uint32_t object_index)
1939 {
1940 unsigned long canonical_size;
1941 enum object_type type;
1942 struct object_info oi = {.typep = &type, .sizep = &canonical_size};
1943
1944 if (IN_PACK(entry)) {
1945 struct packed_git *p = IN_PACK(entry);
1946 struct pack_window *w_curs = NULL;
1947 int have_base = 0;
1948 struct object_id base_ref;
1949 struct object_entry *base_entry;
1950 unsigned long used, used_0;
1951 unsigned long avail;
1952 off_t ofs;
1953 unsigned char *buf, c;
1954 enum object_type type;
1955 unsigned long in_pack_size;
1956
1957 buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
1958
1959 /*
1960 * We want in_pack_type even if we do not reuse delta
1961 * since non-delta representations could still be reused.
1962 */
1963 used = unpack_object_header_buffer(buf, avail,
1964 &type,
1965 &in_pack_size);
1966 if (used == 0)
1967 goto give_up;
1968
1969 if (type < 0)
1970 BUG("invalid type %d", type);
1971 entry->in_pack_type = type;
1972
1973 /*
1974 * Determine if this is a delta and if so whether we can
1975 * reuse it or not. Otherwise let's find out as cheaply as
1976 * possible what the actual type and size for this object is.
1977 */
1978 switch (entry->in_pack_type) {
1979 default:
1980 /* Not a delta hence we've already got all we need. */
1981 oe_set_type(entry, entry->in_pack_type);
1982 SET_SIZE(entry, in_pack_size);
1983 entry->in_pack_header_size = used;
1984 if (oe_type(entry) < OBJ_COMMIT || oe_type(entry) > OBJ_BLOB)
1985 goto give_up;
1986 unuse_pack(&w_curs);
1987 return;
1988 case OBJ_REF_DELTA:
1989 if (reuse_delta && !entry->preferred_base) {
1990 oidread(&base_ref,
1991 use_pack(p, &w_curs,
1992 entry->in_pack_offset + used,
1993 NULL));
1994 have_base = 1;
1995 }
1996 entry->in_pack_header_size = used + the_hash_algo->rawsz;
1997 break;
1998 case OBJ_OFS_DELTA:
1999 buf = use_pack(p, &w_curs,
2000 entry->in_pack_offset + used, NULL);
2001 used_0 = 0;
2002 c = buf[used_0++];
2003 ofs = c & 127;
2004 while (c & 128) {
2005 ofs += 1;
2006 if (!ofs || MSB(ofs, 7)) {
2007 error(_("delta base offset overflow in pack for %s"),
2008 oid_to_hex(&entry->idx.oid));
2009 goto give_up;
2010 }
2011 c = buf[used_0++];
2012 ofs = (ofs << 7) + (c & 127);
2013 }
2014 ofs = entry->in_pack_offset - ofs;
2015 if (ofs <= 0 || ofs >= entry->in_pack_offset) {
2016 error(_("delta base offset out of bound for %s"),
2017 oid_to_hex(&entry->idx.oid));
2018 goto give_up;
2019 }
2020 if (reuse_delta && !entry->preferred_base) {
2021 uint32_t pos;
2022 if (offset_to_pack_pos(p, ofs, &pos) < 0)
2023 goto give_up;
2024 if (!nth_packed_object_id(&base_ref, p,
2025 pack_pos_to_index(p, pos)))
2026 have_base = 1;
2027 }
2028 entry->in_pack_header_size = used + used_0;
2029 break;
2030 }
2031
2032 if (have_base &&
2033 can_reuse_delta(&base_ref, entry, &base_entry)) {
2034 oe_set_type(entry, entry->in_pack_type);
2035 SET_SIZE(entry, in_pack_size); /* delta size */
2036 SET_DELTA_SIZE(entry, in_pack_size);
2037
2038 if (base_entry) {
2039 SET_DELTA(entry, base_entry);
2040 entry->delta_sibling_idx = base_entry->delta_child_idx;
2041 SET_DELTA_CHILD(base_entry, entry);
2042 } else {
2043 SET_DELTA_EXT(entry, &base_ref);
2044 }
2045
2046 unuse_pack(&w_curs);
2047 return;
2048 }
2049
2050 if (oe_type(entry)) {
2051 off_t delta_pos;
2052
2053 /*
2054 * This must be a delta and we already know what the
2055 * final object type is. Let's extract the actual
2056 * object size from the delta header.
2057 */
2058 delta_pos = entry->in_pack_offset + entry->in_pack_header_size;
2059 canonical_size = get_size_from_delta(p, &w_curs, delta_pos);
2060 if (canonical_size == 0)
2061 goto give_up;
2062 SET_SIZE(entry, canonical_size);
2063 unuse_pack(&w_curs);
2064 return;
2065 }
2066
2067 /*
2068 * No choice but to fall back to the recursive delta walk
2069 * with oid_object_info() to find about the object type
2070 * at this point...
2071 */
2072 give_up:
2073 unuse_pack(&w_curs);
2074 }
2075
2076 if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi,
2077 OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0) {
2078 if (has_promisor_remote()) {
2079 prefetch_to_pack(object_index);
2080 if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi,
2081 OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0)
2082 type = -1;
2083 } else {
2084 type = -1;
2085 }
2086 }
2087 oe_set_type(entry, type);
2088 if (entry->type_valid) {
2089 SET_SIZE(entry, canonical_size);
2090 } else {
2091 /*
2092 * Bad object type is checked in prepare_pack(). This is
2093 * to permit a missing preferred base object to be ignored
2094 * as a preferred base. Doing so can result in a larger
2095 * pack file, but the transfer will still take place.
2096 */
2097 }
2098 }
2099
2100 static int pack_offset_sort(const void *_a, const void *_b)
2101 {
2102 const struct object_entry *a = *(struct object_entry **)_a;
2103 const struct object_entry *b = *(struct object_entry **)_b;
2104 const struct packed_git *a_in_pack = IN_PACK(a);
2105 const struct packed_git *b_in_pack = IN_PACK(b);
2106
2107 /* avoid filesystem trashing with loose objects */
2108 if (!a_in_pack && !b_in_pack)
2109 return oidcmp(&a->idx.oid, &b->idx.oid);
2110
2111 if (a_in_pack < b_in_pack)
2112 return -1;
2113 if (a_in_pack > b_in_pack)
2114 return 1;
2115 return a->in_pack_offset < b->in_pack_offset ? -1 :
2116 (a->in_pack_offset > b->in_pack_offset);
2117 }
2118
2119 /*
2120 * Drop an on-disk delta we were planning to reuse. Naively, this would
2121 * just involve blanking out the "delta" field, but we have to deal
2122 * with some extra book-keeping:
2123 *
2124 * 1. Removing ourselves from the delta_sibling linked list.
2125 *
2126 * 2. Updating our size/type to the non-delta representation. These were
2127 * either not recorded initially (size) or overwritten with the delta type
2128 * (type) when check_object() decided to reuse the delta.
2129 *
2130 * 3. Resetting our delta depth, as we are now a base object.
2131 */
2132 static void drop_reused_delta(struct object_entry *entry)
2133 {
2134 unsigned *idx = &to_pack.objects[entry->delta_idx - 1].delta_child_idx;
2135 struct object_info oi = OBJECT_INFO_INIT;
2136 enum object_type type;
2137 unsigned long size;
2138
2139 while (*idx) {
2140 struct object_entry *oe = &to_pack.objects[*idx - 1];
2141
2142 if (oe == entry)
2143 *idx = oe->delta_sibling_idx;
2144 else
2145 idx = &oe->delta_sibling_idx;
2146 }
2147 SET_DELTA(entry, NULL);
2148 entry->depth = 0;
2149
2150 oi.sizep = &size;
2151 oi.typep = &type;
2152 if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) {
2153 /*
2154 * We failed to get the info from this pack for some reason;
2155 * fall back to oid_object_info, which may find another copy.
2156 * And if that fails, the error will be recorded in oe_type(entry)
2157 * and dealt with in prepare_pack().
2158 */
2159 oe_set_type(entry,
2160 oid_object_info(the_repository, &entry->idx.oid, &size));
2161 } else {
2162 oe_set_type(entry, type);
2163 }
2164 SET_SIZE(entry, size);
2165 }
2166
2167 /*
2168 * Follow the chain of deltas from this entry onward, throwing away any links
2169 * that cause us to hit a cycle (as determined by the DFS state flags in
2170 * the entries).
2171 *
2172 * We also detect too-long reused chains that would violate our --depth
2173 * limit.
2174 */
2175 static void break_delta_chains(struct object_entry *entry)
2176 {
2177 /*
2178 * The actual depth of each object we will write is stored as an int,
2179 * as it cannot exceed our int "depth" limit. But before we break
2180 * changes based no that limit, we may potentially go as deep as the
2181 * number of objects, which is elsewhere bounded to a uint32_t.
2182 */
2183 uint32_t total_depth;
2184 struct object_entry *cur, *next;
2185
2186 for (cur = entry, total_depth = 0;
2187 cur;
2188 cur = DELTA(cur), total_depth++) {
2189 if (cur->dfs_state == DFS_DONE) {
2190 /*
2191 * We've already seen this object and know it isn't
2192 * part of a cycle. We do need to append its depth
2193 * to our count.
2194 */
2195 total_depth += cur->depth;
2196 break;
2197 }
2198
2199 /*
2200 * We break cycles before looping, so an ACTIVE state (or any
2201 * other cruft which made its way into the state variable)
2202 * is a bug.
2203 */
2204 if (cur->dfs_state != DFS_NONE)
2205 BUG("confusing delta dfs state in first pass: %d",
2206 cur->dfs_state);
2207
2208 /*
2209 * Now we know this is the first time we've seen the object. If
2210 * it's not a delta, we're done traversing, but we'll mark it
2211 * done to save time on future traversals.
2212 */
2213 if (!DELTA(cur)) {
2214 cur->dfs_state = DFS_DONE;
2215 break;
2216 }
2217
2218 /*
2219 * Mark ourselves as active and see if the next step causes
2220 * us to cycle to another active object. It's important to do
2221 * this _before_ we loop, because it impacts where we make the
2222 * cut, and thus how our total_depth counter works.
2223 * E.g., We may see a partial loop like:
2224 *
2225 * A -> B -> C -> D -> B
2226 *
2227 * Cutting B->C breaks the cycle. But now the depth of A is
2228 * only 1, and our total_depth counter is at 3. The size of the
2229 * error is always one less than the size of the cycle we
2230 * broke. Commits C and D were "lost" from A's chain.
2231 *
2232 * If we instead cut D->B, then the depth of A is correct at 3.
2233 * We keep all commits in the chain that we examined.
2234 */
2235 cur->dfs_state = DFS_ACTIVE;
2236 if (DELTA(cur)->dfs_state == DFS_ACTIVE) {
2237 drop_reused_delta(cur);
2238 cur->dfs_state = DFS_DONE;
2239 break;
2240 }
2241 }
2242
2243 /*
2244 * And now that we've gone all the way to the bottom of the chain, we
2245 * need to clear the active flags and set the depth fields as
2246 * appropriate. Unlike the loop above, which can quit when it drops a
2247 * delta, we need to keep going to look for more depth cuts. So we need
2248 * an extra "next" pointer to keep going after we reset cur->delta.
2249 */
2250 for (cur = entry; cur; cur = next) {
2251 next = DELTA(cur);
2252
2253 /*
2254 * We should have a chain of zero or more ACTIVE states down to
2255 * a final DONE. We can quit after the DONE, because either it
2256 * has no bases, or we've already handled them in a previous
2257 * call.
2258 */
2259 if (cur->dfs_state == DFS_DONE)
2260 break;
2261 else if (cur->dfs_state != DFS_ACTIVE)
2262 BUG("confusing delta dfs state in second pass: %d",
2263 cur->dfs_state);
2264
2265 /*
2266 * If the total_depth is more than depth, then we need to snip
2267 * the chain into two or more smaller chains that don't exceed
2268 * the maximum depth. Most of the resulting chains will contain
2269 * (depth + 1) entries (i.e., depth deltas plus one base), and
2270 * the last chain (i.e., the one containing entry) will contain
2271 * whatever entries are left over, namely
2272 * (total_depth % (depth + 1)) of them.
2273 *
2274 * Since we are iterating towards decreasing depth, we need to
2275 * decrement total_depth as we go, and we need to write to the
2276 * entry what its final depth will be after all of the
2277 * snipping. Since we're snipping into chains of length (depth
2278 * + 1) entries, the final depth of an entry will be its
2279 * original depth modulo (depth + 1). Any time we encounter an
2280 * entry whose final depth is supposed to be zero, we snip it
2281 * from its delta base, thereby making it so.
2282 */
2283 cur->depth = (total_depth--) % (depth + 1);
2284 if (!cur->depth)
2285 drop_reused_delta(cur);
2286
2287 cur->dfs_state = DFS_DONE;
2288 }
2289 }
2290
2291 static void get_object_details(void)
2292 {
2293 uint32_t i;
2294 struct object_entry **sorted_by_offset;
2295
2296 if (progress)
2297 progress_state = start_progress(_("Counting objects"),
2298 to_pack.nr_objects);
2299
2300 CALLOC_ARRAY(sorted_by_offset, to_pack.nr_objects);
2301 for (i = 0; i < to_pack.nr_objects; i++)
2302 sorted_by_offset[i] = to_pack.objects + i;
2303 QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);
2304
2305 for (i = 0; i < to_pack.nr_objects; i++) {
2306 struct object_entry *entry = sorted_by_offset[i];
2307 check_object(entry, i);
2308 if (entry->type_valid &&
2309 oe_size_greater_than(&to_pack, entry, big_file_threshold))
2310 entry->no_try_delta = 1;
2311 display_progress(progress_state, i + 1);
2312 }
2313 stop_progress(&progress_state);
2314
2315 /*
2316 * This must happen in a second pass, since we rely on the delta
2317 * information for the whole list being completed.
2318 */
2319 for (i = 0; i < to_pack.nr_objects; i++)
2320 break_delta_chains(&to_pack.objects[i]);
2321
2322 free(sorted_by_offset);
2323 }
2324
2325 /*
2326 * We search for deltas in a list sorted by type, by filename hash, and then
2327 * by size, so that we see progressively smaller and smaller files.
2328 * That's because we prefer deltas to be from the bigger file
2329 * to the smaller -- deletes are potentially cheaper, but perhaps
2330 * more importantly, the bigger file is likely the more recent
2331 * one. The deepest deltas are therefore the oldest objects which are
2332 * less susceptible to be accessed often.
2333 */
2334 static int type_size_sort(const void *_a, const void *_b)
2335 {
2336 const struct object_entry *a = *(struct object_entry **)_a;
2337 const struct object_entry *b = *(struct object_entry **)_b;
2338 const enum object_type a_type = oe_type(a);
2339 const enum object_type b_type = oe_type(b);
2340 const unsigned long a_size = SIZE(a);
2341 const unsigned long b_size = SIZE(b);
2342
2343 if (a_type > b_type)
2344 return -1;
2345 if (a_type < b_type)
2346 return 1;
2347 if (a->hash > b->hash)
2348 return -1;
2349 if (a->hash < b->hash)
2350 return 1;
2351 if (a->preferred_base > b->preferred_base)
2352 return -1;
2353 if (a->preferred_base < b->preferred_base)
2354 return 1;
2355 if (use_delta_islands) {
2356 const int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid);
2357 if (island_cmp)
2358 return island_cmp;
2359 }
2360 if (a_size > b_size)
2361 return -1;
2362 if (a_size < b_size)
2363 return 1;
2364 return a < b ? -1 : (a > b); /* newest first */
2365 }
2366
2367 struct unpacked {
2368 struct object_entry *entry;
2369 void *data;
2370 struct delta_index *index;
2371 unsigned depth;
2372 };
2373
2374 static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
2375 unsigned long delta_size)
2376 {
2377 if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)
2378 return 0;
2379
2380 if (delta_size < cache_max_small_delta_size)
2381 return 1;
2382
2383 /* cache delta, if objects are large enough compared to delta size */
2384 if ((src_size >> 20) + (trg_size >> 21) > (delta_size >> 10))
2385 return 1;
2386
2387 return 0;
2388 }
2389
2390 /* Protect delta_cache_size */
2391 static pthread_mutex_t cache_mutex;
2392 #define cache_lock() pthread_mutex_lock(&cache_mutex)
2393 #define cache_unlock() pthread_mutex_unlock(&cache_mutex)
2394
2395 /*
2396 * Protect object list partitioning (e.g. struct thread_param) and
2397 * progress_state
2398 */
2399 static pthread_mutex_t progress_mutex;
2400 #define progress_lock() pthread_mutex_lock(&progress_mutex)
2401 #define progress_unlock() pthread_mutex_unlock(&progress_mutex)
2402
2403 /*
2404 * Access to struct object_entry is unprotected since each thread owns
2405 * a portion of the main object list. Just don't access object entries
2406 * ahead in the list because they can be stolen and would need
2407 * progress_mutex for protection.
2408 */
2409
2410 static inline int oe_size_less_than(struct packing_data *pack,
2411 const struct object_entry *lhs,
2412 unsigned long rhs)
2413 {
2414 if (lhs->size_valid)
2415 return lhs->size_ < rhs;
2416 if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */
2417 return 0;
2418 return oe_get_size_slow(pack, lhs) < rhs;
2419 }
2420
2421 static inline void oe_set_tree_depth(struct packing_data *pack,
2422 struct object_entry *e,
2423 unsigned int tree_depth)
2424 {
2425 if (!pack->tree_depth)
2426 CALLOC_ARRAY(pack->tree_depth, pack->nr_alloc);
2427 pack->tree_depth[e - pack->objects] = tree_depth;
2428 }
2429
2430 /*
2431 * Return the size of the object without doing any delta
2432 * reconstruction (so non-deltas are true object sizes, but deltas
2433 * return the size of the delta data).
2434 */
2435 unsigned long oe_get_size_slow(struct packing_data *pack,
2436 const struct object_entry *e)
2437 {
2438 struct packed_git *p;
2439 struct pack_window *w_curs;
2440 unsigned char *buf;
2441 enum object_type type;
2442 unsigned long used, avail, size;
2443
2444 if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {
2445 packing_data_lock(&to_pack);
2446 if (oid_object_info(the_repository, &e->idx.oid, &size) < 0)
2447 die(_("unable to get size of %s"),
2448 oid_to_hex(&e->idx.oid));
2449 packing_data_unlock(&to_pack);
2450 return size;
2451 }
2452
2453 p = oe_in_pack(pack, e);
2454 if (!p)
2455 BUG("when e->type is a delta, it must belong to a pack");
2456
2457 packing_data_lock(&to_pack);
2458 w_curs = NULL;
2459 buf = use_pack(p, &w_curs, e->in_pack_offset, &avail);
2460 used = unpack_object_header_buffer(buf, avail, &type, &size);
2461 if (used == 0)
2462 die(_("unable to parse object header of %s"),
2463 oid_to_hex(&e->idx.oid));
2464
2465 unuse_pack(&w_curs);
2466 packing_data_unlock(&to_pack);
2467 return size;
2468 }
2469
2470 static int try_delta(struct unpacked *trg, struct unpacked *src,
2471 unsigned max_depth, unsigned long *mem_usage)
2472 {
2473 struct object_entry *trg_entry = trg->entry;
2474 struct object_entry *src_entry = src->entry;
2475 unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
2476 unsigned ref_depth;
2477 enum object_type type;
2478 void *delta_buf;
2479
2480 /* Don't bother doing diffs between different types */
2481 if (oe_type(trg_entry) != oe_type(src_entry))
2482 return -1;
2483
2484 /*
2485 * We do not bother to try a delta that we discarded on an
2486 * earlier try, but only when reusing delta data. Note that
2487 * src_entry that is marked as the preferred_base should always
2488 * be considered, as even if we produce a suboptimal delta against
2489 * it, we will still save the transfer cost, as we already know
2490 * the other side has it and we won't send src_entry at all.
2491 */
2492 if (reuse_delta && IN_PACK(trg_entry) &&
2493 IN_PACK(trg_entry) == IN_PACK(src_entry) &&
2494 !src_entry->preferred_base &&
2495 trg_entry->in_pack_type != OBJ_REF_DELTA &&
2496 trg_entry->in_pack_type != OBJ_OFS_DELTA)
2497 return 0;
2498
2499 /* Let's not bust the allowed depth. */
2500 if (src->depth >= max_depth)
2501 return 0;
2502
2503 /* Now some size filtering heuristics. */
2504 trg_size = SIZE(trg_entry);
2505 if (!DELTA(trg_entry)) {
2506 max_size = trg_size/2 - the_hash_algo->rawsz;
2507 ref_depth = 1;
2508 } else {
2509 max_size = DELTA_SIZE(trg_entry);
2510 ref_depth = trg->depth;
2511 }
2512 max_size = (uint64_t)max_size * (max_depth - src->depth) /
2513 (max_depth - ref_depth + 1);
2514 if (max_size == 0)
2515 return 0;
2516 src_size = SIZE(src_entry);
2517 sizediff = src_size < trg_size ? trg_size - src_size : 0;
2518 if (sizediff >= max_size)
2519 return 0;
2520 if (trg_size < src_size / 32)
2521 return 0;
2522
2523 if (!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))
2524 return 0;
2525
2526 /* Load data if not already done */
2527 if (!trg->data) {
2528 packing_data_lock(&to_pack);
2529 trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz);
2530 packing_data_unlock(&to_pack);
2531 if (!trg->data)
2532 die(_("object %s cannot be read"),
2533 oid_to_hex(&trg_entry->idx.oid));
2534 if (sz != trg_size)
2535 die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
2536 oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz,
2537 (uintmax_t)trg_size);
2538 *mem_usage += sz;
2539 }
2540 if (!src->data) {
2541 packing_data_lock(&to_pack);
2542 src->data = read_object_file(&src_entry->idx.oid, &type, &sz);
2543 packing_data_unlock(&to_pack);
2544 if (!src->data) {
2545 if (src_entry->preferred_base) {
2546 static int warned = 0;
2547 if (!warned++)
2548 warning(_("object %s cannot be read"),
2549 oid_to_hex(&src_entry->idx.oid));
2550 /*
2551 * Those objects are not included in the
2552 * resulting pack. Be resilient and ignore
2553 * them if they can't be read, in case the
2554 * pack could be created nevertheless.
2555 */
2556 return 0;
2557 }
2558 die(_("object %s cannot be read"),
2559 oid_to_hex(&src_entry->idx.oid));
2560 }
2561 if (sz != src_size)
2562 die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
2563 oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz,
2564 (uintmax_t)src_size);
2565 *mem_usage += sz;
2566 }
2567 if (!src->index) {
2568 src->index = create_delta_index(src->data, src_size);
2569 if (!src->index) {
2570 static int warned = 0;
2571 if (!warned++)
2572 warning(_("suboptimal pack - out of memory"));
2573 return 0;
2574 }
2575 *mem_usage += sizeof_delta_index(src->index);
2576 }
2577
2578 delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
2579 if (!delta_buf)
2580 return 0;
2581
2582 if (DELTA(trg_entry)) {
2583 /* Prefer only shallower same-sized deltas. */
2584 if (delta_size == DELTA_SIZE(trg_entry) &&
2585 src->depth + 1 >= trg->depth) {
2586 free(delta_buf);
2587 return 0;
2588 }
2589 }
2590
2591 /*
2592 * Handle memory allocation outside of the cache
2593 * accounting lock. Compiler will optimize the strangeness
2594 * away when NO_PTHREADS is defined.
2595 */
2596 free(trg_entry->delta_data);
2597 cache_lock();
2598 if (trg_entry->delta_data) {
2599 delta_cache_size -= DELTA_SIZE(trg_entry);
2600 trg_entry->delta_data = NULL;
2601 }
2602 if (delta_cacheable(src_size, trg_size, delta_size)) {
2603 delta_cache_size += delta_size;
2604 cache_unlock();
2605 trg_entry->delta_data = xrealloc(delta_buf, delta_size);
2606 } else {
2607 cache_unlock();
2608 free(delta_buf);
2609 }
2610
2611 SET_DELTA(trg_entry, src_entry);
2612 SET_DELTA_SIZE(trg_entry, delta_size);
2613 trg->depth = src->depth + 1;
2614
2615 return 1;
2616 }
2617
2618 static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
2619 {
2620 struct object_entry *child = DELTA_CHILD(me);
2621 unsigned int m = n;
2622 while (child) {
2623 const unsigned int c = check_delta_limit(child, n + 1);
2624 if (m < c)
2625 m = c;
2626 child = DELTA_SIBLING(child);
2627 }
2628 return m;
2629 }
2630
2631 static unsigned long free_unpacked(struct unpacked *n)
2632 {
2633 unsigned long freed_mem = sizeof_delta_index(n->index);
2634 free_delta_index(n->index);
2635 n->index = NULL;
2636 if (n->data) {
2637 freed_mem += SIZE(n->entry);
2638 FREE_AND_NULL(n->data);
2639 }
2640 n->entry = NULL;
2641 n->depth = 0;
2642 return freed_mem;
2643 }
2644
2645 static void find_deltas(struct object_entry **list, unsigned *list_size,
2646 int window, int depth, unsigned *processed)
2647 {
2648 uint32_t i, idx = 0, count = 0;
2649 struct unpacked *array;
2650 unsigned long mem_usage = 0;
2651
2652 CALLOC_ARRAY(array, window);
2653
2654 for (;;) {
2655 struct object_entry *entry;
2656 struct unpacked *n = array + idx;
2657 int j, max_depth, best_base = -1;
2658
2659 progress_lock();
2660 if (!*list_size) {
2661 progress_unlock();
2662 break;
2663 }
2664 entry = *list++;
2665 (*list_size)--;
2666 if (!entry->preferred_base) {
2667 (*processed)++;
2668 display_progress(progress_state, *processed);
2669 }
2670 progress_unlock();
2671
2672 mem_usage -= free_unpacked(n);
2673 n->entry = entry;
2674
2675 while (window_memory_limit &&
2676 mem_usage > window_memory_limit &&
2677 count > 1) {
2678 const uint32_t tail = (idx + window - count) % window;
2679 mem_usage -= free_unpacked(array + tail);
2680 count--;
2681 }
2682
2683 /* We do not compute delta to *create* objects we are not
2684 * going to pack.
2685 */
2686 if (entry->preferred_base)
2687 goto next;
2688
2689 /*
2690 * If the current object is at pack edge, take the depth the
2691 * objects that depend on the current object into account
2692 * otherwise they would become too deep.
2693 */
2694 max_depth = depth;
2695 if (DELTA_CHILD(entry)) {
2696 max_depth -= check_delta_limit(entry, 0);
2697 if (max_depth <= 0)
2698 goto next;
2699 }
2700
2701 j = window;
2702 while (--j > 0) {
2703 int ret;
2704 uint32_t other_idx = idx + j;
2705 struct unpacked *m;
2706 if (other_idx >= window)
2707 other_idx -= window;
2708 m = array + other_idx;
2709 if (!m->entry)
2710 break;
2711 ret = try_delta(n, m, max_depth, &mem_usage);
2712 if (ret < 0)
2713 break;
2714 else if (ret > 0)
2715 best_base = other_idx;
2716 }
2717
2718 /*
2719 * If we decided to cache the delta data, then it is best
2720 * to compress it right away. First because we have to do
2721 * it anyway, and doing it here while we're threaded will
2722 * save a lot of time in the non threaded write phase,
2723 * as well as allow for caching more deltas within
2724 * the same cache size limit.
2725 * ...
2726 * But only if not writing to stdout, since in that case
2727 * the network is most likely throttling writes anyway,
2728 * and therefore it is best to go to the write phase ASAP
2729 * instead, as we can afford spending more time compressing
2730 * between writes at that moment.
2731 */
2732 if (entry->delta_data && !pack_to_stdout) {
2733 unsigned long size;
2734
2735 size = do_compress(&entry->delta_data, DELTA_SIZE(entry));
2736 if (size < (1U << OE_Z_DELTA_BITS)) {
2737 entry->z_delta_size = size;
2738 cache_lock();
2739 delta_cache_size -= DELTA_SIZE(entry);
2740 delta_cache_size += entry->z_delta_size;
2741 cache_unlock();
2742 } else {
2743 FREE_AND_NULL(entry->delta_data);
2744 entry->z_delta_size = 0;
2745 }
2746 }
2747
2748 /* if we made n a delta, and if n is already at max
2749 * depth, leaving it in the window is pointless. we
2750 * should evict it first.
2751 */
2752 if (DELTA(entry) && max_depth <= n->depth)
2753 continue;
2754
2755 /*
2756 * Move the best delta base up in the window, after the
2757 * currently deltified object, to keep it longer. It will
2758 * be the first base object to be attempted next.
2759 */
2760 if (DELTA(entry)) {
2761 struct unpacked swap = array[best_base];
2762 int dist = (window + idx - best_base) % window;
2763 int dst = best_base;
2764 while (dist--) {
2765 int src = (dst + 1) % window;
2766 array[dst] = array[src];
2767 dst = src;
2768 }
2769 array[dst] = swap;
2770 }
2771
2772 next:
2773 idx++;
2774 if (count + 1 < window)
2775 count++;
2776 if (idx >= window)
2777 idx = 0;
2778 }
2779
2780 for (i = 0; i < window; ++i) {
2781 free_delta_index(array[i].index);
2782 free(array[i].data);
2783 }
2784 free(array);
2785 }
2786
2787 /*
2788 * The main object list is split into smaller lists, each is handed to
2789 * one worker.
2790 *
2791 * The main thread waits on the condition that (at least) one of the workers
2792 * has stopped working (which is indicated in the .working member of
2793 * struct thread_params).
2794 *
2795 * When a work thread has completed its work, it sets .working to 0 and
2796 * signals the main thread and waits on the condition that .data_ready
2797 * becomes 1.
2798 *
2799 * The main thread steals half of the work from the worker that has
2800 * most work left to hand it to the idle worker.
2801 */
2802
2803 struct thread_params {
2804 pthread_t thread;
2805 struct object_entry **list;
2806 unsigned list_size;
2807 unsigned remaining;
2808 int window;
2809 int depth;
2810 int working;
2811 int data_ready;
2812 pthread_mutex_t mutex;
2813 pthread_cond_t cond;
2814 unsigned *processed;
2815 };
2816
2817 static pthread_cond_t progress_cond;
2818
2819 /*
2820 * Mutex and conditional variable can't be statically-initialized on Windows.
2821 */
2822 static void init_threaded_search(void)
2823 {
2824 pthread_mutex_init(&cache_mutex, NULL);
2825 pthread_mutex_init(&progress_mutex, NULL);
2826 pthread_cond_init(&progress_cond, NULL);
2827 }
2828
2829 static void cleanup_threaded_search(void)
2830 {
2831 pthread_cond_destroy(&progress_cond);
2832 pthread_mutex_destroy(&cache_mutex);
2833 pthread_mutex_destroy(&progress_mutex);
2834 }
2835
2836 static void *threaded_find_deltas(void *arg)
2837 {
2838 struct thread_params *me = arg;
2839
2840 progress_lock();
2841 while (me->remaining) {
2842 progress_unlock();
2843
2844 find_deltas(me->list, &me->remaining,
2845 me->window, me->depth, me->processed);
2846
2847 progress_lock();
2848 me->working = 0;
2849 pthread_cond_signal(&progress_cond);
2850 progress_unlock();
2851
2852 /*
2853 * We must not set ->data_ready before we wait on the
2854 * condition because the main thread may have set it to 1
2855 * before we get here. In order to be sure that new
2856 * work is available if we see 1 in ->data_ready, it
2857 * was initialized to 0 before this thread was spawned
2858 * and we reset it to 0 right away.
2859 */
2860 pthread_mutex_lock(&me->mutex);
2861 while (!me->data_ready)
2862 pthread_cond_wait(&me->cond, &me->mutex);
2863 me->data_ready = 0;
2864 pthread_mutex_unlock(&me->mutex);
2865
2866 progress_lock();
2867 }
2868 progress_unlock();
2869 /* leave ->working 1 so that this doesn't get more work assigned */
2870 return NULL;
2871 }
2872
2873 static void ll_find_deltas(struct object_entry **list, unsigned list_size,
2874 int window, int depth, unsigned *processed)
2875 {
2876 struct thread_params *p;
2877 int i, ret, active_threads = 0;
2878
2879 init_threaded_search();
2880
2881 if (delta_search_threads <= 1) {
2882 find_deltas(list, &list_size, window, depth, processed);
2883 cleanup_threaded_search();
2884 return;
2885 }
2886 if (progress > pack_to_stdout)
2887 fprintf_ln(stderr, _("Delta compression using up to %d threads"),
2888 delta_search_threads);
2889 CALLOC_ARRAY(p, delta_search_threads);
2890
2891 /* Partition the work amongst work threads. */
2892 for (i = 0; i < delta_search_threads; i++) {
2893 unsigned sub_size = list_size / (delta_search_threads - i);
2894
2895 /* don't use too small segments or no deltas will be found */
2896 if (sub_size < 2*window && i+1 < delta_search_threads)
2897 sub_size = 0;
2898
2899 p[i].window = window;
2900 p[i].depth = depth;
2901 p[i].processed = processed;
2902 p[i].working = 1;
2903 p[i].data_ready = 0;
2904
2905 /* try to split chunks on "path" boundaries */
2906 while (sub_size && sub_size < list_size &&
2907 list[sub_size]->hash &&
2908 list[sub_size]->hash == list[sub_size-1]->hash)
2909 sub_size++;
2910
2911 p[i].list = list;
2912 p[i].list_size = sub_size;
2913 p[i].remaining = sub_size;
2914
2915 list += sub_size;
2916 list_size -= sub_size;
2917 }
2918
2919 /* Start work threads. */
2920 for (i = 0; i < delta_search_threads; i++) {
2921 if (!p[i].list_size)
2922 continue;
2923 pthread_mutex_init(&p[i].mutex, NULL);
2924 pthread_cond_init(&p[i].cond, NULL);
2925 ret = pthread_create(&p[i].thread, NULL,
2926 threaded_find_deltas, &p[i]);
2927 if (ret)
2928 die(_("unable to create thread: %s"), strerror(ret));
2929 active_threads++;
2930 }
2931
2932 /*
2933 * Now let's wait for work completion. Each time a thread is done
2934 * with its work, we steal half of the remaining work from the
2935 * thread with the largest number of unprocessed objects and give
2936 * it to that newly idle thread. This ensure good load balancing
2937 * until the remaining object list segments are simply too short
2938 * to be worth splitting anymore.
2939 */
2940 while (active_threads) {
2941 struct thread_params *target = NULL;
2942 struct thread_params *victim = NULL;
2943 unsigned sub_size = 0;
2944
2945 progress_lock();
2946 for (;;) {
2947 for (i = 0; !target && i < delta_search_threads; i++)
2948 if (!p[i].working)
2949 target = &p[i];
2950 if (target)
2951 break;
2952 pthread_cond_wait(&progress_cond, &progress_mutex);
2953 }
2954
2955 for (i = 0; i < delta_search_threads; i++)
2956 if (p[i].remaining > 2*window &&
2957 (!victim || victim->remaining < p[i].remaining))
2958 victim = &p[i];
2959 if (victim) {
2960 sub_size = victim->remaining / 2;
2961 list = victim->list + victim->list_size - sub_size;
2962 while (sub_size && list[0]->hash &&
2963 list[0]->hash == list[-1]->hash) {
2964 list++;
2965 sub_size--;
2966 }
2967 if (!sub_size) {
2968 /*
2969 * It is possible for some "paths" to have
2970 * so many objects that no hash boundary
2971 * might be found. Let's just steal the
2972 * exact half in that case.
2973 */
2974 sub_size = victim->remaining / 2;
2975 list -= sub_size;
2976 }
2977 target->list = list;
2978 victim->list_size -= sub_size;
2979 victim->remaining -= sub_size;
2980 }
2981 target->list_size = sub_size;
2982 target->remaining = sub_size;
2983 target->working = 1;
2984 progress_unlock();
2985
2986 pthread_mutex_lock(&target->mutex);
2987 target->data_ready = 1;
2988 pthread_cond_signal(&target->cond);
2989 pthread_mutex_unlock(&target->mutex);
2990
2991 if (!sub_size) {
2992 pthread_join(target->thread, NULL);
2993 pthread_cond_destroy(&target->cond);
2994 pthread_mutex_destroy(&target->mutex);
2995 active_threads--;
2996 }
2997 }
2998 cleanup_threaded_search();
2999 free(p);
3000 }
3001
3002 static int obj_is_packed(const struct object_id *oid)
3003 {
3004 return packlist_find(&to_pack, oid) ||
3005 (reuse_packfile_bitmap &&
3006 bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid));
3007 }
3008
3009 static void add_tag_chain(const struct object_id *oid)
3010 {
3011 struct tag *tag;
3012
3013 /*
3014 * We catch duplicates already in add_object_entry(), but we'd
3015 * prefer to do this extra check to avoid having to parse the
3016 * tag at all if we already know that it's being packed (e.g., if
3017 * it was included via bitmaps, we would not have parsed it
3018 * previously).
3019 */
3020 if (obj_is_packed(oid))
3021 return;
3022
3023 tag = lookup_tag(the_repository, oid);
3024 while (1) {
3025 if (!tag || parse_tag(tag) || !tag->tagged)
3026 die(_("unable to pack objects reachable from tag %s"),
3027 oid_to_hex(oid));
3028
3029 add_object_entry(&tag->object.oid, OBJ_TAG, NULL, 0);
3030
3031 if (tag->tagged->type != OBJ_TAG)
3032 return;
3033
3034 tag = (struct tag *)tag->tagged;
3035 }
3036 }
3037
3038 static int add_ref_tag(const char *tag UNUSED, const struct object_id *oid,
3039 int flag UNUSED, void *cb_data UNUSED)
3040 {
3041 struct object_id peeled;
3042
3043 if (!peel_iterated_oid(oid, &peeled) && obj_is_packed(&peeled))
3044 add_tag_chain(oid);
3045 return 0;
3046 }
3047
3048 static void prepare_pack(int window, int depth)
3049 {
3050 struct object_entry **delta_list;
3051 uint32_t i, nr_deltas;
3052 unsigned n;
3053
3054 if (use_delta_islands)
3055 resolve_tree_islands(the_repository, progress, &to_pack);
3056
3057 get_object_details();
3058
3059 /*
3060 * If we're locally repacking then we need to be doubly careful
3061 * from now on in order to make sure no stealth corruption gets
3062 * propagated to the new pack. Clients receiving streamed packs
3063 * should validate everything they get anyway so no need to incur
3064 * the additional cost here in that case.
3065 */
3066 if (!pack_to_stdout)
3067 do_check_packed_object_crc = 1;
3068
3069 if (!to_pack.nr_objects || !window || !depth)
3070 return;
3071
3072 ALLOC_ARRAY(delta_list, to_pack.nr_objects);
3073 nr_deltas = n = 0;
3074
3075 for (i = 0; i < to_pack.nr_objects; i++) {
3076 struct object_entry *entry = to_pack.objects + i;
3077
3078 if (DELTA(entry))
3079 /* This happens if we decided to reuse existing
3080 * delta from a pack. "reuse_delta &&" is implied.
3081 */
3082 continue;
3083
3084 if (!entry->type_valid ||
3085 oe_size_less_than(&to_pack, entry, 50))
3086 continue;
3087
3088 if (entry->no_try_delta)
3089 continue;
3090
3091 if (!entry->preferred_base) {
3092 nr_deltas++;
3093 if (oe_type(entry) < 0)
3094 die(_("unable to get type of object %s"),
3095 oid_to_hex(&entry->idx.oid));
3096 } else {
3097 if (oe_type(entry) < 0) {
3098 /*
3099 * This object is not found, but we
3100 * don't have to include it anyway.
3101 */
3102 continue;
3103 }
3104 }
3105
3106 delta_list[n++] = entry;
3107 }
3108
3109 if (nr_deltas && n > 1) {
3110 unsigned nr_done = 0;
3111
3112 if (progress)
3113 progress_state = start_progress(_("Compressing objects"),
3114 nr_deltas);
3115 QSORT(delta_list, n, type_size_sort);
3116 ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
3117 stop_progress(&progress_state);
3118 if (nr_done != nr_deltas)
3119 die(_("inconsistency with delta count"));
3120 }
3121 free(delta_list);
3122 }
3123
3124 static int git_pack_config(const char *k, const char *v, void *cb)
3125 {
3126 if (!strcmp(k, "pack.window")) {
3127 window = git_config_int(k, v);
3128 return 0;
3129 }
3130 if (!strcmp(k, "pack.windowmemory")) {
3131 window_memory_limit = git_config_ulong(k, v);
3132 return 0;
3133 }
3134 if (!strcmp(k, "pack.depth")) {
3135 depth = git_config_int(k, v);
3136 return 0;
3137 }
3138 if (!strcmp(k, "pack.deltacachesize")) {
3139 max_delta_cache_size = git_config_int(k, v);
3140 return 0;
3141 }
3142 if (!strcmp(k, "pack.deltacachelimit")) {
3143 cache_max_small_delta_size = git_config_int(k, v);
3144 return 0;
3145 }
3146 if (!strcmp(k, "pack.writebitmaphashcache")) {
3147 if (git_config_bool(k, v))
3148 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;
3149 else
3150 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;
3151 }
3152
3153 if (!strcmp(k, "pack.writebitmaplookuptable")) {
3154 if (git_config_bool(k, v))
3155 write_bitmap_options |= BITMAP_OPT_LOOKUP_TABLE;
3156 else
3157 write_bitmap_options &= ~BITMAP_OPT_LOOKUP_TABLE;
3158 }
3159
3160 if (!strcmp(k, "pack.usebitmaps")) {
3161 use_bitmap_index_default = git_config_bool(k, v);
3162 return 0;
3163 }
3164 if (!strcmp(k, "pack.allowpackreuse")) {
3165 allow_pack_reuse = git_config_bool(k, v);
3166 return 0;
3167 }
3168 if (!strcmp(k, "pack.threads")) {
3169 delta_search_threads = git_config_int(k, v);
3170 if (delta_search_threads < 0)
3171 die(_("invalid number of threads specified (%d)"),
3172 delta_search_threads);
3173 if (!HAVE_THREADS && delta_search_threads != 1) {
3174 warning(_("no threads support, ignoring %s"), k);
3175 delta_search_threads = 0;
3176 }
3177 return 0;
3178 }
3179 if (!strcmp(k, "pack.indexversion")) {
3180 pack_idx_opts.version = git_config_int(k, v);
3181 if (pack_idx_opts.version > 2)
3182 die(_("bad pack.indexVersion=%"PRIu32),
3183 pack_idx_opts.version);
3184 return 0;
3185 }
3186 if (!strcmp(k, "pack.writereverseindex")) {
3187 if (git_config_bool(k, v))
3188 pack_idx_opts.flags |= WRITE_REV;
3189 else
3190 pack_idx_opts.flags &= ~WRITE_REV;
3191 return 0;
3192 }
3193 if (!strcmp(k, "uploadpack.blobpackfileuri")) {
3194 struct configured_exclusion *ex = xmalloc(sizeof(*ex));
3195 const char *oid_end, *pack_end;
3196 /*
3197 * Stores the pack hash. This is not a true object ID, but is
3198 * of the same form.
3199 */
3200 struct object_id pack_hash;
3201
3202 if (parse_oid_hex(v, &ex->e.oid, &oid_end) ||
3203 *oid_end != ' ' ||
3204 parse_oid_hex(oid_end + 1, &pack_hash, &pack_end) ||
3205 *pack_end != ' ')
3206 die(_("value of uploadpack.blobpackfileuri must be "
3207 "of the form '<object-hash> <pack-hash> <uri>' (got '%s')"), v);
3208 if (oidmap_get(&configured_exclusions, &ex->e.oid))
3209 die(_("object already configured in another "
3210 "uploadpack.blobpackfileuri (got '%s')"), v);
3211 ex->pack_hash_hex = xcalloc(1, pack_end - oid_end);
3212 memcpy(ex->pack_hash_hex, oid_end + 1, pack_end - oid_end - 1);
3213 ex->uri = xstrdup(pack_end + 1);
3214 oidmap_put(&configured_exclusions, ex);
3215 }
3216 return git_default_config(k, v, cb);
3217 }
3218
3219 /* Counters for trace2 output when in --stdin-packs mode. */
3220 static int stdin_packs_found_nr;
3221 static int stdin_packs_hints_nr;
3222
3223 static int add_object_entry_from_pack(const struct object_id *oid,
3224 struct packed_git *p,
3225 uint32_t pos,
3226 void *_data)
3227 {
3228 off_t ofs;
3229 enum object_type type = OBJ_NONE;
3230
3231 display_progress(progress_state, ++nr_seen);
3232
3233 if (have_duplicate_entry(oid, 0))
3234 return 0;
3235
3236 ofs = nth_packed_object_offset(p, pos);
3237 if (!want_object_in_pack(oid, 0, &p, &ofs))
3238 return 0;
3239
3240 if (p) {
3241 struct rev_info *revs = _data;
3242 struct object_info oi = OBJECT_INFO_INIT;
3243
3244 oi.typep = &type;
3245 if (packed_object_info(the_repository, p, ofs, &oi) < 0) {
3246 die(_("could not get type of object %s in pack %s"),
3247 oid_to_hex(oid), p->pack_name);
3248 } else if (type == OBJ_COMMIT) {
3249 /*
3250 * commits in included packs are used as starting points for the
3251 * subsequent revision walk
3252 */
3253 add_pending_oid(revs, NULL, oid, 0);
3254 }
3255
3256 stdin_packs_found_nr++;
3257 }
3258
3259 create_object_entry(oid, type, 0, 0, 0, p, ofs);
3260
3261 return 0;
3262 }
3263
3264 static void show_commit_pack_hint(struct commit *commit, void *_data)
3265 {
3266 /* nothing to do; commits don't have a namehash */
3267 }
3268
3269 static void show_object_pack_hint(struct object *object, const char *name,
3270 void *_data)
3271 {
3272 struct object_entry *oe = packlist_find(&to_pack, &object->oid);
3273 if (!oe)
3274 return;
3275
3276 /*
3277 * Our 'to_pack' list was constructed by iterating all objects packed in
3278 * included packs, and so doesn't have a non-zero hash field that you
3279 * would typically pick up during a reachability traversal.
3280 *
3281 * Make a best-effort attempt to fill in the ->hash and ->no_try_delta
3282 * here using a now in order to perhaps improve the delta selection
3283 * process.
3284 */
3285 oe->hash = pack_name_hash(name);
3286 oe->no_try_delta = name && no_try_delta(name);
3287
3288 stdin_packs_hints_nr++;
3289 }
3290
3291 static int pack_mtime_cmp(const void *_a, const void *_b)
3292 {
3293 struct packed_git *a = ((const struct string_list_item*)_a)->util;
3294 struct packed_git *b = ((const struct string_list_item*)_b)->util;
3295
3296 /*
3297 * order packs by descending mtime so that objects are laid out
3298 * roughly as newest-to-oldest
3299 */
3300 if (a->mtime < b->mtime)
3301 return 1;
3302 else if (b->mtime < a->mtime)
3303 return -1;
3304 else
3305 return 0;
3306 }
3307
3308 static void read_packs_list_from_stdin(void)
3309 {
3310 struct strbuf buf = STRBUF_INIT;
3311 struct string_list include_packs = STRING_LIST_INIT_DUP;
3312 struct string_list exclude_packs = STRING_LIST_INIT_DUP;
3313 struct string_list_item *item = NULL;
3314
3315 struct packed_git *p;
3316 struct rev_info revs;
3317
3318 repo_init_revisions(the_repository, &revs, NULL);
3319 /*
3320 * Use a revision walk to fill in the namehash of objects in the include
3321 * packs. To save time, we'll avoid traversing through objects that are
3322 * in excluded packs.
3323 *
3324 * That may cause us to avoid populating all of the namehash fields of
3325 * all included objects, but our goal is best-effort, since this is only
3326 * an optimization during delta selection.
3327 */
3328 revs.no_kept_objects = 1;
3329 revs.keep_pack_cache_flags |= IN_CORE_KEEP_PACKS;
3330 revs.blob_objects = 1;
3331 revs.tree_objects = 1;
3332 revs.tag_objects = 1;
3333 revs.ignore_missing_links = 1;
3334
3335 while (strbuf_getline(&buf, stdin) != EOF) {
3336 if (!buf.len)
3337 continue;
3338
3339 if (*buf.buf == '^')
3340 string_list_append(&exclude_packs, buf.buf + 1);
3341 else
3342 string_list_append(&include_packs, buf.buf);
3343
3344 strbuf_reset(&buf);
3345 }
3346
3347 string_list_sort(&include_packs);
3348 string_list_sort(&exclude_packs);
3349
3350 for (p = get_all_packs(the_repository); p; p = p->next) {
3351 const char *pack_name = pack_basename(p);
3352
3353 item = string_list_lookup(&include_packs, pack_name);
3354 if (!item)
3355 item = string_list_lookup(&exclude_packs, pack_name);
3356
3357 if (item)
3358 item->util = p;
3359 }
3360
3361 /*
3362 * Arguments we got on stdin may not even be packs. First
3363 * check that to avoid segfaulting later on in
3364 * e.g. pack_mtime_cmp(), excluded packs are handled below.
3365 *
3366 * Since we first parsed our STDIN and then sorted the input
3367 * lines the pack we error on will be whatever line happens to
3368 * sort first. This is lazy, it's enough that we report one
3369 * bad case here, we don't need to report the first/last one,
3370 * or all of them.
3371 */
3372 for_each_string_list_item(item, &include_packs) {
3373 struct packed_git *p = item->util;
3374 if (!p)
3375 die(_("could not find pack '%s'"), item->string);
3376 if (!is_pack_valid(p))
3377 die(_("packfile %s cannot be accessed"), p->pack_name);
3378 }
3379
3380 /*
3381 * Then, handle all of the excluded packs, marking them as
3382 * kept in-core so that later calls to add_object_entry()
3383 * discards any objects that are also found in excluded packs.
3384 */
3385 for_each_string_list_item(item, &exclude_packs) {
3386 struct packed_git *p = item->util;
3387 if (!p)
3388 die(_("could not find pack '%s'"), item->string);
3389 p->pack_keep_in_core = 1;
3390 }
3391
3392 /*
3393 * Order packs by ascending mtime; use QSORT directly to access the
3394 * string_list_item's ->util pointer, which string_list_sort() does not
3395 * provide.
3396 */
3397 QSORT(include_packs.items, include_packs.nr, pack_mtime_cmp);
3398
3399 for_each_string_list_item(item, &include_packs) {
3400 struct packed_git *p = item->util;
3401 for_each_object_in_pack(p,
3402 add_object_entry_from_pack,
3403 &revs,
3404 FOR_EACH_OBJECT_PACK_ORDER);
3405 }
3406
3407 if (prepare_revision_walk(&revs))
3408 die(_("revision walk setup failed"));
3409 traverse_commit_list(&revs,
3410 show_commit_pack_hint,
3411 show_object_pack_hint,
3412 NULL);
3413
3414 trace2_data_intmax("pack-objects", the_repository, "stdin_packs_found",
3415 stdin_packs_found_nr);
3416 trace2_data_intmax("pack-objects", the_repository, "stdin_packs_hints",
3417 stdin_packs_hints_nr);
3418
3419 strbuf_release(&buf);
3420 string_list_clear(&include_packs, 0);
3421 string_list_clear(&exclude_packs, 0);
3422 }
3423
3424 static void add_cruft_object_entry(const struct object_id *oid, enum object_type type,
3425 struct packed_git *pack, off_t offset,
3426 const char *name, uint32_t mtime)
3427 {
3428 struct object_entry *entry;
3429
3430 display_progress(progress_state, ++nr_seen);
3431
3432 entry = packlist_find(&to_pack, oid);
3433 if (entry) {
3434 if (name) {
3435 entry->hash = pack_name_hash(name);
3436 entry->no_try_delta = no_try_delta(name);
3437 }
3438 } else {
3439 if (!want_object_in_pack(oid, 0, &pack, &offset))
3440 return;
3441 if (!pack && type == OBJ_BLOB && !has_loose_object(oid)) {
3442 /*
3443 * If a traversed tree has a missing blob then we want
3444 * to avoid adding that missing object to our pack.
3445 *
3446 * This only applies to missing blobs, not trees,
3447 * because the traversal needs to parse sub-trees but
3448 * not blobs.
3449 *
3450 * Note we only perform this check when we couldn't
3451 * already find the object in a pack, so we're really
3452 * limited to "ensure non-tip blobs which don't exist in
3453 * packs do exist via loose objects". Confused?
3454 */
3455 return;
3456 }
3457
3458 entry = create_object_entry(oid, type, pack_name_hash(name),
3459 0, name && no_try_delta(name),
3460 pack, offset);
3461 }
3462
3463 if (mtime > oe_cruft_mtime(&to_pack, entry))
3464 oe_set_cruft_mtime(&to_pack, entry, mtime);
3465 return;
3466 }
3467
3468 static void show_cruft_object(struct object *obj, const char *name, void *data)
3469 {
3470 /*
3471 * if we did not record it earlier, it's at least as old as our
3472 * expiration value. Rather than find it exactly, just use that
3473 * value. This may bump it forward from its real mtime, but it
3474 * will still be "too old" next time we run with the same
3475 * expiration.
3476 *
3477 * if obj does appear in the packing list, this call is a noop (or may
3478 * set the namehash).
3479 */
3480 add_cruft_object_entry(&obj->oid, obj->type, NULL, 0, name, cruft_expiration);
3481 }
3482
3483 static void show_cruft_commit(struct commit *commit, void *data)
3484 {
3485 show_cruft_object((struct object*)commit, NULL, data);
3486 }
3487
3488 static int cruft_include_check_obj(struct object *obj, void *data)
3489 {
3490 return !has_object_kept_pack(&obj->oid, IN_CORE_KEEP_PACKS);
3491 }
3492
3493 static int cruft_include_check(struct commit *commit, void *data)
3494 {
3495 return cruft_include_check_obj((struct object*)commit, data);
3496 }
3497
3498 static void set_cruft_mtime(const struct object *object,
3499 struct packed_git *pack,
3500 off_t offset, time_t mtime)
3501 {
3502 add_cruft_object_entry(&object->oid, object->type, pack, offset, NULL,
3503 mtime);
3504 }
3505
3506 static void mark_pack_kept_in_core(struct string_list *packs, unsigned keep)
3507 {
3508 struct string_list_item *item = NULL;
3509 for_each_string_list_item(item, packs) {
3510 struct packed_git *p = item->util;
3511 if (!p)
3512 die(_("could not find pack '%s'"), item->string);
3513 p->pack_keep_in_core = keep;
3514 }
3515 }
3516
3517 static void add_unreachable_loose_objects(void);
3518 static void add_objects_in_unpacked_packs(void);
3519
3520 static void enumerate_cruft_objects(void)
3521 {
3522 if (progress)
3523 progress_state = start_progress(_("Enumerating cruft objects"), 0);
3524
3525 add_objects_in_unpacked_packs();
3526 add_unreachable_loose_objects();
3527
3528 stop_progress(&progress_state);
3529 }
3530
3531 static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs)
3532 {
3533 struct packed_git *p;
3534 struct rev_info revs;
3535 int ret;
3536
3537 repo_init_revisions(the_repository, &revs, NULL);
3538
3539 revs.tag_objects = 1;
3540 revs.tree_objects = 1;
3541 revs.blob_objects = 1;
3542
3543 revs.include_check = cruft_include_check;
3544 revs.include_check_obj = cruft_include_check_obj;
3545
3546 revs.ignore_missing_links = 1;
3547
3548 if (progress)
3549 progress_state = start_progress(_("Enumerating cruft objects"), 0);
3550 ret = add_unseen_recent_objects_to_traversal(&revs, cruft_expiration,
3551 set_cruft_mtime, 1);
3552 stop_progress(&progress_state);
3553
3554 if (ret)
3555 die(_("unable to add cruft objects"));
3556
3557 /*
3558 * Re-mark only the fresh packs as kept so that objects in
3559 * unknown packs do not halt the reachability traversal early.
3560 */
3561 for (p = get_all_packs(the_repository); p; p = p->next)
3562 p->pack_keep_in_core = 0;
3563 mark_pack_kept_in_core(fresh_packs, 1);
3564
3565 if (prepare_revision_walk(&revs))
3566 die(_("revision walk setup failed"));
3567 if (progress)
3568 progress_state = start_progress(_("Traversing cruft objects"), 0);
3569 nr_seen = 0;
3570 traverse_commit_list(&revs, show_cruft_commit, show_cruft_object, NULL);
3571
3572 stop_progress(&progress_state);
3573 }
3574
3575 static void read_cruft_objects(void)
3576 {
3577 struct strbuf buf = STRBUF_INIT;
3578 struct string_list discard_packs = STRING_LIST_INIT_DUP;
3579 struct string_list fresh_packs = STRING_LIST_INIT_DUP;
3580 struct packed_git *p;
3581
3582 ignore_packed_keep_in_core = 1;
3583
3584 while (strbuf_getline(&buf, stdin) != EOF) {
3585 if (!buf.len)
3586 continue;
3587
3588 if (*buf.buf == '-')
3589 string_list_append(&discard_packs, buf.buf + 1);
3590 else
3591 string_list_append(&fresh_packs, buf.buf);
3592 strbuf_reset(&buf);
3593 }
3594
3595 string_list_sort(&discard_packs);
3596 string_list_sort(&fresh_packs);
3597
3598 for (p = get_all_packs(the_repository); p; p = p->next) {
3599 const char *pack_name = pack_basename(p);
3600 struct string_list_item *item;
3601
3602 item = string_list_lookup(&fresh_packs, pack_name);
3603 if (!item)
3604 item = string_list_lookup(&discard_packs, pack_name);
3605
3606 if (item) {
3607 item->util = p;
3608 } else {
3609 /*
3610 * This pack wasn't mentioned in either the "fresh" or
3611 * "discard" list, so the caller didn't know about it.
3612 *
3613 * Mark it as kept so that its objects are ignored by
3614 * add_unseen_recent_objects_to_traversal(). We'll
3615 * unmark it before starting the traversal so it doesn't
3616 * halt the traversal early.
3617 */
3618 p->pack_keep_in_core = 1;
3619 }
3620 }
3621
3622 mark_pack_kept_in_core(&fresh_packs, 1);
3623 mark_pack_kept_in_core(&discard_packs, 0);
3624
3625 if (cruft_expiration)
3626 enumerate_and_traverse_cruft_objects(&fresh_packs);
3627 else
3628 enumerate_cruft_objects();
3629
3630 strbuf_release(&buf);
3631 string_list_clear(&discard_packs, 0);
3632 string_list_clear(&fresh_packs, 0);
3633 }
3634
3635 static void read_object_list_from_stdin(void)
3636 {
3637 char line[GIT_MAX_HEXSZ + 1 + PATH_MAX + 2];
3638 struct object_id oid;
3639 const char *p;
3640
3641 for (;;) {
3642 if (!fgets(line, sizeof(line), stdin)) {
3643 if (feof(stdin))
3644 break;
3645 if (!ferror(stdin))
3646 BUG("fgets returned NULL, not EOF, not error!");
3647 if (errno != EINTR)
3648 die_errno("fgets");
3649 clearerr(stdin);
3650 continue;
3651 }
3652 if (line[0] == '-') {
3653 if (get_oid_hex(line+1, &oid))
3654 die(_("expected edge object ID, got garbage:\n %s"),
3655 line);
3656 add_preferred_base(&oid);
3657 continue;
3658 }
3659 if (parse_oid_hex(line, &oid, &p))
3660 die(_("expected object ID, got garbage:\n %s"), line);
3661
3662 add_preferred_base_object(p + 1);
3663 add_object_entry(&oid, OBJ_NONE, p + 1, 0);
3664 }
3665 }
3666
3667 static void show_commit(struct commit *commit, void *data)
3668 {
3669 add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL, 0);
3670
3671 if (write_bitmap_index)
3672 index_commit_for_bitmap(commit);
3673
3674 if (use_delta_islands)
3675 propagate_island_marks(commit);
3676 }
3677
3678 static void show_object(struct object *obj, const char *name, void *data)
3679 {
3680 add_preferred_base_object(name);
3681 add_object_entry(&obj->oid, obj->type, name, 0);
3682
3683 if (use_delta_islands) {
3684 const char *p;
3685 unsigned depth;
3686 struct object_entry *ent;
3687
3688 /* the empty string is a root tree, which is depth 0 */
3689 depth = *name ? 1 : 0;
3690 for (p = strchr(name, '/'); p; p = strchr(p + 1, '/'))
3691 depth++;
3692
3693 ent = packlist_find(&to_pack, &obj->oid);
3694 if (ent && depth > oe_tree_depth(&to_pack, ent))
3695 oe_set_tree_depth(&to_pack, ent, depth);
3696 }
3697 }
3698
3699 static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)
3700 {
3701 assert(arg_missing_action == MA_ALLOW_ANY);
3702
3703 /*
3704 * Quietly ignore ALL missing objects. This avoids problems with
3705 * staging them now and getting an odd error later.
3706 */
3707 if (!has_object(the_repository, &obj->oid, 0))
3708 return;
3709
3710 show_object(obj, name, data);
3711 }
3712
3713 static void show_object__ma_allow_promisor(struct object *obj, const char *name, void *data)
3714 {
3715 assert(arg_missing_action == MA_ALLOW_PROMISOR);
3716
3717 /*
3718 * Quietly ignore EXPECTED missing objects. This avoids problems with
3719 * staging them now and getting an odd error later.
3720 */
3721 if (!has_object(the_repository, &obj->oid, 0) && is_promisor_object(&obj->oid))
3722 return;
3723
3724 show_object(obj, name, data);
3725 }
3726
3727 static int option_parse_missing_action(const struct option *opt,
3728 const char *arg, int unset)
3729 {
3730 assert(arg);
3731 assert(!unset);
3732
3733 if (!strcmp(arg, "error")) {
3734 arg_missing_action = MA_ERROR;
3735 fn_show_object = show_object;
3736 return 0;
3737 }
3738
3739 if (!strcmp(arg, "allow-any")) {
3740 arg_missing_action = MA_ALLOW_ANY;
3741 fetch_if_missing = 0;
3742 fn_show_object = show_object__ma_allow_any;
3743 return 0;
3744 }
3745
3746 if (!strcmp(arg, "allow-promisor")) {
3747 arg_missing_action = MA_ALLOW_PROMISOR;
3748 fetch_if_missing = 0;
3749 fn_show_object = show_object__ma_allow_promisor;
3750 return 0;
3751 }
3752
3753 die(_("invalid value for '%s': '%s'"), "--missing", arg);
3754 return 0;
3755 }
3756
3757 static void show_edge(struct commit *commit)
3758 {
3759 add_preferred_base(&commit->object.oid);
3760 }
3761
3762 static int add_object_in_unpacked_pack(const struct object_id *oid,
3763 struct packed_git *pack,
3764 uint32_t pos,
3765 void *_data)
3766 {
3767 if (cruft) {
3768 off_t offset;
3769 time_t mtime;
3770
3771 if (pack->is_cruft) {
3772 if (load_pack_mtimes(pack) < 0)
3773 die(_("could not load cruft pack .mtimes"));
3774 mtime = nth_packed_mtime(pack, pos);
3775 } else {
3776 mtime = pack->mtime;
3777 }
3778 offset = nth_packed_object_offset(pack, pos);
3779
3780 add_cruft_object_entry(oid, OBJ_NONE, pack, offset,
3781 NULL, mtime);
3782 } else {
3783 add_object_entry(oid, OBJ_NONE, "", 0);
3784 }
3785 return 0;
3786 }
3787
3788 static void add_objects_in_unpacked_packs(void)
3789 {
3790 if (for_each_packed_object(add_object_in_unpacked_pack, NULL,
3791 FOR_EACH_OBJECT_PACK_ORDER |
3792 FOR_EACH_OBJECT_LOCAL_ONLY |
3793 FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
3794 FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
3795 die(_("cannot open pack index"));
3796 }
3797
3798 static int add_loose_object(const struct object_id *oid, const char *path,
3799 void *data)
3800 {
3801 enum object_type type = oid_object_info(the_repository, oid, NULL);
3802
3803 if (type < 0) {
3804 warning(_("loose object at %s could not be examined"), path);
3805 return 0;
3806 }
3807
3808 if (cruft) {
3809 struct stat st;
3810 if (stat(path, &st) < 0) {
3811 if (errno == ENOENT)
3812 return 0;
3813 return error_errno("unable to stat %s", oid_to_hex(oid));
3814 }
3815
3816 add_cruft_object_entry(oid, type, NULL, 0, NULL,
3817 st.st_mtime);
3818 } else {
3819 add_object_entry(oid, type, "", 0);
3820 }
3821 return 0;
3822 }
3823
3824 /*
3825 * We actually don't even have to worry about reachability here.
3826 * add_object_entry will weed out duplicates, so we just add every
3827 * loose object we find.
3828 */
3829 static void add_unreachable_loose_objects(void)
3830 {
3831 for_each_loose_file_in_objdir(get_object_directory(),
3832 add_loose_object,
3833 NULL, NULL, NULL);
3834 }
3835
3836 static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
3837 {
3838 static struct packed_git *last_found = (void *)1;
3839 struct packed_git *p;
3840
3841 p = (last_found != (void *)1) ? last_found :
3842 get_all_packs(the_repository);
3843
3844 while (p) {
3845 if ((!p->pack_local || p->pack_keep ||
3846 p->pack_keep_in_core) &&
3847 find_pack_entry_one(oid->hash, p)) {
3848 last_found = p;
3849 return 1;
3850 }
3851 if (p == last_found)
3852 p = get_all_packs(the_repository);
3853 else
3854 p = p->next;
3855 if (p == last_found)
3856 p = p->next;
3857 }
3858 return 0;
3859 }
3860
3861 /*
3862 * Store a list of sha1s that are should not be discarded
3863 * because they are either written too recently, or are
3864 * reachable from another object that was.
3865 *
3866 * This is filled by get_object_list.
3867 */
3868 static struct oid_array recent_objects;
3869
3870 static int loosened_object_can_be_discarded(const struct object_id *oid,
3871 timestamp_t mtime)
3872 {
3873 if (!unpack_unreachable_expiration)
3874 return 0;
3875 if (mtime > unpack_unreachable_expiration)
3876 return 0;
3877 if (oid_array_lookup(&recent_objects, oid) >= 0)
3878 return 0;
3879 return 1;
3880 }
3881
3882 static void loosen_unused_packed_objects(void)
3883 {
3884 struct packed_git *p;
3885 uint32_t i;
3886 uint32_t loosened_objects_nr = 0;
3887 struct object_id oid;
3888
3889 for (p = get_all_packs(the_repository); p; p = p->next) {
3890 if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
3891 continue;
3892
3893 if (open_pack_index(p))
3894 die(_("cannot open pack index"));
3895
3896 for (i = 0; i < p->num_objects; i++) {
3897 nth_packed_object_id(&oid, p, i);
3898 if (!packlist_find(&to_pack, &oid) &&
3899 !has_sha1_pack_kept_or_nonlocal(&oid) &&
3900 !loosened_object_can_be_discarded(&oid, p->mtime)) {
3901 if (force_object_loose(&oid, p->mtime))
3902 die(_("unable to force loose object"));
3903 loosened_objects_nr++;
3904 }
3905 }
3906 }
3907
3908 trace2_data_intmax("pack-objects", the_repository,
3909 "loosen_unused_packed_objects/loosened", loosened_objects_nr);
3910 }
3911
3912 /*
3913 * This tracks any options which pack-reuse code expects to be on, or which a
3914 * reader of the pack might not understand, and which would therefore prevent
3915 * blind reuse of what we have on disk.
3916 */
3917 static int pack_options_allow_reuse(void)
3918 {
3919 return allow_pack_reuse &&
3920 pack_to_stdout &&
3921 !ignore_packed_keep_on_disk &&
3922 !ignore_packed_keep_in_core &&
3923 (!local || !have_non_local_packs) &&
3924 !incremental;
3925 }
3926
3927 static int get_object_list_from_bitmap(struct rev_info *revs)
3928 {
3929 if (!(bitmap_git = prepare_bitmap_walk(revs, 0)))
3930 return -1;
3931
3932 if (pack_options_allow_reuse() &&
3933 !reuse_partial_packfile_from_bitmap(
3934 bitmap_git,
3935 &reuse_packfile,
3936 &reuse_packfile_objects,
3937 &reuse_packfile_bitmap)) {
3938 assert(reuse_packfile_objects);
3939 nr_result += reuse_packfile_objects;
3940 nr_seen += reuse_packfile_objects;
3941 display_progress(progress_state, nr_seen);
3942 }
3943
3944 traverse_bitmap_commit_list(bitmap_git, revs,
3945 &add_object_entry_from_bitmap);
3946 return 0;
3947 }
3948
3949 static void record_recent_object(struct object *obj,
3950 const char *name,
3951 void *data)
3952 {
3953 oid_array_append(&recent_objects, &obj->oid);
3954 }
3955
3956 static void record_recent_commit(struct commit *commit, void *data)
3957 {
3958 oid_array_append(&recent_objects, &commit->object.oid);
3959 }
3960
3961 static int mark_bitmap_preferred_tip(const char *refname,
3962 const struct object_id *oid,
3963 int flags UNUSED,
3964 void *data UNUSED)
3965 {
3966 struct object_id peeled;
3967 struct object *object;
3968
3969 if (!peel_iterated_oid(oid, &peeled))
3970 oid = &peeled;
3971
3972 object = parse_object_or_die(oid, refname);
3973 if (object->type == OBJ_COMMIT)
3974 object->flags |= NEEDS_BITMAP;
3975
3976 return 0;
3977 }
3978
3979 static void mark_bitmap_preferred_tips(void)
3980 {
3981 struct string_list_item *item;
3982 const struct string_list *preferred_tips;
3983
3984 preferred_tips = bitmap_preferred_tips(the_repository);
3985 if (!preferred_tips)
3986 return;
3987
3988 for_each_string_list_item(item, preferred_tips) {
3989 for_each_ref_in(item->string, mark_bitmap_preferred_tip, NULL);
3990 }
3991 }
3992
3993 static void get_object_list(struct rev_info *revs, int ac, const char **av)
3994 {
3995 struct setup_revision_opt s_r_opt = {
3996 .allow_exclude_promisor_objects = 1,
3997 };
3998 char line[1000];
3999 int flags = 0;
4000 int save_warning;
4001
4002 save_commit_buffer = 0;
4003 setup_revisions(ac, av, revs, &s_r_opt);
4004
4005 /* make sure shallows are read */
4006 is_repository_shallow(the_repository);
4007
4008 save_warning = warn_on_object_refname_ambiguity;
4009 warn_on_object_refname_ambiguity = 0;
4010
4011 while (fgets(line, sizeof(line), stdin) != NULL) {
4012 int len = strlen(line);
4013 if (len && line[len - 1] == '\n')
4014 line[--len] = 0;
4015 if (!len)
4016 break;
4017 if (*line == '-') {
4018 if (!strcmp(line, "--not")) {
4019 flags ^= UNINTERESTING;
4020 write_bitmap_index = 0;
4021 continue;
4022 }
4023 if (starts_with(line, "--shallow ")) {
4024 struct object_id oid;
4025 if (get_oid_hex(line + 10, &oid))
4026 die("not an object name '%s'", line + 10);
4027 register_shallow(the_repository, &oid);
4028 use_bitmap_index = 0;
4029 continue;
4030 }
4031 die(_("not a rev '%s'"), line);
4032 }
4033 if (handle_revision_arg(line, revs, flags, REVARG_CANNOT_BE_FILENAME))
4034 die(_("bad revision '%s'"), line);
4035 }
4036
4037 warn_on_object_refname_ambiguity = save_warning;
4038
4039 if (use_bitmap_index && !get_object_list_from_bitmap(revs))
4040 return;
4041
4042 if (use_delta_islands)
4043 load_delta_islands(the_repository, progress);
4044
4045 if (write_bitmap_index)
4046 mark_bitmap_preferred_tips();
4047
4048 if (prepare_revision_walk(revs))
4049 die(_("revision walk setup failed"));
4050 mark_edges_uninteresting(revs, show_edge, sparse);
4051
4052 if (!fn_show_object)
4053 fn_show_object = show_object;
4054 traverse_commit_list(revs,
4055 show_commit, fn_show_object,
4056 NULL);
4057
4058 if (unpack_unreachable_expiration) {
4059 revs->ignore_missing_links = 1;
4060 if (add_unseen_recent_objects_to_traversal(revs,
4061 unpack_unreachable_expiration, NULL, 0))
4062 die(_("unable to add recent objects"));
4063 if (prepare_revision_walk(revs))
4064 die(_("revision walk setup failed"));
4065 traverse_commit_list(revs, record_recent_commit,
4066 record_recent_object, NULL);
4067 }
4068
4069 if (keep_unreachable)
4070 add_objects_in_unpacked_packs();
4071 if (pack_loose_unreachable)
4072 add_unreachable_loose_objects();
4073 if (unpack_unreachable)
4074 loosen_unused_packed_objects();
4075
4076 oid_array_clear(&recent_objects);
4077 }
4078
4079 static void add_extra_kept_packs(const struct string_list *names)
4080 {
4081 struct packed_git *p;
4082
4083 if (!names->nr)
4084 return;
4085
4086 for (p = get_all_packs(the_repository); p; p = p->next) {
4087 const char *name = basename(p->pack_name);
4088 int i;
4089
4090 if (!p->pack_local)
4091 continue;
4092
4093 for (i = 0; i < names->nr; i++)
4094 if (!fspathcmp(name, names->items[i].string))
4095 break;
4096
4097 if (i < names->nr) {
4098 p->pack_keep_in_core = 1;
4099 ignore_packed_keep_in_core = 1;
4100 continue;
4101 }
4102 }
4103 }
4104
4105 static int option_parse_index_version(const struct option *opt,
4106 const char *arg, int unset)
4107 {
4108 char *c;
4109 const char *val = arg;
4110
4111 BUG_ON_OPT_NEG(unset);
4112
4113 pack_idx_opts.version = strtoul(val, &c, 10);
4114 if (pack_idx_opts.version > 2)
4115 die(_("unsupported index version %s"), val);
4116 if (*c == ',' && c[1])
4117 pack_idx_opts.off32_limit = strtoul(c+1, &c, 0);
4118 if (*c || pack_idx_opts.off32_limit & 0x80000000)
4119 die(_("bad index version '%s'"), val);
4120 return 0;
4121 }
4122
4123 static int option_parse_unpack_unreachable(const struct option *opt,
4124 const char *arg, int unset)
4125 {
4126 if (unset) {
4127 unpack_unreachable = 0;
4128 unpack_unreachable_expiration = 0;
4129 }
4130 else {
4131 unpack_unreachable = 1;
4132 if (arg)
4133 unpack_unreachable_expiration = approxidate(arg);
4134 }
4135 return 0;
4136 }
4137
4138 static int option_parse_cruft_expiration(const struct option *opt,
4139 const char *arg, int unset)
4140 {
4141 if (unset) {
4142 cruft = 0;
4143 cruft_expiration = 0;
4144 } else {
4145 cruft = 1;
4146 if (arg)
4147 cruft_expiration = approxidate(arg);
4148 }
4149 return 0;
4150 }
4151
4152 struct po_filter_data {
4153 unsigned have_revs:1;
4154 struct rev_info revs;
4155 };
4156
4157 static struct list_objects_filter_options *po_filter_revs_init(void *value)
4158 {
4159 struct po_filter_data *data = value;
4160
4161 repo_init_revisions(the_repository, &data->revs, NULL);
4162 data->have_revs = 1;
4163
4164 return &data->revs.filter;
4165 }
4166
4167 int cmd_pack_objects(int argc, const char **argv, const char *prefix)
4168 {
4169 int use_internal_rev_list = 0;
4170 int shallow = 0;
4171 int all_progress_implied = 0;
4172 struct strvec rp = STRVEC_INIT;
4173 int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0;
4174 int rev_list_index = 0;
4175 int stdin_packs = 0;
4176 struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
4177 struct po_filter_data pfd = { .have_revs = 0 };
4178
4179 struct option pack_objects_options[] = {
4180 OPT_SET_INT('q', "quiet", &progress,
4181 N_("do not show progress meter"), 0),
4182 OPT_SET_INT(0, "progress", &progress,
4183 N_("show progress meter"), 1),
4184 OPT_SET_INT(0, "all-progress", &progress,
4185 N_("show progress meter during object writing phase"), 2),
4186 OPT_BOOL(0, "all-progress-implied",
4187 &all_progress_implied,
4188 N_("similar to --all-progress when progress meter is shown")),
4189 OPT_CALLBACK_F(0, "index-version", NULL, N_("<version>[,<offset>]"),
4190 N_("write the pack index file in the specified idx format version"),
4191 PARSE_OPT_NONEG, option_parse_index_version),
4192 OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit,
4193 N_("maximum size of each output pack file")),
4194 OPT_BOOL(0, "local", &local,
4195 N_("ignore borrowed objects from alternate object store")),
4196 OPT_BOOL(0, "incremental", &incremental,
4197 N_("ignore packed objects")),
4198 OPT_INTEGER(0, "window", &window,
4199 N_("limit pack window by objects")),
4200 OPT_MAGNITUDE(0, "window-memory", &window_memory_limit,
4201 N_("limit pack window by memory in addition to object limit")),
4202 OPT_INTEGER(0, "depth", &depth,
4203 N_("maximum length of delta chain allowed in the resulting pack")),
4204 OPT_BOOL(0, "reuse-delta", &reuse_delta,
4205 N_("reuse existing deltas")),
4206 OPT_BOOL(0, "reuse-object", &reuse_object,
4207 N_("reuse existing objects")),
4208 OPT_BOOL(0, "delta-base-offset", &allow_ofs_delta,
4209 N_("use OFS_DELTA objects")),
4210 OPT_INTEGER(0, "threads", &delta_search_threads,
4211 N_("use threads when searching for best delta matches")),
4212 OPT_BOOL(0, "non-empty", &non_empty,
4213 N_("do not create an empty pack output")),
4214 OPT_BOOL(0, "revs", &use_internal_rev_list,
4215 N_("read revision arguments from standard input")),
4216 OPT_SET_INT_F(0, "unpacked", &rev_list_unpacked,
4217 N_("limit the objects to those that are not yet packed"),
4218 1, PARSE_OPT_NONEG),
4219 OPT_SET_INT_F(0, "all", &rev_list_all,
4220 N_("include objects reachable from any reference"),
4221 1, PARSE_OPT_NONEG),
4222 OPT_SET_INT_F(0, "reflog", &rev_list_reflog,
4223 N_("include objects referred by reflog entries"),
4224 1, PARSE_OPT_NONEG),
4225 OPT_SET_INT_F(0, "indexed-objects", &rev_list_index,
4226 N_("include objects referred to by the index"),
4227 1, PARSE_OPT_NONEG),
4228 OPT_BOOL(0, "stdin-packs", &stdin_packs,
4229 N_("read packs from stdin")),
4230 OPT_BOOL(0, "stdout", &pack_to_stdout,
4231 N_("output pack to stdout")),
4232 OPT_BOOL(0, "include-tag", &include_tag,
4233 N_("include tag objects that refer to objects to be packed")),
4234 OPT_BOOL(0, "keep-unreachable", &keep_unreachable,
4235 N_("keep unreachable objects")),
4236 OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable,
4237 N_("pack loose unreachable objects")),
4238 OPT_CALLBACK_F(0, "unpack-unreachable", NULL, N_("time"),
4239 N_("unpack unreachable objects newer than <time>"),
4240 PARSE_OPT_OPTARG, option_parse_unpack_unreachable),
4241 OPT_BOOL(0, "cruft", &cruft, N_("create a cruft pack")),
4242 OPT_CALLBACK_F(0, "cruft-expiration", NULL, N_("time"),
4243 N_("expire cruft objects older than <time>"),
4244 PARSE_OPT_OPTARG, option_parse_cruft_expiration),
4245 OPT_BOOL(0, "sparse", &sparse,
4246 N_("use the sparse reachability algorithm")),
4247 OPT_BOOL(0, "thin", &thin,
4248 N_("create thin packs")),
4249 OPT_BOOL(0, "shallow", &shallow,
4250 N_("create packs suitable for shallow fetches")),
4251 OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep_on_disk,
4252 N_("ignore packs that have companion .keep file")),
4253 OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
4254 N_("ignore this pack")),
4255 OPT_INTEGER(0, "compression", &pack_compression_level,
4256 N_("pack compression level")),
4257 OPT_SET_INT(0, "keep-true-parents", &grafts_replace_parents,
4258 N_("do not hide commits by grafts"), 0),
4259 OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index,
4260 N_("use a bitmap index if available to speed up counting objects")),
4261 OPT_SET_INT(0, "write-bitmap-index", &write_bitmap_index,
4262 N_("write a bitmap index together with the pack index"),
4263 WRITE_BITMAP_TRUE),
4264 OPT_SET_INT_F(0, "write-bitmap-index-quiet",
4265 &write_bitmap_index,
4266 N_("write a bitmap index if possible"),
4267 WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN),
4268 OPT_PARSE_LIST_OBJECTS_FILTER_INIT(&pfd, po_filter_revs_init),
4269 OPT_CALLBACK_F(0, "missing", NULL, N_("action"),
4270 N_("handling for missing objects"), PARSE_OPT_NONEG,
4271 option_parse_missing_action),
4272 OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
4273 N_("do not pack objects in promisor packfiles")),
4274 OPT_BOOL(0, "delta-islands", &use_delta_islands,
4275 N_("respect islands during delta compression")),
4276 OPT_STRING_LIST(0, "uri-protocol", &uri_protocols,
4277 N_("protocol"),
4278 N_("exclude any configured uploadpack.blobpackfileuri with this protocol")),
4279 OPT_END(),
4280 };
4281
4282 if (DFS_NUM_STATES > (1 << OE_DFS_STATE_BITS))
4283 BUG("too many dfs states, increase OE_DFS_STATE_BITS");
4284
4285 read_replace_refs = 0;
4286
4287 sparse = git_env_bool("GIT_TEST_PACK_SPARSE", -1);
4288 if (the_repository->gitdir) {
4289 prepare_repo_settings(the_repository);
4290 if (sparse < 0)
4291 sparse = the_repository->settings.pack_use_sparse;
4292 }
4293
4294 reset_pack_idx_option(&pack_idx_opts);
4295 git_config(git_pack_config, NULL);
4296 if (git_env_bool(GIT_TEST_WRITE_REV_INDEX, 0))
4297 pack_idx_opts.flags |= WRITE_REV;
4298
4299 progress = isatty(2);
4300 argc = parse_options(argc, argv, prefix, pack_objects_options,
4301 pack_usage, 0);
4302
4303 if (argc) {
4304 base_name = argv[0];
4305 argc--;
4306 }
4307 if (pack_to_stdout != !base_name || argc)
4308 usage_with_options(pack_usage, pack_objects_options);
4309
4310 if (depth < 0)
4311 depth = 0;
4312 if (depth >= (1 << OE_DEPTH_BITS)) {
4313 warning(_("delta chain depth %d is too deep, forcing %d"),
4314 depth, (1 << OE_DEPTH_BITS) - 1);
4315 depth = (1 << OE_DEPTH_BITS) - 1;
4316 }
4317 if (cache_max_small_delta_size >= (1U << OE_Z_DELTA_BITS)) {
4318 warning(_("pack.deltaCacheLimit is too high, forcing %d"),
4319 (1U << OE_Z_DELTA_BITS) - 1);
4320 cache_max_small_delta_size = (1U << OE_Z_DELTA_BITS) - 1;
4321 }
4322 if (window < 0)
4323 window = 0;
4324
4325 strvec_push(&rp, "pack-objects");
4326 if (thin) {
4327 use_internal_rev_list = 1;
4328 strvec_push(&rp, shallow
4329 ? "--objects-edge-aggressive"
4330 : "--objects-edge");
4331 } else
4332 strvec_push(&rp, "--objects");
4333
4334 if (rev_list_all) {
4335 use_internal_rev_list = 1;
4336 strvec_push(&rp, "--all");
4337 }
4338 if (rev_list_reflog) {
4339 use_internal_rev_list = 1;
4340 strvec_push(&rp, "--reflog");
4341 }
4342 if (rev_list_index) {
4343 use_internal_rev_list = 1;
4344 strvec_push(&rp, "--indexed-objects");
4345 }
4346 if (rev_list_unpacked && !stdin_packs) {
4347 use_internal_rev_list = 1;
4348 strvec_push(&rp, "--unpacked");
4349 }
4350
4351 if (exclude_promisor_objects) {
4352 use_internal_rev_list = 1;
4353 fetch_if_missing = 0;
4354 strvec_push(&rp, "--exclude-promisor-objects");
4355 }
4356 if (unpack_unreachable || keep_unreachable || pack_loose_unreachable)
4357 use_internal_rev_list = 1;
4358
4359 if (!reuse_object)
4360 reuse_delta = 0;
4361 if (pack_compression_level == -1)
4362 pack_compression_level = Z_DEFAULT_COMPRESSION;
4363 else if (pack_compression_level < 0 || pack_compression_level > Z_BEST_COMPRESSION)
4364 die(_("bad pack compression level %d"), pack_compression_level);
4365
4366 if (!delta_search_threads) /* --threads=0 means autodetect */
4367 delta_search_threads = online_cpus();
4368
4369 if (!HAVE_THREADS && delta_search_threads != 1)
4370 warning(_("no threads support, ignoring --threads"));
4371 if (!pack_to_stdout && !pack_size_limit && !cruft)
4372 pack_size_limit = pack_size_limit_cfg;
4373 if (pack_to_stdout && pack_size_limit)
4374 die(_("--max-pack-size cannot be used to build a pack for transfer"));
4375 if (pack_size_limit && pack_size_limit < 1024*1024) {
4376 warning(_("minimum pack size limit is 1 MiB"));
4377 pack_size_limit = 1024*1024;
4378 }
4379
4380 if (!pack_to_stdout && thin)
4381 die(_("--thin cannot be used to build an indexable pack"));
4382
4383 if (keep_unreachable && unpack_unreachable)
4384 die(_("options '%s' and '%s' cannot be used together"), "--keep-unreachable", "--unpack-unreachable");
4385 if (!rev_list_all || !rev_list_reflog || !rev_list_index)
4386 unpack_unreachable_expiration = 0;
4387
4388 if (pfd.have_revs && pfd.revs.filter.choice) {
4389 if (!pack_to_stdout)
4390 die(_("cannot use --filter without --stdout"));
4391 if (stdin_packs)
4392 die(_("cannot use --filter with --stdin-packs"));
4393 }
4394
4395 if (stdin_packs && use_internal_rev_list)
4396 die(_("cannot use internal rev list with --stdin-packs"));
4397
4398 if (cruft) {
4399 if (use_internal_rev_list)
4400 die(_("cannot use internal rev list with --cruft"));
4401 if (stdin_packs)
4402 die(_("cannot use --stdin-packs with --cruft"));
4403 if (pack_size_limit)
4404 die(_("cannot use --max-pack-size with --cruft"));
4405 }
4406
4407 /*
4408 * "soft" reasons not to use bitmaps - for on-disk repack by default we want
4409 *
4410 * - to produce good pack (with bitmap index not-yet-packed objects are
4411 * packed in suboptimal order).
4412 *
4413 * - to use more robust pack-generation codepath (avoiding possible
4414 * bugs in bitmap code and possible bitmap index corruption).
4415 */
4416 if (!pack_to_stdout)
4417 use_bitmap_index_default = 0;
4418
4419 if (use_bitmap_index < 0)
4420 use_bitmap_index = use_bitmap_index_default;
4421
4422 /* "hard" reasons not to use bitmaps; these just won't work at all */
4423 if (!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) || is_repository_shallow(the_repository))
4424 use_bitmap_index = 0;
4425
4426 if (pack_to_stdout || !rev_list_all)
4427 write_bitmap_index = 0;
4428
4429 if (use_delta_islands)
4430 strvec_push(&rp, "--topo-order");
4431
4432 if (progress && all_progress_implied)
4433 progress = 2;
4434
4435 add_extra_kept_packs(&keep_pack_list);
4436 if (ignore_packed_keep_on_disk) {
4437 struct packed_git *p;
4438 for (p = get_all_packs(the_repository); p; p = p->next)
4439 if (p->pack_local && p->pack_keep)
4440 break;
4441 if (!p) /* no keep-able packs found */
4442 ignore_packed_keep_on_disk = 0;
4443 }
4444 if (local) {
4445 /*
4446 * unlike ignore_packed_keep_on_disk above, we do not
4447 * want to unset "local" based on looking at packs, as
4448 * it also covers non-local objects
4449 */
4450 struct packed_git *p;
4451 for (p = get_all_packs(the_repository); p; p = p->next) {
4452 if (!p->pack_local) {
4453 have_non_local_packs = 1;
4454 break;
4455 }
4456 }
4457 }
4458
4459 trace2_region_enter("pack-objects", "enumerate-objects",
4460 the_repository);
4461 prepare_packing_data(the_repository, &to_pack);
4462
4463 if (progress && !cruft)
4464 progress_state = start_progress(_("Enumerating objects"), 0);
4465 if (stdin_packs) {
4466 /* avoids adding objects in excluded packs */
4467 ignore_packed_keep_in_core = 1;
4468 read_packs_list_from_stdin();
4469 if (rev_list_unpacked)
4470 add_unreachable_loose_objects();
4471 } else if (cruft) {
4472 read_cruft_objects();
4473 } else if (!use_internal_rev_list) {
4474 read_object_list_from_stdin();
4475 } else if (pfd.have_revs) {
4476 get_object_list(&pfd.revs, rp.nr, rp.v);
4477 release_revisions(&pfd.revs);
4478 } else {
4479 struct rev_info revs;
4480
4481 repo_init_revisions(the_repository, &revs, NULL);
4482 get_object_list(&revs, rp.nr, rp.v);
4483 release_revisions(&revs);
4484 }
4485 cleanup_preferred_base();
4486 if (include_tag && nr_result)
4487 for_each_tag_ref(add_ref_tag, NULL);
4488 stop_progress(&progress_state);
4489 trace2_region_leave("pack-objects", "enumerate-objects",
4490 the_repository);
4491
4492 if (non_empty && !nr_result)
4493 goto cleanup;
4494 if (nr_result) {
4495 trace2_region_enter("pack-objects", "prepare-pack",
4496 the_repository);
4497 prepare_pack(window, depth);
4498 trace2_region_leave("pack-objects", "prepare-pack",
4499 the_repository);
4500 }
4501
4502 trace2_region_enter("pack-objects", "write-pack-file", the_repository);
4503 write_excluded_by_configs();
4504 write_pack_file();
4505 trace2_region_leave("pack-objects", "write-pack-file", the_repository);
4506
4507 if (progress)
4508 fprintf_ln(stderr,
4509 _("Total %"PRIu32" (delta %"PRIu32"),"
4510 " reused %"PRIu32" (delta %"PRIu32"),"
4511 " pack-reused %"PRIu32),
4512 written, written_delta, reused, reused_delta,
4513 reuse_packfile_objects);
4514
4515 cleanup:
4516 strvec_clear(&rp);
4517
4518 return 0;
4519 }