2 #include "bulk-checkin.h"
4 #include "environment.h"
8 #include "object-store.h"
14 #include "replace-object.h"
17 #include "tree-walk.h"
22 static int dry_run
, quiet
, recover
, has_errors
, strict
;
23 static const char unpack_usage
[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
25 /* We always read in 4kB chunks. */
26 static unsigned char buffer
[4096];
27 static unsigned int offset
, len
;
28 static off_t consumed_bytes
;
29 static off_t max_input_size
;
30 static git_hash_ctx ctx
;
31 static struct fsck_options fsck_options
= FSCK_OPTIONS_STRICT
;
32 static struct progress
*progress
;
35 * When running under --strict mode, objects whose reachability are
36 * suspect are kept in core without getting written in the object
44 static struct decoration obj_decorate
;
46 static struct obj_buffer
*lookup_object_buffer(struct object
*base
)
48 return lookup_decoration(&obj_decorate
, base
);
51 static void add_object_buffer(struct object
*object
, char *buffer
, unsigned long size
)
53 struct obj_buffer
*obj
;
57 if (add_decoration(&obj_decorate
, object
, obj
))
58 die("object %s tried to add buffer twice!", oid_to_hex(&object
->oid
));
62 * Make sure at least "min" bytes are available in the buffer, and
63 * return the pointer to the buffer.
65 static void *fill(int min
)
68 return buffer
+ offset
;
69 if (min
> sizeof(buffer
))
70 die("cannot fill %d bytes", min
);
72 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
73 memmove(buffer
, buffer
+ offset
, len
);
77 ssize_t ret
= xread(0, buffer
+ len
, sizeof(buffer
) - len
);
81 die_errno("read error on input");
88 static void use(int bytes
)
91 die("used more bytes than were available");
95 /* make sure off_t is sufficiently large not to wrap */
96 if (signed_add_overflows(consumed_bytes
, bytes
))
97 die("pack too large for current definition of off_t");
98 consumed_bytes
+= bytes
;
99 if (max_input_size
&& consumed_bytes
> max_input_size
)
100 die(_("pack exceeds maximum allowed size"));
101 display_throughput(progress
, consumed_bytes
);
105 * Decompress zstream from the standard input into a newly
106 * allocated buffer of specified size and return the buffer.
107 * The caller is responsible to free the returned buffer.
109 * But for dry_run mode, "get_data()" is only used to check the
110 * integrity of data, and the returned buffer is not used at all.
111 * Therefore, in dry_run mode, "get_data()" will release the small
112 * allocated buffer which is reused to hold temporary zstream output
113 * and return NULL instead of returning garbage data.
115 static void *get_data(unsigned long size
)
118 unsigned long bufsize
= dry_run
&& size
> 8192 ? 8192 : size
;
119 void *buf
= xmallocz(bufsize
);
121 memset(&stream
, 0, sizeof(stream
));
123 stream
.next_out
= buf
;
124 stream
.avail_out
= bufsize
;
125 stream
.next_in
= fill(1);
126 stream
.avail_in
= len
;
127 git_inflate_init(&stream
);
130 int ret
= git_inflate(&stream
, 0);
131 use(len
- stream
.avail_in
);
132 if (stream
.total_out
== size
&& ret
== Z_STREAM_END
)
135 error("inflate returned %d", ret
);
142 stream
.next_in
= fill(1);
143 stream
.avail_in
= len
;
145 /* reuse the buffer in dry_run mode */
146 stream
.next_out
= buf
;
147 stream
.avail_out
= bufsize
> size
- stream
.total_out
?
148 size
- stream
.total_out
:
152 git_inflate_end(&stream
);
159 struct object_id base_oid
;
164 struct delta_info
*next
;
167 static struct delta_info
*delta_list
;
169 static void add_delta_to_list(unsigned nr
, const struct object_id
*base_oid
,
171 void *delta
, unsigned long size
)
173 struct delta_info
*info
= xmalloc(sizeof(*info
));
175 oidcpy(&info
->base_oid
, base_oid
);
176 info
->base_offset
= base_offset
;
180 info
->next
= delta_list
;
186 struct object_id oid
;
190 /* Remember to update object flag allocation in object.h */
191 #define FLAG_OPEN (1u<<20)
192 #define FLAG_WRITTEN (1u<<21)
194 static struct obj_info
*obj_list
;
195 static unsigned nr_objects
;
198 * Called only from check_object() after it verified this object
201 static void write_cached_object(struct object
*obj
, struct obj_buffer
*obj_buf
)
203 struct object_id oid
;
205 if (write_object_file(obj_buf
->buffer
, obj_buf
->size
,
206 obj
->type
, &oid
) < 0)
207 die("failed to write object %s", oid_to_hex(&obj
->oid
));
208 obj
->flags
|= FLAG_WRITTEN
;
212 * At the very end of the processing, write_rest() scans the objects
213 * that have reachability requirements and calls this function.
214 * Verify its reachability and validity recursively and write it out.
216 static int check_object(struct object
*obj
, enum object_type type
,
217 void *data
, struct fsck_options
*options
)
219 struct obj_buffer
*obj_buf
;
224 if (obj
->flags
& FLAG_WRITTEN
)
227 if (type
!= OBJ_ANY
&& obj
->type
!= type
)
228 die("object type mismatch");
230 if (!(obj
->flags
& FLAG_OPEN
)) {
232 int type
= oid_object_info(the_repository
, &obj
->oid
, &size
);
233 if (type
!= obj
->type
|| type
<= 0)
234 die("object of unexpected type");
235 obj
->flags
|= FLAG_WRITTEN
;
239 obj_buf
= lookup_object_buffer(obj
);
241 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj
->oid
));
242 if (fsck_object(obj
, obj_buf
->buffer
, obj_buf
->size
, &fsck_options
))
243 die("fsck error in packed object");
244 fsck_options
.walk
= check_object
;
245 if (fsck_walk(obj
, NULL
, &fsck_options
))
246 die("Error on reachable objects of %s", oid_to_hex(&obj
->oid
));
247 write_cached_object(obj
, obj_buf
);
251 static void write_rest(void)
254 for (i
= 0; i
< nr_objects
; i
++) {
256 check_object(obj_list
[i
].obj
, OBJ_ANY
, NULL
, NULL
);
260 static void added_object(unsigned nr
, enum object_type type
,
261 void *data
, unsigned long size
);
264 * Write out nr-th object from the list, now we know the contents
265 * of it. Under --strict, this buffers structured objects in-core,
266 * to be checked at the end.
268 static void write_object(unsigned nr
, enum object_type type
,
269 void *buf
, unsigned long size
)
272 if (write_object_file(buf
, size
, type
,
273 &obj_list
[nr
].oid
) < 0)
274 die("failed to write object");
275 added_object(nr
, type
, buf
, size
);
277 obj_list
[nr
].obj
= NULL
;
278 } else if (type
== OBJ_BLOB
) {
280 if (write_object_file(buf
, size
, type
,
281 &obj_list
[nr
].oid
) < 0)
282 die("failed to write object");
283 added_object(nr
, type
, buf
, size
);
286 blob
= lookup_blob(the_repository
, &obj_list
[nr
].oid
);
288 blob
->object
.flags
|= FLAG_WRITTEN
;
290 die("invalid blob object");
291 obj_list
[nr
].obj
= NULL
;
295 hash_object_file(the_hash_algo
, buf
, size
, type
,
297 added_object(nr
, type
, buf
, size
);
298 obj
= parse_object_buffer(the_repository
, &obj_list
[nr
].oid
,
302 die("invalid %s", type_name(type
));
303 add_object_buffer(obj
, buf
, size
);
304 obj
->flags
|= FLAG_OPEN
;
305 obj_list
[nr
].obj
= obj
;
309 static void resolve_delta(unsigned nr
, enum object_type type
,
310 void *base
, unsigned long base_size
,
311 void *delta
, unsigned long delta_size
)
314 unsigned long result_size
;
316 result
= patch_delta(base
, base_size
,
320 die("failed to apply delta");
322 write_object(nr
, type
, result
, result_size
);
326 * We now know the contents of an object (which is nr-th in the pack);
327 * resolve all the deltified objects that are based on it.
329 static void added_object(unsigned nr
, enum object_type type
,
330 void *data
, unsigned long size
)
332 struct delta_info
**p
= &delta_list
;
333 struct delta_info
*info
;
335 while ((info
= *p
) != NULL
) {
336 if (oideq(&info
->base_oid
, &obj_list
[nr
].oid
) ||
337 info
->base_offset
== obj_list
[nr
].offset
) {
340 resolve_delta(info
->nr
, type
, data
, size
,
341 info
->delta
, info
->size
);
349 static void unpack_non_delta_entry(enum object_type type
, unsigned long size
,
352 void *buf
= get_data(size
);
355 write_object(nr
, type
, buf
, size
);
358 struct input_zstream_data
{
359 git_zstream
*zstream
;
360 unsigned char buf
[8192];
364 static const void *feed_input_zstream(struct input_stream
*in_stream
,
365 unsigned long *readlen
)
367 struct input_zstream_data
*data
= in_stream
->data
;
368 git_zstream
*zstream
= data
->zstream
;
371 if (in_stream
->is_finished
) {
376 zstream
->next_out
= data
->buf
;
377 zstream
->avail_out
= sizeof(data
->buf
);
378 zstream
->next_in
= in
;
379 zstream
->avail_in
= len
;
381 data
->status
= git_inflate(zstream
, 0);
383 in_stream
->is_finished
= data
->status
!= Z_OK
;
384 use(len
- zstream
->avail_in
);
385 *readlen
= sizeof(data
->buf
) - zstream
->avail_out
;
390 static void stream_blob(unsigned long size
, unsigned nr
)
392 git_zstream zstream
= { 0 };
393 struct input_zstream_data data
= { 0 };
394 struct input_stream in_stream
= {
395 .read
= feed_input_zstream
,
398 struct obj_info
*info
= &obj_list
[nr
];
400 data
.zstream
= &zstream
;
401 git_inflate_init(&zstream
);
403 if (stream_loose_object(&in_stream
, size
, &info
->oid
))
404 die(_("failed to write object in stream"));
406 if (data
.status
!= Z_STREAM_END
)
407 die(_("inflate returned (%d)"), data
.status
);
408 git_inflate_end(&zstream
);
411 struct blob
*blob
= lookup_blob(the_repository
, &info
->oid
);
414 die(_("invalid blob object from stream"));
415 blob
->object
.flags
|= FLAG_WRITTEN
;
420 static int resolve_against_held(unsigned nr
, const struct object_id
*base
,
421 void *delta_data
, unsigned long delta_size
)
424 struct obj_buffer
*obj_buffer
;
425 obj
= lookup_object(the_repository
, base
);
428 obj_buffer
= lookup_object_buffer(obj
);
431 resolve_delta(nr
, obj
->type
, obj_buffer
->buffer
,
432 obj_buffer
->size
, delta_data
, delta_size
);
436 static void unpack_delta_entry(enum object_type type
, unsigned long delta_size
,
439 void *delta_data
, *base
;
440 unsigned long base_size
;
441 struct object_id base_oid
;
443 if (type
== OBJ_REF_DELTA
) {
444 oidread(&base_oid
, fill(the_hash_algo
->rawsz
));
445 use(the_hash_algo
->rawsz
);
446 delta_data
= get_data(delta_size
);
449 if (repo_has_object_file(the_repository
, &base_oid
))
450 ; /* Ok we have this one */
451 else if (resolve_against_held(nr
, &base_oid
,
452 delta_data
, delta_size
))
453 return; /* we are done */
455 /* cannot resolve yet --- queue it */
456 oidclr(&obj_list
[nr
].oid
);
457 add_delta_to_list(nr
, &base_oid
, 0, delta_data
, delta_size
);
461 unsigned base_found
= 0;
462 unsigned char *pack
, c
;
464 unsigned lo
, mid
, hi
;
469 base_offset
= c
& 127;
472 if (!base_offset
|| MSB(base_offset
, 7))
473 die("offset value overflow for delta base object");
477 base_offset
= (base_offset
<< 7) + (c
& 127);
479 base_offset
= obj_list
[nr
].offset
- base_offset
;
480 if (base_offset
<= 0 || base_offset
>= obj_list
[nr
].offset
)
481 die("offset value out of bound for delta base object");
483 delta_data
= get_data(delta_size
);
489 mid
= lo
+ (hi
- lo
) / 2;
490 if (base_offset
< obj_list
[mid
].offset
) {
492 } else if (base_offset
> obj_list
[mid
].offset
) {
495 oidcpy(&base_oid
, &obj_list
[mid
].oid
);
496 base_found
= !is_null_oid(&base_oid
);
502 * The delta base object is itself a delta that
503 * has not been resolved yet.
505 oidclr(&obj_list
[nr
].oid
);
506 add_delta_to_list(nr
, null_oid(), base_offset
,
507 delta_data
, delta_size
);
512 if (resolve_against_held(nr
, &base_oid
, delta_data
, delta_size
))
515 base
= repo_read_object_file(the_repository
, &base_oid
, &type
,
518 error("failed to read delta-pack base object %s",
519 oid_to_hex(&base_oid
));
525 resolve_delta(nr
, type
, base
, base_size
, delta_data
, delta_size
);
529 static void unpack_one(unsigned nr
)
533 unsigned long size
, c
;
534 enum object_type type
;
536 obj_list
[nr
].offset
= consumed_bytes
;
548 size
+= (c
& 0x7f) << shift
;
554 if (!dry_run
&& size
> big_file_threshold
) {
555 stream_blob(size
, nr
);
562 unpack_non_delta_entry(type
, size
, nr
);
566 unpack_delta_entry(type
, size
, nr
);
569 error("bad object type %d", type
);
577 static void unpack_all(void)
580 struct pack_header
*hdr
= fill(sizeof(struct pack_header
));
582 nr_objects
= ntohl(hdr
->hdr_entries
);
584 if (ntohl(hdr
->hdr_signature
) != PACK_SIGNATURE
)
585 die("bad pack file");
586 if (!pack_version_ok(hdr
->hdr_version
))
587 die("unknown pack file version %"PRIu32
,
588 ntohl(hdr
->hdr_version
));
589 use(sizeof(struct pack_header
));
592 progress
= start_progress(_("Unpacking objects"), nr_objects
);
593 CALLOC_ARRAY(obj_list
, nr_objects
);
594 begin_odb_transaction();
595 for (i
= 0; i
< nr_objects
; i
++) {
597 display_progress(progress
, i
+ 1);
599 end_odb_transaction();
600 stop_progress(&progress
);
603 die("unresolved deltas left after unpacking");
606 int cmd_unpack_objects(int argc
, const char **argv
, const char *prefix UNUSED
)
609 struct object_id oid
;
611 read_replace_refs
= 0;
613 git_config(git_default_config
, NULL
);
617 for (i
= 1 ; i
< argc
; i
++) {
618 const char *arg
= argv
[i
];
621 if (!strcmp(arg
, "-n")) {
625 if (!strcmp(arg
, "-q")) {
629 if (!strcmp(arg
, "-r")) {
633 if (!strcmp(arg
, "--strict")) {
637 if (skip_prefix(arg
, "--strict=", &arg
)) {
639 fsck_set_msg_types(&fsck_options
, arg
);
642 if (starts_with(arg
, "--pack_header=")) {
643 struct pack_header
*hdr
;
646 hdr
= (struct pack_header
*)buffer
;
647 hdr
->hdr_signature
= htonl(PACK_SIGNATURE
);
648 hdr
->hdr_version
= htonl(strtoul(arg
+ 14, &c
, 10));
651 hdr
->hdr_entries
= htonl(strtoul(c
+ 1, &c
, 10));
657 if (skip_prefix(arg
, "--max-input-size=", &arg
)) {
658 max_input_size
= strtoumax(arg
, NULL
, 10);
664 /* We don't take any non-flag arguments now.. Maybe some day */
667 the_hash_algo
->init_fn(&ctx
);
669 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
670 the_hash_algo
->final_oid_fn(&oid
, &ctx
);
673 if (fsck_finish(&fsck_options
))
674 die(_("fsck error in pack objects"));
676 if (!hasheq(fill(the_hash_algo
->rawsz
), oid
.hash
))
677 die("final sha1 did not match");
678 use(the_hash_algo
->rawsz
);
680 /* Write the last part of the buffer to stdout */
682 int ret
= xwrite(1, buffer
+ offset
, len
);