3 #include "bulk-checkin.h"
6 #include "object-store.h"
12 #include "replace-object.h"
15 #include "tree-walk.h"
20 static int dry_run
, quiet
, recover
, has_errors
, strict
;
21 static const char unpack_usage
[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
23 /* We always read in 4kB chunks. */
24 static unsigned char buffer
[4096];
25 static unsigned int offset
, len
;
26 static off_t consumed_bytes
;
27 static off_t max_input_size
;
28 static git_hash_ctx ctx
;
29 static struct fsck_options fsck_options
= FSCK_OPTIONS_STRICT
;
30 static struct progress
*progress
;
33 * When running under --strict mode, objects whose reachability are
34 * suspect are kept in core without getting written in the object
42 static struct decoration obj_decorate
;
44 static struct obj_buffer
*lookup_object_buffer(struct object
*base
)
46 return lookup_decoration(&obj_decorate
, base
);
49 static void add_object_buffer(struct object
*object
, char *buffer
, unsigned long size
)
51 struct obj_buffer
*obj
;
55 if (add_decoration(&obj_decorate
, object
, obj
))
56 die("object %s tried to add buffer twice!", oid_to_hex(&object
->oid
));
60 * Make sure at least "min" bytes are available in the buffer, and
61 * return the pointer to the buffer.
63 static void *fill(int min
)
66 return buffer
+ offset
;
67 if (min
> sizeof(buffer
))
68 die("cannot fill %d bytes", min
);
70 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
71 memmove(buffer
, buffer
+ offset
, len
);
75 ssize_t ret
= xread(0, buffer
+ len
, sizeof(buffer
) - len
);
79 die_errno("read error on input");
86 static void use(int bytes
)
89 die("used more bytes than were available");
93 /* make sure off_t is sufficiently large not to wrap */
94 if (signed_add_overflows(consumed_bytes
, bytes
))
95 die("pack too large for current definition of off_t");
96 consumed_bytes
+= bytes
;
97 if (max_input_size
&& consumed_bytes
> max_input_size
)
98 die(_("pack exceeds maximum allowed size"));
99 display_throughput(progress
, consumed_bytes
);
103 * Decompress zstream from the standard input into a newly
104 * allocated buffer of specified size and return the buffer.
105 * The caller is responsible to free the returned buffer.
107 * But for dry_run mode, "get_data()" is only used to check the
108 * integrity of data, and the returned buffer is not used at all.
109 * Therefore, in dry_run mode, "get_data()" will release the small
110 * allocated buffer which is reused to hold temporary zstream output
111 * and return NULL instead of returning garbage data.
113 static void *get_data(unsigned long size
)
116 unsigned long bufsize
= dry_run
&& size
> 8192 ? 8192 : size
;
117 void *buf
= xmallocz(bufsize
);
119 memset(&stream
, 0, sizeof(stream
));
121 stream
.next_out
= buf
;
122 stream
.avail_out
= bufsize
;
123 stream
.next_in
= fill(1);
124 stream
.avail_in
= len
;
125 git_inflate_init(&stream
);
128 int ret
= git_inflate(&stream
, 0);
129 use(len
- stream
.avail_in
);
130 if (stream
.total_out
== size
&& ret
== Z_STREAM_END
)
133 error("inflate returned %d", ret
);
140 stream
.next_in
= fill(1);
141 stream
.avail_in
= len
;
143 /* reuse the buffer in dry_run mode */
144 stream
.next_out
= buf
;
145 stream
.avail_out
= bufsize
> size
- stream
.total_out
?
146 size
- stream
.total_out
:
150 git_inflate_end(&stream
);
157 struct object_id base_oid
;
162 struct delta_info
*next
;
165 static struct delta_info
*delta_list
;
167 static void add_delta_to_list(unsigned nr
, const struct object_id
*base_oid
,
169 void *delta
, unsigned long size
)
171 struct delta_info
*info
= xmalloc(sizeof(*info
));
173 oidcpy(&info
->base_oid
, base_oid
);
174 info
->base_offset
= base_offset
;
178 info
->next
= delta_list
;
184 struct object_id oid
;
188 /* Remember to update object flag allocation in object.h */
189 #define FLAG_OPEN (1u<<20)
190 #define FLAG_WRITTEN (1u<<21)
192 static struct obj_info
*obj_list
;
193 static unsigned nr_objects
;
196 * Called only from check_object() after it verified this object
199 static void write_cached_object(struct object
*obj
, struct obj_buffer
*obj_buf
)
201 struct object_id oid
;
203 if (write_object_file(obj_buf
->buffer
, obj_buf
->size
,
204 obj
->type
, &oid
) < 0)
205 die("failed to write object %s", oid_to_hex(&obj
->oid
));
206 obj
->flags
|= FLAG_WRITTEN
;
210 * At the very end of the processing, write_rest() scans the objects
211 * that have reachability requirements and calls this function.
212 * Verify its reachability and validity recursively and write it out.
214 static int check_object(struct object
*obj
, enum object_type type
,
215 void *data
, struct fsck_options
*options
)
217 struct obj_buffer
*obj_buf
;
222 if (obj
->flags
& FLAG_WRITTEN
)
225 if (type
!= OBJ_ANY
&& obj
->type
!= type
)
226 die("object type mismatch");
228 if (!(obj
->flags
& FLAG_OPEN
)) {
230 int type
= oid_object_info(the_repository
, &obj
->oid
, &size
);
231 if (type
!= obj
->type
|| type
<= 0)
232 die("object of unexpected type");
233 obj
->flags
|= FLAG_WRITTEN
;
237 obj_buf
= lookup_object_buffer(obj
);
239 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj
->oid
));
240 if (fsck_object(obj
, obj_buf
->buffer
, obj_buf
->size
, &fsck_options
))
241 die("fsck error in packed object");
242 fsck_options
.walk
= check_object
;
243 if (fsck_walk(obj
, NULL
, &fsck_options
))
244 die("Error on reachable objects of %s", oid_to_hex(&obj
->oid
));
245 write_cached_object(obj
, obj_buf
);
249 static void write_rest(void)
252 for (i
= 0; i
< nr_objects
; i
++) {
254 check_object(obj_list
[i
].obj
, OBJ_ANY
, NULL
, NULL
);
258 static void added_object(unsigned nr
, enum object_type type
,
259 void *data
, unsigned long size
);
262 * Write out nr-th object from the list, now we know the contents
263 * of it. Under --strict, this buffers structured objects in-core,
264 * to be checked at the end.
266 static void write_object(unsigned nr
, enum object_type type
,
267 void *buf
, unsigned long size
)
270 if (write_object_file(buf
, size
, type
,
271 &obj_list
[nr
].oid
) < 0)
272 die("failed to write object");
273 added_object(nr
, type
, buf
, size
);
275 obj_list
[nr
].obj
= NULL
;
276 } else if (type
== OBJ_BLOB
) {
278 if (write_object_file(buf
, size
, type
,
279 &obj_list
[nr
].oid
) < 0)
280 die("failed to write object");
281 added_object(nr
, type
, buf
, size
);
284 blob
= lookup_blob(the_repository
, &obj_list
[nr
].oid
);
286 blob
->object
.flags
|= FLAG_WRITTEN
;
288 die("invalid blob object");
289 obj_list
[nr
].obj
= NULL
;
293 hash_object_file(the_hash_algo
, buf
, size
, type
,
295 added_object(nr
, type
, buf
, size
);
296 obj
= parse_object_buffer(the_repository
, &obj_list
[nr
].oid
,
300 die("invalid %s", type_name(type
));
301 add_object_buffer(obj
, buf
, size
);
302 obj
->flags
|= FLAG_OPEN
;
303 obj_list
[nr
].obj
= obj
;
307 static void resolve_delta(unsigned nr
, enum object_type type
,
308 void *base
, unsigned long base_size
,
309 void *delta
, unsigned long delta_size
)
312 unsigned long result_size
;
314 result
= patch_delta(base
, base_size
,
318 die("failed to apply delta");
320 write_object(nr
, type
, result
, result_size
);
324 * We now know the contents of an object (which is nr-th in the pack);
325 * resolve all the deltified objects that are based on it.
327 static void added_object(unsigned nr
, enum object_type type
,
328 void *data
, unsigned long size
)
330 struct delta_info
**p
= &delta_list
;
331 struct delta_info
*info
;
333 while ((info
= *p
) != NULL
) {
334 if (oideq(&info
->base_oid
, &obj_list
[nr
].oid
) ||
335 info
->base_offset
== obj_list
[nr
].offset
) {
338 resolve_delta(info
->nr
, type
, data
, size
,
339 info
->delta
, info
->size
);
347 static void unpack_non_delta_entry(enum object_type type
, unsigned long size
,
350 void *buf
= get_data(size
);
353 write_object(nr
, type
, buf
, size
);
356 struct input_zstream_data
{
357 git_zstream
*zstream
;
358 unsigned char buf
[8192];
362 static const void *feed_input_zstream(struct input_stream
*in_stream
,
363 unsigned long *readlen
)
365 struct input_zstream_data
*data
= in_stream
->data
;
366 git_zstream
*zstream
= data
->zstream
;
369 if (in_stream
->is_finished
) {
374 zstream
->next_out
= data
->buf
;
375 zstream
->avail_out
= sizeof(data
->buf
);
376 zstream
->next_in
= in
;
377 zstream
->avail_in
= len
;
379 data
->status
= git_inflate(zstream
, 0);
381 in_stream
->is_finished
= data
->status
!= Z_OK
;
382 use(len
- zstream
->avail_in
);
383 *readlen
= sizeof(data
->buf
) - zstream
->avail_out
;
388 static void stream_blob(unsigned long size
, unsigned nr
)
390 git_zstream zstream
= { 0 };
391 struct input_zstream_data data
= { 0 };
392 struct input_stream in_stream
= {
393 .read
= feed_input_zstream
,
396 struct obj_info
*info
= &obj_list
[nr
];
398 data
.zstream
= &zstream
;
399 git_inflate_init(&zstream
);
401 if (stream_loose_object(&in_stream
, size
, &info
->oid
))
402 die(_("failed to write object in stream"));
404 if (data
.status
!= Z_STREAM_END
)
405 die(_("inflate returned (%d)"), data
.status
);
406 git_inflate_end(&zstream
);
409 struct blob
*blob
= lookup_blob(the_repository
, &info
->oid
);
412 die(_("invalid blob object from stream"));
413 blob
->object
.flags
|= FLAG_WRITTEN
;
418 static int resolve_against_held(unsigned nr
, const struct object_id
*base
,
419 void *delta_data
, unsigned long delta_size
)
422 struct obj_buffer
*obj_buffer
;
423 obj
= lookup_object(the_repository
, base
);
426 obj_buffer
= lookup_object_buffer(obj
);
429 resolve_delta(nr
, obj
->type
, obj_buffer
->buffer
,
430 obj_buffer
->size
, delta_data
, delta_size
);
434 static void unpack_delta_entry(enum object_type type
, unsigned long delta_size
,
437 void *delta_data
, *base
;
438 unsigned long base_size
;
439 struct object_id base_oid
;
441 if (type
== OBJ_REF_DELTA
) {
442 oidread(&base_oid
, fill(the_hash_algo
->rawsz
));
443 use(the_hash_algo
->rawsz
);
444 delta_data
= get_data(delta_size
);
447 if (has_object_file(&base_oid
))
448 ; /* Ok we have this one */
449 else if (resolve_against_held(nr
, &base_oid
,
450 delta_data
, delta_size
))
451 return; /* we are done */
453 /* cannot resolve yet --- queue it */
454 oidclr(&obj_list
[nr
].oid
);
455 add_delta_to_list(nr
, &base_oid
, 0, delta_data
, delta_size
);
459 unsigned base_found
= 0;
460 unsigned char *pack
, c
;
462 unsigned lo
, mid
, hi
;
467 base_offset
= c
& 127;
470 if (!base_offset
|| MSB(base_offset
, 7))
471 die("offset value overflow for delta base object");
475 base_offset
= (base_offset
<< 7) + (c
& 127);
477 base_offset
= obj_list
[nr
].offset
- base_offset
;
478 if (base_offset
<= 0 || base_offset
>= obj_list
[nr
].offset
)
479 die("offset value out of bound for delta base object");
481 delta_data
= get_data(delta_size
);
487 mid
= lo
+ (hi
- lo
) / 2;
488 if (base_offset
< obj_list
[mid
].offset
) {
490 } else if (base_offset
> obj_list
[mid
].offset
) {
493 oidcpy(&base_oid
, &obj_list
[mid
].oid
);
494 base_found
= !is_null_oid(&base_oid
);
500 * The delta base object is itself a delta that
501 * has not been resolved yet.
503 oidclr(&obj_list
[nr
].oid
);
504 add_delta_to_list(nr
, null_oid(), base_offset
,
505 delta_data
, delta_size
);
510 if (resolve_against_held(nr
, &base_oid
, delta_data
, delta_size
))
513 base
= read_object_file(&base_oid
, &type
, &base_size
);
515 error("failed to read delta-pack base object %s",
516 oid_to_hex(&base_oid
));
522 resolve_delta(nr
, type
, base
, base_size
, delta_data
, delta_size
);
526 static void unpack_one(unsigned nr
)
530 unsigned long size
, c
;
531 enum object_type type
;
533 obj_list
[nr
].offset
= consumed_bytes
;
545 size
+= (c
& 0x7f) << shift
;
551 if (!dry_run
&& size
> big_file_threshold
) {
552 stream_blob(size
, nr
);
559 unpack_non_delta_entry(type
, size
, nr
);
563 unpack_delta_entry(type
, size
, nr
);
566 error("bad object type %d", type
);
574 static void unpack_all(void)
577 struct pack_header
*hdr
= fill(sizeof(struct pack_header
));
579 nr_objects
= ntohl(hdr
->hdr_entries
);
581 if (ntohl(hdr
->hdr_signature
) != PACK_SIGNATURE
)
582 die("bad pack file");
583 if (!pack_version_ok(hdr
->hdr_version
))
584 die("unknown pack file version %"PRIu32
,
585 ntohl(hdr
->hdr_version
));
586 use(sizeof(struct pack_header
));
589 progress
= start_progress(_("Unpacking objects"), nr_objects
);
590 CALLOC_ARRAY(obj_list
, nr_objects
);
591 begin_odb_transaction();
592 for (i
= 0; i
< nr_objects
; i
++) {
594 display_progress(progress
, i
+ 1);
596 end_odb_transaction();
597 stop_progress(&progress
);
600 die("unresolved deltas left after unpacking");
603 int cmd_unpack_objects(int argc
, const char **argv
, const char *prefix
)
606 struct object_id oid
;
608 read_replace_refs
= 0;
610 git_config(git_default_config
, NULL
);
614 for (i
= 1 ; i
< argc
; i
++) {
615 const char *arg
= argv
[i
];
618 if (!strcmp(arg
, "-n")) {
622 if (!strcmp(arg
, "-q")) {
626 if (!strcmp(arg
, "-r")) {
630 if (!strcmp(arg
, "--strict")) {
634 if (skip_prefix(arg
, "--strict=", &arg
)) {
636 fsck_set_msg_types(&fsck_options
, arg
);
639 if (starts_with(arg
, "--pack_header=")) {
640 struct pack_header
*hdr
;
643 hdr
= (struct pack_header
*)buffer
;
644 hdr
->hdr_signature
= htonl(PACK_SIGNATURE
);
645 hdr
->hdr_version
= htonl(strtoul(arg
+ 14, &c
, 10));
648 hdr
->hdr_entries
= htonl(strtoul(c
+ 1, &c
, 10));
654 if (skip_prefix(arg
, "--max-input-size=", &arg
)) {
655 max_input_size
= strtoumax(arg
, NULL
, 10);
661 /* We don't take any non-flag arguments now.. Maybe some day */
664 the_hash_algo
->init_fn(&ctx
);
666 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
667 the_hash_algo
->final_oid_fn(&oid
, &ctx
);
670 if (fsck_finish(&fsck_options
))
671 die(_("fsck error in pack objects"));
673 if (!hasheq(fill(the_hash_algo
->rawsz
), oid
.hash
))
674 die("final sha1 did not match");
675 use(the_hash_algo
->rawsz
);
677 /* Write the last part of the buffer to stdout */
679 int ret
= xwrite(1, buffer
+ offset
, len
);