3 #include "bulk-checkin.h"
5 #include "object-store.h"
13 #include "tree-walk.h"
18 static int dry_run
, quiet
, recover
, has_errors
, strict
;
19 static const char unpack_usage
[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
21 /* We always read in 4kB chunks. */
22 static unsigned char buffer
[4096];
23 static unsigned int offset
, len
;
24 static off_t consumed_bytes
;
25 static off_t max_input_size
;
26 static git_hash_ctx ctx
;
27 static struct fsck_options fsck_options
= FSCK_OPTIONS_STRICT
;
28 static struct progress
*progress
;
31 * When running under --strict mode, objects whose reachability are
32 * suspect are kept in core without getting written in the object
40 static struct decoration obj_decorate
;
42 static struct obj_buffer
*lookup_object_buffer(struct object
*base
)
44 return lookup_decoration(&obj_decorate
, base
);
47 static void add_object_buffer(struct object
*object
, char *buffer
, unsigned long size
)
49 struct obj_buffer
*obj
;
53 if (add_decoration(&obj_decorate
, object
, obj
))
54 die("object %s tried to add buffer twice!", oid_to_hex(&object
->oid
));
58 * Make sure at least "min" bytes are available in the buffer, and
59 * return the pointer to the buffer.
61 static void *fill(int min
)
64 return buffer
+ offset
;
65 if (min
> sizeof(buffer
))
66 die("cannot fill %d bytes", min
);
68 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
69 memmove(buffer
, buffer
+ offset
, len
);
73 ssize_t ret
= xread(0, buffer
+ len
, sizeof(buffer
) - len
);
77 die_errno("read error on input");
84 static void use(int bytes
)
87 die("used more bytes than were available");
91 /* make sure off_t is sufficiently large not to wrap */
92 if (signed_add_overflows(consumed_bytes
, bytes
))
93 die("pack too large for current definition of off_t");
94 consumed_bytes
+= bytes
;
95 if (max_input_size
&& consumed_bytes
> max_input_size
)
96 die(_("pack exceeds maximum allowed size"));
97 display_throughput(progress
, consumed_bytes
);
100 static void *get_data(unsigned long size
)
103 void *buf
= xmallocz(size
);
105 memset(&stream
, 0, sizeof(stream
));
107 stream
.next_out
= buf
;
108 stream
.avail_out
= size
;
109 stream
.next_in
= fill(1);
110 stream
.avail_in
= len
;
111 git_inflate_init(&stream
);
114 int ret
= git_inflate(&stream
, 0);
115 use(len
- stream
.avail_in
);
116 if (stream
.total_out
== size
&& ret
== Z_STREAM_END
)
119 error("inflate returned %d", ret
);
126 stream
.next_in
= fill(1);
127 stream
.avail_in
= len
;
129 git_inflate_end(&stream
);
134 struct object_id base_oid
;
139 struct delta_info
*next
;
142 static struct delta_info
*delta_list
;
144 static void add_delta_to_list(unsigned nr
, const struct object_id
*base_oid
,
146 void *delta
, unsigned long size
)
148 struct delta_info
*info
= xmalloc(sizeof(*info
));
150 oidcpy(&info
->base_oid
, base_oid
);
151 info
->base_offset
= base_offset
;
155 info
->next
= delta_list
;
161 struct object_id oid
;
165 /* Remember to update object flag allocation in object.h */
166 #define FLAG_OPEN (1u<<20)
167 #define FLAG_WRITTEN (1u<<21)
169 static struct obj_info
*obj_list
;
170 static unsigned nr_objects
;
173 * Called only from check_object() after it verified this object
176 static void write_cached_object(struct object
*obj
, struct obj_buffer
*obj_buf
)
178 struct object_id oid
;
180 if (write_object_file(obj_buf
->buffer
, obj_buf
->size
,
181 obj
->type
, &oid
) < 0)
182 die("failed to write object %s", oid_to_hex(&obj
->oid
));
183 obj
->flags
|= FLAG_WRITTEN
;
187 * At the very end of the processing, write_rest() scans the objects
188 * that have reachability requirements and calls this function.
189 * Verify its reachability and validity recursively and write it out.
191 static int check_object(struct object
*obj
, enum object_type type
,
192 void *data
, struct fsck_options
*options
)
194 struct obj_buffer
*obj_buf
;
199 if (obj
->flags
& FLAG_WRITTEN
)
202 if (type
!= OBJ_ANY
&& obj
->type
!= type
)
203 die("object type mismatch");
205 if (!(obj
->flags
& FLAG_OPEN
)) {
207 int type
= oid_object_info(the_repository
, &obj
->oid
, &size
);
208 if (type
!= obj
->type
|| type
<= 0)
209 die("object of unexpected type");
210 obj
->flags
|= FLAG_WRITTEN
;
214 obj_buf
= lookup_object_buffer(obj
);
216 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj
->oid
));
217 if (fsck_object(obj
, obj_buf
->buffer
, obj_buf
->size
, &fsck_options
))
218 die("fsck error in packed object");
219 fsck_options
.walk
= check_object
;
220 if (fsck_walk(obj
, NULL
, &fsck_options
))
221 die("Error on reachable objects of %s", oid_to_hex(&obj
->oid
));
222 write_cached_object(obj
, obj_buf
);
226 static void write_rest(void)
229 for (i
= 0; i
< nr_objects
; i
++) {
231 check_object(obj_list
[i
].obj
, OBJ_ANY
, NULL
, NULL
);
235 static void added_object(unsigned nr
, enum object_type type
,
236 void *data
, unsigned long size
);
239 * Write out nr-th object from the list, now we know the contents
240 * of it. Under --strict, this buffers structured objects in-core,
241 * to be checked at the end.
243 static void write_object(unsigned nr
, enum object_type type
,
244 void *buf
, unsigned long size
)
247 if (write_object_file(buf
, size
, type
,
248 &obj_list
[nr
].oid
) < 0)
249 die("failed to write object");
250 added_object(nr
, type
, buf
, size
);
252 obj_list
[nr
].obj
= NULL
;
253 } else if (type
== OBJ_BLOB
) {
255 if (write_object_file(buf
, size
, type
,
256 &obj_list
[nr
].oid
) < 0)
257 die("failed to write object");
258 added_object(nr
, type
, buf
, size
);
261 blob
= lookup_blob(the_repository
, &obj_list
[nr
].oid
);
263 blob
->object
.flags
|= FLAG_WRITTEN
;
265 die("invalid blob object");
266 obj_list
[nr
].obj
= NULL
;
270 hash_object_file(the_hash_algo
, buf
, size
, type
,
272 added_object(nr
, type
, buf
, size
);
273 obj
= parse_object_buffer(the_repository
, &obj_list
[nr
].oid
,
277 die("invalid %s", type_name(type
));
278 add_object_buffer(obj
, buf
, size
);
279 obj
->flags
|= FLAG_OPEN
;
280 obj_list
[nr
].obj
= obj
;
284 static void resolve_delta(unsigned nr
, enum object_type type
,
285 void *base
, unsigned long base_size
,
286 void *delta
, unsigned long delta_size
)
289 unsigned long result_size
;
291 result
= patch_delta(base
, base_size
,
295 die("failed to apply delta");
297 write_object(nr
, type
, result
, result_size
);
301 * We now know the contents of an object (which is nr-th in the pack);
302 * resolve all the deltified objects that are based on it.
304 static void added_object(unsigned nr
, enum object_type type
,
305 void *data
, unsigned long size
)
307 struct delta_info
**p
= &delta_list
;
308 struct delta_info
*info
;
310 while ((info
= *p
) != NULL
) {
311 if (oideq(&info
->base_oid
, &obj_list
[nr
].oid
) ||
312 info
->base_offset
== obj_list
[nr
].offset
) {
315 resolve_delta(info
->nr
, type
, data
, size
,
316 info
->delta
, info
->size
);
324 static void unpack_non_delta_entry(enum object_type type
, unsigned long size
,
327 void *buf
= get_data(size
);
330 write_object(nr
, type
, buf
, size
);
335 static int resolve_against_held(unsigned nr
, const struct object_id
*base
,
336 void *delta_data
, unsigned long delta_size
)
339 struct obj_buffer
*obj_buffer
;
340 obj
= lookup_object(the_repository
, base
);
343 obj_buffer
= lookup_object_buffer(obj
);
346 resolve_delta(nr
, obj
->type
, obj_buffer
->buffer
,
347 obj_buffer
->size
, delta_data
, delta_size
);
351 static void unpack_delta_entry(enum object_type type
, unsigned long delta_size
,
354 void *delta_data
, *base
;
355 unsigned long base_size
;
356 struct object_id base_oid
;
358 if (type
== OBJ_REF_DELTA
) {
359 oidread(&base_oid
, fill(the_hash_algo
->rawsz
));
360 use(the_hash_algo
->rawsz
);
361 delta_data
= get_data(delta_size
);
362 if (dry_run
|| !delta_data
) {
366 if (has_object_file(&base_oid
))
367 ; /* Ok we have this one */
368 else if (resolve_against_held(nr
, &base_oid
,
369 delta_data
, delta_size
))
370 return; /* we are done */
372 /* cannot resolve yet --- queue it */
373 oidclr(&obj_list
[nr
].oid
);
374 add_delta_to_list(nr
, &base_oid
, 0, delta_data
, delta_size
);
378 unsigned base_found
= 0;
379 unsigned char *pack
, c
;
381 unsigned lo
, mid
, hi
;
386 base_offset
= c
& 127;
389 if (!base_offset
|| MSB(base_offset
, 7))
390 die("offset value overflow for delta base object");
394 base_offset
= (base_offset
<< 7) + (c
& 127);
396 base_offset
= obj_list
[nr
].offset
- base_offset
;
397 if (base_offset
<= 0 || base_offset
>= obj_list
[nr
].offset
)
398 die("offset value out of bound for delta base object");
400 delta_data
= get_data(delta_size
);
401 if (dry_run
|| !delta_data
) {
408 mid
= lo
+ (hi
- lo
) / 2;
409 if (base_offset
< obj_list
[mid
].offset
) {
411 } else if (base_offset
> obj_list
[mid
].offset
) {
414 oidcpy(&base_oid
, &obj_list
[mid
].oid
);
415 base_found
= !is_null_oid(&base_oid
);
421 * The delta base object is itself a delta that
422 * has not been resolved yet.
424 oidclr(&obj_list
[nr
].oid
);
425 add_delta_to_list(nr
, null_oid(), base_offset
,
426 delta_data
, delta_size
);
431 if (resolve_against_held(nr
, &base_oid
, delta_data
, delta_size
))
434 base
= read_object_file(&base_oid
, &type
, &base_size
);
436 error("failed to read delta-pack base object %s",
437 oid_to_hex(&base_oid
));
443 resolve_delta(nr
, type
, base
, base_size
, delta_data
, delta_size
);
447 static void unpack_one(unsigned nr
)
451 unsigned long size
, c
;
452 enum object_type type
;
454 obj_list
[nr
].offset
= consumed_bytes
;
466 size
+= (c
& 0x7f) << shift
;
475 unpack_non_delta_entry(type
, size
, nr
);
479 unpack_delta_entry(type
, size
, nr
);
482 error("bad object type %d", type
);
490 static void unpack_all(void)
493 struct pack_header
*hdr
= fill(sizeof(struct pack_header
));
495 nr_objects
= ntohl(hdr
->hdr_entries
);
497 if (ntohl(hdr
->hdr_signature
) != PACK_SIGNATURE
)
498 die("bad pack file");
499 if (!pack_version_ok(hdr
->hdr_version
))
500 die("unknown pack file version %"PRIu32
,
501 ntohl(hdr
->hdr_version
));
502 use(sizeof(struct pack_header
));
505 progress
= start_progress(_("Unpacking objects"), nr_objects
);
506 CALLOC_ARRAY(obj_list
, nr_objects
);
507 begin_odb_transaction();
508 for (i
= 0; i
< nr_objects
; i
++) {
510 display_progress(progress
, i
+ 1);
512 end_odb_transaction();
513 stop_progress(&progress
);
516 die("unresolved deltas left after unpacking");
519 int cmd_unpack_objects(int argc
, const char **argv
, const char *prefix
)
522 struct object_id oid
;
524 read_replace_refs
= 0;
526 git_config(git_default_config
, NULL
);
530 for (i
= 1 ; i
< argc
; i
++) {
531 const char *arg
= argv
[i
];
534 if (!strcmp(arg
, "-n")) {
538 if (!strcmp(arg
, "-q")) {
542 if (!strcmp(arg
, "-r")) {
546 if (!strcmp(arg
, "--strict")) {
550 if (skip_prefix(arg
, "--strict=", &arg
)) {
552 fsck_set_msg_types(&fsck_options
, arg
);
555 if (starts_with(arg
, "--pack_header=")) {
556 struct pack_header
*hdr
;
559 hdr
= (struct pack_header
*)buffer
;
560 hdr
->hdr_signature
= htonl(PACK_SIGNATURE
);
561 hdr
->hdr_version
= htonl(strtoul(arg
+ 14, &c
, 10));
564 hdr
->hdr_entries
= htonl(strtoul(c
+ 1, &c
, 10));
570 if (skip_prefix(arg
, "--max-input-size=", &arg
)) {
571 max_input_size
= strtoumax(arg
, NULL
, 10);
577 /* We don't take any non-flag arguments now.. Maybe some day */
580 the_hash_algo
->init_fn(&ctx
);
582 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
583 the_hash_algo
->final_oid_fn(&oid
, &ctx
);
586 if (fsck_finish(&fsck_options
))
587 die(_("fsck error in pack objects"));
589 if (!hasheq(fill(the_hash_algo
->rawsz
), oid
.hash
))
590 die("final sha1 did not match");
591 use(the_hash_algo
->rawsz
);
593 /* Write the last part of the buffer to stdout */
595 int ret
= xwrite(1, buffer
+ offset
, len
);