]> git.ipfire.org Git - thirdparty/git.git/blob - builtin/unpack-objects.c
clone: allow "--bare" with "-o"
[thirdparty/git.git] / builtin / unpack-objects.c
1 #include "builtin.h"
2 #include "cache.h"
3 #include "bulk-checkin.h"
4 #include "config.h"
5 #include "object-store.h"
6 #include "object.h"
7 #include "delta.h"
8 #include "pack.h"
9 #include "blob.h"
10 #include "commit.h"
11 #include "tag.h"
12 #include "tree.h"
13 #include "tree-walk.h"
14 #include "progress.h"
15 #include "decorate.h"
16 #include "fsck.h"
17
18 static int dry_run, quiet, recover, has_errors, strict;
19 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
20
21 /* We always read in 4kB chunks. */
22 static unsigned char buffer[4096];
23 static unsigned int offset, len;
24 static off_t consumed_bytes;
25 static off_t max_input_size;
26 static git_hash_ctx ctx;
27 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
28 static struct progress *progress;
29
30 /*
31 * When running under --strict mode, objects whose reachability are
32 * suspect are kept in core without getting written in the object
33 * store.
34 */
35 struct obj_buffer {
36 char *buffer;
37 unsigned long size;
38 };
39
40 static struct decoration obj_decorate;
41
42 static struct obj_buffer *lookup_object_buffer(struct object *base)
43 {
44 return lookup_decoration(&obj_decorate, base);
45 }
46
47 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
48 {
49 struct obj_buffer *obj;
50 CALLOC_ARRAY(obj, 1);
51 obj->buffer = buffer;
52 obj->size = size;
53 if (add_decoration(&obj_decorate, object, obj))
54 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
55 }
56
57 /*
58 * Make sure at least "min" bytes are available in the buffer, and
59 * return the pointer to the buffer.
60 */
61 static void *fill(int min)
62 {
63 if (min <= len)
64 return buffer + offset;
65 if (min > sizeof(buffer))
66 die("cannot fill %d bytes", min);
67 if (offset) {
68 the_hash_algo->update_fn(&ctx, buffer, offset);
69 memmove(buffer, buffer + offset, len);
70 offset = 0;
71 }
72 do {
73 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
74 if (ret <= 0) {
75 if (!ret)
76 die("early EOF");
77 die_errno("read error on input");
78 }
79 len += ret;
80 } while (len < min);
81 return buffer;
82 }
83
84 static void use(int bytes)
85 {
86 if (bytes > len)
87 die("used more bytes than were available");
88 len -= bytes;
89 offset += bytes;
90
91 /* make sure off_t is sufficiently large not to wrap */
92 if (signed_add_overflows(consumed_bytes, bytes))
93 die("pack too large for current definition of off_t");
94 consumed_bytes += bytes;
95 if (max_input_size && consumed_bytes > max_input_size)
96 die(_("pack exceeds maximum allowed size"));
97 display_throughput(progress, consumed_bytes);
98 }
99
100 static void *get_data(unsigned long size)
101 {
102 git_zstream stream;
103 void *buf = xmallocz(size);
104
105 memset(&stream, 0, sizeof(stream));
106
107 stream.next_out = buf;
108 stream.avail_out = size;
109 stream.next_in = fill(1);
110 stream.avail_in = len;
111 git_inflate_init(&stream);
112
113 for (;;) {
114 int ret = git_inflate(&stream, 0);
115 use(len - stream.avail_in);
116 if (stream.total_out == size && ret == Z_STREAM_END)
117 break;
118 if (ret != Z_OK) {
119 error("inflate returned %d", ret);
120 FREE_AND_NULL(buf);
121 if (!recover)
122 exit(1);
123 has_errors = 1;
124 break;
125 }
126 stream.next_in = fill(1);
127 stream.avail_in = len;
128 }
129 git_inflate_end(&stream);
130 return buf;
131 }
132
133 struct delta_info {
134 struct object_id base_oid;
135 unsigned nr;
136 off_t base_offset;
137 unsigned long size;
138 void *delta;
139 struct delta_info *next;
140 };
141
142 static struct delta_info *delta_list;
143
144 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
145 off_t base_offset,
146 void *delta, unsigned long size)
147 {
148 struct delta_info *info = xmalloc(sizeof(*info));
149
150 oidcpy(&info->base_oid, base_oid);
151 info->base_offset = base_offset;
152 info->size = size;
153 info->delta = delta;
154 info->nr = nr;
155 info->next = delta_list;
156 delta_list = info;
157 }
158
159 struct obj_info {
160 off_t offset;
161 struct object_id oid;
162 struct object *obj;
163 };
164
165 /* Remember to update object flag allocation in object.h */
166 #define FLAG_OPEN (1u<<20)
167 #define FLAG_WRITTEN (1u<<21)
168
169 static struct obj_info *obj_list;
170 static unsigned nr_objects;
171
172 /*
173 * Called only from check_object() after it verified this object
174 * is Ok.
175 */
176 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
177 {
178 struct object_id oid;
179
180 if (write_object_file(obj_buf->buffer, obj_buf->size,
181 obj->type, &oid) < 0)
182 die("failed to write object %s", oid_to_hex(&obj->oid));
183 obj->flags |= FLAG_WRITTEN;
184 }
185
186 /*
187 * At the very end of the processing, write_rest() scans the objects
188 * that have reachability requirements and calls this function.
189 * Verify its reachability and validity recursively and write it out.
190 */
191 static int check_object(struct object *obj, enum object_type type,
192 void *data, struct fsck_options *options)
193 {
194 struct obj_buffer *obj_buf;
195
196 if (!obj)
197 return 1;
198
199 if (obj->flags & FLAG_WRITTEN)
200 return 0;
201
202 if (type != OBJ_ANY && obj->type != type)
203 die("object type mismatch");
204
205 if (!(obj->flags & FLAG_OPEN)) {
206 unsigned long size;
207 int type = oid_object_info(the_repository, &obj->oid, &size);
208 if (type != obj->type || type <= 0)
209 die("object of unexpected type");
210 obj->flags |= FLAG_WRITTEN;
211 return 0;
212 }
213
214 obj_buf = lookup_object_buffer(obj);
215 if (!obj_buf)
216 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
217 if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
218 die("fsck error in packed object");
219 fsck_options.walk = check_object;
220 if (fsck_walk(obj, NULL, &fsck_options))
221 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
222 write_cached_object(obj, obj_buf);
223 return 0;
224 }
225
226 static void write_rest(void)
227 {
228 unsigned i;
229 for (i = 0; i < nr_objects; i++) {
230 if (obj_list[i].obj)
231 check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
232 }
233 }
234
235 static void added_object(unsigned nr, enum object_type type,
236 void *data, unsigned long size);
237
238 /*
239 * Write out nr-th object from the list, now we know the contents
240 * of it. Under --strict, this buffers structured objects in-core,
241 * to be checked at the end.
242 */
243 static void write_object(unsigned nr, enum object_type type,
244 void *buf, unsigned long size)
245 {
246 if (!strict) {
247 if (write_object_file(buf, size, type,
248 &obj_list[nr].oid) < 0)
249 die("failed to write object");
250 added_object(nr, type, buf, size);
251 free(buf);
252 obj_list[nr].obj = NULL;
253 } else if (type == OBJ_BLOB) {
254 struct blob *blob;
255 if (write_object_file(buf, size, type,
256 &obj_list[nr].oid) < 0)
257 die("failed to write object");
258 added_object(nr, type, buf, size);
259 free(buf);
260
261 blob = lookup_blob(the_repository, &obj_list[nr].oid);
262 if (blob)
263 blob->object.flags |= FLAG_WRITTEN;
264 else
265 die("invalid blob object");
266 obj_list[nr].obj = NULL;
267 } else {
268 struct object *obj;
269 int eaten;
270 hash_object_file(the_hash_algo, buf, size, type,
271 &obj_list[nr].oid);
272 added_object(nr, type, buf, size);
273 obj = parse_object_buffer(the_repository, &obj_list[nr].oid,
274 type, size, buf,
275 &eaten);
276 if (!obj)
277 die("invalid %s", type_name(type));
278 add_object_buffer(obj, buf, size);
279 obj->flags |= FLAG_OPEN;
280 obj_list[nr].obj = obj;
281 }
282 }
283
284 static void resolve_delta(unsigned nr, enum object_type type,
285 void *base, unsigned long base_size,
286 void *delta, unsigned long delta_size)
287 {
288 void *result;
289 unsigned long result_size;
290
291 result = patch_delta(base, base_size,
292 delta, delta_size,
293 &result_size);
294 if (!result)
295 die("failed to apply delta");
296 free(delta);
297 write_object(nr, type, result, result_size);
298 }
299
300 /*
301 * We now know the contents of an object (which is nr-th in the pack);
302 * resolve all the deltified objects that are based on it.
303 */
304 static void added_object(unsigned nr, enum object_type type,
305 void *data, unsigned long size)
306 {
307 struct delta_info **p = &delta_list;
308 struct delta_info *info;
309
310 while ((info = *p) != NULL) {
311 if (oideq(&info->base_oid, &obj_list[nr].oid) ||
312 info->base_offset == obj_list[nr].offset) {
313 *p = info->next;
314 p = &delta_list;
315 resolve_delta(info->nr, type, data, size,
316 info->delta, info->size);
317 free(info);
318 continue;
319 }
320 p = &info->next;
321 }
322 }
323
324 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
325 unsigned nr)
326 {
327 void *buf = get_data(size);
328
329 if (!dry_run && buf)
330 write_object(nr, type, buf, size);
331 else
332 free(buf);
333 }
334
335 static int resolve_against_held(unsigned nr, const struct object_id *base,
336 void *delta_data, unsigned long delta_size)
337 {
338 struct object *obj;
339 struct obj_buffer *obj_buffer;
340 obj = lookup_object(the_repository, base);
341 if (!obj)
342 return 0;
343 obj_buffer = lookup_object_buffer(obj);
344 if (!obj_buffer)
345 return 0;
346 resolve_delta(nr, obj->type, obj_buffer->buffer,
347 obj_buffer->size, delta_data, delta_size);
348 return 1;
349 }
350
351 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
352 unsigned nr)
353 {
354 void *delta_data, *base;
355 unsigned long base_size;
356 struct object_id base_oid;
357
358 if (type == OBJ_REF_DELTA) {
359 oidread(&base_oid, fill(the_hash_algo->rawsz));
360 use(the_hash_algo->rawsz);
361 delta_data = get_data(delta_size);
362 if (dry_run || !delta_data) {
363 free(delta_data);
364 return;
365 }
366 if (has_object_file(&base_oid))
367 ; /* Ok we have this one */
368 else if (resolve_against_held(nr, &base_oid,
369 delta_data, delta_size))
370 return; /* we are done */
371 else {
372 /* cannot resolve yet --- queue it */
373 oidclr(&obj_list[nr].oid);
374 add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
375 return;
376 }
377 } else {
378 unsigned base_found = 0;
379 unsigned char *pack, c;
380 off_t base_offset;
381 unsigned lo, mid, hi;
382
383 pack = fill(1);
384 c = *pack;
385 use(1);
386 base_offset = c & 127;
387 while (c & 128) {
388 base_offset += 1;
389 if (!base_offset || MSB(base_offset, 7))
390 die("offset value overflow for delta base object");
391 pack = fill(1);
392 c = *pack;
393 use(1);
394 base_offset = (base_offset << 7) + (c & 127);
395 }
396 base_offset = obj_list[nr].offset - base_offset;
397 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
398 die("offset value out of bound for delta base object");
399
400 delta_data = get_data(delta_size);
401 if (dry_run || !delta_data) {
402 free(delta_data);
403 return;
404 }
405 lo = 0;
406 hi = nr;
407 while (lo < hi) {
408 mid = lo + (hi - lo) / 2;
409 if (base_offset < obj_list[mid].offset) {
410 hi = mid;
411 } else if (base_offset > obj_list[mid].offset) {
412 lo = mid + 1;
413 } else {
414 oidcpy(&base_oid, &obj_list[mid].oid);
415 base_found = !is_null_oid(&base_oid);
416 break;
417 }
418 }
419 if (!base_found) {
420 /*
421 * The delta base object is itself a delta that
422 * has not been resolved yet.
423 */
424 oidclr(&obj_list[nr].oid);
425 add_delta_to_list(nr, null_oid(), base_offset,
426 delta_data, delta_size);
427 return;
428 }
429 }
430
431 if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
432 return;
433
434 base = read_object_file(&base_oid, &type, &base_size);
435 if (!base) {
436 error("failed to read delta-pack base object %s",
437 oid_to_hex(&base_oid));
438 if (!recover)
439 exit(1);
440 has_errors = 1;
441 return;
442 }
443 resolve_delta(nr, type, base, base_size, delta_data, delta_size);
444 free(base);
445 }
446
447 static void unpack_one(unsigned nr)
448 {
449 unsigned shift;
450 unsigned char *pack;
451 unsigned long size, c;
452 enum object_type type;
453
454 obj_list[nr].offset = consumed_bytes;
455
456 pack = fill(1);
457 c = *pack;
458 use(1);
459 type = (c >> 4) & 7;
460 size = (c & 15);
461 shift = 4;
462 while (c & 0x80) {
463 pack = fill(1);
464 c = *pack;
465 use(1);
466 size += (c & 0x7f) << shift;
467 shift += 7;
468 }
469
470 switch (type) {
471 case OBJ_COMMIT:
472 case OBJ_TREE:
473 case OBJ_BLOB:
474 case OBJ_TAG:
475 unpack_non_delta_entry(type, size, nr);
476 return;
477 case OBJ_REF_DELTA:
478 case OBJ_OFS_DELTA:
479 unpack_delta_entry(type, size, nr);
480 return;
481 default:
482 error("bad object type %d", type);
483 has_errors = 1;
484 if (recover)
485 return;
486 exit(1);
487 }
488 }
489
490 static void unpack_all(void)
491 {
492 int i;
493 struct pack_header *hdr = fill(sizeof(struct pack_header));
494
495 nr_objects = ntohl(hdr->hdr_entries);
496
497 if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
498 die("bad pack file");
499 if (!pack_version_ok(hdr->hdr_version))
500 die("unknown pack file version %"PRIu32,
501 ntohl(hdr->hdr_version));
502 use(sizeof(struct pack_header));
503
504 if (!quiet)
505 progress = start_progress(_("Unpacking objects"), nr_objects);
506 CALLOC_ARRAY(obj_list, nr_objects);
507 begin_odb_transaction();
508 for (i = 0; i < nr_objects; i++) {
509 unpack_one(i);
510 display_progress(progress, i + 1);
511 }
512 end_odb_transaction();
513 stop_progress(&progress);
514
515 if (delta_list)
516 die("unresolved deltas left after unpacking");
517 }
518
519 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
520 {
521 int i;
522 struct object_id oid;
523
524 read_replace_refs = 0;
525
526 git_config(git_default_config, NULL);
527
528 quiet = !isatty(2);
529
530 for (i = 1 ; i < argc; i++) {
531 const char *arg = argv[i];
532
533 if (*arg == '-') {
534 if (!strcmp(arg, "-n")) {
535 dry_run = 1;
536 continue;
537 }
538 if (!strcmp(arg, "-q")) {
539 quiet = 1;
540 continue;
541 }
542 if (!strcmp(arg, "-r")) {
543 recover = 1;
544 continue;
545 }
546 if (!strcmp(arg, "--strict")) {
547 strict = 1;
548 continue;
549 }
550 if (skip_prefix(arg, "--strict=", &arg)) {
551 strict = 1;
552 fsck_set_msg_types(&fsck_options, arg);
553 continue;
554 }
555 if (starts_with(arg, "--pack_header=")) {
556 struct pack_header *hdr;
557 char *c;
558
559 hdr = (struct pack_header *)buffer;
560 hdr->hdr_signature = htonl(PACK_SIGNATURE);
561 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
562 if (*c != ',')
563 die("bad %s", arg);
564 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
565 if (*c)
566 die("bad %s", arg);
567 len = sizeof(*hdr);
568 continue;
569 }
570 if (skip_prefix(arg, "--max-input-size=", &arg)) {
571 max_input_size = strtoumax(arg, NULL, 10);
572 continue;
573 }
574 usage(unpack_usage);
575 }
576
577 /* We don't take any non-flag arguments now.. Maybe some day */
578 usage(unpack_usage);
579 }
580 the_hash_algo->init_fn(&ctx);
581 unpack_all();
582 the_hash_algo->update_fn(&ctx, buffer, offset);
583 the_hash_algo->final_oid_fn(&oid, &ctx);
584 if (strict) {
585 write_rest();
586 if (fsck_finish(&fsck_options))
587 die(_("fsck error in pack objects"));
588 }
589 if (!hasheq(fill(the_hash_algo->rawsz), oid.hash))
590 die("final sha1 did not match");
591 use(the_hash_algo->rawsz);
592
593 /* Write the last part of the buffer to stdout */
594 while (len) {
595 int ret = xwrite(1, buffer + offset, len);
596 if (ret <= 0)
597 break;
598 len -= ret;
599 offset += ret;
600 }
601
602 /* All done */
603 return has_errors;
604 }