builtin/unpack-objects.c

   1 #include "builtin.h"
   2 #include "cache.h"
   3 #include "bulk-checkin.h"
   4 #include "config.h"
   5 #include "object-store.h"
   6 #include "object.h"
   7 #include "delta.h"
   8 #include "pack.h"
   9 #include "blob.h"
  10 #include "commit.h"
  11 #include "tag.h"
  12 #include "tree.h"
  13 #include "tree-walk.h"
  14 #include "progress.h"
  15 #include "decorate.h"
  16 #include "fsck.h"
  17
  18 static int dry_run, quiet, recover, has_errors, strict;
  19 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
  20
  21 /* We always read in 4kB chunks. */
  22 static unsigned char buffer[4096];
  23 static unsigned int offset, len;
  24 static off_t consumed_bytes;
  25 static off_t max_input_size;
  26 static git_hash_ctx ctx;
  27 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
  28 static struct progress *progress;
  29
  30 /*
  31  * When running under --strict mode, objects whose reachability are
  32  * suspect are kept in core without getting written in the object
  33  * store.
  34  */
  35 struct obj_buffer {
  36         char *buffer;
  37         unsigned long size;
  38 };
  39
  40 static struct decoration obj_decorate;
  41
  42 static struct obj_buffer *lookup_object_buffer(struct object *base)
  43 {
  44         return lookup_decoration(&obj_decorate, base);
  45 }
  46
  47 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
  48 {
  49         struct obj_buffer *obj;
  50         CALLOC_ARRAY(obj, 1);
  51         obj->buffer = buffer;
  52         obj->size = size;
  53         if (add_decoration(&obj_decorate, object, obj))
  54                 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
  55 }
  56
  57 /*
  58  * Make sure at least "min" bytes are available in the buffer, and
  59  * return the pointer to the buffer.
  60  */
  61 static void *fill(int min)
  62 {
  63         if (min <= len)
  64                 return buffer + offset;
  65         if (min > sizeof(buffer))
  66                 die("cannot fill %d bytes", min);
  67         if (offset) {
  68                 the_hash_algo->update_fn(&ctx, buffer, offset);
  69                 memmove(buffer, buffer + offset, len);
  70                 offset = 0;
  71         }
  72         do {
  73                 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
  74                 if (ret <= 0) {
  75                         if (!ret)
  76                                 die("early EOF");
  77                         die_errno("read error on input");
  78                 }
  79                 len += ret;
  80         } while (len < min);
  81         return buffer;
  82 }
  83
  84 static void use(int bytes)
  85 {
  86         if (bytes > len)
  87                 die("used more bytes than were available");
  88         len -= bytes;
  89         offset += bytes;
  90
  91         /* make sure off_t is sufficiently large not to wrap */
  92         if (signed_add_overflows(consumed_bytes, bytes))
  93                 die("pack too large for current definition of off_t");
  94         consumed_bytes += bytes;
  95         if (max_input_size && consumed_bytes > max_input_size)
  96                 die(_("pack exceeds maximum allowed size"));
  97         display_throughput(progress, consumed_bytes);
  98 }
  99
 100 static void *get_data(unsigned long size)
 101 {
 102         git_zstream stream;
 103         void *buf = xmallocz(size);
 104
 105         memset(&stream, 0, sizeof(stream));
 106
 107         stream.next_out = buf;
 108         stream.avail_out = size;
 109         stream.next_in = fill(1);
 110         stream.avail_in = len;
 111         git_inflate_init(&stream);
 112
 113         for (;;) {
 114                 int ret = git_inflate(&stream, 0);
 115                 use(len - stream.avail_in);
 116                 if (stream.total_out == size && ret == Z_STREAM_END)
 117                         break;
 118                 if (ret != Z_OK) {
 119                         error("inflate returned %d", ret);
 120                         FREE_AND_NULL(buf);
 121                         if (!recover)
 122                                 exit(1);
 123                         has_errors = 1;
 124                         break;
 125                 }
 126                 stream.next_in = fill(1);
 127                 stream.avail_in = len;
 128         }
 129         git_inflate_end(&stream);
 130         return buf;
 131 }
 132
 133 struct delta_info {
 134         struct object_id base_oid;
 135         unsigned nr;
 136         off_t base_offset;
 137         unsigned long size;
 138         void *delta;
 139         struct delta_info *next;
 140 };
 141
 142 static struct delta_info *delta_list;
 143
 144 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
 145                               off_t base_offset,
 146                               void *delta, unsigned long size)
 147 {
 148         struct delta_info *info = xmalloc(sizeof(*info));
 149
 150         oidcpy(&info->base_oid, base_oid);
 151         info->base_offset = base_offset;
 152         info->size = size;
 153         info->delta = delta;
 154         info->nr = nr;
 155         info->next = delta_list;
 156         delta_list = info;
 157 }
 158
 159 struct obj_info {
 160         off_t offset;
 161         struct object_id oid;
 162         struct object *obj;
 163 };
 164
 165 /* Remember to update object flag allocation in object.h */
 166 #define FLAG_OPEN (1u<<20)
 167 #define FLAG_WRITTEN (1u<<21)
 168
 169 static struct obj_info *obj_list;
 170 static unsigned nr_objects;
 171
 172 /*
 173  * Called only from check_object() after it verified this object
 174  * is Ok.
 175  */
 176 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
 177 {
 178         struct object_id oid;
 179
 180         if (write_object_file(obj_buf->buffer, obj_buf->size,
 181                               obj->type, &oid) < 0)
 182                 die("failed to write object %s", oid_to_hex(&obj->oid));
 183         obj->flags |= FLAG_WRITTEN;
 184 }
 185
 186 /*
 187  * At the very end of the processing, write_rest() scans the objects
 188  * that have reachability requirements and calls this function.
 189  * Verify its reachability and validity recursively and write it out.
 190  */
 191 static int check_object(struct object *obj, enum object_type type,
 192                         void *data, struct fsck_options *options)
 193 {
 194         struct obj_buffer *obj_buf;
 195
 196         if (!obj)
 197                 return 1;
 198
 199         if (obj->flags & FLAG_WRITTEN)
 200                 return 0;
 201
 202         if (type != OBJ_ANY && obj->type != type)
 203                 die("object type mismatch");
 204
 205         if (!(obj->flags & FLAG_OPEN)) {
 206                 unsigned long size;
 207                 int type = oid_object_info(the_repository, &obj->oid, &size);
 208                 if (type != obj->type || type <= 0)
 209                         die("object of unexpected type");
 210                 obj->flags |= FLAG_WRITTEN;
 211                 return 0;
 212         }
 213
 214         obj_buf = lookup_object_buffer(obj);
 215         if (!obj_buf)
 216                 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
 217         if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
 218                 die("fsck error in packed object");
 219         fsck_options.walk = check_object;
 220         if (fsck_walk(obj, NULL, &fsck_options))
 221                 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
 222         write_cached_object(obj, obj_buf);
 223         return 0;
 224 }
 225
 226 static void write_rest(void)
 227 {
 228         unsigned i;
 229         for (i = 0; i < nr_objects; i++) {
 230                 if (obj_list[i].obj)
 231                         check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
 232         }
 233 }
 234
 235 static void added_object(unsigned nr, enum object_type type,
 236                          void *data, unsigned long size);
 237
 238 /*
 239  * Write out nr-th object from the list, now we know the contents
 240  * of it.  Under --strict, this buffers structured objects in-core,
 241  * to be checked at the end.
 242  */
 243 static void write_object(unsigned nr, enum object_type type,
 244                          void *buf, unsigned long size)
 245 {
 246         if (!strict) {
 247                 if (write_object_file(buf, size, type,
 248                                       &obj_list[nr].oid) < 0)
 249                         die("failed to write object");
 250                 added_object(nr, type, buf, size);
 251                 free(buf);
 252                 obj_list[nr].obj = NULL;
 253         } else if (type == OBJ_BLOB) {
 254                 struct blob *blob;
 255                 if (write_object_file(buf, size, type,
 256                                       &obj_list[nr].oid) < 0)
 257                         die("failed to write object");
 258                 added_object(nr, type, buf, size);
 259                 free(buf);
 260
 261                 blob = lookup_blob(the_repository, &obj_list[nr].oid);
 262                 if (blob)
 263                         blob->object.flags |= FLAG_WRITTEN;
 264                 else
 265                         die("invalid blob object");
 266                 obj_list[nr].obj = NULL;
 267         } else {
 268                 struct object *obj;
 269                 int eaten;
 270                 hash_object_file(the_hash_algo, buf, size, type,
 271                                  &obj_list[nr].oid);
 272                 added_object(nr, type, buf, size);
 273                 obj = parse_object_buffer(the_repository, &obj_list[nr].oid,
 274                                           type, size, buf,
 275                                           &eaten);
 276                 if (!obj)
 277                         die("invalid %s", type_name(type));
 278                 add_object_buffer(obj, buf, size);
 279                 obj->flags |= FLAG_OPEN;
 280                 obj_list[nr].obj = obj;
 281         }
 282 }
 283
 284 static void resolve_delta(unsigned nr, enum object_type type,
 285                           void *base, unsigned long base_size,
 286                           void *delta, unsigned long delta_size)
 287 {
 288         void *result;
 289         unsigned long result_size;
 290
 291         result = patch_delta(base, base_size,
 292                              delta, delta_size,
 293                              &result_size);
 294         if (!result)
 295                 die("failed to apply delta");
 296         free(delta);
 297         write_object(nr, type, result, result_size);
 298 }
 299
 300 /*
 301  * We now know the contents of an object (which is nr-th in the pack);
 302  * resolve all the deltified objects that are based on it.
 303  */
 304 static void added_object(unsigned nr, enum object_type type,
 305                          void *data, unsigned long size)
 306 {
 307         struct delta_info **p = &delta_list;
 308         struct delta_info *info;
 309
 310         while ((info = *p) != NULL) {
 311                 if (oideq(&info->base_oid, &obj_list[nr].oid) ||
 312                     info->base_offset == obj_list[nr].offset) {
 313                         *p = info->next;
 314                         p = &delta_list;
 315                         resolve_delta(info->nr, type, data, size,
 316                                       info->delta, info->size);
 317                         free(info);
 318                         continue;
 319                 }
 320                 p = &info->next;
 321         }
 322 }
 323
 324 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 325                                    unsigned nr)
 326 {
 327         void *buf = get_data(size);
 328
 329         if (!dry_run && buf)
 330                 write_object(nr, type, buf, size);
 331         else
 332                 free(buf);
 333 }
 334
 335 static int resolve_against_held(unsigned nr, const struct object_id *base,
 336                                 void *delta_data, unsigned long delta_size)
 337 {
 338         struct object *obj;
 339         struct obj_buffer *obj_buffer;
 340         obj = lookup_object(the_repository, base);
 341         if (!obj)
 342                 return 0;
 343         obj_buffer = lookup_object_buffer(obj);
 344         if (!obj_buffer)
 345                 return 0;
 346         resolve_delta(nr, obj->type, obj_buffer->buffer,
 347                       obj_buffer->size, delta_data, delta_size);
 348         return 1;
 349 }
 350
 351 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 352                                unsigned nr)
 353 {
 354         void *delta_data, *base;
 355         unsigned long base_size;
 356         struct object_id base_oid;
 357
 358         if (type == OBJ_REF_DELTA) {
 359                 oidread(&base_oid, fill(the_hash_algo->rawsz));
 360                 use(the_hash_algo->rawsz);
 361                 delta_data = get_data(delta_size);
 362                 if (dry_run || !delta_data) {
 363                         free(delta_data);
 364                         return;
 365                 }
 366                 if (has_object_file(&base_oid))
 367                         ; /* Ok we have this one */
 368                 else if (resolve_against_held(nr, &base_oid,
 369                                               delta_data, delta_size))
 370                         return; /* we are done */
 371                 else {
 372                         /* cannot resolve yet --- queue it */
 373                         oidclr(&obj_list[nr].oid);
 374                         add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
 375                         return;
 376                 }
 377         } else {
 378                 unsigned base_found = 0;
 379                 unsigned char *pack, c;
 380                 off_t base_offset;
 381                 unsigned lo, mid, hi;
 382
 383                 pack = fill(1);
 384                 c = *pack;
 385                 use(1);
 386                 base_offset = c & 127;
 387                 while (c & 128) {
 388                         base_offset += 1;
 389                         if (!base_offset || MSB(base_offset, 7))
 390                                 die("offset value overflow for delta base object");
 391                         pack = fill(1);
 392                         c = *pack;
 393                         use(1);
 394                         base_offset = (base_offset << 7) + (c & 127);
 395                 }
 396                 base_offset = obj_list[nr].offset - base_offset;
 397                 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
 398                         die("offset value out of bound for delta base object");
 399
 400                 delta_data = get_data(delta_size);
 401                 if (dry_run || !delta_data) {
 402                         free(delta_data);
 403                         return;
 404                 }
 405                 lo = 0;
 406                 hi = nr;
 407                 while (lo < hi) {
 408                         mid = lo + (hi - lo) / 2;
 409                         if (base_offset < obj_list[mid].offset) {
 410                                 hi = mid;
 411                         } else if (base_offset > obj_list[mid].offset) {
 412                                 lo = mid + 1;
 413                         } else {
 414                                 oidcpy(&base_oid, &obj_list[mid].oid);
 415                                 base_found = !is_null_oid(&base_oid);
 416                                 break;
 417                         }
 418                 }
 419                 if (!base_found) {
 420                         /*
 421                          * The delta base object is itself a delta that
 422                          * has not been resolved yet.
 423                          */
 424                         oidclr(&obj_list[nr].oid);
 425                         add_delta_to_list(nr, null_oid(), base_offset,
 426                                           delta_data, delta_size);
 427                         return;
 428                 }
 429         }
 430
 431         if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
 432                 return;
 433
 434         base = read_object_file(&base_oid, &type, &base_size);
 435         if (!base) {
 436                 error("failed to read delta-pack base object %s",
 437                       oid_to_hex(&base_oid));
 438                 if (!recover)
 439                         exit(1);
 440                 has_errors = 1;
 441                 return;
 442         }
 443         resolve_delta(nr, type, base, base_size, delta_data, delta_size);
 444         free(base);
 445 }
 446
 447 static void unpack_one(unsigned nr)
 448 {
 449         unsigned shift;
 450         unsigned char *pack;
 451         unsigned long size, c;
 452         enum object_type type;
 453
 454         obj_list[nr].offset = consumed_bytes;
 455
 456         pack = fill(1);
 457         c = *pack;
 458         use(1);
 459         type = (c >> 4) & 7;
 460         size = (c & 15);
 461         shift = 4;
 462         while (c & 0x80) {
 463                 pack = fill(1);
 464                 c = *pack;
 465                 use(1);
 466                 size += (c & 0x7f) << shift;
 467                 shift += 7;
 468         }
 469
 470         switch (type) {
 471         case OBJ_COMMIT:
 472         case OBJ_TREE:
 473         case OBJ_BLOB:
 474         case OBJ_TAG:
 475                 unpack_non_delta_entry(type, size, nr);
 476                 return;
 477         case OBJ_REF_DELTA:
 478         case OBJ_OFS_DELTA:
 479                 unpack_delta_entry(type, size, nr);
 480                 return;
 481         default:
 482                 error("bad object type %d", type);
 483                 has_errors = 1;
 484                 if (recover)
 485                         return;
 486                 exit(1);
 487         }
 488 }
 489
 490 static void unpack_all(void)
 491 {
 492         int i;
 493         struct pack_header *hdr = fill(sizeof(struct pack_header));
 494
 495         nr_objects = ntohl(hdr->hdr_entries);
 496
 497         if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
 498                 die("bad pack file");
 499         if (!pack_version_ok(hdr->hdr_version))
 500                 die("unknown pack file version %"PRIu32,
 501                         ntohl(hdr->hdr_version));
 502         use(sizeof(struct pack_header));
 503
 504         if (!quiet)
 505                 progress = start_progress(_("Unpacking objects"), nr_objects);
 506         CALLOC_ARRAY(obj_list, nr_objects);
 507         begin_odb_transaction();
 508         for (i = 0; i < nr_objects; i++) {
 509                 unpack_one(i);
 510                 display_progress(progress, i + 1);
 511         }
 512         end_odb_transaction();
 513         stop_progress(&progress);
 514
 515         if (delta_list)
 516                 die("unresolved deltas left after unpacking");
 517 }
 518
 519 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
 520 {
 521         int i;
 522         struct object_id oid;
 523
 524         read_replace_refs = 0;
 525
 526         git_config(git_default_config, NULL);
 527
 528         quiet = !isatty(2);
 529
 530         for (i = 1 ; i < argc; i++) {
 531                 const char *arg = argv[i];
 532
 533                 if (*arg == '-') {
 534                         if (!strcmp(arg, "-n")) {
 535                                 dry_run = 1;
 536                                 continue;
 537                         }
 538                         if (!strcmp(arg, "-q")) {
 539                                 quiet = 1;
 540                                 continue;
 541                         }
 542                         if (!strcmp(arg, "-r")) {
 543                                 recover = 1;
 544                                 continue;
 545                         }
 546                         if (!strcmp(arg, "--strict")) {
 547                                 strict = 1;
 548                                 continue;
 549                         }
 550                         if (skip_prefix(arg, "--strict=", &arg)) {
 551                                 strict = 1;
 552                                 fsck_set_msg_types(&fsck_options, arg);
 553                                 continue;
 554                         }
 555                         if (starts_with(arg, "--pack_header=")) {
 556                                 struct pack_header *hdr;
 557                                 char *c;
 558
 559                                 hdr = (struct pack_header *)buffer;
 560                                 hdr->hdr_signature = htonl(PACK_SIGNATURE);
 561                                 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
 562                                 if (*c != ',')
 563                                         die("bad %s", arg);
 564                                 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
 565                                 if (*c)
 566                                         die("bad %s", arg);
 567                                 len = sizeof(*hdr);
 568                                 continue;
 569                         }
 570                         if (skip_prefix(arg, "--max-input-size=", &arg)) {
 571                                 max_input_size = strtoumax(arg, NULL, 10);
 572                                 continue;
 573                         }
 574                         usage(unpack_usage);
 575                 }
 576
 577                 /* We don't take any non-flag arguments now.. Maybe some day */
 578                 usage(unpack_usage);
 579         }
 580         the_hash_algo->init_fn(&ctx);
 581         unpack_all();
 582         the_hash_algo->update_fn(&ctx, buffer, offset);
 583         the_hash_algo->final_oid_fn(&oid, &ctx);
 584         if (strict) {
 585                 write_rest();
 586                 if (fsck_finish(&fsck_options))
 587                         die(_("fsck error in pack objects"));
 588         }
 589         if (!hasheq(fill(the_hash_algo->rawsz), oid.hash))
 590                 die("final sha1 did not match");
 591         use(the_hash_algo->rawsz);
 592
 593         /* Write the last part of the buffer to stdout */
 594         while (len) {
 595                 int ret = xwrite(1, buffer + offset, len);
 596                 if (ret <= 0)
 597                         break;
 598                 len -= ret;
 599                 offset += ret;
 600         }
 601
 602         /* All done */
 603         return has_errors;
 604 }