fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  47
  48 static void rev_list_push(struct commit *commit, int mark)
  49 {
  50         if (!(commit->object.flags & mark)) {
  51                 commit->object.flags |= mark;
  52
  53                 if (parse_commit(commit))
  54                         return;
  55
  56                 prio_queue_put(&rev_list, commit);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  74                                    int flag, void *cb_data)
  75 {
  76         return rev_list_insert_ref(refname, oid->hash);
  77 }
  78
  79 static int clear_marks(const char *refname, const struct object_id *oid,
  80                        int flag, void *cb_data)
  81 {
  82         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
  83
  84         if (o && o->type == OBJ_COMMIT)
  85                 clear_commit_marks((struct commit *)o,
  86                                    COMMON | COMMON_REF | SEEN | POPPED);
  87         return 0;
  88 }
  89
  90 /*
  91    This function marks a rev and its ancestors as common.
  92    In some cases, it is desirable to mark only the ancestors (for example
  93    when only the server does not yet know that they are common).
  94 */
  95
  96 static void mark_common(struct commit *commit,
  97                 int ancestors_only, int dont_parse)
  98 {
  99         if (commit != NULL && !(commit->object.flags & COMMON)) {
 100                 struct object *o = (struct object *)commit;
 101
 102                 if (!ancestors_only)
 103                         o->flags |= COMMON;
 104
 105                 if (!(o->flags & SEEN))
 106                         rev_list_push(commit, SEEN);
 107                 else {
 108                         struct commit_list *parents;
 109
 110                         if (!ancestors_only && !(o->flags & POPPED))
 111                                 non_common_revs--;
 112                         if (!o->parsed && !dont_parse)
 113                                 if (parse_commit(commit))
 114                                         return;
 115
 116                         for (parents = commit->parents;
 117                                         parents;
 118                                         parents = parents->next)
 119                                 mark_common(parents->item, 0, dont_parse);
 120                 }
 121         }
 122 }
 123
 124 /*
 125   Get the next rev to send, ignoring the common.
 126 */
 127
 128 static const unsigned char *get_rev(void)
 129 {
 130         struct commit *commit = NULL;
 131
 132         while (commit == NULL) {
 133                 unsigned int mark;
 134                 struct commit_list *parents;
 135
 136                 if (rev_list.nr == 0 || non_common_revs == 0)
 137                         return NULL;
 138
 139                 commit = prio_queue_get(&rev_list);
 140                 parse_commit(commit);
 141                 parents = commit->parents;
 142
 143                 commit->object.flags |= POPPED;
 144                 if (!(commit->object.flags & COMMON))
 145                         non_common_revs--;
 146
 147                 if (commit->object.flags & COMMON) {
 148                         /* do not send "have", and ignore ancestors */
 149                         commit = NULL;
 150                         mark = COMMON | SEEN;
 151                 } else if (commit->object.flags & COMMON_REF)
 152                         /* send "have", and ignore ancestors */
 153                         mark = COMMON | SEEN;
 154                 else
 155                         /* send "have", also for its ancestors */
 156                         mark = SEEN;
 157
 158                 while (parents) {
 159                         if (!(parents->item->object.flags & SEEN))
 160                                 rev_list_push(parents->item, mark);
 161                         if (mark & COMMON)
 162                                 mark_common(parents->item, 1, 0);
 163                         parents = parents->next;
 164                 }
 165         }
 166
 167         return commit->object.sha1;
 168 }
 169
 170 enum ack_type {
 171         NAK = 0,
 172         ACK,
 173         ACK_continue,
 174         ACK_common,
 175         ACK_ready
 176 };
 177
 178 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 179 {
 180         if (args->stateless_rpc && args->depth > 0) {
 181                 /* If we sent a depth we will get back "duplicate"
 182                  * shallow and unshallow commands every time there
 183                  * is a block of have lines exchanged.
 184                  */
 185                 char *line;
 186                 while ((line = packet_read_line(fd, NULL))) {
 187                         if (starts_with(line, "shallow "))
 188                                 continue;
 189                         if (starts_with(line, "unshallow "))
 190                                 continue;
 191                         die("git fetch-pack: expected shallow list");
 192                 }
 193         }
 194 }
 195
 196 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 197 {
 198         int len;
 199         char *line = packet_read_line(fd, &len);
 200         const char *arg;
 201
 202         if (!len)
 203                 die("git fetch-pack: expected ACK/NAK, got EOF");
 204         if (!strcmp(line, "NAK"))
 205                 return NAK;
 206         if (skip_prefix(line, "ACK ", &arg)) {
 207                 if (!get_sha1_hex(arg, result_sha1)) {
 208                         arg += 40;
 209                         len -= arg - line;
 210                         if (len < 1)
 211                                 return ACK;
 212                         if (strstr(arg, "continue"))
 213                                 return ACK_continue;
 214                         if (strstr(arg, "common"))
 215                                 return ACK_common;
 216                         if (strstr(arg, "ready"))
 217                                 return ACK_ready;
 218                         return ACK;
 219                 }
 220         }
 221         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 222 }
 223
 224 static void send_request(struct fetch_pack_args *args,
 225                          int fd, struct strbuf *buf)
 226 {
 227         if (args->stateless_rpc) {
 228                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 229                 packet_flush(fd);
 230         } else
 231                 write_or_die(fd, buf->buf, buf->len);
 232 }
 233
 234 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 235 {
 236         rev_list_insert_ref(NULL, ref->old_sha1);
 237 }
 238
 239 #define INITIAL_FLUSH 16
 240 #define PIPESAFE_FLUSH 32
 241 #define LARGE_FLUSH 1024
 242
 243 static int next_flush(struct fetch_pack_args *args, int count)
 244 {
 245         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 246
 247         if (count < flush_limit)
 248                 count <<= 1;
 249         else
 250                 count += flush_limit;
 251         return count;
 252 }
 253
 254 static int find_common(struct fetch_pack_args *args,
 255                        int fd[2], unsigned char *result_sha1,
 256                        struct ref *refs)
 257 {
 258         int fetching;
 259         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 260         const unsigned char *sha1;
 261         unsigned in_vain = 0;
 262         int got_continue = 0;
 263         int got_ready = 0;
 264         struct strbuf req_buf = STRBUF_INIT;
 265         size_t state_len = 0;
 266
 267         if (args->stateless_rpc && multi_ack == 1)
 268                 die("--stateless-rpc requires multi_ack_detailed");
 269         if (marked)
 270                 for_each_ref(clear_marks, NULL);
 271         marked = 1;
 272
 273         for_each_ref(rev_list_insert_ref_oid, NULL);
 274         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 275
 276         fetching = 0;
 277         for ( ; refs ; refs = refs->next) {
 278                 unsigned char *remote = refs->old_sha1;
 279                 const char *remote_hex;
 280                 struct object *o;
 281
 282                 /*
 283                  * If that object is complete (i.e. it is an ancestor of a
 284                  * local ref), we tell them we have it but do not have to
 285                  * tell them about its ancestors, which they already know
 286                  * about.
 287                  *
 288                  * We use lookup_object here because we are only
 289                  * interested in the case we *know* the object is
 290                  * reachable and we have already scanned it.
 291                  */
 292                 if (((o = lookup_object(remote)) != NULL) &&
 293                                 (o->flags & COMPLETE)) {
 294                         continue;
 295                 }
 296
 297                 remote_hex = sha1_to_hex(remote);
 298                 if (!fetching) {
 299                         struct strbuf c = STRBUF_INIT;
 300                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 301                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 302                         if (no_done)            strbuf_addstr(&c, " no-done");
 303                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 304                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 305                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 306                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 307                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 308                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 309                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 310                                                             git_user_agent_sanitized());
 311                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 312                         strbuf_release(&c);
 313                 } else
 314                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 315                 fetching++;
 316         }
 317
 318         if (!fetching) {
 319                 strbuf_release(&req_buf);
 320                 packet_flush(fd[1]);
 321                 return 1;
 322         }
 323
 324         if (is_repository_shallow())
 325                 write_shallow_commits(&req_buf, 1, NULL);
 326         if (args->depth > 0)
 327                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 328         packet_buf_flush(&req_buf);
 329         state_len = req_buf.len;
 330
 331         if (args->depth > 0) {
 332                 char *line;
 333                 const char *arg;
 334                 unsigned char sha1[20];
 335
 336                 send_request(args, fd[1], &req_buf);
 337                 while ((line = packet_read_line(fd[0], NULL))) {
 338                         if (skip_prefix(line, "shallow ", &arg)) {
 339                                 if (get_sha1_hex(arg, sha1))
 340                                         die("invalid shallow line: %s", line);
 341                                 register_shallow(sha1);
 342                                 continue;
 343                         }
 344                         if (skip_prefix(line, "unshallow ", &arg)) {
 345                                 if (get_sha1_hex(arg, sha1))
 346                                         die("invalid unshallow line: %s", line);
 347                                 if (!lookup_object(sha1))
 348                                         die("object not found: %s", line);
 349                                 /* make sure that it is parsed as shallow */
 350                                 if (!parse_object(sha1))
 351                                         die("error in object: %s", line);
 352                                 if (unregister_shallow(sha1))
 353                                         die("no shallow found: %s", line);
 354                                 continue;
 355                         }
 356                         die("expected shallow/unshallow, got %s", line);
 357                 }
 358         } else if (!args->stateless_rpc)
 359                 send_request(args, fd[1], &req_buf);
 360
 361         if (!args->stateless_rpc) {
 362                 /* If we aren't using the stateless-rpc interface
 363                  * we don't need to retain the headers.
 364                  */
 365                 strbuf_setlen(&req_buf, 0);
 366                 state_len = 0;
 367         }
 368
 369         flushes = 0;
 370         retval = -1;
 371         while ((sha1 = get_rev())) {
 372                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 373                 if (args->verbose)
 374                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 375                 in_vain++;
 376                 if (flush_at <= ++count) {
 377                         int ack;
 378
 379                         packet_buf_flush(&req_buf);
 380                         send_request(args, fd[1], &req_buf);
 381                         strbuf_setlen(&req_buf, state_len);
 382                         flushes++;
 383                         flush_at = next_flush(args, count);
 384
 385                         /*
 386                          * We keep one window "ahead" of the other side, and
 387                          * will wait for an ACK only on the next one
 388                          */
 389                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 390                                 continue;
 391
 392                         consume_shallow_list(args, fd[0]);
 393                         do {
 394                                 ack = get_ack(fd[0], result_sha1);
 395                                 if (args->verbose && ack)
 396                                         fprintf(stderr, "got ack %d %s\n", ack,
 397                                                         sha1_to_hex(result_sha1));
 398                                 switch (ack) {
 399                                 case ACK:
 400                                         flushes = 0;
 401                                         multi_ack = 0;
 402                                         retval = 0;
 403                                         goto done;
 404                                 case ACK_common:
 405                                 case ACK_ready:
 406                                 case ACK_continue: {
 407                                         struct commit *commit =
 408                                                 lookup_commit(result_sha1);
 409                                         if (!commit)
 410                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 411                                         if (args->stateless_rpc
 412                                          && ack == ACK_common
 413                                          && !(commit->object.flags & COMMON)) {
 414                                                 /* We need to replay the have for this object
 415                                                  * on the next RPC request so the peer knows
 416                                                  * it is in common with us.
 417                                                  */
 418                                                 const char *hex = sha1_to_hex(result_sha1);
 419                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 420                                                 state_len = req_buf.len;
 421                                         }
 422                                         mark_common(commit, 0, 1);
 423                                         retval = 0;
 424                                         in_vain = 0;
 425                                         got_continue = 1;
 426                                         if (ack == ACK_ready) {
 427                                                 clear_prio_queue(&rev_list);
 428                                                 got_ready = 1;
 429                                         }
 430                                         break;
 431                                         }
 432                                 }
 433                         } while (ack);
 434                         flushes--;
 435                         if (got_continue && MAX_IN_VAIN < in_vain) {
 436                                 if (args->verbose)
 437                                         fprintf(stderr, "giving up\n");
 438                                 break; /* give up */
 439                         }
 440                 }
 441         }
 442 done:
 443         if (!got_ready || !no_done) {
 444                 packet_buf_write(&req_buf, "done\n");
 445                 send_request(args, fd[1], &req_buf);
 446         }
 447         if (args->verbose)
 448                 fprintf(stderr, "done\n");
 449         if (retval != 0) {
 450                 multi_ack = 0;
 451                 flushes++;
 452         }
 453         strbuf_release(&req_buf);
 454
 455         if (!got_ready || !no_done)
 456                 consume_shallow_list(args, fd[0]);
 457         while (flushes || multi_ack) {
 458                 int ack = get_ack(fd[0], result_sha1);
 459                 if (ack) {
 460                         if (args->verbose)
 461                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 462                                         sha1_to_hex(result_sha1));
 463                         if (ack == ACK)
 464                                 return 0;
 465                         multi_ack = 1;
 466                         continue;
 467                 }
 468                 flushes--;
 469         }
 470         /* it is no error to fetch into a completely empty repo */
 471         return count ? retval : 0;
 472 }
 473
 474 static struct commit_list *complete;
 475
 476 static int mark_complete(const unsigned char *sha1)
 477 {
 478         struct object *o = parse_object(sha1);
 479
 480         while (o && o->type == OBJ_TAG) {
 481                 struct tag *t = (struct tag *) o;
 482                 if (!t->tagged)
 483                         break; /* broken repository */
 484                 o->flags |= COMPLETE;
 485                 o = parse_object(t->tagged->sha1);
 486         }
 487         if (o && o->type == OBJ_COMMIT) {
 488                 struct commit *commit = (struct commit *)o;
 489                 if (!(commit->object.flags & COMPLETE)) {
 490                         commit->object.flags |= COMPLETE;
 491                         commit_list_insert(commit, &complete);
 492                 }
 493         }
 494         return 0;
 495 }
 496
 497 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 498                              int flag, void *cb_data)
 499 {
 500         return mark_complete(oid->hash);
 501 }
 502
 503 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 504                                          unsigned long cutoff)
 505 {
 506         while (complete && cutoff <= complete->item->date) {
 507                 if (args->verbose)
 508                         fprintf(stderr, "Marking %s as complete\n",
 509                                 sha1_to_hex(complete->item->object.sha1));
 510                 pop_most_recent_commit(&complete, COMPLETE);
 511         }
 512 }
 513
 514 static void filter_refs(struct fetch_pack_args *args,
 515                         struct ref **refs,
 516                         struct ref **sought, int nr_sought)
 517 {
 518         struct ref *newlist = NULL;
 519         struct ref **newtail = &newlist;
 520         struct ref *ref, *next;
 521         int i;
 522
 523         i = 0;
 524         for (ref = *refs; ref; ref = next) {
 525                 int keep = 0;
 526                 next = ref->next;
 527
 528                 if (starts_with(ref->name, "refs/") &&
 529                     check_refname_format(ref->name, 0))
 530                         ; /* trash */
 531                 else {
 532                         while (i < nr_sought) {
 533                                 int cmp = strcmp(ref->name, sought[i]->name);
 534                                 if (cmp < 0)
 535                                         break; /* definitely do not have it */
 536                                 else if (cmp == 0) {
 537                                         keep = 1; /* definitely have it */
 538                                         sought[i]->matched = 1;
 539                                 }
 540                                 i++;
 541                         }
 542                 }
 543
 544                 if (!keep && args->fetch_all &&
 545                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 546                         keep = 1;
 547
 548                 if (keep) {
 549                         *newtail = ref;
 550                         ref->next = NULL;
 551                         newtail = &ref->next;
 552                 } else {
 553                         free(ref);
 554                 }
 555         }
 556
 557         /* Append unmatched requests to the list */
 558         if (allow_tip_sha1_in_want) {
 559                 for (i = 0; i < nr_sought; i++) {
 560                         unsigned char sha1[20];
 561
 562                         ref = sought[i];
 563                         if (ref->matched)
 564                                 continue;
 565                         if (get_sha1_hex(ref->name, sha1) ||
 566                             ref->name[40] != '\0' ||
 567                             hashcmp(sha1, ref->old_sha1))
 568                                 continue;
 569
 570                         ref->matched = 1;
 571                         *newtail = copy_ref(ref);
 572                         newtail = &(*newtail)->next;
 573                 }
 574         }
 575         *refs = newlist;
 576 }
 577
 578 static void mark_alternate_complete(const struct ref *ref, void *unused)
 579 {
 580         mark_complete(ref->old_sha1);
 581 }
 582
 583 static int everything_local(struct fetch_pack_args *args,
 584                             struct ref **refs,
 585                             struct ref **sought, int nr_sought)
 586 {
 587         struct ref *ref;
 588         int retval;
 589         unsigned long cutoff = 0;
 590
 591         save_commit_buffer = 0;
 592
 593         for (ref = *refs; ref; ref = ref->next) {
 594                 struct object *o;
 595
 596                 if (!has_sha1_file(ref->old_sha1))
 597                         continue;
 598
 599                 o = parse_object(ref->old_sha1);
 600                 if (!o)
 601                         continue;
 602
 603                 /* We already have it -- which may mean that we were
 604                  * in sync with the other side at some time after
 605                  * that (it is OK if we guess wrong here).
 606                  */
 607                 if (o->type == OBJ_COMMIT) {
 608                         struct commit *commit = (struct commit *)o;
 609                         if (!cutoff || cutoff < commit->date)
 610                                 cutoff = commit->date;
 611                 }
 612         }
 613
 614         if (!args->depth) {
 615                 for_each_ref(mark_complete_oid, NULL);
 616                 for_each_alternate_ref(mark_alternate_complete, NULL);
 617                 commit_list_sort_by_date(&complete);
 618                 if (cutoff)
 619                         mark_recent_complete_commits(args, cutoff);
 620         }
 621
 622         /*
 623          * Mark all complete remote refs as common refs.
 624          * Don't mark them common yet; the server has to be told so first.
 625          */
 626         for (ref = *refs; ref; ref = ref->next) {
 627                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 628                                              NULL, 0);
 629
 630                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 631                         continue;
 632
 633                 if (!(o->flags & SEEN)) {
 634                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 635
 636                         mark_common((struct commit *)o, 1, 1);
 637                 }
 638         }
 639
 640         filter_refs(args, refs, sought, nr_sought);
 641
 642         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 643                 const unsigned char *remote = ref->old_sha1;
 644                 struct object *o;
 645
 646                 o = lookup_object(remote);
 647                 if (!o || !(o->flags & COMPLETE)) {
 648                         retval = 0;
 649                         if (!args->verbose)
 650                                 continue;
 651                         fprintf(stderr,
 652                                 "want %s (%s)\n", sha1_to_hex(remote),
 653                                 ref->name);
 654                         continue;
 655                 }
 656                 if (!args->verbose)
 657                         continue;
 658                 fprintf(stderr,
 659                         "already have %s (%s)\n", sha1_to_hex(remote),
 660                         ref->name);
 661         }
 662         return retval;
 663 }
 664
 665 static int sideband_demux(int in, int out, void *data)
 666 {
 667         int *xd = data;
 668
 669         int ret = recv_sideband("fetch-pack", xd[0], out);
 670         close(out);
 671         return ret;
 672 }
 673
 674 static int get_pack(struct fetch_pack_args *args,
 675                     int xd[2], char **pack_lockfile)
 676 {
 677         struct async demux;
 678         const char *argv[22];
 679         char keep_arg[256];
 680         char hdr_arg[256];
 681         const char **av, *cmd_name;
 682         int do_keep = args->keep_pack;
 683         struct child_process cmd = CHILD_PROCESS_INIT;
 684         int ret;
 685
 686         memset(&demux, 0, sizeof(demux));
 687         if (use_sideband) {
 688                 /* xd[] is talking with upload-pack; subprocess reads from
 689                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 690                  * through demux->out.
 691                  */
 692                 demux.proc = sideband_demux;
 693                 demux.data = xd;
 694                 demux.out = -1;
 695                 if (start_async(&demux))
 696                         die("fetch-pack: unable to fork off sideband"
 697                             " demultiplexer");
 698         }
 699         else
 700                 demux.out = xd[0];
 701
 702         cmd.argv = argv;
 703         av = argv;
 704         *hdr_arg = 0;
 705         if (!args->keep_pack && unpack_limit) {
 706                 struct pack_header header;
 707
 708                 if (read_pack_header(demux.out, &header))
 709                         die("protocol error: bad pack header");
 710                 snprintf(hdr_arg, sizeof(hdr_arg),
 711                          "--pack_header=%"PRIu32",%"PRIu32,
 712                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 713                 if (ntohl(header.hdr_entries) < unpack_limit)
 714                         do_keep = 0;
 715                 else
 716                         do_keep = 1;
 717         }
 718
 719         if (alternate_shallow_file) {
 720                 *av++ = "--shallow-file";
 721                 *av++ = alternate_shallow_file;
 722         }
 723
 724         if (do_keep) {
 725                 if (pack_lockfile)
 726                         cmd.out = -1;
 727                 *av++ = cmd_name = "index-pack";
 728                 *av++ = "--stdin";
 729                 if (!args->quiet && !args->no_progress)
 730                         *av++ = "-v";
 731                 if (args->use_thin_pack)
 732                         *av++ = "--fix-thin";
 733                 if (args->lock_pack || unpack_limit) {
 734                         int s = sprintf(keep_arg,
 735                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 736                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 737                                 strcpy(keep_arg + s, "localhost");
 738                         *av++ = keep_arg;
 739                 }
 740                 if (args->check_self_contained_and_connected)
 741                         *av++ = "--check-self-contained-and-connected";
 742         }
 743         else {
 744                 *av++ = cmd_name = "unpack-objects";
 745                 if (args->quiet || args->no_progress)
 746                         *av++ = "-q";
 747                 args->check_self_contained_and_connected = 0;
 748         }
 749         if (*hdr_arg)
 750                 *av++ = hdr_arg;
 751         if (fetch_fsck_objects >= 0
 752             ? fetch_fsck_objects
 753             : transfer_fsck_objects >= 0
 754             ? transfer_fsck_objects
 755             : 0)
 756                 *av++ = "--strict";
 757         *av++ = NULL;
 758
 759         cmd.in = demux.out;
 760         cmd.git_cmd = 1;
 761         if (start_command(&cmd))
 762                 die("fetch-pack: unable to fork off %s", cmd_name);
 763         if (do_keep && pack_lockfile) {
 764                 *pack_lockfile = index_pack_lockfile(cmd.out);
 765                 close(cmd.out);
 766         }
 767
 768         if (!use_sideband)
 769                 /* Closed by start_command() */
 770                 xd[0] = -1;
 771
 772         ret = finish_command(&cmd);
 773         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 774                 args->self_contained_and_connected =
 775                         args->check_self_contained_and_connected &&
 776                         ret == 0;
 777         else
 778                 die("%s failed", cmd_name);
 779         if (use_sideband && finish_async(&demux))
 780                 die("error in sideband demultiplexer");
 781         return 0;
 782 }
 783
 784 static int cmp_ref_by_name(const void *a_, const void *b_)
 785 {
 786         const struct ref *a = *((const struct ref **)a_);
 787         const struct ref *b = *((const struct ref **)b_);
 788         return strcmp(a->name, b->name);
 789 }
 790
 791 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 792                                  int fd[2],
 793                                  const struct ref *orig_ref,
 794                                  struct ref **sought, int nr_sought,
 795                                  struct shallow_info *si,
 796                                  char **pack_lockfile)
 797 {
 798         struct ref *ref = copy_ref_list(orig_ref);
 799         unsigned char sha1[20];
 800         const char *agent_feature;
 801         int agent_len;
 802
 803         sort_ref_list(&ref, ref_compare_name);
 804         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 805
 806         if (is_repository_shallow() && !server_supports("shallow"))
 807                 die("Server does not support shallow clients");
 808         if (server_supports("multi_ack_detailed")) {
 809                 if (args->verbose)
 810                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 811                 multi_ack = 2;
 812                 if (server_supports("no-done")) {
 813                         if (args->verbose)
 814                                 fprintf(stderr, "Server supports no-done\n");
 815                         if (args->stateless_rpc)
 816                                 no_done = 1;
 817                 }
 818         }
 819         else if (server_supports("multi_ack")) {
 820                 if (args->verbose)
 821                         fprintf(stderr, "Server supports multi_ack\n");
 822                 multi_ack = 1;
 823         }
 824         if (server_supports("side-band-64k")) {
 825                 if (args->verbose)
 826                         fprintf(stderr, "Server supports side-band-64k\n");
 827                 use_sideband = 2;
 828         }
 829         else if (server_supports("side-band")) {
 830                 if (args->verbose)
 831                         fprintf(stderr, "Server supports side-band\n");
 832                 use_sideband = 1;
 833         }
 834         if (server_supports("allow-tip-sha1-in-want")) {
 835                 if (args->verbose)
 836                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 837                 allow_tip_sha1_in_want = 1;
 838         }
 839         if (!server_supports("thin-pack"))
 840                 args->use_thin_pack = 0;
 841         if (!server_supports("no-progress"))
 842                 args->no_progress = 0;
 843         if (!server_supports("include-tag"))
 844                 args->include_tag = 0;
 845         if (server_supports("ofs-delta")) {
 846                 if (args->verbose)
 847                         fprintf(stderr, "Server supports ofs-delta\n");
 848         } else
 849                 prefer_ofs_delta = 0;
 850
 851         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 852                 agent_supported = 1;
 853                 if (args->verbose && agent_len)
 854                         fprintf(stderr, "Server version is %.*s\n",
 855                                 agent_len, agent_feature);
 856         }
 857
 858         if (everything_local(args, &ref, sought, nr_sought)) {
 859                 packet_flush(fd[1]);
 860                 goto all_done;
 861         }
 862         if (find_common(args, fd, sha1, ref) < 0)
 863                 if (!args->keep_pack)
 864                         /* When cloning, it is not unusual to have
 865                          * no common commit.
 866                          */
 867                         warning("no common commits");
 868
 869         if (args->stateless_rpc)
 870                 packet_flush(fd[1]);
 871         if (args->depth > 0)
 872                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 873                                         NULL);
 874         else if (si->nr_ours || si->nr_theirs)
 875                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 876         else
 877                 alternate_shallow_file = NULL;
 878         if (get_pack(args, fd, pack_lockfile))
 879                 die("git fetch-pack: fetch failed.");
 880
 881  all_done:
 882         return ref;
 883 }
 884
 885 static void fetch_pack_config(void)
 886 {
 887         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 888         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 889         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 890         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 891         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 892
 893         git_config(git_default_config, NULL);
 894 }
 895
 896 static void fetch_pack_setup(void)
 897 {
 898         static int did_setup;
 899         if (did_setup)
 900                 return;
 901         fetch_pack_config();
 902         if (0 <= transfer_unpack_limit)
 903                 unpack_limit = transfer_unpack_limit;
 904         else if (0 <= fetch_unpack_limit)
 905                 unpack_limit = fetch_unpack_limit;
 906         did_setup = 1;
 907 }
 908
 909 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 910 {
 911         struct string_list names = STRING_LIST_INIT_NODUP;
 912         int src, dst;
 913
 914         for (src = dst = 0; src < nr; src++) {
 915                 struct string_list_item *item;
 916                 item = string_list_insert(&names, ref[src]->name);
 917                 if (item->util)
 918                         continue; /* already have it */
 919                 item->util = ref[src];
 920                 if (src != dst)
 921                         ref[dst] = ref[src];
 922                 dst++;
 923         }
 924         for (src = dst; src < nr; src++)
 925                 ref[src] = NULL;
 926         string_list_clear(&names, 0);
 927         return dst;
 928 }
 929
 930 static void update_shallow(struct fetch_pack_args *args,
 931                            struct ref **sought, int nr_sought,
 932                            struct shallow_info *si)
 933 {
 934         struct sha1_array ref = SHA1_ARRAY_INIT;
 935         int *status;
 936         int i;
 937
 938         if (args->depth > 0 && alternate_shallow_file) {
 939                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 940                         unlink_or_warn(git_path("shallow"));
 941                         rollback_lock_file(&shallow_lock);
 942                 } else
 943                         commit_lock_file(&shallow_lock);
 944                 return;
 945         }
 946
 947         if (!si->shallow || !si->shallow->nr)
 948                 return;
 949
 950         if (args->cloning) {
 951                 /*
 952                  * remote is shallow, but this is a clone, there are
 953                  * no objects in repo to worry about. Accept any
 954                  * shallow points that exist in the pack (iow in repo
 955                  * after get_pack() and reprepare_packed_git())
 956                  */
 957                 struct sha1_array extra = SHA1_ARRAY_INIT;
 958                 unsigned char (*sha1)[20] = si->shallow->sha1;
 959                 for (i = 0; i < si->shallow->nr; i++)
 960                         if (has_sha1_file(sha1[i]))
 961                                 sha1_array_append(&extra, sha1[i]);
 962                 if (extra.nr) {
 963                         setup_alternate_shallow(&shallow_lock,
 964                                                 &alternate_shallow_file,
 965                                                 &extra);
 966                         commit_lock_file(&shallow_lock);
 967                 }
 968                 sha1_array_clear(&extra);
 969                 return;
 970         }
 971
 972         if (!si->nr_ours && !si->nr_theirs)
 973                 return;
 974
 975         remove_nonexistent_theirs_shallow(si);
 976         if (!si->nr_ours && !si->nr_theirs)
 977                 return;
 978         for (i = 0; i < nr_sought; i++)
 979                 sha1_array_append(&ref, sought[i]->old_sha1);
 980         si->ref = &ref;
 981
 982         if (args->update_shallow) {
 983                 /*
 984                  * remote is also shallow, .git/shallow may be updated
 985                  * so all refs can be accepted. Make sure we only add
 986                  * shallow roots that are actually reachable from new
 987                  * refs.
 988                  */
 989                 struct sha1_array extra = SHA1_ARRAY_INIT;
 990                 unsigned char (*sha1)[20] = si->shallow->sha1;
 991                 assign_shallow_commits_to_refs(si, NULL, NULL);
 992                 if (!si->nr_ours && !si->nr_theirs) {
 993                         sha1_array_clear(&ref);
 994                         return;
 995                 }
 996                 for (i = 0; i < si->nr_ours; i++)
 997                         sha1_array_append(&extra, sha1[si->ours[i]]);
 998                 for (i = 0; i < si->nr_theirs; i++)
 999                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1000                 setup_alternate_shallow(&shallow_lock,
1001                                         &alternate_shallow_file,
1002                                         &extra);
1003                 commit_lock_file(&shallow_lock);
1004                 sha1_array_clear(&extra);
1005                 sha1_array_clear(&ref);
1006                 return;
1007         }
1008
1009         /*
1010          * remote is also shallow, check what ref is safe to update
1011          * without updating .git/shallow
1012          */
1013         status = xcalloc(nr_sought, sizeof(*status));
1014         assign_shallow_commits_to_refs(si, NULL, status);
1015         if (si->nr_ours || si->nr_theirs) {
1016                 for (i = 0; i < nr_sought; i++)
1017                         if (status[i])
1018                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1019         }
1020         free(status);
1021         sha1_array_clear(&ref);
1022 }
1023
1024 struct ref *fetch_pack(struct fetch_pack_args *args,
1025                        int fd[], struct child_process *conn,
1026                        const struct ref *ref,
1027                        const char *dest,
1028                        struct ref **sought, int nr_sought,
1029                        struct sha1_array *shallow,
1030                        char **pack_lockfile)
1031 {
1032         struct ref *ref_cpy;
1033         struct shallow_info si;
1034
1035         fetch_pack_setup();
1036         if (nr_sought)
1037                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1038
1039         if (!ref) {
1040                 packet_flush(fd[1]);
1041                 die("no matching remote head");
1042         }
1043         prepare_shallow_info(&si, shallow);
1044         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1045                                 &si, pack_lockfile);
1046         reprepare_packed_git();
1047         update_shallow(args, sought, nr_sought, &si);
1048         clear_shallow_info(&si);
1049         return ref_cpy;
1050 }