fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13 #include "version.h"
  14 #include "prio-queue.h"
  15
  16 static int transfer_unpack_limit = -1;
  17 static int fetch_unpack_limit = -1;
  18 static int unpack_limit = 100;
  19 static int prefer_ofs_delta = 1;
  20 static int no_done;
  21 static int fetch_fsck_objects = -1;
  22 static int transfer_fsck_objects = -1;
  23 static int agent_supported;
  24 static struct lock_file shallow_lock;
  25 static const char *alternate_shallow_file;
  26
  27 #define COMPLETE        (1U << 0)
  28 #define COMMON          (1U << 1)
  29 #define COMMON_REF      (1U << 2)
  30 #define SEEN            (1U << 3)
  31 #define POPPED          (1U << 4)
  32
  33 static int marked;
  34
  35 /*
  36  * After sending this many "have"s if we do not get any new ACK , we
  37  * give up traversing our history.
  38  */
  39 #define MAX_IN_VAIN 256
  40
  41 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  42 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  43
  44 static void rev_list_push(struct commit *commit, int mark)
  45 {
  46         if (!(commit->object.flags & mark)) {
  47                 commit->object.flags |= mark;
  48
  49                 if (!(commit->object.parsed))
  50                         if (parse_commit(commit))
  51                                 return;
  52
  53                 prio_queue_put(&rev_list, commit);
  54
  55                 if (!(commit->object.flags & COMMON))
  56                         non_common_revs++;
  57         }
  58 }
  59
  60 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  61 {
  62         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  63
  64         if (o && o->type == OBJ_COMMIT)
  65                 rev_list_push((struct commit *)o, SEEN);
  66
  67         return 0;
  68 }
  69
  70 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  71 {
  72         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  73
  74         if (o && o->type == OBJ_COMMIT)
  75                 clear_commit_marks((struct commit *)o,
  76                                    COMMON | COMMON_REF | SEEN | POPPED);
  77         return 0;
  78 }
  79
  80 /*
  81    This function marks a rev and its ancestors as common.
  82    In some cases, it is desirable to mark only the ancestors (for example
  83    when only the server does not yet know that they are common).
  84 */
  85
  86 static void mark_common(struct commit *commit,
  87                 int ancestors_only, int dont_parse)
  88 {
  89         if (commit != NULL && !(commit->object.flags & COMMON)) {
  90                 struct object *o = (struct object *)commit;
  91
  92                 if (!ancestors_only)
  93                         o->flags |= COMMON;
  94
  95                 if (!(o->flags & SEEN))
  96                         rev_list_push(commit, SEEN);
  97                 else {
  98                         struct commit_list *parents;
  99
 100                         if (!ancestors_only && !(o->flags & POPPED))
 101                                 non_common_revs--;
 102                         if (!o->parsed && !dont_parse)
 103                                 if (parse_commit(commit))
 104                                         return;
 105
 106                         for (parents = commit->parents;
 107                                         parents;
 108                                         parents = parents->next)
 109                                 mark_common(parents->item, 0, dont_parse);
 110                 }
 111         }
 112 }
 113
 114 /*
 115   Get the next rev to send, ignoring the common.
 116 */
 117
 118 static const unsigned char *get_rev(void)
 119 {
 120         struct commit *commit = NULL;
 121
 122         while (commit == NULL) {
 123                 unsigned int mark;
 124                 struct commit_list *parents;
 125
 126                 if (rev_list.nr == 0 || non_common_revs == 0)
 127                         return NULL;
 128
 129                 commit = prio_queue_get(&rev_list);
 130                 if (!commit->object.parsed)
 131                         parse_commit(commit);
 132                 parents = commit->parents;
 133
 134                 commit->object.flags |= POPPED;
 135                 if (!(commit->object.flags & COMMON))
 136                         non_common_revs--;
 137
 138                 if (commit->object.flags & COMMON) {
 139                         /* do not send "have", and ignore ancestors */
 140                         commit = NULL;
 141                         mark = COMMON | SEEN;
 142                 } else if (commit->object.flags & COMMON_REF)
 143                         /* send "have", and ignore ancestors */
 144                         mark = COMMON | SEEN;
 145                 else
 146                         /* send "have", also for its ancestors */
 147                         mark = SEEN;
 148
 149                 while (parents) {
 150                         if (!(parents->item->object.flags & SEEN))
 151                                 rev_list_push(parents->item, mark);
 152                         if (mark & COMMON)
 153                                 mark_common(parents->item, 1, 0);
 154                         parents = parents->next;
 155                 }
 156         }
 157
 158         return commit->object.sha1;
 159 }
 160
 161 enum ack_type {
 162         NAK = 0,
 163         ACK,
 164         ACK_continue,
 165         ACK_common,
 166         ACK_ready
 167 };
 168
 169 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 170 {
 171         if (args->stateless_rpc && args->depth > 0) {
 172                 /* If we sent a depth we will get back "duplicate"
 173                  * shallow and unshallow commands every time there
 174                  * is a block of have lines exchanged.
 175                  */
 176                 char *line;
 177                 while ((line = packet_read_line(fd, NULL))) {
 178                         if (!prefixcmp(line, "shallow "))
 179                                 continue;
 180                         if (!prefixcmp(line, "unshallow "))
 181                                 continue;
 182                         die("git fetch-pack: expected shallow list");
 183                 }
 184         }
 185 }
 186
 187 struct write_shallow_data {
 188         struct strbuf *out;
 189         int use_pack_protocol;
 190         int count;
 191 };
 192
 193 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 194 {
 195         struct write_shallow_data *data = cb_data;
 196         const char *hex = sha1_to_hex(graft->sha1);
 197         data->count++;
 198         if (data->use_pack_protocol)
 199                 packet_buf_write(data->out, "shallow %s", hex);
 200         else {
 201                 strbuf_addstr(data->out, hex);
 202                 strbuf_addch(data->out, '\n');
 203         }
 204         return 0;
 205 }
 206
 207 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 208 {
 209         struct write_shallow_data data;
 210         data.out = out;
 211         data.use_pack_protocol = use_pack_protocol;
 212         data.count = 0;
 213         for_each_commit_graft(write_one_shallow, &data);
 214         return data.count;
 215 }
 216
 217 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 218 {
 219         int len;
 220         char *line = packet_read_line(fd, &len);
 221
 222         if (!len)
 223                 die("git fetch-pack: expected ACK/NAK, got EOF");
 224         if (!strcmp(line, "NAK"))
 225                 return NAK;
 226         if (!prefixcmp(line, "ACK ")) {
 227                 if (!get_sha1_hex(line+4, result_sha1)) {
 228                         if (len < 45)
 229                                 return ACK;
 230                         if (strstr(line+45, "continue"))
 231                                 return ACK_continue;
 232                         if (strstr(line+45, "common"))
 233                                 return ACK_common;
 234                         if (strstr(line+45, "ready"))
 235                                 return ACK_ready;
 236                         return ACK;
 237                 }
 238         }
 239         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 240 }
 241
 242 static void send_request(struct fetch_pack_args *args,
 243                          int fd, struct strbuf *buf)
 244 {
 245         if (args->stateless_rpc) {
 246                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 247                 packet_flush(fd);
 248         } else
 249                 write_or_die(fd, buf->buf, buf->len);
 250 }
 251
 252 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 253 {
 254         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 255 }
 256
 257 #define INITIAL_FLUSH 16
 258 #define PIPESAFE_FLUSH 32
 259 #define LARGE_FLUSH 1024
 260
 261 static int next_flush(struct fetch_pack_args *args, int count)
 262 {
 263         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 264
 265         if (count < flush_limit)
 266                 count <<= 1;
 267         else
 268                 count += flush_limit;
 269         return count;
 270 }
 271
 272 static int find_common(struct fetch_pack_args *args,
 273                        int fd[2], unsigned char *result_sha1,
 274                        struct ref *refs)
 275 {
 276         int fetching;
 277         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 278         const unsigned char *sha1;
 279         unsigned in_vain = 0;
 280         int got_continue = 0;
 281         int got_ready = 0;
 282         struct strbuf req_buf = STRBUF_INIT;
 283         size_t state_len = 0;
 284
 285         if (args->stateless_rpc && multi_ack == 1)
 286                 die("--stateless-rpc requires multi_ack_detailed");
 287         if (marked)
 288                 for_each_ref(clear_marks, NULL);
 289         marked = 1;
 290
 291         for_each_ref(rev_list_insert_ref, NULL);
 292         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 293
 294         fetching = 0;
 295         for ( ; refs ; refs = refs->next) {
 296                 unsigned char *remote = refs->old_sha1;
 297                 const char *remote_hex;
 298                 struct object *o;
 299
 300                 /*
 301                  * If that object is complete (i.e. it is an ancestor of a
 302                  * local ref), we tell them we have it but do not have to
 303                  * tell them about its ancestors, which they already know
 304                  * about.
 305                  *
 306                  * We use lookup_object here because we are only
 307                  * interested in the case we *know* the object is
 308                  * reachable and we have already scanned it.
 309                  */
 310                 if (((o = lookup_object(remote)) != NULL) &&
 311                                 (o->flags & COMPLETE)) {
 312                         continue;
 313                 }
 314
 315                 remote_hex = sha1_to_hex(remote);
 316                 if (!fetching) {
 317                         struct strbuf c = STRBUF_INIT;
 318                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 319                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 320                         if (no_done)            strbuf_addstr(&c, " no-done");
 321                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 322                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 323                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 324                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 325                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 326                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 327                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 328                                                             git_user_agent_sanitized());
 329                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 330                         strbuf_release(&c);
 331                 } else
 332                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 333                 fetching++;
 334         }
 335
 336         if (!fetching) {
 337                 strbuf_release(&req_buf);
 338                 packet_flush(fd[1]);
 339                 return 1;
 340         }
 341
 342         if (is_repository_shallow())
 343                 write_shallow_commits(&req_buf, 1);
 344         if (args->depth > 0)
 345                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 346         packet_buf_flush(&req_buf);
 347         state_len = req_buf.len;
 348
 349         if (args->depth > 0) {
 350                 char *line;
 351                 unsigned char sha1[20];
 352
 353                 send_request(args, fd[1], &req_buf);
 354                 while ((line = packet_read_line(fd[0], NULL))) {
 355                         if (!prefixcmp(line, "shallow ")) {
 356                                 if (get_sha1_hex(line + 8, sha1))
 357                                         die("invalid shallow line: %s", line);
 358                                 register_shallow(sha1);
 359                                 continue;
 360                         }
 361                         if (!prefixcmp(line, "unshallow ")) {
 362                                 if (get_sha1_hex(line + 10, sha1))
 363                                         die("invalid unshallow line: %s", line);
 364                                 if (!lookup_object(sha1))
 365                                         die("object not found: %s", line);
 366                                 /* make sure that it is parsed as shallow */
 367                                 if (!parse_object(sha1))
 368                                         die("error in object: %s", line);
 369                                 if (unregister_shallow(sha1))
 370                                         die("no shallow found: %s", line);
 371                                 continue;
 372                         }
 373                         die("expected shallow/unshallow, got %s", line);
 374                 }
 375         } else if (!args->stateless_rpc)
 376                 send_request(args, fd[1], &req_buf);
 377
 378         if (!args->stateless_rpc) {
 379                 /* If we aren't using the stateless-rpc interface
 380                  * we don't need to retain the headers.
 381                  */
 382                 strbuf_setlen(&req_buf, 0);
 383                 state_len = 0;
 384         }
 385
 386         flushes = 0;
 387         retval = -1;
 388         while ((sha1 = get_rev())) {
 389                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 390                 if (args->verbose)
 391                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 392                 in_vain++;
 393                 if (flush_at <= ++count) {
 394                         int ack;
 395
 396                         packet_buf_flush(&req_buf);
 397                         send_request(args, fd[1], &req_buf);
 398                         strbuf_setlen(&req_buf, state_len);
 399                         flushes++;
 400                         flush_at = next_flush(args, count);
 401
 402                         /*
 403                          * We keep one window "ahead" of the other side, and
 404                          * will wait for an ACK only on the next one
 405                          */
 406                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 407                                 continue;
 408
 409                         consume_shallow_list(args, fd[0]);
 410                         do {
 411                                 ack = get_ack(fd[0], result_sha1);
 412                                 if (args->verbose && ack)
 413                                         fprintf(stderr, "got ack %d %s\n", ack,
 414                                                         sha1_to_hex(result_sha1));
 415                                 switch (ack) {
 416                                 case ACK:
 417                                         flushes = 0;
 418                                         multi_ack = 0;
 419                                         retval = 0;
 420                                         goto done;
 421                                 case ACK_common:
 422                                 case ACK_ready:
 423                                 case ACK_continue: {
 424                                         struct commit *commit =
 425                                                 lookup_commit(result_sha1);
 426                                         if (!commit)
 427                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 428                                         if (args->stateless_rpc
 429                                          && ack == ACK_common
 430                                          && !(commit->object.flags & COMMON)) {
 431                                                 /* We need to replay the have for this object
 432                                                  * on the next RPC request so the peer knows
 433                                                  * it is in common with us.
 434                                                  */
 435                                                 const char *hex = sha1_to_hex(result_sha1);
 436                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 437                                                 state_len = req_buf.len;
 438                                         }
 439                                         mark_common(commit, 0, 1);
 440                                         retval = 0;
 441                                         in_vain = 0;
 442                                         got_continue = 1;
 443                                         if (ack == ACK_ready) {
 444                                                 clear_prio_queue(&rev_list);
 445                                                 got_ready = 1;
 446                                         }
 447                                         break;
 448                                         }
 449                                 }
 450                         } while (ack);
 451                         flushes--;
 452                         if (got_continue && MAX_IN_VAIN < in_vain) {
 453                                 if (args->verbose)
 454                                         fprintf(stderr, "giving up\n");
 455                                 break; /* give up */
 456                         }
 457                 }
 458         }
 459 done:
 460         if (!got_ready || !no_done) {
 461                 packet_buf_write(&req_buf, "done\n");
 462                 send_request(args, fd[1], &req_buf);
 463         }
 464         if (args->verbose)
 465                 fprintf(stderr, "done\n");
 466         if (retval != 0) {
 467                 multi_ack = 0;
 468                 flushes++;
 469         }
 470         strbuf_release(&req_buf);
 471
 472         consume_shallow_list(args, fd[0]);
 473         while (flushes || multi_ack) {
 474                 int ack = get_ack(fd[0], result_sha1);
 475                 if (ack) {
 476                         if (args->verbose)
 477                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 478                                         sha1_to_hex(result_sha1));
 479                         if (ack == ACK)
 480                                 return 0;
 481                         multi_ack = 1;
 482                         continue;
 483                 }
 484                 flushes--;
 485         }
 486         /* it is no error to fetch into a completely empty repo */
 487         return count ? retval : 0;
 488 }
 489
 490 static struct commit_list *complete;
 491
 492 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 493 {
 494         struct object *o = parse_object(sha1);
 495
 496         while (o && o->type == OBJ_TAG) {
 497                 struct tag *t = (struct tag *) o;
 498                 if (!t->tagged)
 499                         break; /* broken repository */
 500                 o->flags |= COMPLETE;
 501                 o = parse_object(t->tagged->sha1);
 502         }
 503         if (o && o->type == OBJ_COMMIT) {
 504                 struct commit *commit = (struct commit *)o;
 505                 if (!(commit->object.flags & COMPLETE)) {
 506                         commit->object.flags |= COMPLETE;
 507                         commit_list_insert(commit, &complete);
 508                 }
 509         }
 510         return 0;
 511 }
 512
 513 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 514                                          unsigned long cutoff)
 515 {
 516         while (complete && cutoff <= complete->item->date) {
 517                 if (args->verbose)
 518                         fprintf(stderr, "Marking %s as complete\n",
 519                                 sha1_to_hex(complete->item->object.sha1));
 520                 pop_most_recent_commit(&complete, COMPLETE);
 521         }
 522 }
 523
 524 static void filter_refs(struct fetch_pack_args *args,
 525                         struct ref **refs,
 526                         struct ref **sought, int nr_sought)
 527 {
 528         struct ref *newlist = NULL;
 529         struct ref **newtail = &newlist;
 530         struct ref *ref, *next;
 531         int i;
 532
 533         i = 0;
 534         for (ref = *refs; ref; ref = next) {
 535                 int keep = 0;
 536                 next = ref->next;
 537
 538                 if (!memcmp(ref->name, "refs/", 5) &&
 539                     check_refname_format(ref->name + 5, 0))
 540                         ; /* trash */
 541                 else {
 542                         while (i < nr_sought) {
 543                                 int cmp = strcmp(ref->name, sought[i]->name);
 544                                 if (cmp < 0)
 545                                         break; /* definitely do not have it */
 546                                 else if (cmp == 0) {
 547                                         keep = 1; /* definitely have it */
 548                                         sought[i]->matched = 1;
 549                                 }
 550                                 i++;
 551                         }
 552                 }
 553
 554                 if (!keep && args->fetch_all &&
 555                     (!args->depth || prefixcmp(ref->name, "refs/tags/")))
 556                         keep = 1;
 557
 558                 if (keep) {
 559                         *newtail = ref;
 560                         ref->next = NULL;
 561                         newtail = &ref->next;
 562                 } else {
 563                         free(ref);
 564                 }
 565         }
 566
 567         /* Append unmatched requests to the list */
 568         if (allow_tip_sha1_in_want) {
 569                 for (i = 0; i < nr_sought; i++) {
 570                         ref = sought[i];
 571                         if (ref->matched)
 572                                 continue;
 573                         if (get_sha1_hex(ref->name, ref->old_sha1))
 574                                 continue;
 575
 576                         ref->matched = 1;
 577                         *newtail = ref;
 578                         ref->next = NULL;
 579                         newtail = &ref->next;
 580                 }
 581         }
 582         *refs = newlist;
 583 }
 584
 585 static void mark_alternate_complete(const struct ref *ref, void *unused)
 586 {
 587         mark_complete(NULL, ref->old_sha1, 0, NULL);
 588 }
 589
 590 static int everything_local(struct fetch_pack_args *args,
 591                             struct ref **refs,
 592                             struct ref **sought, int nr_sought)
 593 {
 594         struct ref *ref;
 595         int retval;
 596         unsigned long cutoff = 0;
 597
 598         save_commit_buffer = 0;
 599
 600         for (ref = *refs; ref; ref = ref->next) {
 601                 struct object *o;
 602
 603                 if (!has_sha1_file(ref->old_sha1))
 604                         continue;
 605
 606                 o = parse_object(ref->old_sha1);
 607                 if (!o)
 608                         continue;
 609
 610                 /* We already have it -- which may mean that we were
 611                  * in sync with the other side at some time after
 612                  * that (it is OK if we guess wrong here).
 613                  */
 614                 if (o->type == OBJ_COMMIT) {
 615                         struct commit *commit = (struct commit *)o;
 616                         if (!cutoff || cutoff < commit->date)
 617                                 cutoff = commit->date;
 618                 }
 619         }
 620
 621         if (!args->depth) {
 622                 for_each_ref(mark_complete, NULL);
 623                 for_each_alternate_ref(mark_alternate_complete, NULL);
 624                 commit_list_sort_by_date(&complete);
 625                 if (cutoff)
 626                         mark_recent_complete_commits(args, cutoff);
 627         }
 628
 629         /*
 630          * Mark all complete remote refs as common refs.
 631          * Don't mark them common yet; the server has to be told so first.
 632          */
 633         for (ref = *refs; ref; ref = ref->next) {
 634                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 635                                              NULL, 0);
 636
 637                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 638                         continue;
 639
 640                 if (!(o->flags & SEEN)) {
 641                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 642
 643                         mark_common((struct commit *)o, 1, 1);
 644                 }
 645         }
 646
 647         filter_refs(args, refs, sought, nr_sought);
 648
 649         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 650                 const unsigned char *remote = ref->old_sha1;
 651                 unsigned char local[20];
 652                 struct object *o;
 653
 654                 o = lookup_object(remote);
 655                 if (!o || !(o->flags & COMPLETE)) {
 656                         retval = 0;
 657                         if (!args->verbose)
 658                                 continue;
 659                         fprintf(stderr,
 660                                 "want %s (%s)\n", sha1_to_hex(remote),
 661                                 ref->name);
 662                         continue;
 663                 }
 664
 665                 hashcpy(ref->new_sha1, local);
 666                 if (!args->verbose)
 667                         continue;
 668                 fprintf(stderr,
 669                         "already have %s (%s)\n", sha1_to_hex(remote),
 670                         ref->name);
 671         }
 672         return retval;
 673 }
 674
 675 static int sideband_demux(int in, int out, void *data)
 676 {
 677         int *xd = data;
 678
 679         int ret = recv_sideband("fetch-pack", xd[0], out);
 680         close(out);
 681         return ret;
 682 }
 683
 684 static int get_pack(struct fetch_pack_args *args,
 685                     int xd[2], char **pack_lockfile)
 686 {
 687         struct async demux;
 688         const char *argv[22];
 689         char keep_arg[256];
 690         char hdr_arg[256];
 691         const char **av;
 692         int do_keep = args->keep_pack;
 693         struct child_process cmd;
 694         int ret;
 695
 696         memset(&demux, 0, sizeof(demux));
 697         if (use_sideband) {
 698                 /* xd[] is talking with upload-pack; subprocess reads from
 699                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 700                  * through demux->out.
 701                  */
 702                 demux.proc = sideband_demux;
 703                 demux.data = xd;
 704                 demux.out = -1;
 705                 if (start_async(&demux))
 706                         die("fetch-pack: unable to fork off sideband"
 707                             " demultiplexer");
 708         }
 709         else
 710                 demux.out = xd[0];
 711
 712         memset(&cmd, 0, sizeof(cmd));
 713         cmd.argv = argv;
 714         av = argv;
 715         *hdr_arg = 0;
 716         if (!args->keep_pack && unpack_limit) {
 717                 struct pack_header header;
 718
 719                 if (read_pack_header(demux.out, &header))
 720                         die("protocol error: bad pack header");
 721                 snprintf(hdr_arg, sizeof(hdr_arg),
 722                          "--pack_header=%"PRIu32",%"PRIu32,
 723                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 724                 if (ntohl(header.hdr_entries) < unpack_limit)
 725                         do_keep = 0;
 726                 else
 727                         do_keep = 1;
 728         }
 729
 730         if (alternate_shallow_file) {
 731                 *av++ = "--shallow-file";
 732                 *av++ = alternate_shallow_file;
 733         }
 734
 735         if (do_keep) {
 736                 if (pack_lockfile)
 737                         cmd.out = -1;
 738                 *av++ = "index-pack";
 739                 *av++ = "--stdin";
 740                 if (!args->quiet && !args->no_progress)
 741                         *av++ = "-v";
 742                 if (args->use_thin_pack)
 743                         *av++ = "--fix-thin";
 744                 if (args->lock_pack || unpack_limit) {
 745                         int s = sprintf(keep_arg,
 746                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 747                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 748                                 strcpy(keep_arg + s, "localhost");
 749                         *av++ = keep_arg;
 750                 }
 751                 if (args->check_self_contained_and_connected)
 752                         *av++ = "--check-self-contained-and-connected";
 753         }
 754         else {
 755                 *av++ = "unpack-objects";
 756                 if (args->quiet || args->no_progress)
 757                         *av++ = "-q";
 758                 args->check_self_contained_and_connected = 0;
 759         }
 760         if (*hdr_arg)
 761                 *av++ = hdr_arg;
 762         if (fetch_fsck_objects >= 0
 763             ? fetch_fsck_objects
 764             : transfer_fsck_objects >= 0
 765             ? transfer_fsck_objects
 766             : 0)
 767                 *av++ = "--strict";
 768         *av++ = NULL;
 769
 770         cmd.in = demux.out;
 771         cmd.git_cmd = 1;
 772         if (start_command(&cmd))
 773                 die("fetch-pack: unable to fork off %s", argv[0]);
 774         if (do_keep && pack_lockfile) {
 775                 *pack_lockfile = index_pack_lockfile(cmd.out);
 776                 close(cmd.out);
 777         }
 778
 779         if (!use_sideband)
 780                 /* Closed by start_command() */
 781                 xd[0] = -1;
 782
 783         ret = finish_command(&cmd);
 784         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 785                 args->self_contained_and_connected =
 786                         args->check_self_contained_and_connected &&
 787                         ret == 0;
 788         else
 789                 die("%s failed", argv[0]);
 790         if (use_sideband && finish_async(&demux))
 791                 die("error in sideband demultiplexer");
 792         return 0;
 793 }
 794
 795 static int cmp_ref_by_name(const void *a_, const void *b_)
 796 {
 797         const struct ref *a = *((const struct ref **)a_);
 798         const struct ref *b = *((const struct ref **)b_);
 799         return strcmp(a->name, b->name);
 800 }
 801
 802 static void setup_alternate_shallow(void)
 803 {
 804         struct strbuf sb = STRBUF_INIT;
 805         int fd;
 806
 807         check_shallow_file_for_update();
 808         fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
 809                                        LOCK_DIE_ON_ERROR);
 810         if (write_shallow_commits(&sb, 0)) {
 811                 if (write_in_full(fd, sb.buf, sb.len) != sb.len)
 812                         die_errno("failed to write to %s", shallow_lock.filename);
 813                 alternate_shallow_file = shallow_lock.filename;
 814         } else
 815                 /*
 816                  * is_repository_shallow() sees empty string as "no
 817                  * shallow file".
 818                  */
 819                 alternate_shallow_file = "";
 820         strbuf_release(&sb);
 821 }
 822
 823 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 824                                  int fd[2],
 825                                  const struct ref *orig_ref,
 826                                  struct ref **sought, int nr_sought,
 827                                  char **pack_lockfile)
 828 {
 829         struct ref *ref = copy_ref_list(orig_ref);
 830         unsigned char sha1[20];
 831         const char *agent_feature;
 832         int agent_len;
 833
 834         sort_ref_list(&ref, ref_compare_name);
 835         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 836
 837         if (is_repository_shallow() && !server_supports("shallow"))
 838                 die("Server does not support shallow clients");
 839         if (server_supports("multi_ack_detailed")) {
 840                 if (args->verbose)
 841                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 842                 multi_ack = 2;
 843                 if (server_supports("no-done")) {
 844                         if (args->verbose)
 845                                 fprintf(stderr, "Server supports no-done\n");
 846                         if (args->stateless_rpc)
 847                                 no_done = 1;
 848                 }
 849         }
 850         else if (server_supports("multi_ack")) {
 851                 if (args->verbose)
 852                         fprintf(stderr, "Server supports multi_ack\n");
 853                 multi_ack = 1;
 854         }
 855         if (server_supports("side-band-64k")) {
 856                 if (args->verbose)
 857                         fprintf(stderr, "Server supports side-band-64k\n");
 858                 use_sideband = 2;
 859         }
 860         else if (server_supports("side-band")) {
 861                 if (args->verbose)
 862                         fprintf(stderr, "Server supports side-band\n");
 863                 use_sideband = 1;
 864         }
 865         if (server_supports("allow-tip-sha1-in-want")) {
 866                 if (args->verbose)
 867                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 868                 allow_tip_sha1_in_want = 1;
 869         }
 870         if (!server_supports("thin-pack"))
 871                 args->use_thin_pack = 0;
 872         if (!server_supports("no-progress"))
 873                 args->no_progress = 0;
 874         if (!server_supports("include-tag"))
 875                 args->include_tag = 0;
 876         if (server_supports("ofs-delta")) {
 877                 if (args->verbose)
 878                         fprintf(stderr, "Server supports ofs-delta\n");
 879         } else
 880                 prefer_ofs_delta = 0;
 881
 882         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 883                 agent_supported = 1;
 884                 if (args->verbose && agent_len)
 885                         fprintf(stderr, "Server version is %.*s\n",
 886                                 agent_len, agent_feature);
 887         }
 888
 889         if (everything_local(args, &ref, sought, nr_sought)) {
 890                 packet_flush(fd[1]);
 891                 goto all_done;
 892         }
 893         if (find_common(args, fd, sha1, ref) < 0)
 894                 if (!args->keep_pack)
 895                         /* When cloning, it is not unusual to have
 896                          * no common commit.
 897                          */
 898                         warning("no common commits");
 899
 900         if (args->stateless_rpc)
 901                 packet_flush(fd[1]);
 902         if (args->depth > 0)
 903                 setup_alternate_shallow();
 904         else
 905                 alternate_shallow_file = NULL;
 906         if (get_pack(args, fd, pack_lockfile))
 907                 die("git fetch-pack: fetch failed.");
 908
 909  all_done:
 910         return ref;
 911 }
 912
 913 static int fetch_pack_config(const char *var, const char *value, void *cb)
 914 {
 915         if (strcmp(var, "fetch.unpacklimit") == 0) {
 916                 fetch_unpack_limit = git_config_int(var, value);
 917                 return 0;
 918         }
 919
 920         if (strcmp(var, "transfer.unpacklimit") == 0) {
 921                 transfer_unpack_limit = git_config_int(var, value);
 922                 return 0;
 923         }
 924
 925         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 926                 prefer_ofs_delta = git_config_bool(var, value);
 927                 return 0;
 928         }
 929
 930         if (!strcmp(var, "fetch.fsckobjects")) {
 931                 fetch_fsck_objects = git_config_bool(var, value);
 932                 return 0;
 933         }
 934
 935         if (!strcmp(var, "transfer.fsckobjects")) {
 936                 transfer_fsck_objects = git_config_bool(var, value);
 937                 return 0;
 938         }
 939
 940         return git_default_config(var, value, cb);
 941 }
 942
 943 static void fetch_pack_setup(void)
 944 {
 945         static int did_setup;
 946         if (did_setup)
 947                 return;
 948         git_config(fetch_pack_config, NULL);
 949         if (0 <= transfer_unpack_limit)
 950                 unpack_limit = transfer_unpack_limit;
 951         else if (0 <= fetch_unpack_limit)
 952                 unpack_limit = fetch_unpack_limit;
 953         did_setup = 1;
 954 }
 955
 956 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 957 {
 958         struct string_list names = STRING_LIST_INIT_NODUP;
 959         int src, dst;
 960
 961         for (src = dst = 0; src < nr; src++) {
 962                 struct string_list_item *item;
 963                 item = string_list_insert(&names, ref[src]->name);
 964                 if (item->util)
 965                         continue; /* already have it */
 966                 item->util = ref[src];
 967                 if (src != dst)
 968                         ref[dst] = ref[src];
 969                 dst++;
 970         }
 971         for (src = dst; src < nr; src++)
 972                 ref[src] = NULL;
 973         string_list_clear(&names, 0);
 974         return dst;
 975 }
 976
 977 struct ref *fetch_pack(struct fetch_pack_args *args,
 978                        int fd[], struct child_process *conn,
 979                        const struct ref *ref,
 980                        const char *dest,
 981                        struct ref **sought, int nr_sought,
 982                        char **pack_lockfile)
 983 {
 984         struct ref *ref_cpy;
 985
 986         fetch_pack_setup();
 987         if (nr_sought)
 988                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
 989
 990         if (!ref) {
 991                 packet_flush(fd[1]);
 992                 die("no matching remote head");
 993         }
 994         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
 995
 996         if (args->depth > 0 && alternate_shallow_file) {
 997                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 998                         unlink_or_warn(git_path("shallow"));
 999                         rollback_lock_file(&shallow_lock);
1000                 } else
1001                         commit_lock_file(&shallow_lock);
1002         }
1003
1004         reprepare_packed_git();
1005         return ref_cpy;
1006 }