bundle-uri.c

   1 #include "cache.h"
   2 #include "bundle-uri.h"
   3 #include "bundle.h"
   4 #include "object-store.h"
   5 #include "refs.h"
   6 #include "run-command.h"
   7 #include "hashmap.h"
   8 #include "pkt-line.h"
   9 #include "config.h"
  10 #include "remote.h"
  11
  12 static struct {
  13         enum bundle_list_heuristic heuristic;
  14         const char *name;
  15 } heuristics[BUNDLE_HEURISTIC__COUNT] = {
  16         { BUNDLE_HEURISTIC_NONE, ""},
  17         { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" },
  18 };
  19
  20 static int compare_bundles(const void *hashmap_cmp_fn_data,
  21                            const struct hashmap_entry *he1,
  22                            const struct hashmap_entry *he2,
  23                            const void *id)
  24 {
  25         const struct remote_bundle_info *e1 =
  26                 container_of(he1, const struct remote_bundle_info, ent);
  27         const struct remote_bundle_info *e2 =
  28                 container_of(he2, const struct remote_bundle_info, ent);
  29
  30         return strcmp(e1->id, id ? (const char *)id : e2->id);
  31 }
  32
  33 void init_bundle_list(struct bundle_list *list)
  34 {
  35         memset(list, 0, sizeof(*list));
  36
  37         /* Implied defaults. */
  38         list->mode = BUNDLE_MODE_ALL;
  39         list->version = 1;
  40
  41         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  42 }
  43
  44 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  45                                     void *data)
  46 {
  47         FREE_AND_NULL(bundle->id);
  48         FREE_AND_NULL(bundle->uri);
  49         FREE_AND_NULL(bundle->file);
  50         bundle->unbundled = 0;
  51         return 0;
  52 }
  53
  54 void clear_bundle_list(struct bundle_list *list)
  55 {
  56         if (!list)
  57                 return;
  58
  59         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  60         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  61         free(list->baseURI);
  62 }
  63
  64 int for_all_bundles_in_list(struct bundle_list *list,
  65                             bundle_iterator iter,
  66                             void *data)
  67 {
  68         struct remote_bundle_info *info;
  69         struct hashmap_iter i;
  70
  71         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  72                 int result = iter(info, data);
  73
  74                 if (result)
  75                         return result;
  76         }
  77
  78         return 0;
  79 }
  80
  81 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  82 {
  83         FILE *fp = data;
  84         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  85         fprintf(fp, "\turi = %s\n", info->uri);
  86
  87         if (info->creationToken)
  88                 fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken);
  89         return 0;
  90 }
  91
  92 void print_bundle_list(FILE *fp, struct bundle_list *list)
  93 {
  94         const char *mode;
  95
  96         switch (list->mode) {
  97         case BUNDLE_MODE_ALL:
  98                 mode = "all";
  99                 break;
 100
 101         case BUNDLE_MODE_ANY:
 102                 mode = "any";
 103                 break;
 104
 105         case BUNDLE_MODE_NONE:
 106         default:
 107                 mode = "<unknown>";
 108         }
 109
 110         fprintf(fp, "[bundle]\n");
 111         fprintf(fp, "\tversion = %d\n", list->version);
 112         fprintf(fp, "\tmode = %s\n", mode);
 113
 114         if (list->heuristic) {
 115                 int i;
 116                 for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 117                         if (heuristics[i].heuristic == list->heuristic) {
 118                                 printf("\theuristic = %s\n",
 119                                        heuristics[list->heuristic].name);
 120                                 break;
 121                         }
 122                 }
 123         }
 124
 125         for_all_bundles_in_list(list, summarize_bundle, fp);
 126 }
 127
 128 /**
 129  * Given a key-value pair, update the state of the given bundle list.
 130  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 131  * is not understood or the value is malformed.
 132  */
 133 static int bundle_list_update(const char *key, const char *value,
 134                               struct bundle_list *list)
 135 {
 136         struct strbuf id = STRBUF_INIT;
 137         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 138         struct remote_bundle_info *bundle;
 139         const char *subsection, *subkey;
 140         size_t subsection_len;
 141
 142         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 143                 return -1;
 144
 145         if (!subsection_len) {
 146                 if (!strcmp(subkey, "version")) {
 147                         int version;
 148                         if (!git_parse_int(value, &version))
 149                                 return -1;
 150                         if (version != 1)
 151                                 return -1;
 152
 153                         list->version = version;
 154                         return 0;
 155                 }
 156
 157                 if (!strcmp(subkey, "mode")) {
 158                         if (!strcmp(value, "all"))
 159                                 list->mode = BUNDLE_MODE_ALL;
 160                         else if (!strcmp(value, "any"))
 161                                 list->mode = BUNDLE_MODE_ANY;
 162                         else
 163                                 return -1;
 164                         return 0;
 165                 }
 166
 167                 if (!strcmp(subkey, "heuristic")) {
 168                         int i;
 169                         for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 170                                 if (heuristics[i].heuristic &&
 171                                     heuristics[i].name &&
 172                                     !strcmp(value, heuristics[i].name)) {
 173                                         list->heuristic = heuristics[i].heuristic;
 174                                         return 0;
 175                                 }
 176                         }
 177
 178                         /* Ignore unknown heuristics. */
 179                         return 0;
 180                 }
 181
 182                 /* Ignore other unknown global keys. */
 183                 return 0;
 184         }
 185
 186         strbuf_add(&id, subsection, subsection_len);
 187
 188         /*
 189          * Check for an existing bundle with this <id>, or create one
 190          * if necessary.
 191          */
 192         lookup.id = id.buf;
 193         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 194         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 195                 CALLOC_ARRAY(bundle, 1);
 196                 bundle->id = strbuf_detach(&id, NULL);
 197                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 198                 hashmap_add(&list->bundles, &bundle->ent);
 199         }
 200         strbuf_release(&id);
 201
 202         if (!strcmp(subkey, "uri")) {
 203                 if (bundle->uri)
 204                         return -1;
 205                 bundle->uri = relative_url(list->baseURI, value, NULL);
 206                 return 0;
 207         }
 208
 209         if (!strcmp(subkey, "creationtoken")) {
 210                 if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1)
 211                         warning(_("could not parse bundle list key %s with value '%s'"),
 212                                 "creationToken", value);
 213                 return 0;
 214         }
 215
 216         /*
 217          * At this point, we ignore any information that we don't
 218          * understand, assuming it to be hints for a heuristic the client
 219          * does not currently understand.
 220          */
 221         return 0;
 222 }
 223
 224 static int config_to_bundle_list(const char *key, const char *value, void *data)
 225 {
 226         struct bundle_list *list = data;
 227         return bundle_list_update(key, value, list);
 228 }
 229
 230 int bundle_uri_parse_config_format(const char *uri,
 231                                    const char *filename,
 232                                    struct bundle_list *list)
 233 {
 234         int result;
 235         struct config_options opts = {
 236                 .error_action = CONFIG_ERROR_ERROR,
 237         };
 238
 239         if (!list->baseURI) {
 240                 struct strbuf baseURI = STRBUF_INIT;
 241                 strbuf_addstr(&baseURI, uri);
 242
 243                 /*
 244                  * If the URI does not end with a trailing slash, then
 245                  * remove the filename portion of the path. This is
 246                  * important for relative URIs.
 247                  */
 248                 strbuf_strip_file_from_path(&baseURI);
 249                 list->baseURI = strbuf_detach(&baseURI, NULL);
 250         }
 251         result = git_config_from_file_with_options(config_to_bundle_list,
 252                                                    filename, list,
 253                                                    &opts);
 254
 255         if (!result && list->mode == BUNDLE_MODE_NONE) {
 256                 warning(_("bundle list at '%s' has no mode"), uri);
 257                 result = 1;
 258         }
 259
 260         return result;
 261 }
 262
 263 static char *find_temp_filename(void)
 264 {
 265         int fd;
 266         struct strbuf name = STRBUF_INIT;
 267         /*
 268          * Find a temporary filename that is available. This is briefly
 269          * racy, but unlikely to collide.
 270          */
 271         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 272         if (fd < 0) {
 273                 warning(_("failed to create temporary file"));
 274                 return NULL;
 275         }
 276
 277         close(fd);
 278         unlink(name.buf);
 279         return strbuf_detach(&name, NULL);
 280 }
 281
 282 static int download_https_uri_to_file(const char *file, const char *uri)
 283 {
 284         int result = 0;
 285         struct child_process cp = CHILD_PROCESS_INIT;
 286         FILE *child_in = NULL, *child_out = NULL;
 287         struct strbuf line = STRBUF_INIT;
 288         int found_get = 0;
 289
 290         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 291         cp.err = -1;
 292         cp.in = -1;
 293         cp.out = -1;
 294
 295         if (start_command(&cp))
 296                 return 1;
 297
 298         child_in = fdopen(cp.in, "w");
 299         if (!child_in) {
 300                 result = 1;
 301                 goto cleanup;
 302         }
 303
 304         child_out = fdopen(cp.out, "r");
 305         if (!child_out) {
 306                 result = 1;
 307                 goto cleanup;
 308         }
 309
 310         fprintf(child_in, "capabilities\n");
 311         fflush(child_in);
 312
 313         while (!strbuf_getline(&line, child_out)) {
 314                 if (!line.len)
 315                         break;
 316                 if (!strcmp(line.buf, "get"))
 317                         found_get = 1;
 318         }
 319         strbuf_release(&line);
 320
 321         if (!found_get) {
 322                 result = error(_("insufficient capabilities"));
 323                 goto cleanup;
 324         }
 325
 326         fprintf(child_in, "get %s %s\n\n", uri, file);
 327
 328 cleanup:
 329         if (child_in)
 330                 fclose(child_in);
 331         if (finish_command(&cp))
 332                 return 1;
 333         if (child_out)
 334                 fclose(child_out);
 335         return result;
 336 }
 337
 338 static int copy_uri_to_file(const char *filename, const char *uri)
 339 {
 340         const char *out;
 341
 342         if (starts_with(uri, "https:") ||
 343             starts_with(uri, "http:"))
 344                 return download_https_uri_to_file(filename, uri);
 345
 346         if (skip_prefix(uri, "file://", &out))
 347                 uri = out;
 348
 349         /* Copy as a file */
 350         return copy_file(filename, uri, 0);
 351 }
 352
 353 static int unbundle_from_file(struct repository *r, const char *file)
 354 {
 355         int result = 0;
 356         int bundle_fd;
 357         struct bundle_header header = BUNDLE_HEADER_INIT;
 358         struct string_list_item *refname;
 359         struct strbuf bundle_ref = STRBUF_INIT;
 360         size_t bundle_prefix_len;
 361
 362         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 363                 return 1;
 364
 365         /*
 366          * Skip the reachability walk here, since we will be adding
 367          * a reachable ref pointing to the new tips, which will reach
 368          * the prerequisite commits.
 369          */
 370         if ((result = unbundle(r, &header, bundle_fd, NULL,
 371                                VERIFY_BUNDLE_QUIET)))
 372                 return 1;
 373
 374         /*
 375          * Convert all refs/heads/ from the bundle into refs/bundles/
 376          * in the local repository.
 377          */
 378         strbuf_addstr(&bundle_ref, "refs/bundles/");
 379         bundle_prefix_len = bundle_ref.len;
 380
 381         for_each_string_list_item(refname, &header.references) {
 382                 struct object_id *oid = refname->util;
 383                 struct object_id old_oid;
 384                 const char *branch_name;
 385                 int has_old;
 386
 387                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 388                         continue;
 389
 390                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 391                 strbuf_addstr(&bundle_ref, branch_name);
 392
 393                 has_old = !read_ref(bundle_ref.buf, &old_oid);
 394                 update_ref("fetched bundle", bundle_ref.buf, oid,
 395                            has_old ? &old_oid : NULL,
 396                            REF_SKIP_OID_VERIFICATION,
 397                            UPDATE_REFS_MSG_ON_ERR);
 398         }
 399
 400         bundle_header_release(&header);
 401         return result;
 402 }
 403
 404 struct bundle_list_context {
 405         struct repository *r;
 406         struct bundle_list *list;
 407         enum bundle_list_mode mode;
 408         int count;
 409         int depth;
 410 };
 411
 412 /*
 413  * This early definition is necessary because we use indirect recursion:
 414  *
 415  * While iterating through a bundle list that was downloaded as part
 416  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 417  * again, but with depth + 1.
 418  */
 419 static int fetch_bundle_uri_internal(struct repository *r,
 420                                      struct remote_bundle_info *bundle,
 421                                      int depth,
 422                                      struct bundle_list *list);
 423
 424 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 425 {
 426         int res;
 427         struct bundle_list_context *ctx = data;
 428
 429         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 430                 return 0;
 431
 432         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 433
 434         /*
 435          * Only increment count if the download succeeded. If our mode is
 436          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 437          * list in case they work instead.
 438          */
 439         if (!res)
 440                 ctx->count++;
 441
 442         /*
 443          * To be opportunistic as possible, we continue iterating and
 444          * download as many bundles as we can, so we can apply the ones
 445          * that work, even in BUNDLE_MODE_ALL mode.
 446          */
 447         return 0;
 448 }
 449
 450 struct bundles_for_sorting {
 451         struct remote_bundle_info **items;
 452         size_t alloc;
 453         size_t nr;
 454 };
 455
 456 static int append_bundle(struct remote_bundle_info *bundle, void *data)
 457 {
 458         struct bundles_for_sorting *list = data;
 459         list->items[list->nr++] = bundle;
 460         return 0;
 461 }
 462
 463 /**
 464  * For use in QSORT() to get a list sorted by creationToken
 465  * in decreasing order.
 466  */
 467 static int compare_creation_token_decreasing(const void *va, const void *vb)
 468 {
 469         const struct remote_bundle_info * const *a = va;
 470         const struct remote_bundle_info * const *b = vb;
 471
 472         if ((*a)->creationToken > (*b)->creationToken)
 473                 return -1;
 474         if ((*a)->creationToken < (*b)->creationToken)
 475                 return 1;
 476         return 0;
 477 }
 478
 479 static int fetch_bundles_by_token(struct repository *r,
 480                                   struct bundle_list *list)
 481 {
 482         int cur;
 483         int move_direction = 0;
 484         const char *creationTokenStr;
 485         uint64_t maxCreationToken = 0, newMaxCreationToken = 0;
 486         struct bundle_list_context ctx = {
 487                 .r = r,
 488                 .list = list,
 489                 .mode = list->mode,
 490         };
 491         struct bundles_for_sorting bundles = {
 492                 .alloc = hashmap_get_size(&list->bundles),
 493         };
 494
 495         ALLOC_ARRAY(bundles.items, bundles.alloc);
 496
 497         for_all_bundles_in_list(list, append_bundle, &bundles);
 498
 499         if (!bundles.nr) {
 500                 free(bundles.items);
 501                 return 0;
 502         }
 503
 504         QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 505
 506         /*
 507          * If fetch.bundleCreationToken exists, parses to a uint64t, and
 508          * is not strictly smaller than the maximum creation token in the
 509          * bundle list, then do not download any bundles.
 510          */
 511         if (!repo_config_get_value(r,
 512                                    "fetch.bundlecreationtoken",
 513                                    &creationTokenStr) &&
 514             sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) == 1 &&
 515             bundles.items[0]->creationToken <= maxCreationToken) {
 516                 free(bundles.items);
 517                 return 0;
 518         }
 519
 520         /*
 521          * Attempt to download and unbundle the minimum number of bundles by
 522          * creationToken in decreasing order. If we fail to unbundle (after
 523          * a successful download) then move to the next non-downloaded bundle
 524          * and attempt downloading. Once we succeed in applying a bundle,
 525          * move to the previous unapplied bundle and attempt to unbundle it
 526          * again.
 527          *
 528          * In the case of a fresh clone, we will likely download all of the
 529          * bundles before successfully unbundling the oldest one, then the
 530          * rest of the bundles unbundle successfully in increasing order
 531          * of creationToken.
 532          *
 533          * If there are existing objects, then this process may terminate
 534          * early when all required commits from "new" bundles exist in the
 535          * repo's object store.
 536          */
 537         cur = 0;
 538         while (cur >= 0 && cur < bundles.nr) {
 539                 struct remote_bundle_info *bundle = bundles.items[cur];
 540
 541                 /*
 542                  * If we need to dig into bundles below the previous
 543                  * creation token value, then likely we are in an erroneous
 544                  * state due to missing or invalid bundles. Halt the process
 545                  * instead of continuing to download extra data.
 546                  */
 547                 if (bundle->creationToken <= maxCreationToken)
 548                         break;
 549
 550                 if (!bundle->file) {
 551                         /*
 552                          * Not downloaded yet. Try downloading.
 553                          *
 554                          * Note that bundle->file is non-NULL if a download
 555                          * was attempted, even if it failed to download.
 556                          */
 557                         if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
 558                                 /* Mark as unbundled so we do not retry. */
 559                                 bundle->unbundled = 1;
 560
 561                                 /* Try looking deeper in the list. */
 562                                 move_direction = 1;
 563                                 goto move;
 564                         }
 565
 566                         /* We expect bundles when using creationTokens. */
 567                         if (!is_bundle(bundle->file, 1)) {
 568                                 warning(_("file downloaded from '%s' is not a bundle"),
 569                                         bundle->uri);
 570                                 break;
 571                         }
 572                 }
 573
 574                 if (bundle->file && !bundle->unbundled) {
 575                         /*
 576                          * This was downloaded, but not successfully
 577                          * unbundled. Try unbundling again.
 578                          */
 579                         if (unbundle_from_file(ctx.r, bundle->file)) {
 580                                 /* Try looking deeper in the list. */
 581                                 move_direction = 1;
 582                         } else {
 583                                 /*
 584                                  * Succeeded in unbundle. Retry bundles
 585                                  * that previously failed to unbundle.
 586                                  */
 587                                 move_direction = -1;
 588                                 bundle->unbundled = 1;
 589
 590                                 if (bundle->creationToken > newMaxCreationToken)
 591                                         newMaxCreationToken = bundle->creationToken;
 592                         }
 593                 }
 594
 595                 /*
 596                  * Else case: downloaded and unbundled successfully.
 597                  * Skip this by moving in the same direction as the
 598                  * previous step.
 599                  */
 600
 601 move:
 602                 /* Move in the specified direction and repeat. */
 603                 cur += move_direction;
 604         }
 605
 606         /*
 607          * We succeed if the loop terminates because 'cur' drops below
 608          * zero. The other case is that we terminate because 'cur'
 609          * reaches the end of the list, so we have a failure no matter
 610          * which bundles we apply from the list.
 611          */
 612         if (cur < 0) {
 613                 struct strbuf value = STRBUF_INIT;
 614                 strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken);
 615                 if (repo_config_set_multivar_gently(ctx.r,
 616                                                     "fetch.bundleCreationToken",
 617                                                     value.buf, NULL, 0))
 618                         warning(_("failed to store maximum creation token"));
 619
 620                 strbuf_release(&value);
 621         }
 622
 623         free(bundles.items);
 624         return cur >= 0;
 625 }
 626
 627 static int download_bundle_list(struct repository *r,
 628                                 struct bundle_list *local_list,
 629                                 struct bundle_list *global_list,
 630                                 int depth)
 631 {
 632         struct bundle_list_context ctx = {
 633                 .r = r,
 634                 .list = global_list,
 635                 .depth = depth + 1,
 636                 .mode = local_list->mode,
 637         };
 638
 639         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 640 }
 641
 642 static int fetch_bundle_list_in_config_format(struct repository *r,
 643                                               struct bundle_list *global_list,
 644                                               struct remote_bundle_info *bundle,
 645                                               int depth)
 646 {
 647         int result;
 648         struct bundle_list list_from_bundle;
 649
 650         init_bundle_list(&list_from_bundle);
 651
 652         if ((result = bundle_uri_parse_config_format(bundle->uri,
 653                                                      bundle->file,
 654                                                      &list_from_bundle)))
 655                 goto cleanup;
 656
 657         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 658                 warning(_("unrecognized bundle mode from URI '%s'"),
 659                         bundle->uri);
 660                 result = -1;
 661                 goto cleanup;
 662         }
 663
 664         /*
 665          * If this list uses the creationToken heuristic, then the URIs
 666          * it advertises are expected to be bundles, not nested lists.
 667          * We can drop 'global_list' and 'depth'.
 668          */
 669         if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
 670                 result = fetch_bundles_by_token(r, &list_from_bundle);
 671                 global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
 672         } else if ((result = download_bundle_list(r, &list_from_bundle,
 673                                            global_list, depth)))
 674                 goto cleanup;
 675
 676 cleanup:
 677         clear_bundle_list(&list_from_bundle);
 678         return result;
 679 }
 680
 681 /**
 682  * This limits the recursion on fetch_bundle_uri_internal() when following
 683  * bundle lists.
 684  */
 685 static int max_bundle_uri_depth = 4;
 686
 687 /**
 688  * Recursively download all bundles advertised at the given URI
 689  * to files. If the file is a bundle, then add it to the given
 690  * 'list'. Otherwise, expect a bundle list and recurse on the
 691  * URIs in that list according to the list mode (ANY or ALL).
 692  */
 693 static int fetch_bundle_uri_internal(struct repository *r,
 694                                      struct remote_bundle_info *bundle,
 695                                      int depth,
 696                                      struct bundle_list *list)
 697 {
 698         int result = 0;
 699         struct remote_bundle_info *bcopy;
 700
 701         if (depth >= max_bundle_uri_depth) {
 702                 warning(_("exceeded bundle URI recursion limit (%d)"),
 703                         max_bundle_uri_depth);
 704                 return -1;
 705         }
 706
 707         if (!bundle->file &&
 708             !(bundle->file = find_temp_filename())) {
 709                 result = -1;
 710                 goto cleanup;
 711         }
 712
 713         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 714                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 715                 goto cleanup;
 716         }
 717
 718         if ((result = !is_bundle(bundle->file, 1))) {
 719                 result = fetch_bundle_list_in_config_format(
 720                                 r, list, bundle, depth);
 721                 if (result)
 722                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 723                                 bundle->uri);
 724                 goto cleanup;
 725         }
 726
 727         /* Copy the bundle and insert it into the global list. */
 728         CALLOC_ARRAY(bcopy, 1);
 729         bcopy->id = xstrdup(bundle->id);
 730         bcopy->file = xstrdup(bundle->file);
 731         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 732         hashmap_add(&list->bundles, &bcopy->ent);
 733
 734 cleanup:
 735         if (result && bundle->file)
 736                 unlink(bundle->file);
 737         return result;
 738 }
 739
 740 /**
 741  * This loop iterator breaks the loop with nonzero return code on the
 742  * first successful unbundling of a bundle.
 743  */
 744 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 745 {
 746         struct repository *r = data;
 747
 748         if (!info->file || info->unbundled)
 749                 return 0;
 750
 751         if (!unbundle_from_file(r, info->file)) {
 752                 info->unbundled = 1;
 753                 return 1;
 754         }
 755
 756         return 0;
 757 }
 758
 759 static int unbundle_all_bundles(struct repository *r,
 760                                 struct bundle_list *list)
 761 {
 762         /*
 763          * Iterate through all bundles looking for ones that can
 764          * successfully unbundle. If any succeed, then perhaps another
 765          * will succeed in the next attempt.
 766          *
 767          * Keep in mind that a non-zero result for the loop here means
 768          * the loop terminated early on a successful unbundling, which
 769          * signals that we can try again.
 770          */
 771         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 772
 773         return 0;
 774 }
 775
 776 static int unlink_bundle(struct remote_bundle_info *info, void *data)
 777 {
 778         if (info->file)
 779                 unlink_or_warn(info->file);
 780         return 0;
 781 }
 782
 783 int fetch_bundle_uri(struct repository *r, const char *uri,
 784                      int *has_heuristic)
 785 {
 786         int result;
 787         struct bundle_list list;
 788         struct remote_bundle_info bundle = {
 789                 .uri = xstrdup(uri),
 790                 .id = xstrdup(""),
 791         };
 792
 793         init_bundle_list(&list);
 794
 795         /* If a bundle is added to this global list, then it is required. */
 796         list.mode = BUNDLE_MODE_ALL;
 797
 798         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 799                 goto cleanup;
 800
 801         result = unbundle_all_bundles(r, &list);
 802
 803 cleanup:
 804         if (has_heuristic)
 805                 *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE);
 806         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 807         clear_bundle_list(&list);
 808         clear_remote_bundle_info(&bundle, NULL);
 809         return result;
 810 }
 811
 812 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 813 {
 814         int result;
 815         struct bundle_list global_list;
 816
 817         /*
 818          * If the creationToken heuristic is used, then the URIs
 819          * advertised by 'list' are not nested lists and instead
 820          * direct bundles. We do not need to use global_list.
 821          */
 822         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 823                 return fetch_bundles_by_token(r, list);
 824
 825         init_bundle_list(&global_list);
 826
 827         /* If a bundle is added to this global list, then it is required. */
 828         global_list.mode = BUNDLE_MODE_ALL;
 829
 830         if ((result = download_bundle_list(r, list, &global_list, 0)))
 831                 goto cleanup;
 832
 833         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 834                 result = fetch_bundles_by_token(r, list);
 835         else
 836                 result = unbundle_all_bundles(r, &global_list);
 837
 838 cleanup:
 839         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 840         clear_bundle_list(&global_list);
 841         return result;
 842 }
 843
 844 /**
 845  * API for serve.c.
 846  */
 847
 848 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 849 {
 850         static int advertise_bundle_uri = -1;
 851
 852         if (advertise_bundle_uri != -1)
 853                 goto cached;
 854
 855         advertise_bundle_uri = 0;
 856         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 857
 858 cached:
 859         return advertise_bundle_uri;
 860 }
 861
 862 static int config_to_packet_line(const char *key, const char *value, void *data)
 863 {
 864         struct packet_reader *writer = data;
 865
 866         if (starts_with(key, "bundle."))
 867                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 868
 869         return 0;
 870 }
 871
 872 int bundle_uri_command(struct repository *r,
 873                        struct packet_reader *request)
 874 {
 875         struct packet_writer writer;
 876         packet_writer_init(&writer, 1);
 877
 878         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 879                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 880         if (request->status != PACKET_READ_FLUSH)
 881                 die(_("bundle-uri: expected flush after arguments"));
 882
 883         /*
 884          * Read all "bundle.*" config lines to the client as key=value
 885          * packet lines.
 886          */
 887         git_config(config_to_packet_line, &writer);
 888
 889         packet_writer_flush(&writer);
 890
 891         return 0;
 892 }
 893
 894 /**
 895  * General API for {transport,connect}.c etc.
 896  */
 897 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 898 {
 899         int result;
 900         const char *equals;
 901         struct strbuf key = STRBUF_INIT;
 902
 903         if (!strlen(line))
 904                 return error(_("bundle-uri: got an empty line"));
 905
 906         equals = strchr(line, '=');
 907
 908         if (!equals)
 909                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 910         if (line == equals || !*(equals + 1))
 911                 return error(_("bundle-uri: line has empty key or value"));
 912
 913         strbuf_add(&key, line, equals - line);
 914         result = bundle_list_update(key.buf, equals + 1, list);
 915         strbuf_release(&key);
 916
 917         return result;
 918 }