bundle-uri.c

   1 #include "cache.h"
   2 #include "bundle-uri.h"
   3 #include "bundle.h"
   4 #include "gettext.h"
   5 #include "object-store.h"
   6 #include "refs.h"
   7 #include "run-command.h"
   8 #include "hashmap.h"
   9 #include "pkt-line.h"
  10 #include "config.h"
  11 #include "remote.h"
  12
  13 static struct {
  14         enum bundle_list_heuristic heuristic;
  15         const char *name;
  16 } heuristics[BUNDLE_HEURISTIC__COUNT] = {
  17         { BUNDLE_HEURISTIC_NONE, ""},
  18         { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" },
  19 };
  20
  21 static int compare_bundles(const void *hashmap_cmp_fn_data,
  22                            const struct hashmap_entry *he1,
  23                            const struct hashmap_entry *he2,
  24                            const void *id)
  25 {
  26         const struct remote_bundle_info *e1 =
  27                 container_of(he1, const struct remote_bundle_info, ent);
  28         const struct remote_bundle_info *e2 =
  29                 container_of(he2, const struct remote_bundle_info, ent);
  30
  31         return strcmp(e1->id, id ? (const char *)id : e2->id);
  32 }
  33
  34 void init_bundle_list(struct bundle_list *list)
  35 {
  36         memset(list, 0, sizeof(*list));
  37
  38         /* Implied defaults. */
  39         list->mode = BUNDLE_MODE_ALL;
  40         list->version = 1;
  41
  42         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  43 }
  44
  45 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  46                                     void *data)
  47 {
  48         FREE_AND_NULL(bundle->id);
  49         FREE_AND_NULL(bundle->uri);
  50         FREE_AND_NULL(bundle->file);
  51         bundle->unbundled = 0;
  52         return 0;
  53 }
  54
  55 void clear_bundle_list(struct bundle_list *list)
  56 {
  57         if (!list)
  58                 return;
  59
  60         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  61         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  62         free(list->baseURI);
  63 }
  64
  65 int for_all_bundles_in_list(struct bundle_list *list,
  66                             bundle_iterator iter,
  67                             void *data)
  68 {
  69         struct remote_bundle_info *info;
  70         struct hashmap_iter i;
  71
  72         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  73                 int result = iter(info, data);
  74
  75                 if (result)
  76                         return result;
  77         }
  78
  79         return 0;
  80 }
  81
  82 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  83 {
  84         FILE *fp = data;
  85         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  86         fprintf(fp, "\turi = %s\n", info->uri);
  87
  88         if (info->creationToken)
  89                 fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken);
  90         return 0;
  91 }
  92
  93 void print_bundle_list(FILE *fp, struct bundle_list *list)
  94 {
  95         const char *mode;
  96
  97         switch (list->mode) {
  98         case BUNDLE_MODE_ALL:
  99                 mode = "all";
 100                 break;
 101
 102         case BUNDLE_MODE_ANY:
 103                 mode = "any";
 104                 break;
 105
 106         case BUNDLE_MODE_NONE:
 107         default:
 108                 mode = "<unknown>";
 109         }
 110
 111         fprintf(fp, "[bundle]\n");
 112         fprintf(fp, "\tversion = %d\n", list->version);
 113         fprintf(fp, "\tmode = %s\n", mode);
 114
 115         if (list->heuristic) {
 116                 int i;
 117                 for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 118                         if (heuristics[i].heuristic == list->heuristic) {
 119                                 printf("\theuristic = %s\n",
 120                                        heuristics[list->heuristic].name);
 121                                 break;
 122                         }
 123                 }
 124         }
 125
 126         for_all_bundles_in_list(list, summarize_bundle, fp);
 127 }
 128
 129 /**
 130  * Given a key-value pair, update the state of the given bundle list.
 131  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 132  * is not understood or the value is malformed.
 133  */
 134 static int bundle_list_update(const char *key, const char *value,
 135                               struct bundle_list *list)
 136 {
 137         struct strbuf id = STRBUF_INIT;
 138         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 139         struct remote_bundle_info *bundle;
 140         const char *subsection, *subkey;
 141         size_t subsection_len;
 142
 143         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 144                 return -1;
 145
 146         if (!subsection_len) {
 147                 if (!strcmp(subkey, "version")) {
 148                         int version;
 149                         if (!git_parse_int(value, &version))
 150                                 return -1;
 151                         if (version != 1)
 152                                 return -1;
 153
 154                         list->version = version;
 155                         return 0;
 156                 }
 157
 158                 if (!strcmp(subkey, "mode")) {
 159                         if (!strcmp(value, "all"))
 160                                 list->mode = BUNDLE_MODE_ALL;
 161                         else if (!strcmp(value, "any"))
 162                                 list->mode = BUNDLE_MODE_ANY;
 163                         else
 164                                 return -1;
 165                         return 0;
 166                 }
 167
 168                 if (!strcmp(subkey, "heuristic")) {
 169                         int i;
 170                         for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 171                                 if (heuristics[i].heuristic &&
 172                                     heuristics[i].name &&
 173                                     !strcmp(value, heuristics[i].name)) {
 174                                         list->heuristic = heuristics[i].heuristic;
 175                                         return 0;
 176                                 }
 177                         }
 178
 179                         /* Ignore unknown heuristics. */
 180                         return 0;
 181                 }
 182
 183                 /* Ignore other unknown global keys. */
 184                 return 0;
 185         }
 186
 187         strbuf_add(&id, subsection, subsection_len);
 188
 189         /*
 190          * Check for an existing bundle with this <id>, or create one
 191          * if necessary.
 192          */
 193         lookup.id = id.buf;
 194         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 195         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 196                 CALLOC_ARRAY(bundle, 1);
 197                 bundle->id = strbuf_detach(&id, NULL);
 198                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 199                 hashmap_add(&list->bundles, &bundle->ent);
 200         }
 201         strbuf_release(&id);
 202
 203         if (!strcmp(subkey, "uri")) {
 204                 if (bundle->uri)
 205                         return -1;
 206                 bundle->uri = relative_url(list->baseURI, value, NULL);
 207                 return 0;
 208         }
 209
 210         if (!strcmp(subkey, "creationtoken")) {
 211                 if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1)
 212                         warning(_("could not parse bundle list key %s with value '%s'"),
 213                                 "creationToken", value);
 214                 return 0;
 215         }
 216
 217         /*
 218          * At this point, we ignore any information that we don't
 219          * understand, assuming it to be hints for a heuristic the client
 220          * does not currently understand.
 221          */
 222         return 0;
 223 }
 224
 225 static int config_to_bundle_list(const char *key, const char *value, void *data)
 226 {
 227         struct bundle_list *list = data;
 228         return bundle_list_update(key, value, list);
 229 }
 230
 231 int bundle_uri_parse_config_format(const char *uri,
 232                                    const char *filename,
 233                                    struct bundle_list *list)
 234 {
 235         int result;
 236         struct config_options opts = {
 237                 .error_action = CONFIG_ERROR_ERROR,
 238         };
 239
 240         if (!list->baseURI) {
 241                 struct strbuf baseURI = STRBUF_INIT;
 242                 strbuf_addstr(&baseURI, uri);
 243
 244                 /*
 245                  * If the URI does not end with a trailing slash, then
 246                  * remove the filename portion of the path. This is
 247                  * important for relative URIs.
 248                  */
 249                 strbuf_strip_file_from_path(&baseURI);
 250                 list->baseURI = strbuf_detach(&baseURI, NULL);
 251         }
 252         result = git_config_from_file_with_options(config_to_bundle_list,
 253                                                    filename, list,
 254                                                    &opts);
 255
 256         if (!result && list->mode == BUNDLE_MODE_NONE) {
 257                 warning(_("bundle list at '%s' has no mode"), uri);
 258                 result = 1;
 259         }
 260
 261         return result;
 262 }
 263
 264 static char *find_temp_filename(void)
 265 {
 266         int fd;
 267         struct strbuf name = STRBUF_INIT;
 268         /*
 269          * Find a temporary filename that is available. This is briefly
 270          * racy, but unlikely to collide.
 271          */
 272         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 273         if (fd < 0) {
 274                 warning(_("failed to create temporary file"));
 275                 return NULL;
 276         }
 277
 278         close(fd);
 279         unlink(name.buf);
 280         return strbuf_detach(&name, NULL);
 281 }
 282
 283 static int download_https_uri_to_file(const char *file, const char *uri)
 284 {
 285         int result = 0;
 286         struct child_process cp = CHILD_PROCESS_INIT;
 287         FILE *child_in = NULL, *child_out = NULL;
 288         struct strbuf line = STRBUF_INIT;
 289         int found_get = 0;
 290
 291         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 292         cp.err = -1;
 293         cp.in = -1;
 294         cp.out = -1;
 295
 296         if (start_command(&cp))
 297                 return 1;
 298
 299         child_in = fdopen(cp.in, "w");
 300         if (!child_in) {
 301                 result = 1;
 302                 goto cleanup;
 303         }
 304
 305         child_out = fdopen(cp.out, "r");
 306         if (!child_out) {
 307                 result = 1;
 308                 goto cleanup;
 309         }
 310
 311         fprintf(child_in, "capabilities\n");
 312         fflush(child_in);
 313
 314         while (!strbuf_getline(&line, child_out)) {
 315                 if (!line.len)
 316                         break;
 317                 if (!strcmp(line.buf, "get"))
 318                         found_get = 1;
 319         }
 320         strbuf_release(&line);
 321
 322         if (!found_get) {
 323                 result = error(_("insufficient capabilities"));
 324                 goto cleanup;
 325         }
 326
 327         fprintf(child_in, "get %s %s\n\n", uri, file);
 328
 329 cleanup:
 330         if (child_in)
 331                 fclose(child_in);
 332         if (finish_command(&cp))
 333                 return 1;
 334         if (child_out)
 335                 fclose(child_out);
 336         return result;
 337 }
 338
 339 static int copy_uri_to_file(const char *filename, const char *uri)
 340 {
 341         const char *out;
 342
 343         if (starts_with(uri, "https:") ||
 344             starts_with(uri, "http:"))
 345                 return download_https_uri_to_file(filename, uri);
 346
 347         if (skip_prefix(uri, "file://", &out))
 348                 uri = out;
 349
 350         /* Copy as a file */
 351         return copy_file(filename, uri, 0);
 352 }
 353
 354 static int unbundle_from_file(struct repository *r, const char *file)
 355 {
 356         int result = 0;
 357         int bundle_fd;
 358         struct bundle_header header = BUNDLE_HEADER_INIT;
 359         struct string_list_item *refname;
 360         struct strbuf bundle_ref = STRBUF_INIT;
 361         size_t bundle_prefix_len;
 362
 363         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 364                 return 1;
 365
 366         /*
 367          * Skip the reachability walk here, since we will be adding
 368          * a reachable ref pointing to the new tips, which will reach
 369          * the prerequisite commits.
 370          */
 371         if ((result = unbundle(r, &header, bundle_fd, NULL,
 372                                VERIFY_BUNDLE_QUIET)))
 373                 return 1;
 374
 375         /*
 376          * Convert all refs/heads/ from the bundle into refs/bundles/
 377          * in the local repository.
 378          */
 379         strbuf_addstr(&bundle_ref, "refs/bundles/");
 380         bundle_prefix_len = bundle_ref.len;
 381
 382         for_each_string_list_item(refname, &header.references) {
 383                 struct object_id *oid = refname->util;
 384                 struct object_id old_oid;
 385                 const char *branch_name;
 386                 int has_old;
 387
 388                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 389                         continue;
 390
 391                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 392                 strbuf_addstr(&bundle_ref, branch_name);
 393
 394                 has_old = !read_ref(bundle_ref.buf, &old_oid);
 395                 update_ref("fetched bundle", bundle_ref.buf, oid,
 396                            has_old ? &old_oid : NULL,
 397                            REF_SKIP_OID_VERIFICATION,
 398                            UPDATE_REFS_MSG_ON_ERR);
 399         }
 400
 401         bundle_header_release(&header);
 402         return result;
 403 }
 404
 405 struct bundle_list_context {
 406         struct repository *r;
 407         struct bundle_list *list;
 408         enum bundle_list_mode mode;
 409         int count;
 410         int depth;
 411 };
 412
 413 /*
 414  * This early definition is necessary because we use indirect recursion:
 415  *
 416  * While iterating through a bundle list that was downloaded as part
 417  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 418  * again, but with depth + 1.
 419  */
 420 static int fetch_bundle_uri_internal(struct repository *r,
 421                                      struct remote_bundle_info *bundle,
 422                                      int depth,
 423                                      struct bundle_list *list);
 424
 425 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 426 {
 427         int res;
 428         struct bundle_list_context *ctx = data;
 429
 430         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 431                 return 0;
 432
 433         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 434
 435         /*
 436          * Only increment count if the download succeeded. If our mode is
 437          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 438          * list in case they work instead.
 439          */
 440         if (!res)
 441                 ctx->count++;
 442
 443         /*
 444          * To be opportunistic as possible, we continue iterating and
 445          * download as many bundles as we can, so we can apply the ones
 446          * that work, even in BUNDLE_MODE_ALL mode.
 447          */
 448         return 0;
 449 }
 450
 451 struct bundles_for_sorting {
 452         struct remote_bundle_info **items;
 453         size_t alloc;
 454         size_t nr;
 455 };
 456
 457 static int append_bundle(struct remote_bundle_info *bundle, void *data)
 458 {
 459         struct bundles_for_sorting *list = data;
 460         list->items[list->nr++] = bundle;
 461         return 0;
 462 }
 463
 464 /**
 465  * For use in QSORT() to get a list sorted by creationToken
 466  * in decreasing order.
 467  */
 468 static int compare_creation_token_decreasing(const void *va, const void *vb)
 469 {
 470         const struct remote_bundle_info * const *a = va;
 471         const struct remote_bundle_info * const *b = vb;
 472
 473         if ((*a)->creationToken > (*b)->creationToken)
 474                 return -1;
 475         if ((*a)->creationToken < (*b)->creationToken)
 476                 return 1;
 477         return 0;
 478 }
 479
 480 static int fetch_bundles_by_token(struct repository *r,
 481                                   struct bundle_list *list)
 482 {
 483         int cur;
 484         int move_direction = 0;
 485         const char *creationTokenStr;
 486         uint64_t maxCreationToken = 0, newMaxCreationToken = 0;
 487         struct bundle_list_context ctx = {
 488                 .r = r,
 489                 .list = list,
 490                 .mode = list->mode,
 491         };
 492         struct bundles_for_sorting bundles = {
 493                 .alloc = hashmap_get_size(&list->bundles),
 494         };
 495
 496         ALLOC_ARRAY(bundles.items, bundles.alloc);
 497
 498         for_all_bundles_in_list(list, append_bundle, &bundles);
 499
 500         if (!bundles.nr) {
 501                 free(bundles.items);
 502                 return 0;
 503         }
 504
 505         QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 506
 507         /*
 508          * If fetch.bundleCreationToken exists, parses to a uint64t, and
 509          * is not strictly smaller than the maximum creation token in the
 510          * bundle list, then do not download any bundles.
 511          */
 512         if (!repo_config_get_value(r,
 513                                    "fetch.bundlecreationtoken",
 514                                    &creationTokenStr) &&
 515             sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) == 1 &&
 516             bundles.items[0]->creationToken <= maxCreationToken) {
 517                 free(bundles.items);
 518                 return 0;
 519         }
 520
 521         /*
 522          * Attempt to download and unbundle the minimum number of bundles by
 523          * creationToken in decreasing order. If we fail to unbundle (after
 524          * a successful download) then move to the next non-downloaded bundle
 525          * and attempt downloading. Once we succeed in applying a bundle,
 526          * move to the previous unapplied bundle and attempt to unbundle it
 527          * again.
 528          *
 529          * In the case of a fresh clone, we will likely download all of the
 530          * bundles before successfully unbundling the oldest one, then the
 531          * rest of the bundles unbundle successfully in increasing order
 532          * of creationToken.
 533          *
 534          * If there are existing objects, then this process may terminate
 535          * early when all required commits from "new" bundles exist in the
 536          * repo's object store.
 537          */
 538         cur = 0;
 539         while (cur >= 0 && cur < bundles.nr) {
 540                 struct remote_bundle_info *bundle = bundles.items[cur];
 541
 542                 /*
 543                  * If we need to dig into bundles below the previous
 544                  * creation token value, then likely we are in an erroneous
 545                  * state due to missing or invalid bundles. Halt the process
 546                  * instead of continuing to download extra data.
 547                  */
 548                 if (bundle->creationToken <= maxCreationToken)
 549                         break;
 550
 551                 if (!bundle->file) {
 552                         /*
 553                          * Not downloaded yet. Try downloading.
 554                          *
 555                          * Note that bundle->file is non-NULL if a download
 556                          * was attempted, even if it failed to download.
 557                          */
 558                         if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
 559                                 /* Mark as unbundled so we do not retry. */
 560                                 bundle->unbundled = 1;
 561
 562                                 /* Try looking deeper in the list. */
 563                                 move_direction = 1;
 564                                 goto move;
 565                         }
 566
 567                         /* We expect bundles when using creationTokens. */
 568                         if (!is_bundle(bundle->file, 1)) {
 569                                 warning(_("file downloaded from '%s' is not a bundle"),
 570                                         bundle->uri);
 571                                 break;
 572                         }
 573                 }
 574
 575                 if (bundle->file && !bundle->unbundled) {
 576                         /*
 577                          * This was downloaded, but not successfully
 578                          * unbundled. Try unbundling again.
 579                          */
 580                         if (unbundle_from_file(ctx.r, bundle->file)) {
 581                                 /* Try looking deeper in the list. */
 582                                 move_direction = 1;
 583                         } else {
 584                                 /*
 585                                  * Succeeded in unbundle. Retry bundles
 586                                  * that previously failed to unbundle.
 587                                  */
 588                                 move_direction = -1;
 589                                 bundle->unbundled = 1;
 590
 591                                 if (bundle->creationToken > newMaxCreationToken)
 592                                         newMaxCreationToken = bundle->creationToken;
 593                         }
 594                 }
 595
 596                 /*
 597                  * Else case: downloaded and unbundled successfully.
 598                  * Skip this by moving in the same direction as the
 599                  * previous step.
 600                  */
 601
 602 move:
 603                 /* Move in the specified direction and repeat. */
 604                 cur += move_direction;
 605         }
 606
 607         /*
 608          * We succeed if the loop terminates because 'cur' drops below
 609          * zero. The other case is that we terminate because 'cur'
 610          * reaches the end of the list, so we have a failure no matter
 611          * which bundles we apply from the list.
 612          */
 613         if (cur < 0) {
 614                 struct strbuf value = STRBUF_INIT;
 615                 strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken);
 616                 if (repo_config_set_multivar_gently(ctx.r,
 617                                                     "fetch.bundleCreationToken",
 618                                                     value.buf, NULL, 0))
 619                         warning(_("failed to store maximum creation token"));
 620
 621                 strbuf_release(&value);
 622         }
 623
 624         free(bundles.items);
 625         return cur >= 0;
 626 }
 627
 628 static int download_bundle_list(struct repository *r,
 629                                 struct bundle_list *local_list,
 630                                 struct bundle_list *global_list,
 631                                 int depth)
 632 {
 633         struct bundle_list_context ctx = {
 634                 .r = r,
 635                 .list = global_list,
 636                 .depth = depth + 1,
 637                 .mode = local_list->mode,
 638         };
 639
 640         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 641 }
 642
 643 static int fetch_bundle_list_in_config_format(struct repository *r,
 644                                               struct bundle_list *global_list,
 645                                               struct remote_bundle_info *bundle,
 646                                               int depth)
 647 {
 648         int result;
 649         struct bundle_list list_from_bundle;
 650
 651         init_bundle_list(&list_from_bundle);
 652
 653         if ((result = bundle_uri_parse_config_format(bundle->uri,
 654                                                      bundle->file,
 655                                                      &list_from_bundle)))
 656                 goto cleanup;
 657
 658         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 659                 warning(_("unrecognized bundle mode from URI '%s'"),
 660                         bundle->uri);
 661                 result = -1;
 662                 goto cleanup;
 663         }
 664
 665         /*
 666          * If this list uses the creationToken heuristic, then the URIs
 667          * it advertises are expected to be bundles, not nested lists.
 668          * We can drop 'global_list' and 'depth'.
 669          */
 670         if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
 671                 result = fetch_bundles_by_token(r, &list_from_bundle);
 672                 global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
 673         } else if ((result = download_bundle_list(r, &list_from_bundle,
 674                                            global_list, depth)))
 675                 goto cleanup;
 676
 677 cleanup:
 678         clear_bundle_list(&list_from_bundle);
 679         return result;
 680 }
 681
 682 /**
 683  * This limits the recursion on fetch_bundle_uri_internal() when following
 684  * bundle lists.
 685  */
 686 static int max_bundle_uri_depth = 4;
 687
 688 /**
 689  * Recursively download all bundles advertised at the given URI
 690  * to files. If the file is a bundle, then add it to the given
 691  * 'list'. Otherwise, expect a bundle list and recurse on the
 692  * URIs in that list according to the list mode (ANY or ALL).
 693  */
 694 static int fetch_bundle_uri_internal(struct repository *r,
 695                                      struct remote_bundle_info *bundle,
 696                                      int depth,
 697                                      struct bundle_list *list)
 698 {
 699         int result = 0;
 700         struct remote_bundle_info *bcopy;
 701
 702         if (depth >= max_bundle_uri_depth) {
 703                 warning(_("exceeded bundle URI recursion limit (%d)"),
 704                         max_bundle_uri_depth);
 705                 return -1;
 706         }
 707
 708         if (!bundle->file &&
 709             !(bundle->file = find_temp_filename())) {
 710                 result = -1;
 711                 goto cleanup;
 712         }
 713
 714         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 715                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 716                 goto cleanup;
 717         }
 718
 719         if ((result = !is_bundle(bundle->file, 1))) {
 720                 result = fetch_bundle_list_in_config_format(
 721                                 r, list, bundle, depth);
 722                 if (result)
 723                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 724                                 bundle->uri);
 725                 goto cleanup;
 726         }
 727
 728         /* Copy the bundle and insert it into the global list. */
 729         CALLOC_ARRAY(bcopy, 1);
 730         bcopy->id = xstrdup(bundle->id);
 731         bcopy->file = xstrdup(bundle->file);
 732         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 733         hashmap_add(&list->bundles, &bcopy->ent);
 734
 735 cleanup:
 736         if (result && bundle->file)
 737                 unlink(bundle->file);
 738         return result;
 739 }
 740
 741 /**
 742  * This loop iterator breaks the loop with nonzero return code on the
 743  * first successful unbundling of a bundle.
 744  */
 745 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 746 {
 747         struct repository *r = data;
 748
 749         if (!info->file || info->unbundled)
 750                 return 0;
 751
 752         if (!unbundle_from_file(r, info->file)) {
 753                 info->unbundled = 1;
 754                 return 1;
 755         }
 756
 757         return 0;
 758 }
 759
 760 static int unbundle_all_bundles(struct repository *r,
 761                                 struct bundle_list *list)
 762 {
 763         /*
 764          * Iterate through all bundles looking for ones that can
 765          * successfully unbundle. If any succeed, then perhaps another
 766          * will succeed in the next attempt.
 767          *
 768          * Keep in mind that a non-zero result for the loop here means
 769          * the loop terminated early on a successful unbundling, which
 770          * signals that we can try again.
 771          */
 772         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 773
 774         return 0;
 775 }
 776
 777 static int unlink_bundle(struct remote_bundle_info *info, void *data)
 778 {
 779         if (info->file)
 780                 unlink_or_warn(info->file);
 781         return 0;
 782 }
 783
 784 int fetch_bundle_uri(struct repository *r, const char *uri,
 785                      int *has_heuristic)
 786 {
 787         int result;
 788         struct bundle_list list;
 789         struct remote_bundle_info bundle = {
 790                 .uri = xstrdup(uri),
 791                 .id = xstrdup(""),
 792         };
 793
 794         init_bundle_list(&list);
 795
 796         /* If a bundle is added to this global list, then it is required. */
 797         list.mode = BUNDLE_MODE_ALL;
 798
 799         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 800                 goto cleanup;
 801
 802         result = unbundle_all_bundles(r, &list);
 803
 804 cleanup:
 805         if (has_heuristic)
 806                 *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE);
 807         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 808         clear_bundle_list(&list);
 809         clear_remote_bundle_info(&bundle, NULL);
 810         return result;
 811 }
 812
 813 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 814 {
 815         int result;
 816         struct bundle_list global_list;
 817
 818         /*
 819          * If the creationToken heuristic is used, then the URIs
 820          * advertised by 'list' are not nested lists and instead
 821          * direct bundles. We do not need to use global_list.
 822          */
 823         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 824                 return fetch_bundles_by_token(r, list);
 825
 826         init_bundle_list(&global_list);
 827
 828         /* If a bundle is added to this global list, then it is required. */
 829         global_list.mode = BUNDLE_MODE_ALL;
 830
 831         if ((result = download_bundle_list(r, list, &global_list, 0)))
 832                 goto cleanup;
 833
 834         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 835                 result = fetch_bundles_by_token(r, list);
 836         else
 837                 result = unbundle_all_bundles(r, &global_list);
 838
 839 cleanup:
 840         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 841         clear_bundle_list(&global_list);
 842         return result;
 843 }
 844
 845 /**
 846  * API for serve.c.
 847  */
 848
 849 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 850 {
 851         static int advertise_bundle_uri = -1;
 852
 853         if (advertise_bundle_uri != -1)
 854                 goto cached;
 855
 856         advertise_bundle_uri = 0;
 857         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 858
 859 cached:
 860         return advertise_bundle_uri;
 861 }
 862
 863 static int config_to_packet_line(const char *key, const char *value, void *data)
 864 {
 865         struct packet_reader *writer = data;
 866
 867         if (starts_with(key, "bundle."))
 868                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 869
 870         return 0;
 871 }
 872
 873 int bundle_uri_command(struct repository *r,
 874                        struct packet_reader *request)
 875 {
 876         struct packet_writer writer;
 877         packet_writer_init(&writer, 1);
 878
 879         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 880                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 881         if (request->status != PACKET_READ_FLUSH)
 882                 die(_("bundle-uri: expected flush after arguments"));
 883
 884         /*
 885          * Read all "bundle.*" config lines to the client as key=value
 886          * packet lines.
 887          */
 888         repo_config(r, config_to_packet_line, &writer);
 889
 890         packet_writer_flush(&writer);
 891
 892         return 0;
 893 }
 894
 895 /**
 896  * General API for {transport,connect}.c etc.
 897  */
 898 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 899 {
 900         int result;
 901         const char *equals;
 902         struct strbuf key = STRBUF_INIT;
 903
 904         if (!strlen(line))
 905                 return error(_("bundle-uri: got an empty line"));
 906
 907         equals = strchr(line, '=');
 908
 909         if (!equals)
 910                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 911         if (line == equals || !*(equals + 1))
 912                 return error(_("bundle-uri: line has empty key or value"));
 913
 914         strbuf_add(&key, line, equals - line);
 915         result = bundle_list_update(key.buf, equals + 1, list);
 916         strbuf_release(&key);
 917
 918         return result;
 919 }