bundle-uri.c

   1 #include "git-compat-util.h"
   2 #include "bundle-uri.h"
   3 #include "bundle.h"
   4 #include "copy.h"
   5 #include "environment.h"
   6 #include "gettext.h"
   7 #include "object-store-ll.h"
   8 #include "refs.h"
   9 #include "run-command.h"
  10 #include "hashmap.h"
  11 #include "pkt-line.h"
  12 #include "config.h"
  13 #include "remote.h"
  14
  15 static struct {
  16         enum bundle_list_heuristic heuristic;
  17         const char *name;
  18 } heuristics[BUNDLE_HEURISTIC__COUNT] = {
  19         { BUNDLE_HEURISTIC_NONE, ""},
  20         { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" },
  21 };
  22
  23 static int compare_bundles(const void *hashmap_cmp_fn_data UNUSED,
  24                            const struct hashmap_entry *he1,
  25                            const struct hashmap_entry *he2,
  26                            const void *id)
  27 {
  28         const struct remote_bundle_info *e1 =
  29                 container_of(he1, const struct remote_bundle_info, ent);
  30         const struct remote_bundle_info *e2 =
  31                 container_of(he2, const struct remote_bundle_info, ent);
  32
  33         return strcmp(e1->id, id ? (const char *)id : e2->id);
  34 }
  35
  36 void init_bundle_list(struct bundle_list *list)
  37 {
  38         memset(list, 0, sizeof(*list));
  39
  40         /* Implied defaults. */
  41         list->mode = BUNDLE_MODE_ALL;
  42         list->version = 1;
  43
  44         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  45 }
  46
  47 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  48                                     void *data UNUSED)
  49 {
  50         FREE_AND_NULL(bundle->id);
  51         FREE_AND_NULL(bundle->uri);
  52         FREE_AND_NULL(bundle->file);
  53         bundle->unbundled = 0;
  54         return 0;
  55 }
  56
  57 void clear_bundle_list(struct bundle_list *list)
  58 {
  59         if (!list)
  60                 return;
  61
  62         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  63         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  64         free(list->baseURI);
  65 }
  66
  67 int for_all_bundles_in_list(struct bundle_list *list,
  68                             bundle_iterator iter,
  69                             void *data)
  70 {
  71         struct remote_bundle_info *info;
  72         struct hashmap_iter i;
  73
  74         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  75                 int result = iter(info, data);
  76
  77                 if (result)
  78                         return result;
  79         }
  80
  81         return 0;
  82 }
  83
  84 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  85 {
  86         FILE *fp = data;
  87         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  88         fprintf(fp, "\turi = %s\n", info->uri);
  89
  90         if (info->creationToken)
  91                 fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken);
  92         return 0;
  93 }
  94
  95 void print_bundle_list(FILE *fp, struct bundle_list *list)
  96 {
  97         const char *mode;
  98
  99         switch (list->mode) {
 100         case BUNDLE_MODE_ALL:
 101                 mode = "all";
 102                 break;
 103
 104         case BUNDLE_MODE_ANY:
 105                 mode = "any";
 106                 break;
 107
 108         case BUNDLE_MODE_NONE:
 109         default:
 110                 mode = "<unknown>";
 111         }
 112
 113         fprintf(fp, "[bundle]\n");
 114         fprintf(fp, "\tversion = %d\n", list->version);
 115         fprintf(fp, "\tmode = %s\n", mode);
 116
 117         if (list->heuristic) {
 118                 int i;
 119                 for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 120                         if (heuristics[i].heuristic == list->heuristic) {
 121                                 printf("\theuristic = %s\n",
 122                                        heuristics[list->heuristic].name);
 123                                 break;
 124                         }
 125                 }
 126         }
 127
 128         for_all_bundles_in_list(list, summarize_bundle, fp);
 129 }
 130
 131 /**
 132  * Given a key-value pair, update the state of the given bundle list.
 133  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 134  * is not understood or the value is malformed.
 135  */
 136 static int bundle_list_update(const char *key, const char *value,
 137                               struct bundle_list *list)
 138 {
 139         struct strbuf id = STRBUF_INIT;
 140         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 141         struct remote_bundle_info *bundle;
 142         const char *subsection, *subkey;
 143         size_t subsection_len;
 144
 145         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 146                 return -1;
 147
 148         if (!subsection_len) {
 149                 if (!strcmp(subkey, "version")) {
 150                         int version;
 151                         if (!git_parse_int(value, &version))
 152                                 return -1;
 153                         if (version != 1)
 154                                 return -1;
 155
 156                         list->version = version;
 157                         return 0;
 158                 }
 159
 160                 if (!strcmp(subkey, "mode")) {
 161                         if (!strcmp(value, "all"))
 162                                 list->mode = BUNDLE_MODE_ALL;
 163                         else if (!strcmp(value, "any"))
 164                                 list->mode = BUNDLE_MODE_ANY;
 165                         else
 166                                 return -1;
 167                         return 0;
 168                 }
 169
 170                 if (!strcmp(subkey, "heuristic")) {
 171                         int i;
 172                         for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 173                                 if (heuristics[i].heuristic &&
 174                                     heuristics[i].name &&
 175                                     !strcmp(value, heuristics[i].name)) {
 176                                         list->heuristic = heuristics[i].heuristic;
 177                                         return 0;
 178                                 }
 179                         }
 180
 181                         /* Ignore unknown heuristics. */
 182                         return 0;
 183                 }
 184
 185                 /* Ignore other unknown global keys. */
 186                 return 0;
 187         }
 188
 189         strbuf_add(&id, subsection, subsection_len);
 190
 191         /*
 192          * Check for an existing bundle with this <id>, or create one
 193          * if necessary.
 194          */
 195         lookup.id = id.buf;
 196         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 197         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 198                 CALLOC_ARRAY(bundle, 1);
 199                 bundle->id = strbuf_detach(&id, NULL);
 200                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 201                 hashmap_add(&list->bundles, &bundle->ent);
 202         }
 203         strbuf_release(&id);
 204
 205         if (!strcmp(subkey, "uri")) {
 206                 if (bundle->uri)
 207                         return -1;
 208                 bundle->uri = relative_url(list->baseURI, value, NULL);
 209                 return 0;
 210         }
 211
 212         if (!strcmp(subkey, "creationtoken")) {
 213                 if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1)
 214                         warning(_("could not parse bundle list key %s with value '%s'"),
 215                                 "creationToken", value);
 216                 return 0;
 217         }
 218
 219         /*
 220          * At this point, we ignore any information that we don't
 221          * understand, assuming it to be hints for a heuristic the client
 222          * does not currently understand.
 223          */
 224         return 0;
 225 }
 226
 227 static int config_to_bundle_list(const char *key, const char *value,
 228                                  const struct config_context *ctx UNUSED,
 229                                  void *data)
 230 {
 231         struct bundle_list *list = data;
 232         return bundle_list_update(key, value, list);
 233 }
 234
 235 int bundle_uri_parse_config_format(const char *uri,
 236                                    const char *filename,
 237                                    struct bundle_list *list)
 238 {
 239         int result;
 240         struct config_options opts = {
 241                 .error_action = CONFIG_ERROR_ERROR,
 242         };
 243
 244         if (!list->baseURI) {
 245                 struct strbuf baseURI = STRBUF_INIT;
 246                 strbuf_addstr(&baseURI, uri);
 247
 248                 /*
 249                  * If the URI does not end with a trailing slash, then
 250                  * remove the filename portion of the path. This is
 251                  * important for relative URIs.
 252                  */
 253                 strbuf_strip_file_from_path(&baseURI);
 254                 list->baseURI = strbuf_detach(&baseURI, NULL);
 255         }
 256         result = git_config_from_file_with_options(config_to_bundle_list,
 257                                                    filename, list,
 258                                                    CONFIG_SCOPE_UNKNOWN,
 259                                                    &opts);
 260
 261         if (!result && list->mode == BUNDLE_MODE_NONE) {
 262                 warning(_("bundle list at '%s' has no mode"), uri);
 263                 result = 1;
 264         }
 265
 266         return result;
 267 }
 268
 269 static char *find_temp_filename(void)
 270 {
 271         int fd;
 272         struct strbuf name = STRBUF_INIT;
 273         /*
 274          * Find a temporary filename that is available. This is briefly
 275          * racy, but unlikely to collide.
 276          */
 277         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 278         if (fd < 0) {
 279                 warning(_("failed to create temporary file"));
 280                 return NULL;
 281         }
 282
 283         close(fd);
 284         unlink(name.buf);
 285         return strbuf_detach(&name, NULL);
 286 }
 287
 288 static int download_https_uri_to_file(const char *file, const char *uri)
 289 {
 290         int result = 0;
 291         struct child_process cp = CHILD_PROCESS_INIT;
 292         FILE *child_in = NULL, *child_out = NULL;
 293         struct strbuf line = STRBUF_INIT;
 294         int found_get = 0;
 295
 296         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 297         cp.err = -1;
 298         cp.in = -1;
 299         cp.out = -1;
 300
 301         if (start_command(&cp))
 302                 return 1;
 303
 304         child_in = fdopen(cp.in, "w");
 305         if (!child_in) {
 306                 result = 1;
 307                 goto cleanup;
 308         }
 309
 310         child_out = fdopen(cp.out, "r");
 311         if (!child_out) {
 312                 result = 1;
 313                 goto cleanup;
 314         }
 315
 316         fprintf(child_in, "capabilities\n");
 317         fflush(child_in);
 318
 319         while (!strbuf_getline(&line, child_out)) {
 320                 if (!line.len)
 321                         break;
 322                 if (!strcmp(line.buf, "get"))
 323                         found_get = 1;
 324         }
 325         strbuf_release(&line);
 326
 327         if (!found_get) {
 328                 result = error(_("insufficient capabilities"));
 329                 goto cleanup;
 330         }
 331
 332         fprintf(child_in, "get %s %s\n\n", uri, file);
 333
 334 cleanup:
 335         if (child_in)
 336                 fclose(child_in);
 337         if (finish_command(&cp))
 338                 return 1;
 339         if (child_out)
 340                 fclose(child_out);
 341         return result;
 342 }
 343
 344 static int copy_uri_to_file(const char *filename, const char *uri)
 345 {
 346         const char *out;
 347
 348         if (starts_with(uri, "https:") ||
 349             starts_with(uri, "http:"))
 350                 return download_https_uri_to_file(filename, uri);
 351
 352         if (skip_prefix(uri, "file://", &out))
 353                 uri = out;
 354
 355         /* Copy as a file */
 356         return copy_file(filename, uri, 0);
 357 }
 358
 359 static int unbundle_from_file(struct repository *r, const char *file)
 360 {
 361         int result = 0;
 362         int bundle_fd;
 363         struct bundle_header header = BUNDLE_HEADER_INIT;
 364         struct string_list_item *refname;
 365         struct strbuf bundle_ref = STRBUF_INIT;
 366         size_t bundle_prefix_len;
 367
 368         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 369                 return 1;
 370
 371         /*
 372          * Skip the reachability walk here, since we will be adding
 373          * a reachable ref pointing to the new tips, which will reach
 374          * the prerequisite commits.
 375          */
 376         if ((result = unbundle(r, &header, bundle_fd, NULL,
 377                                VERIFY_BUNDLE_QUIET)))
 378                 return 1;
 379
 380         /*
 381          * Convert all refs/heads/ from the bundle into refs/bundles/
 382          * in the local repository.
 383          */
 384         strbuf_addstr(&bundle_ref, "refs/bundles/");
 385         bundle_prefix_len = bundle_ref.len;
 386
 387         for_each_string_list_item(refname, &header.references) {
 388                 struct object_id *oid = refname->util;
 389                 struct object_id old_oid;
 390                 const char *branch_name;
 391                 int has_old;
 392
 393                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 394                         continue;
 395
 396                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 397                 strbuf_addstr(&bundle_ref, branch_name);
 398
 399                 has_old = !read_ref(bundle_ref.buf, &old_oid);
 400                 update_ref("fetched bundle", bundle_ref.buf, oid,
 401                            has_old ? &old_oid : NULL,
 402                            REF_SKIP_OID_VERIFICATION,
 403                            UPDATE_REFS_MSG_ON_ERR);
 404         }
 405
 406         bundle_header_release(&header);
 407         return result;
 408 }
 409
 410 struct bundle_list_context {
 411         struct repository *r;
 412         struct bundle_list *list;
 413         enum bundle_list_mode mode;
 414         int count;
 415         int depth;
 416 };
 417
 418 /*
 419  * This early definition is necessary because we use indirect recursion:
 420  *
 421  * While iterating through a bundle list that was downloaded as part
 422  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 423  * again, but with depth + 1.
 424  */
 425 static int fetch_bundle_uri_internal(struct repository *r,
 426                                      struct remote_bundle_info *bundle,
 427                                      int depth,
 428                                      struct bundle_list *list);
 429
 430 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 431 {
 432         int res;
 433         struct bundle_list_context *ctx = data;
 434
 435         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 436                 return 0;
 437
 438         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 439
 440         /*
 441          * Only increment count if the download succeeded. If our mode is
 442          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 443          * list in case they work instead.
 444          */
 445         if (!res)
 446                 ctx->count++;
 447
 448         /*
 449          * To be opportunistic as possible, we continue iterating and
 450          * download as many bundles as we can, so we can apply the ones
 451          * that work, even in BUNDLE_MODE_ALL mode.
 452          */
 453         return 0;
 454 }
 455
 456 struct bundles_for_sorting {
 457         struct remote_bundle_info **items;
 458         size_t alloc;
 459         size_t nr;
 460 };
 461
 462 static int append_bundle(struct remote_bundle_info *bundle, void *data)
 463 {
 464         struct bundles_for_sorting *list = data;
 465         list->items[list->nr++] = bundle;
 466         return 0;
 467 }
 468
 469 /**
 470  * For use in QSORT() to get a list sorted by creationToken
 471  * in decreasing order.
 472  */
 473 static int compare_creation_token_decreasing(const void *va, const void *vb)
 474 {
 475         const struct remote_bundle_info * const *a = va;
 476         const struct remote_bundle_info * const *b = vb;
 477
 478         if ((*a)->creationToken > (*b)->creationToken)
 479                 return -1;
 480         if ((*a)->creationToken < (*b)->creationToken)
 481                 return 1;
 482         return 0;
 483 }
 484
 485 static int fetch_bundles_by_token(struct repository *r,
 486                                   struct bundle_list *list)
 487 {
 488         int cur;
 489         int move_direction = 0;
 490         const char *creationTokenStr;
 491         uint64_t maxCreationToken = 0, newMaxCreationToken = 0;
 492         struct bundle_list_context ctx = {
 493                 .r = r,
 494                 .list = list,
 495                 .mode = list->mode,
 496         };
 497         struct bundles_for_sorting bundles = {
 498                 .alloc = hashmap_get_size(&list->bundles),
 499         };
 500
 501         ALLOC_ARRAY(bundles.items, bundles.alloc);
 502
 503         for_all_bundles_in_list(list, append_bundle, &bundles);
 504
 505         if (!bundles.nr) {
 506                 free(bundles.items);
 507                 return 0;
 508         }
 509
 510         QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 511
 512         /*
 513          * If fetch.bundleCreationToken exists, parses to a uint64t, and
 514          * is not strictly smaller than the maximum creation token in the
 515          * bundle list, then do not download any bundles.
 516          */
 517         if (!repo_config_get_value(r,
 518                                    "fetch.bundlecreationtoken",
 519                                    &creationTokenStr) &&
 520             sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) == 1 &&
 521             bundles.items[0]->creationToken <= maxCreationToken) {
 522                 free(bundles.items);
 523                 return 0;
 524         }
 525
 526         /*
 527          * Attempt to download and unbundle the minimum number of bundles by
 528          * creationToken in decreasing order. If we fail to unbundle (after
 529          * a successful download) then move to the next non-downloaded bundle
 530          * and attempt downloading. Once we succeed in applying a bundle,
 531          * move to the previous unapplied bundle and attempt to unbundle it
 532          * again.
 533          *
 534          * In the case of a fresh clone, we will likely download all of the
 535          * bundles before successfully unbundling the oldest one, then the
 536          * rest of the bundles unbundle successfully in increasing order
 537          * of creationToken.
 538          *
 539          * If there are existing objects, then this process may terminate
 540          * early when all required commits from "new" bundles exist in the
 541          * repo's object store.
 542          */
 543         cur = 0;
 544         while (cur >= 0 && cur < bundles.nr) {
 545                 struct remote_bundle_info *bundle = bundles.items[cur];
 546
 547                 /*
 548                  * If we need to dig into bundles below the previous
 549                  * creation token value, then likely we are in an erroneous
 550                  * state due to missing or invalid bundles. Halt the process
 551                  * instead of continuing to download extra data.
 552                  */
 553                 if (bundle->creationToken <= maxCreationToken)
 554                         break;
 555
 556                 if (!bundle->file) {
 557                         /*
 558                          * Not downloaded yet. Try downloading.
 559                          *
 560                          * Note that bundle->file is non-NULL if a download
 561                          * was attempted, even if it failed to download.
 562                          */
 563                         if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
 564                                 /* Mark as unbundled so we do not retry. */
 565                                 bundle->unbundled = 1;
 566
 567                                 /* Try looking deeper in the list. */
 568                                 move_direction = 1;
 569                                 goto move;
 570                         }
 571
 572                         /* We expect bundles when using creationTokens. */
 573                         if (!is_bundle(bundle->file, 1)) {
 574                                 warning(_("file downloaded from '%s' is not a bundle"),
 575                                         bundle->uri);
 576                                 break;
 577                         }
 578                 }
 579
 580                 if (bundle->file && !bundle->unbundled) {
 581                         /*
 582                          * This was downloaded, but not successfully
 583                          * unbundled. Try unbundling again.
 584                          */
 585                         if (unbundle_from_file(ctx.r, bundle->file)) {
 586                                 /* Try looking deeper in the list. */
 587                                 move_direction = 1;
 588                         } else {
 589                                 /*
 590                                  * Succeeded in unbundle. Retry bundles
 591                                  * that previously failed to unbundle.
 592                                  */
 593                                 move_direction = -1;
 594                                 bundle->unbundled = 1;
 595
 596                                 if (bundle->creationToken > newMaxCreationToken)
 597                                         newMaxCreationToken = bundle->creationToken;
 598                         }
 599                 }
 600
 601                 /*
 602                  * Else case: downloaded and unbundled successfully.
 603                  * Skip this by moving in the same direction as the
 604                  * previous step.
 605                  */
 606
 607 move:
 608                 /* Move in the specified direction and repeat. */
 609                 cur += move_direction;
 610         }
 611
 612         /*
 613          * We succeed if the loop terminates because 'cur' drops below
 614          * zero. The other case is that we terminate because 'cur'
 615          * reaches the end of the list, so we have a failure no matter
 616          * which bundles we apply from the list.
 617          */
 618         if (cur < 0) {
 619                 struct strbuf value = STRBUF_INIT;
 620                 strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken);
 621                 if (repo_config_set_multivar_gently(ctx.r,
 622                                                     "fetch.bundleCreationToken",
 623                                                     value.buf, NULL, 0))
 624                         warning(_("failed to store maximum creation token"));
 625
 626                 strbuf_release(&value);
 627         }
 628
 629         free(bundles.items);
 630         return cur >= 0;
 631 }
 632
 633 static int download_bundle_list(struct repository *r,
 634                                 struct bundle_list *local_list,
 635                                 struct bundle_list *global_list,
 636                                 int depth)
 637 {
 638         struct bundle_list_context ctx = {
 639                 .r = r,
 640                 .list = global_list,
 641                 .depth = depth + 1,
 642                 .mode = local_list->mode,
 643         };
 644
 645         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 646 }
 647
 648 static int fetch_bundle_list_in_config_format(struct repository *r,
 649                                               struct bundle_list *global_list,
 650                                               struct remote_bundle_info *bundle,
 651                                               int depth)
 652 {
 653         int result;
 654         struct bundle_list list_from_bundle;
 655
 656         init_bundle_list(&list_from_bundle);
 657
 658         if ((result = bundle_uri_parse_config_format(bundle->uri,
 659                                                      bundle->file,
 660                                                      &list_from_bundle)))
 661                 goto cleanup;
 662
 663         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 664                 warning(_("unrecognized bundle mode from URI '%s'"),
 665                         bundle->uri);
 666                 result = -1;
 667                 goto cleanup;
 668         }
 669
 670         /*
 671          * If this list uses the creationToken heuristic, then the URIs
 672          * it advertises are expected to be bundles, not nested lists.
 673          * We can drop 'global_list' and 'depth'.
 674          */
 675         if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
 676                 result = fetch_bundles_by_token(r, &list_from_bundle);
 677                 global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
 678         } else if ((result = download_bundle_list(r, &list_from_bundle,
 679                                            global_list, depth)))
 680                 goto cleanup;
 681
 682 cleanup:
 683         clear_bundle_list(&list_from_bundle);
 684         return result;
 685 }
 686
 687 /**
 688  * This limits the recursion on fetch_bundle_uri_internal() when following
 689  * bundle lists.
 690  */
 691 static int max_bundle_uri_depth = 4;
 692
 693 /**
 694  * Recursively download all bundles advertised at the given URI
 695  * to files. If the file is a bundle, then add it to the given
 696  * 'list'. Otherwise, expect a bundle list and recurse on the
 697  * URIs in that list according to the list mode (ANY or ALL).
 698  */
 699 static int fetch_bundle_uri_internal(struct repository *r,
 700                                      struct remote_bundle_info *bundle,
 701                                      int depth,
 702                                      struct bundle_list *list)
 703 {
 704         int result = 0;
 705         struct remote_bundle_info *bcopy;
 706
 707         if (depth >= max_bundle_uri_depth) {
 708                 warning(_("exceeded bundle URI recursion limit (%d)"),
 709                         max_bundle_uri_depth);
 710                 return -1;
 711         }
 712
 713         if (!bundle->file &&
 714             !(bundle->file = find_temp_filename())) {
 715                 result = -1;
 716                 goto cleanup;
 717         }
 718
 719         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 720                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 721                 goto cleanup;
 722         }
 723
 724         if ((result = !is_bundle(bundle->file, 1))) {
 725                 result = fetch_bundle_list_in_config_format(
 726                                 r, list, bundle, depth);
 727                 if (result)
 728                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 729                                 bundle->uri);
 730                 goto cleanup;
 731         }
 732
 733         /* Copy the bundle and insert it into the global list. */
 734         CALLOC_ARRAY(bcopy, 1);
 735         bcopy->id = xstrdup(bundle->id);
 736         bcopy->file = xstrdup(bundle->file);
 737         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 738         hashmap_add(&list->bundles, &bcopy->ent);
 739
 740 cleanup:
 741         if (result && bundle->file)
 742                 unlink(bundle->file);
 743         return result;
 744 }
 745
 746 /**
 747  * This loop iterator breaks the loop with nonzero return code on the
 748  * first successful unbundling of a bundle.
 749  */
 750 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 751 {
 752         struct repository *r = data;
 753
 754         if (!info->file || info->unbundled)
 755                 return 0;
 756
 757         if (!unbundle_from_file(r, info->file)) {
 758                 info->unbundled = 1;
 759                 return 1;
 760         }
 761
 762         return 0;
 763 }
 764
 765 static int unbundle_all_bundles(struct repository *r,
 766                                 struct bundle_list *list)
 767 {
 768         /*
 769          * Iterate through all bundles looking for ones that can
 770          * successfully unbundle. If any succeed, then perhaps another
 771          * will succeed in the next attempt.
 772          *
 773          * Keep in mind that a non-zero result for the loop here means
 774          * the loop terminated early on a successful unbundling, which
 775          * signals that we can try again.
 776          */
 777         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 778
 779         return 0;
 780 }
 781
 782 static int unlink_bundle(struct remote_bundle_info *info, void *data UNUSED)
 783 {
 784         if (info->file)
 785                 unlink_or_warn(info->file);
 786         return 0;
 787 }
 788
 789 int fetch_bundle_uri(struct repository *r, const char *uri,
 790                      int *has_heuristic)
 791 {
 792         int result;
 793         struct bundle_list list;
 794         struct remote_bundle_info bundle = {
 795                 .uri = xstrdup(uri),
 796                 .id = xstrdup(""),
 797         };
 798
 799         init_bundle_list(&list);
 800
 801         /*
 802          * Do not fetch an empty bundle URI. An empty bundle URI
 803          * could signal that a configured bundle URI has been disabled.
 804          */
 805         if (!*uri) {
 806                 result = 0;
 807                 goto cleanup;
 808         }
 809
 810         /* If a bundle is added to this global list, then it is required. */
 811         list.mode = BUNDLE_MODE_ALL;
 812
 813         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 814                 goto cleanup;
 815
 816         result = unbundle_all_bundles(r, &list);
 817
 818 cleanup:
 819         if (has_heuristic)
 820                 *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE);
 821         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 822         clear_bundle_list(&list);
 823         clear_remote_bundle_info(&bundle, NULL);
 824         return result;
 825 }
 826
 827 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 828 {
 829         int result;
 830         struct bundle_list global_list;
 831
 832         /*
 833          * If the creationToken heuristic is used, then the URIs
 834          * advertised by 'list' are not nested lists and instead
 835          * direct bundles. We do not need to use global_list.
 836          */
 837         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 838                 return fetch_bundles_by_token(r, list);
 839
 840         init_bundle_list(&global_list);
 841
 842         /* If a bundle is added to this global list, then it is required. */
 843         global_list.mode = BUNDLE_MODE_ALL;
 844
 845         if ((result = download_bundle_list(r, list, &global_list, 0)))
 846                 goto cleanup;
 847
 848         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 849                 result = fetch_bundles_by_token(r, list);
 850         else
 851                 result = unbundle_all_bundles(r, &global_list);
 852
 853 cleanup:
 854         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 855         clear_bundle_list(&global_list);
 856         return result;
 857 }
 858
 859 /**
 860  * API for serve.c.
 861  */
 862
 863 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 864 {
 865         static int advertise_bundle_uri = -1;
 866
 867         if (advertise_bundle_uri != -1)
 868                 goto cached;
 869
 870         advertise_bundle_uri = 0;
 871         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 872
 873 cached:
 874         return advertise_bundle_uri;
 875 }
 876
 877 static int config_to_packet_line(const char *key, const char *value,
 878                                  const struct config_context *ctx UNUSED,
 879                                  void *data)
 880 {
 881         struct packet_reader *writer = data;
 882
 883         if (starts_with(key, "bundle."))
 884                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 885
 886         return 0;
 887 }
 888
 889 int bundle_uri_command(struct repository *r,
 890                        struct packet_reader *request)
 891 {
 892         struct packet_writer writer;
 893         packet_writer_init(&writer, 1);
 894
 895         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 896                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 897         if (request->status != PACKET_READ_FLUSH)
 898                 die(_("bundle-uri: expected flush after arguments"));
 899
 900         /*
 901          * Read all "bundle.*" config lines to the client as key=value
 902          * packet lines.
 903          */
 904         repo_config(r, config_to_packet_line, &writer);
 905
 906         packet_writer_flush(&writer);
 907
 908         return 0;
 909 }
 910
 911 /**
 912  * General API for {transport,connect}.c etc.
 913  */
 914 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 915 {
 916         int result;
 917         const char *equals;
 918         struct strbuf key = STRBUF_INIT;
 919
 920         if (!strlen(line))
 921                 return error(_("bundle-uri: got an empty line"));
 922
 923         equals = strchr(line, '=');
 924
 925         if (!equals)
 926                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 927         if (line == equals || !*(equals + 1))
 928                 return error(_("bundle-uri: line has empty key or value"));
 929
 930         strbuf_add(&key, line, equals - line);
 931         result = bundle_list_update(key.buf, equals + 1, list);
 932         strbuf_release(&key);
 933
 934         return result;
 935 }