bundle-uri.c

   1 #include "cache.h"
   2 #include "bundle-uri.h"
   3 #include "bundle.h"
   4 #include "environment.h"
   5 #include "gettext.h"
   6 #include "object-store.h"
   7 #include "refs.h"
   8 #include "run-command.h"
   9 #include "hashmap.h"
  10 #include "pkt-line.h"
  11 #include "config.h"
  12 #include "remote.h"
  13
  14 static struct {
  15         enum bundle_list_heuristic heuristic;
  16         const char *name;
  17 } heuristics[BUNDLE_HEURISTIC__COUNT] = {
  18         { BUNDLE_HEURISTIC_NONE, ""},
  19         { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" },
  20 };
  21
  22 static int compare_bundles(const void *hashmap_cmp_fn_data,
  23                            const struct hashmap_entry *he1,
  24                            const struct hashmap_entry *he2,
  25                            const void *id)
  26 {
  27         const struct remote_bundle_info *e1 =
  28                 container_of(he1, const struct remote_bundle_info, ent);
  29         const struct remote_bundle_info *e2 =
  30                 container_of(he2, const struct remote_bundle_info, ent);
  31
  32         return strcmp(e1->id, id ? (const char *)id : e2->id);
  33 }
  34
  35 void init_bundle_list(struct bundle_list *list)
  36 {
  37         memset(list, 0, sizeof(*list));
  38
  39         /* Implied defaults. */
  40         list->mode = BUNDLE_MODE_ALL;
  41         list->version = 1;
  42
  43         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  44 }
  45
  46 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  47                                     void *data)
  48 {
  49         FREE_AND_NULL(bundle->id);
  50         FREE_AND_NULL(bundle->uri);
  51         FREE_AND_NULL(bundle->file);
  52         bundle->unbundled = 0;
  53         return 0;
  54 }
  55
  56 void clear_bundle_list(struct bundle_list *list)
  57 {
  58         if (!list)
  59                 return;
  60
  61         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  62         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  63         free(list->baseURI);
  64 }
  65
  66 int for_all_bundles_in_list(struct bundle_list *list,
  67                             bundle_iterator iter,
  68                             void *data)
  69 {
  70         struct remote_bundle_info *info;
  71         struct hashmap_iter i;
  72
  73         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  74                 int result = iter(info, data);
  75
  76                 if (result)
  77                         return result;
  78         }
  79
  80         return 0;
  81 }
  82
  83 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  84 {
  85         FILE *fp = data;
  86         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  87         fprintf(fp, "\turi = %s\n", info->uri);
  88
  89         if (info->creationToken)
  90                 fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken);
  91         return 0;
  92 }
  93
  94 void print_bundle_list(FILE *fp, struct bundle_list *list)
  95 {
  96         const char *mode;
  97
  98         switch (list->mode) {
  99         case BUNDLE_MODE_ALL:
 100                 mode = "all";
 101                 break;
 102
 103         case BUNDLE_MODE_ANY:
 104                 mode = "any";
 105                 break;
 106
 107         case BUNDLE_MODE_NONE:
 108         default:
 109                 mode = "<unknown>";
 110         }
 111
 112         fprintf(fp, "[bundle]\n");
 113         fprintf(fp, "\tversion = %d\n", list->version);
 114         fprintf(fp, "\tmode = %s\n", mode);
 115
 116         if (list->heuristic) {
 117                 int i;
 118                 for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 119                         if (heuristics[i].heuristic == list->heuristic) {
 120                                 printf("\theuristic = %s\n",
 121                                        heuristics[list->heuristic].name);
 122                                 break;
 123                         }
 124                 }
 125         }
 126
 127         for_all_bundles_in_list(list, summarize_bundle, fp);
 128 }
 129
 130 /**
 131  * Given a key-value pair, update the state of the given bundle list.
 132  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 133  * is not understood or the value is malformed.
 134  */
 135 static int bundle_list_update(const char *key, const char *value,
 136                               struct bundle_list *list)
 137 {
 138         struct strbuf id = STRBUF_INIT;
 139         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 140         struct remote_bundle_info *bundle;
 141         const char *subsection, *subkey;
 142         size_t subsection_len;
 143
 144         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 145                 return -1;
 146
 147         if (!subsection_len) {
 148                 if (!strcmp(subkey, "version")) {
 149                         int version;
 150                         if (!git_parse_int(value, &version))
 151                                 return -1;
 152                         if (version != 1)
 153                                 return -1;
 154
 155                         list->version = version;
 156                         return 0;
 157                 }
 158
 159                 if (!strcmp(subkey, "mode")) {
 160                         if (!strcmp(value, "all"))
 161                                 list->mode = BUNDLE_MODE_ALL;
 162                         else if (!strcmp(value, "any"))
 163                                 list->mode = BUNDLE_MODE_ANY;
 164                         else
 165                                 return -1;
 166                         return 0;
 167                 }
 168
 169                 if (!strcmp(subkey, "heuristic")) {
 170                         int i;
 171                         for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 172                                 if (heuristics[i].heuristic &&
 173                                     heuristics[i].name &&
 174                                     !strcmp(value, heuristics[i].name)) {
 175                                         list->heuristic = heuristics[i].heuristic;
 176                                         return 0;
 177                                 }
 178                         }
 179
 180                         /* Ignore unknown heuristics. */
 181                         return 0;
 182                 }
 183
 184                 /* Ignore other unknown global keys. */
 185                 return 0;
 186         }
 187
 188         strbuf_add(&id, subsection, subsection_len);
 189
 190         /*
 191          * Check for an existing bundle with this <id>, or create one
 192          * if necessary.
 193          */
 194         lookup.id = id.buf;
 195         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 196         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 197                 CALLOC_ARRAY(bundle, 1);
 198                 bundle->id = strbuf_detach(&id, NULL);
 199                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 200                 hashmap_add(&list->bundles, &bundle->ent);
 201         }
 202         strbuf_release(&id);
 203
 204         if (!strcmp(subkey, "uri")) {
 205                 if (bundle->uri)
 206                         return -1;
 207                 bundle->uri = relative_url(list->baseURI, value, NULL);
 208                 return 0;
 209         }
 210
 211         if (!strcmp(subkey, "creationtoken")) {
 212                 if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1)
 213                         warning(_("could not parse bundle list key %s with value '%s'"),
 214                                 "creationToken", value);
 215                 return 0;
 216         }
 217
 218         /*
 219          * At this point, we ignore any information that we don't
 220          * understand, assuming it to be hints for a heuristic the client
 221          * does not currently understand.
 222          */
 223         return 0;
 224 }
 225
 226 static int config_to_bundle_list(const char *key, const char *value, void *data)
 227 {
 228         struct bundle_list *list = data;
 229         return bundle_list_update(key, value, list);
 230 }
 231
 232 int bundle_uri_parse_config_format(const char *uri,
 233                                    const char *filename,
 234                                    struct bundle_list *list)
 235 {
 236         int result;
 237         struct config_options opts = {
 238                 .error_action = CONFIG_ERROR_ERROR,
 239         };
 240
 241         if (!list->baseURI) {
 242                 struct strbuf baseURI = STRBUF_INIT;
 243                 strbuf_addstr(&baseURI, uri);
 244
 245                 /*
 246                  * If the URI does not end with a trailing slash, then
 247                  * remove the filename portion of the path. This is
 248                  * important for relative URIs.
 249                  */
 250                 strbuf_strip_file_from_path(&baseURI);
 251                 list->baseURI = strbuf_detach(&baseURI, NULL);
 252         }
 253         result = git_config_from_file_with_options(config_to_bundle_list,
 254                                                    filename, list,
 255                                                    &opts);
 256
 257         if (!result && list->mode == BUNDLE_MODE_NONE) {
 258                 warning(_("bundle list at '%s' has no mode"), uri);
 259                 result = 1;
 260         }
 261
 262         return result;
 263 }
 264
 265 static char *find_temp_filename(void)
 266 {
 267         int fd;
 268         struct strbuf name = STRBUF_INIT;
 269         /*
 270          * Find a temporary filename that is available. This is briefly
 271          * racy, but unlikely to collide.
 272          */
 273         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 274         if (fd < 0) {
 275                 warning(_("failed to create temporary file"));
 276                 return NULL;
 277         }
 278
 279         close(fd);
 280         unlink(name.buf);
 281         return strbuf_detach(&name, NULL);
 282 }
 283
 284 static int download_https_uri_to_file(const char *file, const char *uri)
 285 {
 286         int result = 0;
 287         struct child_process cp = CHILD_PROCESS_INIT;
 288         FILE *child_in = NULL, *child_out = NULL;
 289         struct strbuf line = STRBUF_INIT;
 290         int found_get = 0;
 291
 292         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 293         cp.err = -1;
 294         cp.in = -1;
 295         cp.out = -1;
 296
 297         if (start_command(&cp))
 298                 return 1;
 299
 300         child_in = fdopen(cp.in, "w");
 301         if (!child_in) {
 302                 result = 1;
 303                 goto cleanup;
 304         }
 305
 306         child_out = fdopen(cp.out, "r");
 307         if (!child_out) {
 308                 result = 1;
 309                 goto cleanup;
 310         }
 311
 312         fprintf(child_in, "capabilities\n");
 313         fflush(child_in);
 314
 315         while (!strbuf_getline(&line, child_out)) {
 316                 if (!line.len)
 317                         break;
 318                 if (!strcmp(line.buf, "get"))
 319                         found_get = 1;
 320         }
 321         strbuf_release(&line);
 322
 323         if (!found_get) {
 324                 result = error(_("insufficient capabilities"));
 325                 goto cleanup;
 326         }
 327
 328         fprintf(child_in, "get %s %s\n\n", uri, file);
 329
 330 cleanup:
 331         if (child_in)
 332                 fclose(child_in);
 333         if (finish_command(&cp))
 334                 return 1;
 335         if (child_out)
 336                 fclose(child_out);
 337         return result;
 338 }
 339
 340 static int copy_uri_to_file(const char *filename, const char *uri)
 341 {
 342         const char *out;
 343
 344         if (starts_with(uri, "https:") ||
 345             starts_with(uri, "http:"))
 346                 return download_https_uri_to_file(filename, uri);
 347
 348         if (skip_prefix(uri, "file://", &out))
 349                 uri = out;
 350
 351         /* Copy as a file */
 352         return copy_file(filename, uri, 0);
 353 }
 354
 355 static int unbundle_from_file(struct repository *r, const char *file)
 356 {
 357         int result = 0;
 358         int bundle_fd;
 359         struct bundle_header header = BUNDLE_HEADER_INIT;
 360         struct string_list_item *refname;
 361         struct strbuf bundle_ref = STRBUF_INIT;
 362         size_t bundle_prefix_len;
 363
 364         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 365                 return 1;
 366
 367         /*
 368          * Skip the reachability walk here, since we will be adding
 369          * a reachable ref pointing to the new tips, which will reach
 370          * the prerequisite commits.
 371          */
 372         if ((result = unbundle(r, &header, bundle_fd, NULL,
 373                                VERIFY_BUNDLE_QUIET)))
 374                 return 1;
 375
 376         /*
 377          * Convert all refs/heads/ from the bundle into refs/bundles/
 378          * in the local repository.
 379          */
 380         strbuf_addstr(&bundle_ref, "refs/bundles/");
 381         bundle_prefix_len = bundle_ref.len;
 382
 383         for_each_string_list_item(refname, &header.references) {
 384                 struct object_id *oid = refname->util;
 385                 struct object_id old_oid;
 386                 const char *branch_name;
 387                 int has_old;
 388
 389                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 390                         continue;
 391
 392                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 393                 strbuf_addstr(&bundle_ref, branch_name);
 394
 395                 has_old = !read_ref(bundle_ref.buf, &old_oid);
 396                 update_ref("fetched bundle", bundle_ref.buf, oid,
 397                            has_old ? &old_oid : NULL,
 398                            REF_SKIP_OID_VERIFICATION,
 399                            UPDATE_REFS_MSG_ON_ERR);
 400         }
 401
 402         bundle_header_release(&header);
 403         return result;
 404 }
 405
 406 struct bundle_list_context {
 407         struct repository *r;
 408         struct bundle_list *list;
 409         enum bundle_list_mode mode;
 410         int count;
 411         int depth;
 412 };
 413
 414 /*
 415  * This early definition is necessary because we use indirect recursion:
 416  *
 417  * While iterating through a bundle list that was downloaded as part
 418  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 419  * again, but with depth + 1.
 420  */
 421 static int fetch_bundle_uri_internal(struct repository *r,
 422                                      struct remote_bundle_info *bundle,
 423                                      int depth,
 424                                      struct bundle_list *list);
 425
 426 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 427 {
 428         int res;
 429         struct bundle_list_context *ctx = data;
 430
 431         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 432                 return 0;
 433
 434         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 435
 436         /*
 437          * Only increment count if the download succeeded. If our mode is
 438          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 439          * list in case they work instead.
 440          */
 441         if (!res)
 442                 ctx->count++;
 443
 444         /*
 445          * To be opportunistic as possible, we continue iterating and
 446          * download as many bundles as we can, so we can apply the ones
 447          * that work, even in BUNDLE_MODE_ALL mode.
 448          */
 449         return 0;
 450 }
 451
 452 struct bundles_for_sorting {
 453         struct remote_bundle_info **items;
 454         size_t alloc;
 455         size_t nr;
 456 };
 457
 458 static int append_bundle(struct remote_bundle_info *bundle, void *data)
 459 {
 460         struct bundles_for_sorting *list = data;
 461         list->items[list->nr++] = bundle;
 462         return 0;
 463 }
 464
 465 /**
 466  * For use in QSORT() to get a list sorted by creationToken
 467  * in decreasing order.
 468  */
 469 static int compare_creation_token_decreasing(const void *va, const void *vb)
 470 {
 471         const struct remote_bundle_info * const *a = va;
 472         const struct remote_bundle_info * const *b = vb;
 473
 474         if ((*a)->creationToken > (*b)->creationToken)
 475                 return -1;
 476         if ((*a)->creationToken < (*b)->creationToken)
 477                 return 1;
 478         return 0;
 479 }
 480
 481 static int fetch_bundles_by_token(struct repository *r,
 482                                   struct bundle_list *list)
 483 {
 484         int cur;
 485         int move_direction = 0;
 486         const char *creationTokenStr;
 487         uint64_t maxCreationToken = 0, newMaxCreationToken = 0;
 488         struct bundle_list_context ctx = {
 489                 .r = r,
 490                 .list = list,
 491                 .mode = list->mode,
 492         };
 493         struct bundles_for_sorting bundles = {
 494                 .alloc = hashmap_get_size(&list->bundles),
 495         };
 496
 497         ALLOC_ARRAY(bundles.items, bundles.alloc);
 498
 499         for_all_bundles_in_list(list, append_bundle, &bundles);
 500
 501         if (!bundles.nr) {
 502                 free(bundles.items);
 503                 return 0;
 504         }
 505
 506         QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 507
 508         /*
 509          * If fetch.bundleCreationToken exists, parses to a uint64t, and
 510          * is not strictly smaller than the maximum creation token in the
 511          * bundle list, then do not download any bundles.
 512          */
 513         if (!repo_config_get_value(r,
 514                                    "fetch.bundlecreationtoken",
 515                                    &creationTokenStr) &&
 516             sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) == 1 &&
 517             bundles.items[0]->creationToken <= maxCreationToken) {
 518                 free(bundles.items);
 519                 return 0;
 520         }
 521
 522         /*
 523          * Attempt to download and unbundle the minimum number of bundles by
 524          * creationToken in decreasing order. If we fail to unbundle (after
 525          * a successful download) then move to the next non-downloaded bundle
 526          * and attempt downloading. Once we succeed in applying a bundle,
 527          * move to the previous unapplied bundle and attempt to unbundle it
 528          * again.
 529          *
 530          * In the case of a fresh clone, we will likely download all of the
 531          * bundles before successfully unbundling the oldest one, then the
 532          * rest of the bundles unbundle successfully in increasing order
 533          * of creationToken.
 534          *
 535          * If there are existing objects, then this process may terminate
 536          * early when all required commits from "new" bundles exist in the
 537          * repo's object store.
 538          */
 539         cur = 0;
 540         while (cur >= 0 && cur < bundles.nr) {
 541                 struct remote_bundle_info *bundle = bundles.items[cur];
 542
 543                 /*
 544                  * If we need to dig into bundles below the previous
 545                  * creation token value, then likely we are in an erroneous
 546                  * state due to missing or invalid bundles. Halt the process
 547                  * instead of continuing to download extra data.
 548                  */
 549                 if (bundle->creationToken <= maxCreationToken)
 550                         break;
 551
 552                 if (!bundle->file) {
 553                         /*
 554                          * Not downloaded yet. Try downloading.
 555                          *
 556                          * Note that bundle->file is non-NULL if a download
 557                          * was attempted, even if it failed to download.
 558                          */
 559                         if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
 560                                 /* Mark as unbundled so we do not retry. */
 561                                 bundle->unbundled = 1;
 562
 563                                 /* Try looking deeper in the list. */
 564                                 move_direction = 1;
 565                                 goto move;
 566                         }
 567
 568                         /* We expect bundles when using creationTokens. */
 569                         if (!is_bundle(bundle->file, 1)) {
 570                                 warning(_("file downloaded from '%s' is not a bundle"),
 571                                         bundle->uri);
 572                                 break;
 573                         }
 574                 }
 575
 576                 if (bundle->file && !bundle->unbundled) {
 577                         /*
 578                          * This was downloaded, but not successfully
 579                          * unbundled. Try unbundling again.
 580                          */
 581                         if (unbundle_from_file(ctx.r, bundle->file)) {
 582                                 /* Try looking deeper in the list. */
 583                                 move_direction = 1;
 584                         } else {
 585                                 /*
 586                                  * Succeeded in unbundle. Retry bundles
 587                                  * that previously failed to unbundle.
 588                                  */
 589                                 move_direction = -1;
 590                                 bundle->unbundled = 1;
 591
 592                                 if (bundle->creationToken > newMaxCreationToken)
 593                                         newMaxCreationToken = bundle->creationToken;
 594                         }
 595                 }
 596
 597                 /*
 598                  * Else case: downloaded and unbundled successfully.
 599                  * Skip this by moving in the same direction as the
 600                  * previous step.
 601                  */
 602
 603 move:
 604                 /* Move in the specified direction and repeat. */
 605                 cur += move_direction;
 606         }
 607
 608         /*
 609          * We succeed if the loop terminates because 'cur' drops below
 610          * zero. The other case is that we terminate because 'cur'
 611          * reaches the end of the list, so we have a failure no matter
 612          * which bundles we apply from the list.
 613          */
 614         if (cur < 0) {
 615                 struct strbuf value = STRBUF_INIT;
 616                 strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken);
 617                 if (repo_config_set_multivar_gently(ctx.r,
 618                                                     "fetch.bundleCreationToken",
 619                                                     value.buf, NULL, 0))
 620                         warning(_("failed to store maximum creation token"));
 621
 622                 strbuf_release(&value);
 623         }
 624
 625         free(bundles.items);
 626         return cur >= 0;
 627 }
 628
 629 static int download_bundle_list(struct repository *r,
 630                                 struct bundle_list *local_list,
 631                                 struct bundle_list *global_list,
 632                                 int depth)
 633 {
 634         struct bundle_list_context ctx = {
 635                 .r = r,
 636                 .list = global_list,
 637                 .depth = depth + 1,
 638                 .mode = local_list->mode,
 639         };
 640
 641         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 642 }
 643
 644 static int fetch_bundle_list_in_config_format(struct repository *r,
 645                                               struct bundle_list *global_list,
 646                                               struct remote_bundle_info *bundle,
 647                                               int depth)
 648 {
 649         int result;
 650         struct bundle_list list_from_bundle;
 651
 652         init_bundle_list(&list_from_bundle);
 653
 654         if ((result = bundle_uri_parse_config_format(bundle->uri,
 655                                                      bundle->file,
 656                                                      &list_from_bundle)))
 657                 goto cleanup;
 658
 659         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 660                 warning(_("unrecognized bundle mode from URI '%s'"),
 661                         bundle->uri);
 662                 result = -1;
 663                 goto cleanup;
 664         }
 665
 666         /*
 667          * If this list uses the creationToken heuristic, then the URIs
 668          * it advertises are expected to be bundles, not nested lists.
 669          * We can drop 'global_list' and 'depth'.
 670          */
 671         if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
 672                 result = fetch_bundles_by_token(r, &list_from_bundle);
 673                 global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
 674         } else if ((result = download_bundle_list(r, &list_from_bundle,
 675                                            global_list, depth)))
 676                 goto cleanup;
 677
 678 cleanup:
 679         clear_bundle_list(&list_from_bundle);
 680         return result;
 681 }
 682
 683 /**
 684  * This limits the recursion on fetch_bundle_uri_internal() when following
 685  * bundle lists.
 686  */
 687 static int max_bundle_uri_depth = 4;
 688
 689 /**
 690  * Recursively download all bundles advertised at the given URI
 691  * to files. If the file is a bundle, then add it to the given
 692  * 'list'. Otherwise, expect a bundle list and recurse on the
 693  * URIs in that list according to the list mode (ANY or ALL).
 694  */
 695 static int fetch_bundle_uri_internal(struct repository *r,
 696                                      struct remote_bundle_info *bundle,
 697                                      int depth,
 698                                      struct bundle_list *list)
 699 {
 700         int result = 0;
 701         struct remote_bundle_info *bcopy;
 702
 703         if (depth >= max_bundle_uri_depth) {
 704                 warning(_("exceeded bundle URI recursion limit (%d)"),
 705                         max_bundle_uri_depth);
 706                 return -1;
 707         }
 708
 709         if (!bundle->file &&
 710             !(bundle->file = find_temp_filename())) {
 711                 result = -1;
 712                 goto cleanup;
 713         }
 714
 715         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 716                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 717                 goto cleanup;
 718         }
 719
 720         if ((result = !is_bundle(bundle->file, 1))) {
 721                 result = fetch_bundle_list_in_config_format(
 722                                 r, list, bundle, depth);
 723                 if (result)
 724                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 725                                 bundle->uri);
 726                 goto cleanup;
 727         }
 728
 729         /* Copy the bundle and insert it into the global list. */
 730         CALLOC_ARRAY(bcopy, 1);
 731         bcopy->id = xstrdup(bundle->id);
 732         bcopy->file = xstrdup(bundle->file);
 733         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 734         hashmap_add(&list->bundles, &bcopy->ent);
 735
 736 cleanup:
 737         if (result && bundle->file)
 738                 unlink(bundle->file);
 739         return result;
 740 }
 741
 742 /**
 743  * This loop iterator breaks the loop with nonzero return code on the
 744  * first successful unbundling of a bundle.
 745  */
 746 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 747 {
 748         struct repository *r = data;
 749
 750         if (!info->file || info->unbundled)
 751                 return 0;
 752
 753         if (!unbundle_from_file(r, info->file)) {
 754                 info->unbundled = 1;
 755                 return 1;
 756         }
 757
 758         return 0;
 759 }
 760
 761 static int unbundle_all_bundles(struct repository *r,
 762                                 struct bundle_list *list)
 763 {
 764         /*
 765          * Iterate through all bundles looking for ones that can
 766          * successfully unbundle. If any succeed, then perhaps another
 767          * will succeed in the next attempt.
 768          *
 769          * Keep in mind that a non-zero result for the loop here means
 770          * the loop terminated early on a successful unbundling, which
 771          * signals that we can try again.
 772          */
 773         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 774
 775         return 0;
 776 }
 777
 778 static int unlink_bundle(struct remote_bundle_info *info, void *data)
 779 {
 780         if (info->file)
 781                 unlink_or_warn(info->file);
 782         return 0;
 783 }
 784
 785 int fetch_bundle_uri(struct repository *r, const char *uri,
 786                      int *has_heuristic)
 787 {
 788         int result;
 789         struct bundle_list list;
 790         struct remote_bundle_info bundle = {
 791                 .uri = xstrdup(uri),
 792                 .id = xstrdup(""),
 793         };
 794
 795         init_bundle_list(&list);
 796
 797         /* If a bundle is added to this global list, then it is required. */
 798         list.mode = BUNDLE_MODE_ALL;
 799
 800         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 801                 goto cleanup;
 802
 803         result = unbundle_all_bundles(r, &list);
 804
 805 cleanup:
 806         if (has_heuristic)
 807                 *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE);
 808         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 809         clear_bundle_list(&list);
 810         clear_remote_bundle_info(&bundle, NULL);
 811         return result;
 812 }
 813
 814 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 815 {
 816         int result;
 817         struct bundle_list global_list;
 818
 819         /*
 820          * If the creationToken heuristic is used, then the URIs
 821          * advertised by 'list' are not nested lists and instead
 822          * direct bundles. We do not need to use global_list.
 823          */
 824         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 825                 return fetch_bundles_by_token(r, list);
 826
 827         init_bundle_list(&global_list);
 828
 829         /* If a bundle is added to this global list, then it is required. */
 830         global_list.mode = BUNDLE_MODE_ALL;
 831
 832         if ((result = download_bundle_list(r, list, &global_list, 0)))
 833                 goto cleanup;
 834
 835         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 836                 result = fetch_bundles_by_token(r, list);
 837         else
 838                 result = unbundle_all_bundles(r, &global_list);
 839
 840 cleanup:
 841         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 842         clear_bundle_list(&global_list);
 843         return result;
 844 }
 845
 846 /**
 847  * API for serve.c.
 848  */
 849
 850 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 851 {
 852         static int advertise_bundle_uri = -1;
 853
 854         if (advertise_bundle_uri != -1)
 855                 goto cached;
 856
 857         advertise_bundle_uri = 0;
 858         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 859
 860 cached:
 861         return advertise_bundle_uri;
 862 }
 863
 864 static int config_to_packet_line(const char *key, const char *value, void *data)
 865 {
 866         struct packet_reader *writer = data;
 867
 868         if (starts_with(key, "bundle."))
 869                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 870
 871         return 0;
 872 }
 873
 874 int bundle_uri_command(struct repository *r,
 875                        struct packet_reader *request)
 876 {
 877         struct packet_writer writer;
 878         packet_writer_init(&writer, 1);
 879
 880         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 881                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 882         if (request->status != PACKET_READ_FLUSH)
 883                 die(_("bundle-uri: expected flush after arguments"));
 884
 885         /*
 886          * Read all "bundle.*" config lines to the client as key=value
 887          * packet lines.
 888          */
 889         repo_config(r, config_to_packet_line, &writer);
 890
 891         packet_writer_flush(&writer);
 892
 893         return 0;
 894 }
 895
 896 /**
 897  * General API for {transport,connect}.c etc.
 898  */
 899 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 900 {
 901         int result;
 902         const char *equals;
 903         struct strbuf key = STRBUF_INIT;
 904
 905         if (!strlen(line))
 906                 return error(_("bundle-uri: got an empty line"));
 907
 908         equals = strchr(line, '=');
 909
 910         if (!equals)
 911                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 912         if (line == equals || !*(equals + 1))
 913                 return error(_("bundle-uri: line has empty key or value"));
 914
 915         strbuf_add(&key, line, equals - line);
 916         result = bundle_list_update(key.buf, equals + 1, list);
 917         strbuf_release(&key);
 918
 919         return result;
 920 }