bundle-uri.c

   1 #include "cache.h"
   2 #include "bundle-uri.h"
   3 #include "bundle.h"
   4 #include "object-store.h"
   5 #include "refs.h"
   6 #include "run-command.h"
   7 #include "hashmap.h"
   8 #include "pkt-line.h"
   9 #include "config.h"
  10
  11 static int compare_bundles(const void *hashmap_cmp_fn_data,
  12                            const struct hashmap_entry *he1,
  13                            const struct hashmap_entry *he2,
  14                            const void *id)
  15 {
  16         const struct remote_bundle_info *e1 =
  17                 container_of(he1, const struct remote_bundle_info, ent);
  18         const struct remote_bundle_info *e2 =
  19                 container_of(he2, const struct remote_bundle_info, ent);
  20
  21         return strcmp(e1->id, id ? (const char *)id : e2->id);
  22 }
  23
  24 void init_bundle_list(struct bundle_list *list)
  25 {
  26         memset(list, 0, sizeof(*list));
  27
  28         /* Implied defaults. */
  29         list->mode = BUNDLE_MODE_ALL;
  30         list->version = 1;
  31
  32         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  33 }
  34
  35 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  36                                     void *data)
  37 {
  38         FREE_AND_NULL(bundle->id);
  39         FREE_AND_NULL(bundle->uri);
  40         FREE_AND_NULL(bundle->file);
  41         bundle->unbundled = 0;
  42         return 0;
  43 }
  44
  45 void clear_bundle_list(struct bundle_list *list)
  46 {
  47         if (!list)
  48                 return;
  49
  50         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  51         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  52 }
  53
  54 int for_all_bundles_in_list(struct bundle_list *list,
  55                             bundle_iterator iter,
  56                             void *data)
  57 {
  58         struct remote_bundle_info *info;
  59         struct hashmap_iter i;
  60
  61         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  62                 int result = iter(info, data);
  63
  64                 if (result)
  65                         return result;
  66         }
  67
  68         return 0;
  69 }
  70
  71 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  72 {
  73         FILE *fp = data;
  74         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  75         fprintf(fp, "\turi = %s\n", info->uri);
  76         return 0;
  77 }
  78
  79 void print_bundle_list(FILE *fp, struct bundle_list *list)
  80 {
  81         const char *mode;
  82
  83         switch (list->mode) {
  84         case BUNDLE_MODE_ALL:
  85                 mode = "all";
  86                 break;
  87
  88         case BUNDLE_MODE_ANY:
  89                 mode = "any";
  90                 break;
  91
  92         case BUNDLE_MODE_NONE:
  93         default:
  94                 mode = "<unknown>";
  95         }
  96
  97         fprintf(fp, "[bundle]\n");
  98         fprintf(fp, "\tversion = %d\n", list->version);
  99         fprintf(fp, "\tmode = %s\n", mode);
 100
 101         for_all_bundles_in_list(list, summarize_bundle, fp);
 102 }
 103
 104 /**
 105  * Given a key-value pair, update the state of the given bundle list.
 106  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 107  * is not understood or the value is malformed.
 108  */
 109 static int bundle_list_update(const char *key, const char *value,
 110                               struct bundle_list *list)
 111 {
 112         struct strbuf id = STRBUF_INIT;
 113         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 114         struct remote_bundle_info *bundle;
 115         const char *subsection, *subkey;
 116         size_t subsection_len;
 117
 118         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 119                 return -1;
 120
 121         if (!subsection_len) {
 122                 if (!strcmp(subkey, "version")) {
 123                         int version;
 124                         if (!git_parse_int(value, &version))
 125                                 return -1;
 126                         if (version != 1)
 127                                 return -1;
 128
 129                         list->version = version;
 130                         return 0;
 131                 }
 132
 133                 if (!strcmp(subkey, "mode")) {
 134                         if (!strcmp(value, "all"))
 135                                 list->mode = BUNDLE_MODE_ALL;
 136                         else if (!strcmp(value, "any"))
 137                                 list->mode = BUNDLE_MODE_ANY;
 138                         else
 139                                 return -1;
 140                         return 0;
 141                 }
 142
 143                 /* Ignore other unknown global keys. */
 144                 return 0;
 145         }
 146
 147         strbuf_add(&id, subsection, subsection_len);
 148
 149         /*
 150          * Check for an existing bundle with this <id>, or create one
 151          * if necessary.
 152          */
 153         lookup.id = id.buf;
 154         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 155         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 156                 CALLOC_ARRAY(bundle, 1);
 157                 bundle->id = strbuf_detach(&id, NULL);
 158                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 159                 hashmap_add(&list->bundles, &bundle->ent);
 160         }
 161         strbuf_release(&id);
 162
 163         if (!strcmp(subkey, "uri")) {
 164                 if (bundle->uri)
 165                         return -1;
 166                 bundle->uri = xstrdup(value);
 167                 return 0;
 168         }
 169
 170         /*
 171          * At this point, we ignore any information that we don't
 172          * understand, assuming it to be hints for a heuristic the client
 173          * does not currently understand.
 174          */
 175         return 0;
 176 }
 177
 178 static int config_to_bundle_list(const char *key, const char *value, void *data)
 179 {
 180         struct bundle_list *list = data;
 181         return bundle_list_update(key, value, list);
 182 }
 183
 184 int bundle_uri_parse_config_format(const char *uri,
 185                                    const char *filename,
 186                                    struct bundle_list *list)
 187 {
 188         int result;
 189         struct config_options opts = {
 190                 .error_action = CONFIG_ERROR_ERROR,
 191         };
 192
 193         result = git_config_from_file_with_options(config_to_bundle_list,
 194                                                    filename, list,
 195                                                    &opts);
 196
 197         if (!result && list->mode == BUNDLE_MODE_NONE) {
 198                 warning(_("bundle list at '%s' has no mode"), uri);
 199                 result = 1;
 200         }
 201
 202         return result;
 203 }
 204
 205 static char *find_temp_filename(void)
 206 {
 207         int fd;
 208         struct strbuf name = STRBUF_INIT;
 209         /*
 210          * Find a temporary filename that is available. This is briefly
 211          * racy, but unlikely to collide.
 212          */
 213         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 214         if (fd < 0) {
 215                 warning(_("failed to create temporary file"));
 216                 return NULL;
 217         }
 218
 219         close(fd);
 220         unlink(name.buf);
 221         return strbuf_detach(&name, NULL);
 222 }
 223
 224 static int download_https_uri_to_file(const char *file, const char *uri)
 225 {
 226         int result = 0;
 227         struct child_process cp = CHILD_PROCESS_INIT;
 228         FILE *child_in = NULL, *child_out = NULL;
 229         struct strbuf line = STRBUF_INIT;
 230         int found_get = 0;
 231
 232         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 233         cp.in = -1;
 234         cp.out = -1;
 235
 236         if (start_command(&cp))
 237                 return 1;
 238
 239         child_in = fdopen(cp.in, "w");
 240         if (!child_in) {
 241                 result = 1;
 242                 goto cleanup;
 243         }
 244
 245         child_out = fdopen(cp.out, "r");
 246         if (!child_out) {
 247                 result = 1;
 248                 goto cleanup;
 249         }
 250
 251         fprintf(child_in, "capabilities\n");
 252         fflush(child_in);
 253
 254         while (!strbuf_getline(&line, child_out)) {
 255                 if (!line.len)
 256                         break;
 257                 if (!strcmp(line.buf, "get"))
 258                         found_get = 1;
 259         }
 260         strbuf_release(&line);
 261
 262         if (!found_get) {
 263                 result = error(_("insufficient capabilities"));
 264                 goto cleanup;
 265         }
 266
 267         fprintf(child_in, "get %s %s\n\n", uri, file);
 268
 269 cleanup:
 270         if (child_in)
 271                 fclose(child_in);
 272         if (finish_command(&cp))
 273                 return 1;
 274         if (child_out)
 275                 fclose(child_out);
 276         return result;
 277 }
 278
 279 static int copy_uri_to_file(const char *filename, const char *uri)
 280 {
 281         const char *out;
 282
 283         if (starts_with(uri, "https:") ||
 284             starts_with(uri, "http:"))
 285                 return download_https_uri_to_file(filename, uri);
 286
 287         if (skip_prefix(uri, "file://", &out))
 288                 uri = out;
 289
 290         /* Copy as a file */
 291         return copy_file(filename, uri, 0);
 292 }
 293
 294 static int unbundle_from_file(struct repository *r, const char *file)
 295 {
 296         int result = 0;
 297         int bundle_fd;
 298         struct bundle_header header = BUNDLE_HEADER_INIT;
 299         struct string_list_item *refname;
 300         struct strbuf bundle_ref = STRBUF_INIT;
 301         size_t bundle_prefix_len;
 302
 303         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 304                 return 1;
 305
 306         /*
 307          * Skip the reachability walk here, since we will be adding
 308          * a reachable ref pointing to the new tips, which will reach
 309          * the prerequisite commits.
 310          */
 311         if ((result = unbundle(r, &header, bundle_fd, NULL, 0)))
 312                 return 1;
 313
 314         /*
 315          * Convert all refs/heads/ from the bundle into refs/bundles/
 316          * in the local repository.
 317          */
 318         strbuf_addstr(&bundle_ref, "refs/bundles/");
 319         bundle_prefix_len = bundle_ref.len;
 320
 321         for_each_string_list_item(refname, &header.references) {
 322                 struct object_id *oid = refname->util;
 323                 struct object_id old_oid;
 324                 const char *branch_name;
 325                 int has_old;
 326
 327                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 328                         continue;
 329
 330                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 331                 strbuf_addstr(&bundle_ref, branch_name);
 332
 333                 has_old = !read_ref(bundle_ref.buf, &old_oid);
 334                 update_ref("fetched bundle", bundle_ref.buf, oid,
 335                            has_old ? &old_oid : NULL,
 336                            REF_SKIP_OID_VERIFICATION,
 337                            UPDATE_REFS_MSG_ON_ERR);
 338         }
 339
 340         bundle_header_release(&header);
 341         return result;
 342 }
 343
 344 struct bundle_list_context {
 345         struct repository *r;
 346         struct bundle_list *list;
 347         enum bundle_list_mode mode;
 348         int count;
 349         int depth;
 350 };
 351
 352 /*
 353  * This early definition is necessary because we use indirect recursion:
 354  *
 355  * While iterating through a bundle list that was downloaded as part
 356  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 357  * again, but with depth + 1.
 358  */
 359 static int fetch_bundle_uri_internal(struct repository *r,
 360                                      struct remote_bundle_info *bundle,
 361                                      int depth,
 362                                      struct bundle_list *list);
 363
 364 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 365 {
 366         int res;
 367         struct bundle_list_context *ctx = data;
 368
 369         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 370                 return 0;
 371
 372         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 373
 374         /*
 375          * Only increment count if the download succeeded. If our mode is
 376          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 377          * list in case they work instead.
 378          */
 379         if (!res)
 380                 ctx->count++;
 381
 382         /*
 383          * To be opportunistic as possible, we continue iterating and
 384          * download as many bundles as we can, so we can apply the ones
 385          * that work, even in BUNDLE_MODE_ALL mode.
 386          */
 387         return 0;
 388 }
 389
 390 static int download_bundle_list(struct repository *r,
 391                                 struct bundle_list *local_list,
 392                                 struct bundle_list *global_list,
 393                                 int depth)
 394 {
 395         struct bundle_list_context ctx = {
 396                 .r = r,
 397                 .list = global_list,
 398                 .depth = depth + 1,
 399                 .mode = local_list->mode,
 400         };
 401
 402         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 403 }
 404
 405 static int fetch_bundle_list_in_config_format(struct repository *r,
 406                                               struct bundle_list *global_list,
 407                                               struct remote_bundle_info *bundle,
 408                                               int depth)
 409 {
 410         int result;
 411         struct bundle_list list_from_bundle;
 412
 413         init_bundle_list(&list_from_bundle);
 414
 415         if ((result = bundle_uri_parse_config_format(bundle->uri,
 416                                                      bundle->file,
 417                                                      &list_from_bundle)))
 418                 goto cleanup;
 419
 420         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 421                 warning(_("unrecognized bundle mode from URI '%s'"),
 422                         bundle->uri);
 423                 result = -1;
 424                 goto cleanup;
 425         }
 426
 427         if ((result = download_bundle_list(r, &list_from_bundle,
 428                                            global_list, depth)))
 429                 goto cleanup;
 430
 431 cleanup:
 432         clear_bundle_list(&list_from_bundle);
 433         return result;
 434 }
 435
 436 /**
 437  * This limits the recursion on fetch_bundle_uri_internal() when following
 438  * bundle lists.
 439  */
 440 static int max_bundle_uri_depth = 4;
 441
 442 /**
 443  * Recursively download all bundles advertised at the given URI
 444  * to files. If the file is a bundle, then add it to the given
 445  * 'list'. Otherwise, expect a bundle list and recurse on the
 446  * URIs in that list according to the list mode (ANY or ALL).
 447  */
 448 static int fetch_bundle_uri_internal(struct repository *r,
 449                                      struct remote_bundle_info *bundle,
 450                                      int depth,
 451                                      struct bundle_list *list)
 452 {
 453         int result = 0;
 454         struct remote_bundle_info *bcopy;
 455
 456         if (depth >= max_bundle_uri_depth) {
 457                 warning(_("exceeded bundle URI recursion limit (%d)"),
 458                         max_bundle_uri_depth);
 459                 return -1;
 460         }
 461
 462         if (!bundle->file &&
 463             !(bundle->file = find_temp_filename())) {
 464                 result = -1;
 465                 goto cleanup;
 466         }
 467
 468         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 469                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 470                 goto cleanup;
 471         }
 472
 473         if ((result = !is_bundle(bundle->file, 1))) {
 474                 result = fetch_bundle_list_in_config_format(
 475                                 r, list, bundle, depth);
 476                 if (result)
 477                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 478                                 bundle->uri);
 479                 goto cleanup;
 480         }
 481
 482         /* Copy the bundle and insert it into the global list. */
 483         CALLOC_ARRAY(bcopy, 1);
 484         bcopy->id = xstrdup(bundle->id);
 485         bcopy->file = xstrdup(bundle->file);
 486         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 487         hashmap_add(&list->bundles, &bcopy->ent);
 488
 489 cleanup:
 490         if (result && bundle->file)
 491                 unlink(bundle->file);
 492         return result;
 493 }
 494
 495 /**
 496  * This loop iterator breaks the loop with nonzero return code on the
 497  * first successful unbundling of a bundle.
 498  */
 499 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 500 {
 501         struct repository *r = data;
 502
 503         if (!info->file || info->unbundled)
 504                 return 0;
 505
 506         if (!unbundle_from_file(r, info->file)) {
 507                 info->unbundled = 1;
 508                 return 1;
 509         }
 510
 511         return 0;
 512 }
 513
 514 static int unbundle_all_bundles(struct repository *r,
 515                                 struct bundle_list *list)
 516 {
 517         /*
 518          * Iterate through all bundles looking for ones that can
 519          * successfully unbundle. If any succeed, then perhaps another
 520          * will succeed in the next attempt.
 521          *
 522          * Keep in mind that a non-zero result for the loop here means
 523          * the loop terminated early on a successful unbundling, which
 524          * signals that we can try again.
 525          */
 526         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 527
 528         return 0;
 529 }
 530
 531 static int unlink_bundle(struct remote_bundle_info *info, void *data)
 532 {
 533         if (info->file)
 534                 unlink_or_warn(info->file);
 535         return 0;
 536 }
 537
 538 int fetch_bundle_uri(struct repository *r, const char *uri)
 539 {
 540         int result;
 541         struct bundle_list list;
 542         struct remote_bundle_info bundle = {
 543                 .uri = xstrdup(uri),
 544                 .id = xstrdup(""),
 545         };
 546
 547         init_bundle_list(&list);
 548
 549         /* If a bundle is added to this global list, then it is required. */
 550         list.mode = BUNDLE_MODE_ALL;
 551
 552         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 553                 goto cleanup;
 554
 555         result = unbundle_all_bundles(r, &list);
 556
 557 cleanup:
 558         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 559         clear_bundle_list(&list);
 560         clear_remote_bundle_info(&bundle, NULL);
 561         return result;
 562 }
 563
 564 /**
 565  * General API for {transport,connect}.c etc.
 566  */
 567 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 568 {
 569         int result;
 570         const char *equals;
 571         struct strbuf key = STRBUF_INIT;
 572
 573         if (!strlen(line))
 574                 return error(_("bundle-uri: got an empty line"));
 575
 576         equals = strchr(line, '=');
 577
 578         if (!equals)
 579                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 580         if (line == equals || !*(equals + 1))
 581                 return error(_("bundle-uri: line has empty key or value"));
 582
 583         strbuf_add(&key, line, equals - line);
 584         result = bundle_list_update(key.buf, equals + 1, list);
 585         strbuf_release(&key);
 586
 587         return result;
 588 }