src/import/pull-tar.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <curl/curl.h>
   4 #include <sys/prctl.h>
   5
   6 #include "sd-daemon.h"
   7
   8 #include "alloc-util.h"
   9 #include "btrfs-util.h"
  10 #include "copy.h"
  11 #include "curl-util.h"
  12 #include "fd-util.h"
  13 #include "fs-util.h"
  14 #include "hostname-util.h"
  15 #include "import-common.h"
  16 #include "import-util.h"
  17 #include "install-file.h"
  18 #include "macro.h"
  19 #include "mkdir-label.h"
  20 #include "path-util.h"
  21 #include "process-util.h"
  22 #include "pull-common.h"
  23 #include "pull-job.h"
  24 #include "pull-tar.h"
  25 #include "rm-rf.h"
  26 #include "string-util.h"
  27 #include "strv.h"
  28 #include "tmpfile-util.h"
  29 #include "user-util.h"
  30 #include "utf8.h"
  31 #include "web-util.h"
  32
  33 typedef enum TarProgress {
  34         TAR_DOWNLOADING,
  35         TAR_VERIFYING,
  36         TAR_FINALIZING,
  37         TAR_COPYING,
  38 } TarProgress;
  39
  40 struct TarPull {
  41         sd_event *event;
  42         CurlGlue *glue;
  43
  44         PullFlags flags;
  45         ImportVerify verify;
  46         char *image_root;
  47
  48         PullJob *tar_job;
  49         PullJob *checksum_job;
  50         PullJob *signature_job;
  51         PullJob *settings_job;
  52
  53         TarPullFinished on_finished;
  54         void *userdata;
  55
  56         char *local;
  57
  58         pid_t tar_pid;
  59
  60         char *final_path;
  61         char *temp_path;
  62
  63         char *settings_path;
  64         char *settings_temp_path;
  65
  66         char *checksum;
  67 };
  68
  69 TarPull* tar_pull_unref(TarPull *i) {
  70         if (!i)
  71                 return NULL;
  72
  73         if (i->tar_pid > 1)
  74                 sigkill_wait(i->tar_pid);
  75
  76         pull_job_unref(i->tar_job);
  77         pull_job_unref(i->checksum_job);
  78         pull_job_unref(i->signature_job);
  79         pull_job_unref(i->settings_job);
  80
  81         curl_glue_unref(i->glue);
  82         sd_event_unref(i->event);
  83
  84         rm_rf_subvolume_and_free(i->temp_path);
  85         unlink_and_free(i->settings_temp_path);
  86
  87         free(i->final_path);
  88         free(i->settings_path);
  89         free(i->image_root);
  90         free(i->local);
  91         free(i->checksum);
  92
  93         return mfree(i);
  94 }
  95
  96 int tar_pull_new(
  97                 TarPull **ret,
  98                 sd_event *event,
  99                 const char *image_root,
 100                 TarPullFinished on_finished,
 101                 void *userdata) {
 102
 103         _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL;
 104         _cleanup_(sd_event_unrefp) sd_event *e = NULL;
 105         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 106         _cleanup_free_ char *root = NULL;
 107         int r;
 108
 109         assert(ret);
 110
 111         root = strdup(image_root ?: "/var/lib/machines");
 112         if (!root)
 113                 return -ENOMEM;
 114
 115         if (event)
 116                 e = sd_event_ref(event);
 117         else {
 118                 r = sd_event_default(&e);
 119                 if (r < 0)
 120                         return r;
 121         }
 122
 123         r = curl_glue_new(&g, e);
 124         if (r < 0)
 125                 return r;
 126
 127         i = new(TarPull, 1);
 128         if (!i)
 129                 return -ENOMEM;
 130
 131         *i = (TarPull) {
 132                 .on_finished = on_finished,
 133                 .userdata = userdata,
 134                 .image_root = TAKE_PTR(root),
 135                 .event = TAKE_PTR(e),
 136                 .glue = TAKE_PTR(g),
 137         };
 138
 139         i->glue->on_finished = pull_job_curl_on_finished;
 140         i->glue->userdata = i;
 141
 142         *ret = TAKE_PTR(i);
 143
 144         return 0;
 145 }
 146
 147 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 148         unsigned percent;
 149
 150         assert(i);
 151
 152         switch (p) {
 153
 154         case TAR_DOWNLOADING: {
 155                 unsigned remain = 85;
 156
 157                 percent = 0;
 158
 159                 if (i->checksum_job) {
 160                         percent += i->checksum_job->progress_percent * 5 / 100;
 161                         remain -= 5;
 162                 }
 163
 164                 if (i->signature_job) {
 165                         percent += i->signature_job->progress_percent * 5 / 100;
 166                         remain -= 5;
 167                 }
 168
 169                 if (i->settings_job) {
 170                         percent += i->settings_job->progress_percent * 5 / 100;
 171                         remain -= 5;
 172                 }
 173
 174                 if (i->tar_job)
 175                         percent += i->tar_job->progress_percent * remain / 100;
 176                 break;
 177         }
 178
 179         case TAR_VERIFYING:
 180                 percent = 85;
 181                 break;
 182
 183         case TAR_FINALIZING:
 184                 percent = 90;
 185                 break;
 186
 187         case TAR_COPYING:
 188                 percent = 95;
 189                 break;
 190
 191         default:
 192                 assert_not_reached();
 193         }
 194
 195         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 196         log_debug("Combined progress %u%%", percent);
 197 }
 198
 199 static int tar_pull_determine_path(
 200                 TarPull *i,
 201                 const char *suffix,
 202                 char **field /* input + output (!) */) {
 203         int r;
 204
 205         assert(i);
 206         assert(field);
 207
 208         if (*field)
 209                 return 0;
 210
 211         assert(i->tar_job);
 212
 213         r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field);
 214         if (r < 0)
 215                 return log_oom();
 216
 217         return 1;
 218 }
 219
 220 static int tar_pull_make_local_copy(TarPull *i) {
 221         _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL;
 222         const char *p;
 223         int r;
 224
 225         assert(i);
 226         assert(i->tar_job);
 227
 228         if (!i->local)
 229                 return 0;
 230
 231         assert(i->final_path);
 232
 233         p = prefix_roota(i->image_root, i->local);
 234
 235         r = tempfn_random(p, NULL, &t);
 236         if (r < 0)
 237                 return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p);
 238
 239         if (i->flags & PULL_BTRFS_SUBVOL)
 240                 r = btrfs_subvol_snapshot(
 241                                 i->final_path,
 242                                 t,
 243                                 (i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)|
 244                                 BTRFS_SNAPSHOT_FALLBACK_COPY|
 245                                 BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
 246                                 BTRFS_SNAPSHOT_RECURSIVE);
 247         else
 248                 r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS, NULL);
 249         if (r < 0)
 250                 return log_error_errno(r, "Failed to create local image: %m");
 251
 252         r = install_file(AT_FDCWD, t,
 253                          AT_FDCWD, p,
 254                          (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
 255                          (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
 256                          (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
 257         if (r < 0)
 258                 return log_error_errno(r, "Failed to install local image '%s': %m", p);
 259
 260         t = mfree(t);
 261
 262         log_info("Created new local image '%s'.", i->local);
 263
 264         if (FLAGS_SET(i->flags, PULL_SETTINGS)) {
 265                 const char *local_settings;
 266                 assert(i->settings_job);
 267
 268                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 269                 if (r < 0)
 270                         return r;
 271
 272                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 273
 274                 r = copy_file_atomic(
 275                                 i->settings_path,
 276                                 local_settings,
 277                                 0664,
 278                                 0, 0,
 279                                 COPY_REFLINK |
 280                                 (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
 281                                 (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
 282                 if (r == -EEXIST)
 283                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 284                 else if (r == -ENOENT)
 285                         log_debug_errno(r, "Skipping creation of settings file, since none was found.");
 286                 else if (r < 0)
 287                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 288                 else
 289                         log_info("Created new settings file %s.", local_settings);
 290         }
 291
 292         return 0;
 293 }
 294
 295 static bool tar_pull_is_done(TarPull *i) {
 296         assert(i);
 297         assert(i->tar_job);
 298
 299         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 300                 return false;
 301         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 302                 return false;
 303         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 304                 return false;
 305         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 306                 return false;
 307
 308         return true;
 309 }
 310
 311 static void tar_pull_job_on_finished(PullJob *j) {
 312         TarPull *i;
 313         int r;
 314
 315         assert(j);
 316         assert(j->userdata);
 317
 318         i = j->userdata;
 319
 320         if (j->error != 0) {
 321                 if (j == i->tar_job) {
 322                         if (j->error == ENOMEDIUM) /* HTTP 404 */
 323                                 r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 324                         else
 325                                 r = log_error_errno(j->error, "Failed to retrieve image file.");
 326                         goto finish;
 327                 } else if (j == i->checksum_job) {
 328                         r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 329                         goto finish;
 330                 } else if (j == i->signature_job)
 331                         log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
 332                 else if (j == i->settings_job)
 333                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 334                 else
 335                         assert("unexpected job");
 336         }
 337
 338         /* This is invoked if either the download completed successfully, or the download was skipped because
 339          * we already have the etag. */
 340
 341         if (!tar_pull_is_done(i))
 342                 return;
 343
 344         if (i->signature_job && i->signature_job->error != 0) {
 345                 VerificationStyle style;
 346
 347                 assert(i->checksum_job);
 348
 349                 r = verification_style_from_url(i->checksum_job->url, &style);
 350                 if (r < 0) {
 351                         log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
 352                         goto finish;
 353                 }
 354
 355                 if (style == VERIFICATION_PER_DIRECTORY) { /* A failed signature file download only matters
 356                                                             * in per-directory verification mode, since only
 357                                                             * then the signature is detached, and thus a file
 358                                                             * of its own. */
 359                         r = log_error_errno(i->signature_job->error,
 360                                             "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 361                         goto finish;
 362                 }
 363         }
 364
 365         pull_job_close_disk_fd(i->tar_job);
 366         pull_job_close_disk_fd(i->settings_job);
 367
 368         if (i->tar_pid > 0) {
 369                 r = wait_for_terminate_and_check("tar", TAKE_PID(i->tar_pid), WAIT_LOG);
 370                 if (r < 0)
 371                         goto finish;
 372                 if (r != EXIT_SUCCESS) {
 373                         r = -EIO;
 374                         goto finish;
 375                 }
 376         }
 377
 378         if (!i->tar_job->etag_exists) {
 379                 /* This is a new download, verify it, and move it into place */
 380
 381                 tar_pull_report_progress(i, TAR_VERIFYING);
 382
 383                 r = pull_verify(i->verify,
 384                                 i->checksum,
 385                                 i->tar_job,
 386                                 i->checksum_job,
 387                                 i->signature_job,
 388                                 i->settings_job,
 389                                 /* roothash_job = */ NULL,
 390                                 /* roothash_signature_job = */ NULL,
 391                                 /* verity_job = */ NULL);
 392                 if (r < 0)
 393                         goto finish;
 394         }
 395
 396         if (i->flags & PULL_DIRECT) {
 397                 assert(!i->settings_job);
 398                 assert(i->local);
 399                 assert(!i->temp_path);
 400
 401                 tar_pull_report_progress(i, TAR_FINALIZING);
 402
 403                 r = import_mangle_os_tree(i->local);
 404                 if (r < 0)
 405                         goto finish;
 406
 407                 r = install_file(
 408                                 AT_FDCWD, i->local,
 409                                 AT_FDCWD, NULL,
 410                                 (i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 |
 411                                 (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
 412                 if (r < 0) {
 413                         log_error_errno(r, "Failed to finalize '%s': %m", i->local);
 414                         goto finish;
 415                 }
 416         } else {
 417                 r = tar_pull_determine_path(i, NULL, &i->final_path);
 418                 if (r < 0)
 419                         goto finish;
 420
 421                 if (!i->tar_job->etag_exists) {
 422                         /* This is a new download, verify it, and move it into place */
 423
 424                         assert(i->temp_path);
 425                         assert(i->final_path);
 426
 427                         tar_pull_report_progress(i, TAR_FINALIZING);
 428
 429                         r = import_mangle_os_tree(i->temp_path);
 430                         if (r < 0)
 431                                 goto finish;
 432
 433                         r = install_file(
 434                                         AT_FDCWD, i->temp_path,
 435                                         AT_FDCWD, i->final_path,
 436                                         INSTALL_READ_ONLY|
 437                                         (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
 438                         if (r < 0) {
 439                                 log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
 440                                 goto finish;
 441                         }
 442
 443                         i->temp_path = mfree(i->temp_path);
 444
 445                         if (i->settings_job &&
 446                             i->settings_job->error == 0) {
 447
 448                                 /* Also move the settings file into place, if it exists. Note that we do so only if we also
 449                                  * moved the tar file in place, to keep things strictly in sync. */
 450                                 assert(i->settings_temp_path);
 451
 452                                 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
 453                                  * we should incorporate it in the file name if we can */
 454                                 i->settings_path = mfree(i->settings_path);
 455
 456                                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 457                                 if (r < 0)
 458                                         goto finish;
 459
 460                                 r = install_file(
 461                                                 AT_FDCWD, i->settings_temp_path,
 462                                                 AT_FDCWD, i->settings_path,
 463                                                 INSTALL_READ_ONLY|
 464                                                 (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
 465                                 if (r < 0) {
 466                                         log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
 467                                         goto finish;
 468                                 }
 469
 470                                 i->settings_temp_path = mfree(i->settings_temp_path);
 471                         }
 472                 }
 473
 474                 tar_pull_report_progress(i, TAR_COPYING);
 475
 476                 r = tar_pull_make_local_copy(i);
 477                 if (r < 0)
 478                         goto finish;
 479         }
 480
 481         r = 0;
 482
 483 finish:
 484         if (i->on_finished)
 485                 i->on_finished(i, r, i->userdata);
 486         else
 487                 sd_event_exit(i->event, r);
 488 }
 489
 490 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 491         const char *where;
 492         TarPull *i;
 493         int r;
 494
 495         assert(j);
 496         assert(j->userdata);
 497
 498         i = j->userdata;
 499         assert(i->tar_job == j);
 500         assert(i->tar_pid <= 0);
 501
 502         if (i->flags & PULL_DIRECT)
 503                 where = i->local;
 504         else {
 505                 if (!i->temp_path) {
 506                         r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
 507                         if (r < 0)
 508                                 return log_oom();
 509                 }
 510
 511                 where = i->temp_path;
 512         }
 513
 514         (void) mkdir_parents_label(where, 0700);
 515
 516         if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE))
 517                 (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 518
 519         if (i->flags & PULL_BTRFS_SUBVOL)
 520                 r = btrfs_subvol_make_fallback(where, 0755);
 521         else
 522                 r = RET_NERRNO(mkdir(where, 0755));
 523         if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
 524                                                        * because in that case our temporary path collided */
 525                 r = 0;
 526         if (r < 0)
 527                 return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where);
 528         if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */
 529                 if (!(i->flags & PULL_DIRECT))
 530                         (void) import_assign_pool_quota_and_warn(i->image_root);
 531                 (void) import_assign_pool_quota_and_warn(where);
 532         }
 533
 534         j->disk_fd = import_fork_tar_x(where, &i->tar_pid);
 535         if (j->disk_fd < 0)
 536                 return j->disk_fd;
 537
 538         return 0;
 539 }
 540
 541 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 542         TarPull *i;
 543         int r;
 544
 545         assert(j);
 546         assert(j->userdata);
 547
 548         i = j->userdata;
 549         assert(i->settings_job == j);
 550
 551         if (!i->settings_temp_path) {
 552                 r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path);
 553                 if (r < 0)
 554                         return log_oom();
 555         }
 556
 557         (void) mkdir_parents_label(i->settings_temp_path, 0700);
 558
 559         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 560         if (j->disk_fd < 0)
 561                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 562
 563         return 0;
 564 }
 565
 566 static void tar_pull_job_on_progress(PullJob *j) {
 567         TarPull *i;
 568
 569         assert(j);
 570         assert(j->userdata);
 571
 572         i = j->userdata;
 573
 574         tar_pull_report_progress(i, TAR_DOWNLOADING);
 575 }
 576
 577 int tar_pull_start(
 578                 TarPull *i,
 579                 const char *url,
 580                 const char *local,
 581                 PullFlags flags,
 582                 ImportVerify verify,
 583                 const char *checksum) {
 584
 585         PullJob *j;
 586         int r;
 587
 588         assert(i);
 589         assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
 590         assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
 591         assert((verify < 0) || !checksum);
 592         assert(!(flags & ~PULL_FLAGS_MASK_TAR));
 593         assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT));
 594         assert(!(flags & PULL_SETTINGS) || !checksum);
 595
 596         if (!http_url_is_valid(url) && !file_url_is_valid(url))
 597                 return -EINVAL;
 598
 599         if (local && !pull_validate_local(local, flags))
 600                 return -EINVAL;
 601
 602         if (i->tar_job)
 603                 return -EBUSY;
 604
 605         r = free_and_strdup(&i->local, local);
 606         if (r < 0)
 607                 return r;
 608
 609         r = free_and_strdup(&i->checksum, checksum);
 610         if (r < 0)
 611                 return r;
 612
 613         i->flags = flags;
 614         i->verify = verify;
 615
 616         /* Set up download job for TAR file */
 617         r = pull_job_new(&i->tar_job, url, i->glue, i);
 618         if (r < 0)
 619                 return r;
 620
 621         i->tar_job->on_finished = tar_pull_job_on_finished;
 622         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 623         i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
 624
 625         if (!FLAGS_SET(flags, PULL_DIRECT)) {
 626                 r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 627                 if (r < 0)
 628                         return r;
 629         }
 630
 631         /* Set up download of checksum/signature files */
 632         r = pull_make_verification_jobs(
 633                         &i->checksum_job,
 634                         &i->signature_job,
 635                         verify,
 636                         checksum,
 637                         url,
 638                         i->glue,
 639                         tar_pull_job_on_finished,
 640                         i);
 641         if (r < 0)
 642                 return r;
 643
 644         /* Set up download job for the settings file (.nspawn) */
 645         if (FLAGS_SET(flags, PULL_SETTINGS)) {
 646                 r = pull_make_auxiliary_job(
 647                                 &i->settings_job,
 648                                 url,
 649                                 tar_strip_suffixes,
 650                                 ".nspawn",
 651                                 verify,
 652                                 i->glue,
 653                                 tar_pull_job_on_open_disk_settings,
 654                                 tar_pull_job_on_finished,
 655                                 i);
 656                 if (r < 0)
 657                         return r;
 658         }
 659
 660         FOREACH_POINTER(j,
 661                         i->tar_job,
 662                         i->checksum_job,
 663                         i->signature_job,
 664                         i->settings_job) {
 665
 666                 if (!j)
 667                         continue;
 668
 669                 j->on_progress = tar_pull_job_on_progress;
 670                 j->sync = FLAGS_SET(flags, PULL_SYNC);
 671
 672                 r = pull_job_begin(j);
 673                 if (r < 0)
 674                         return r;
 675         }
 676
 677         return 0;
 678 }