src/import/pull-tar.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <dirent.h>
   4
   5 #include "sd-daemon.h"
   6 #include "sd-event.h"
   7
   8 #include "alloc-util.h"
   9 #include "btrfs-util.h"
  10 #include "copy.h"
  11 #include "curl-util.h"
  12 #include "errno-util.h"
  13 #include "fs-util.h"
  14 #include "import-common.h"
  15 #include "import-util.h"
  16 #include "install-file.h"
  17 #include "log.h"
  18 #include "mkdir-label.h"
  19 #include "path-util.h"
  20 #include "process-util.h"
  21 #include "pull-common.h"
  22 #include "pull-job.h"
  23 #include "pull-tar.h"
  24 #include "rm-rf.h"
  25 #include "string-util.h"
  26 #include "tmpfile-util.h"
  27 #include "web-util.h"
  28
  29 typedef enum TarProgress {
  30         TAR_DOWNLOADING,
  31         TAR_VERIFYING,
  32         TAR_FINALIZING,
  33         TAR_COPYING,
  34 } TarProgress;
  35
  36 typedef struct TarPull {
  37         sd_event *event;
  38         CurlGlue *glue;
  39
  40         ImportFlags flags;
  41         ImportVerify verify;
  42         char *image_root;
  43
  44         PullJob *tar_job;
  45         PullJob *checksum_job;
  46         PullJob *signature_job;
  47         PullJob *settings_job;
  48
  49         TarPullFinished on_finished;
  50         void *userdata;
  51
  52         char *local;
  53
  54         pid_t tar_pid;
  55
  56         char *final_path;
  57         char *temp_path;
  58
  59         char *settings_path;
  60         char *settings_temp_path;
  61
  62         char *checksum;
  63 } TarPull;
  64
  65 TarPull* tar_pull_unref(TarPull *i) {
  66         if (!i)
  67                 return NULL;
  68
  69         if (i->tar_pid > 1)
  70                 sigkill_wait(i->tar_pid);
  71
  72         pull_job_unref(i->tar_job);
  73         pull_job_unref(i->checksum_job);
  74         pull_job_unref(i->signature_job);
  75         pull_job_unref(i->settings_job);
  76
  77         curl_glue_unref(i->glue);
  78         sd_event_unref(i->event);
  79
  80         rm_rf_subvolume_and_free(i->temp_path);
  81         unlink_and_free(i->settings_temp_path);
  82
  83         free(i->final_path);
  84         free(i->settings_path);
  85         free(i->image_root);
  86         free(i->local);
  87         free(i->checksum);
  88
  89         return mfree(i);
  90 }
  91
  92 int tar_pull_new(
  93                 TarPull **ret,
  94                 sd_event *event,
  95                 const char *image_root,
  96                 TarPullFinished on_finished,
  97                 void *userdata) {
  98
  99         _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL;
 100         _cleanup_(sd_event_unrefp) sd_event *e = NULL;
 101         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 102         _cleanup_free_ char *root = NULL;
 103         int r;
 104
 105         assert(image_root);
 106         assert(ret);
 107
 108         root = strdup(image_root);
 109         if (!root)
 110                 return -ENOMEM;
 111
 112         if (event)
 113                 e = sd_event_ref(event);
 114         else {
 115                 r = sd_event_default(&e);
 116                 if (r < 0)
 117                         return r;
 118         }
 119
 120         r = curl_glue_new(&g, e);
 121         if (r < 0)
 122                 return r;
 123
 124         i = new(TarPull, 1);
 125         if (!i)
 126                 return -ENOMEM;
 127
 128         *i = (TarPull) {
 129                 .on_finished = on_finished,
 130                 .userdata = userdata,
 131                 .image_root = TAKE_PTR(root),
 132                 .event = TAKE_PTR(e),
 133                 .glue = TAKE_PTR(g),
 134         };
 135
 136         i->glue->on_finished = pull_job_curl_on_finished;
 137         i->glue->userdata = i;
 138
 139         *ret = TAKE_PTR(i);
 140
 141         return 0;
 142 }
 143
 144 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 145         unsigned percent;
 146
 147         assert(i);
 148
 149         switch (p) {
 150
 151         case TAR_DOWNLOADING: {
 152                 unsigned remain = 85;
 153
 154                 percent = 0;
 155
 156                 if (i->checksum_job) {
 157                         percent += i->checksum_job->progress_percent * 5 / 100;
 158                         remain -= 5;
 159                 }
 160
 161                 if (i->signature_job) {
 162                         percent += i->signature_job->progress_percent * 5 / 100;
 163                         remain -= 5;
 164                 }
 165
 166                 if (i->settings_job) {
 167                         percent += i->settings_job->progress_percent * 5 / 100;
 168                         remain -= 5;
 169                 }
 170
 171                 if (i->tar_job)
 172                         percent += i->tar_job->progress_percent * remain / 100;
 173                 break;
 174         }
 175
 176         case TAR_VERIFYING:
 177                 percent = 85;
 178                 break;
 179
 180         case TAR_FINALIZING:
 181                 percent = 90;
 182                 break;
 183
 184         case TAR_COPYING:
 185                 percent = 95;
 186                 break;
 187
 188         default:
 189                 assert_not_reached();
 190         }
 191
 192         sd_notifyf(false, "X_IMPORT_PROGRESS=%u%%", percent);
 193         log_debug("Combined progress %u%%", percent);
 194 }
 195
 196 static int tar_pull_determine_path(
 197                 TarPull *i,
 198                 const char *suffix,
 199                 char **field /* input + output (!) */) {
 200         int r;
 201
 202         assert(i);
 203         assert(field);
 204
 205         if (*field)
 206                 return 0;
 207
 208         assert(i->tar_job);
 209
 210         r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field);
 211         if (r < 0)
 212                 return log_oom();
 213
 214         return 1;
 215 }
 216
 217 static int tar_pull_make_local_copy(TarPull *i) {
 218         _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL;
 219         _cleanup_free_ char *p = NULL;
 220         const char *source;
 221         int r;
 222
 223         assert(i);
 224         assert(i->tar_job);
 225
 226         if (!i->local)
 227                 return 0;
 228
 229         assert(i->final_path);
 230
 231         p = path_join(i->image_root, i->local);
 232         if (!p)
 233                 return log_oom();
 234
 235         if (FLAGS_SET(i->flags, IMPORT_PULL_KEEP_DOWNLOAD)) {
 236                 r = tempfn_random(p, NULL, &t);
 237                 if (r < 0)
 238                         return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p);
 239
 240                 if (i->flags & IMPORT_BTRFS_SUBVOL)
 241                         r = btrfs_subvol_snapshot_at(
 242                                         AT_FDCWD, i->final_path,
 243                                         AT_FDCWD, t,
 244                                         (i->flags & IMPORT_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)|
 245                                         BTRFS_SNAPSHOT_FALLBACK_COPY|
 246                                         BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
 247                                         BTRFS_SNAPSHOT_RECURSIVE);
 248                 else
 249                         r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS, NULL, NULL);
 250                 if (r < 0)
 251                         return log_error_errno(r, "Failed to create local image: %m");
 252
 253                 source = t;
 254         } else
 255                 source = i->final_path;
 256
 257         r = install_file(AT_FDCWD, source,
 258                          AT_FDCWD, p,
 259                          (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
 260                          (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
 261                          (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0));
 262         if (r < 0)
 263                 return log_error_errno(r, "Failed to install local image '%s': %m", p);
 264
 265         t = mfree(t);
 266
 267         log_info("Created new local image '%s'.", i->local);
 268
 269         if (FLAGS_SET(i->flags, IMPORT_PULL_SETTINGS)) {
 270                 _cleanup_free_ char *local_settings = NULL;
 271                 assert(i->settings_job);
 272
 273                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 274                 if (r < 0)
 275                         return r;
 276
 277                 local_settings = strjoin(i->image_root, "/", i->local, ".nspawn");
 278                 if (!local_settings)
 279                         return log_oom();
 280
 281                 if (FLAGS_SET(i->flags, IMPORT_PULL_KEEP_DOWNLOAD))
 282                         r = copy_file_atomic(
 283                                         i->settings_path,
 284                                         local_settings,
 285                                         0664,
 286                                         COPY_REFLINK |
 287                                         (FLAGS_SET(i->flags, IMPORT_FORCE) ? COPY_REPLACE : 0) |
 288                                         (FLAGS_SET(i->flags, IMPORT_SYNC) ? COPY_FSYNC_FULL : 0));
 289                 else
 290                         r = install_file(AT_FDCWD, i->settings_path,
 291                                          AT_FDCWD, local_settings,
 292                                          (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
 293                                          (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0));
 294                 if (r == -EEXIST)
 295                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 296                 else if (r == -ENOENT)
 297                         log_debug_errno(r, "Skipping creation of settings file, since none was found.");
 298                 else if (r < 0)
 299                         log_warning_errno(r, "Failed to install settings files %s, ignoring: %m", local_settings);
 300                 else
 301                         log_info("Created new settings file %s.", local_settings);
 302         }
 303
 304         return 0;
 305 }
 306
 307 static bool tar_pull_is_done(TarPull *i) {
 308         assert(i);
 309         assert(i->tar_job);
 310
 311         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 312                 return false;
 313         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 314                 return false;
 315         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 316                 return false;
 317         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 318                 return false;
 319
 320         return true;
 321 }
 322
 323 static void tar_pull_job_on_finished(PullJob *j) {
 324         TarPull *i;
 325         int r;
 326
 327         assert(j);
 328         assert(j->userdata);
 329
 330         i = j->userdata;
 331
 332         if (j->error != 0) {
 333                 if (j == i->tar_job) {
 334                         if (j->error == ENOMEDIUM) /* HTTP 404 */
 335                                 r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 336                         else
 337                                 r = log_error_errno(j->error, "Failed to retrieve image file.");
 338                         goto finish;
 339                 } else if (j == i->checksum_job) {
 340                         r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 341                         goto finish;
 342                 } else if (j == i->signature_job)
 343                         log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
 344                 else if (j == i->settings_job)
 345                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 346                 else
 347                         assert("unexpected job");
 348         }
 349
 350         /* This is invoked if either the download completed successfully, or the download was skipped because
 351          * we already have the etag. */
 352
 353         if (!tar_pull_is_done(i))
 354                 return;
 355
 356         if (i->signature_job && i->signature_job->error != 0) {
 357                 VerificationStyle style;
 358
 359                 assert(i->checksum_job);
 360
 361                 r = verification_style_from_url(i->checksum_job->url, &style);
 362                 if (r < 0) {
 363                         log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
 364                         goto finish;
 365                 }
 366
 367                 if (style == VERIFICATION_PER_DIRECTORY) { /* A failed signature file download only matters
 368                                                             * in per-directory verification mode, since only
 369                                                             * then the signature is detached, and thus a file
 370                                                             * of its own. */
 371                         r = log_error_errno(i->signature_job->error,
 372                                             "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 373                         goto finish;
 374                 }
 375         }
 376
 377         pull_job_close_disk_fd(i->tar_job);
 378         pull_job_close_disk_fd(i->settings_job);
 379
 380         if (i->tar_pid > 0) {
 381                 r = wait_for_terminate_and_check("tar", TAKE_PID(i->tar_pid), WAIT_LOG);
 382                 if (r < 0)
 383                         goto finish;
 384                 if (r != EXIT_SUCCESS) {
 385                         r = -EIO;
 386                         goto finish;
 387                 }
 388         }
 389
 390         if (!i->tar_job->etag_exists) {
 391                 /* This is a new download, verify it, and move it into place */
 392
 393                 tar_pull_report_progress(i, TAR_VERIFYING);
 394
 395                 r = pull_verify(i->verify,
 396                                 i->checksum,
 397                                 i->tar_job,
 398                                 i->checksum_job,
 399                                 i->signature_job,
 400                                 i->settings_job,
 401                                 /* roothash_job = */ NULL,
 402                                 /* roothash_signature_job = */ NULL,
 403                                 /* verity_job = */ NULL);
 404                 if (r < 0)
 405                         goto finish;
 406         }
 407
 408         if (i->flags & IMPORT_DIRECT) {
 409                 assert(!i->settings_job);
 410                 assert(i->local);
 411                 assert(!i->temp_path);
 412
 413                 tar_pull_report_progress(i, TAR_FINALIZING);
 414
 415                 r = import_mangle_os_tree(i->local);
 416                 if (r < 0)
 417                         goto finish;
 418
 419                 r = install_file(
 420                                 AT_FDCWD, i->local,
 421                                 AT_FDCWD, NULL,
 422                                 (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
 423                                 (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0));
 424                 if (r < 0) {
 425                         log_error_errno(r, "Failed to finalize '%s': %m", i->local);
 426                         goto finish;
 427                 }
 428         } else {
 429                 r = tar_pull_determine_path(i, NULL, &i->final_path);
 430                 if (r < 0)
 431                         goto finish;
 432
 433                 if (!i->tar_job->etag_exists) {
 434                         /* This is a new download, verify it, and move it into place */
 435
 436                         assert(i->temp_path);
 437                         assert(i->final_path);
 438
 439                         tar_pull_report_progress(i, TAR_FINALIZING);
 440
 441                         r = import_mangle_os_tree(i->temp_path);
 442                         if (r < 0)
 443                                 goto finish;
 444
 445                         r = install_file(
 446                                         AT_FDCWD, i->temp_path,
 447                                         AT_FDCWD, i->final_path,
 448                                         (i->flags & IMPORT_PULL_KEEP_DOWNLOAD ? INSTALL_READ_ONLY : 0) |
 449                                         (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0));
 450                         if (r < 0) {
 451                                 log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
 452                                 goto finish;
 453                         }
 454
 455                         i->temp_path = mfree(i->temp_path);
 456
 457                         if (i->settings_job &&
 458                             i->settings_job->error == 0) {
 459
 460                                 /* Also move the settings file into place, if it exists. Note that we do so only if we also
 461                                  * moved the tar file in place, to keep things strictly in sync. */
 462                                 assert(i->settings_temp_path);
 463
 464                                 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
 465                                  * we should incorporate it in the file name if we can */
 466                                 i->settings_path = mfree(i->settings_path);
 467
 468                                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 469                                 if (r < 0)
 470                                         goto finish;
 471
 472                                 r = install_file(
 473                                                 AT_FDCWD, i->settings_temp_path,
 474                                                 AT_FDCWD, i->settings_path,
 475                                                 INSTALL_READ_ONLY|
 476                                                 (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0));
 477                                 if (r < 0) {
 478                                         log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
 479                                         goto finish;
 480                                 }
 481
 482                                 i->settings_temp_path = mfree(i->settings_temp_path);
 483                         }
 484                 }
 485
 486                 tar_pull_report_progress(i, TAR_COPYING);
 487
 488                 r = tar_pull_make_local_copy(i);
 489                 if (r < 0)
 490                         goto finish;
 491         }
 492
 493         r = 0;
 494
 495 finish:
 496         if (i->on_finished)
 497                 i->on_finished(i, r, i->userdata);
 498         else
 499                 sd_event_exit(i->event, r);
 500 }
 501
 502 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 503         const char *where;
 504         TarPull *i;
 505         int r;
 506
 507         assert(j);
 508         assert(j->userdata);
 509
 510         i = j->userdata;
 511         assert(i->tar_job == j);
 512         assert(i->tar_pid <= 0);
 513
 514         if (i->flags & IMPORT_DIRECT)
 515                 where = i->local;
 516         else {
 517                 if (!i->temp_path) {
 518                         r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
 519                         if (r < 0)
 520                                 return log_oom();
 521                 }
 522
 523                 where = i->temp_path;
 524         }
 525
 526         (void) mkdir_parents_label(where, 0700);
 527
 528         if (FLAGS_SET(i->flags, IMPORT_DIRECT|IMPORT_FORCE))
 529                 (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 530
 531         if (i->flags & IMPORT_BTRFS_SUBVOL)
 532                 r = btrfs_subvol_make_fallback(AT_FDCWD, where, 0755);
 533         else
 534                 r = RET_NERRNO(mkdir(where, 0755));
 535         if (r == -EEXIST && (i->flags & IMPORT_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
 536                                                        * because in that case our temporary path collided */
 537                 r = 0;
 538         if (r < 0)
 539                 return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where);
 540         if (r > 0 && (i->flags & IMPORT_BTRFS_QUOTA)) { /* actually btrfs subvol */
 541                 if (!(i->flags & IMPORT_DIRECT))
 542                         (void) import_assign_pool_quota_and_warn(i->image_root);
 543                 (void) import_assign_pool_quota_and_warn(where);
 544         }
 545
 546         j->disk_fd = import_fork_tar_x(where, &i->tar_pid);
 547         if (j->disk_fd < 0)
 548                 return j->disk_fd;
 549
 550         return 0;
 551 }
 552
 553 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 554         TarPull *i;
 555         int r;
 556
 557         assert(j);
 558         assert(j->userdata);
 559
 560         i = j->userdata;
 561         assert(i->settings_job == j);
 562
 563         if (!i->settings_temp_path) {
 564                 r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path);
 565                 if (r < 0)
 566                         return log_oom();
 567         }
 568
 569         (void) mkdir_parents_label(i->settings_temp_path, 0700);
 570
 571         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 572         if (j->disk_fd < 0)
 573                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 574
 575         return 0;
 576 }
 577
 578 static void tar_pull_job_on_progress(PullJob *j) {
 579         TarPull *i;
 580
 581         assert(j);
 582         assert(j->userdata);
 583
 584         i = j->userdata;
 585
 586         tar_pull_report_progress(i, TAR_DOWNLOADING);
 587 }
 588
 589 int tar_pull_start(
 590                 TarPull *i,
 591                 const char *url,
 592                 const char *local,
 593                 ImportFlags flags,
 594                 ImportVerify verify,
 595                 const char *checksum) {
 596
 597         int r;
 598
 599         assert(i);
 600         assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
 601         assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
 602         assert((verify < 0) || !checksum);
 603         assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_TAR));
 604         assert(!(flags & IMPORT_PULL_SETTINGS) || !(flags & IMPORT_DIRECT));
 605         assert(!(flags & IMPORT_PULL_SETTINGS) || !checksum);
 606
 607         if (!http_url_is_valid(url) && !file_url_is_valid(url))
 608                 return -EINVAL;
 609
 610         if (local && !pull_validate_local(local, flags))
 611                 return -EINVAL;
 612
 613         if (i->tar_job)
 614                 return -EBUSY;
 615
 616         r = free_and_strdup(&i->local, local);
 617         if (r < 0)
 618                 return r;
 619
 620         r = free_and_strdup(&i->checksum, checksum);
 621         if (r < 0)
 622                 return r;
 623
 624         i->flags = flags;
 625         i->verify = verify;
 626
 627         /* Set up download job for TAR file */
 628         r = pull_job_new(&i->tar_job, url, i->glue, i);
 629         if (r < 0)
 630                 return r;
 631
 632         i->tar_job->on_finished = tar_pull_job_on_finished;
 633         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 634         i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
 635
 636         if (!FLAGS_SET(flags, IMPORT_DIRECT)) {
 637                 r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 638                 if (r < 0)
 639                         return r;
 640         }
 641
 642         /* Set up download of checksum/signature files */
 643         r = pull_make_verification_jobs(
 644                         &i->checksum_job,
 645                         &i->signature_job,
 646                         verify,
 647                         checksum,
 648                         url,
 649                         i->glue,
 650                         tar_pull_job_on_finished,
 651                         i);
 652         if (r < 0)
 653                 return r;
 654
 655         /* Set up download job for the settings file (.nspawn) */
 656         if (FLAGS_SET(flags, IMPORT_PULL_SETTINGS)) {
 657                 r = pull_make_auxiliary_job(
 658                                 &i->settings_job,
 659                                 url,
 660                                 tar_strip_suffixes,
 661                                 ".nspawn",
 662                                 verify,
 663                                 i->glue,
 664                                 tar_pull_job_on_open_disk_settings,
 665                                 tar_pull_job_on_finished,
 666                                 i);
 667                 if (r < 0)
 668                         return r;
 669         }
 670
 671         PullJob *j;
 672         FOREACH_ARGUMENT(j,
 673                          i->tar_job,
 674                          i->checksum_job,
 675                          i->signature_job,
 676                          i->settings_job) {
 677
 678                 if (!j)
 679                         continue;
 680
 681                 j->on_progress = tar_pull_job_on_progress;
 682                 j->sync = FLAGS_SET(flags, IMPORT_SYNC);
 683
 684                 r = pull_job_begin(j);
 685                 if (r < 0)
 686                         return r;
 687         }
 688
 689         return 0;
 690 }