src/import/pull-tar.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <curl/curl.h>
   4 #include <sys/prctl.h>
   5
   6 #include "sd-daemon.h"
   7
   8 #include "alloc-util.h"
   9 #include "btrfs-util.h"
  10 #include "copy.h"
  11 #include "curl-util.h"
  12 #include "fd-util.h"
  13 #include "fileio.h"
  14 #include "fs-util.h"
  15 #include "hostname-util.h"
  16 #include "import-common.h"
  17 #include "import-util.h"
  18 #include "macro.h"
  19 #include "mkdir.h"
  20 #include "path-util.h"
  21 #include "process-util.h"
  22 #include "pull-common.h"
  23 #include "pull-job.h"
  24 #include "pull-tar.h"
  25 #include "rm-rf.h"
  26 #include "string-util.h"
  27 #include "strv.h"
  28 #include "utf8.h"
  29 #include "util.h"
  30 #include "web-util.h"
  31
  32 typedef enum TarProgress {
  33         TAR_DOWNLOADING,
  34         TAR_VERIFYING,
  35         TAR_FINALIZING,
  36         TAR_COPYING,
  37 } TarProgress;
  38
  39 struct TarPull {
  40         sd_event *event;
  41         CurlGlue *glue;
  42
  43         char *image_root;
  44
  45         PullJob *tar_job;
  46         PullJob *settings_job;
  47         PullJob *checksum_job;
  48         PullJob *signature_job;
  49
  50         TarPullFinished on_finished;
  51         void *userdata;
  52
  53         char *local;
  54         bool force_local;
  55         bool grow_machine_directory;
  56         bool settings;
  57
  58         pid_t tar_pid;
  59
  60         char *final_path;
  61         char *temp_path;
  62
  63         char *settings_path;
  64         char *settings_temp_path;
  65
  66         ImportVerify verify;
  67 };
  68
  69 TarPull* tar_pull_unref(TarPull *i) {
  70         if (!i)
  71                 return NULL;
  72
  73         if (i->tar_pid > 1) {
  74                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  75                 (void) wait_for_terminate(i->tar_pid, NULL);
  76         }
  77
  78         pull_job_unref(i->tar_job);
  79         pull_job_unref(i->settings_job);
  80         pull_job_unref(i->checksum_job);
  81         pull_job_unref(i->signature_job);
  82
  83         curl_glue_unref(i->glue);
  84         sd_event_unref(i->event);
  85
  86         if (i->temp_path) {
  87                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
  88                 free(i->temp_path);
  89         }
  90
  91         if (i->settings_temp_path) {
  92                 (void) unlink(i->settings_temp_path);
  93                 free(i->settings_temp_path);
  94         }
  95
  96         free(i->final_path);
  97         free(i->settings_path);
  98         free(i->image_root);
  99         free(i->local);
 100
 101         return mfree(i);
 102 }
 103
 104 int tar_pull_new(
 105                 TarPull **ret,
 106                 sd_event *event,
 107                 const char *image_root,
 108                 TarPullFinished on_finished,
 109                 void *userdata) {
 110
 111         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 112         int r;
 113
 114         assert(ret);
 115
 116         i = new0(TarPull, 1);
 117         if (!i)
 118                 return -ENOMEM;
 119
 120         i->on_finished = on_finished;
 121         i->userdata = userdata;
 122
 123         i->image_root = strdup(image_root ?: "/var/lib/machines");
 124         if (!i->image_root)
 125                 return -ENOMEM;
 126
 127         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 128
 129         if (event)
 130                 i->event = sd_event_ref(event);
 131         else {
 132                 r = sd_event_default(&i->event);
 133                 if (r < 0)
 134                         return r;
 135         }
 136
 137         r = curl_glue_new(&i->glue, i->event);
 138         if (r < 0)
 139                 return r;
 140
 141         i->glue->on_finished = pull_job_curl_on_finished;
 142         i->glue->userdata = i;
 143
 144         *ret = TAKE_PTR(i);
 145
 146         return 0;
 147 }
 148
 149 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 150         unsigned percent;
 151
 152         assert(i);
 153
 154         switch (p) {
 155
 156         case TAR_DOWNLOADING: {
 157                 unsigned remain = 85;
 158
 159                 percent = 0;
 160
 161                 if (i->settings_job) {
 162                         percent += i->settings_job->progress_percent * 5 / 100;
 163                         remain -= 5;
 164                 }
 165
 166                 if (i->checksum_job) {
 167                         percent += i->checksum_job->progress_percent * 5 / 100;
 168                         remain -= 5;
 169                 }
 170
 171                 if (i->signature_job) {
 172                         percent += i->signature_job->progress_percent * 5 / 100;
 173                         remain -= 5;
 174                 }
 175
 176                 if (i->tar_job)
 177                         percent += i->tar_job->progress_percent * remain / 100;
 178                 break;
 179         }
 180
 181         case TAR_VERIFYING:
 182                 percent = 85;
 183                 break;
 184
 185         case TAR_FINALIZING:
 186                 percent = 90;
 187                 break;
 188
 189         case TAR_COPYING:
 190                 percent = 95;
 191                 break;
 192
 193         default:
 194                 assert_not_reached("Unknown progress state");
 195         }
 196
 197         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 198         log_debug("Combined progress %u%%", percent);
 199 }
 200
 201 static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) {
 202         int r;
 203
 204         assert(i);
 205         assert(field);
 206
 207         if (*field)
 208                 return 0;
 209
 210         assert(i->tar_job);
 211
 212         r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field);
 213         if (r < 0)
 214                 return log_oom();
 215
 216         return 1;
 217 }
 218
 219 static int tar_pull_make_local_copy(TarPull *i) {
 220         int r;
 221
 222         assert(i);
 223         assert(i->tar_job);
 224
 225         if (!i->local)
 226                 return 0;
 227
 228         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 229         if (r < 0)
 230                 return r;
 231
 232         if (i->settings) {
 233                 const char *local_settings;
 234                 assert(i->settings_job);
 235
 236                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 237                 if (r < 0)
 238                         return r;
 239
 240                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 241
 242                 r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0));
 243                 if (r == -EEXIST)
 244                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 245                 else if (r == -ENOENT)
 246                         log_debug_errno(r, "Skipping creation of settings file, since none was found.");
 247                 else if (r < 0)
 248                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 249                 else
 250                         log_info("Created new settings file %s.", local_settings);
 251         }
 252
 253         return 0;
 254 }
 255
 256 static bool tar_pull_is_done(TarPull *i) {
 257         assert(i);
 258         assert(i->tar_job);
 259
 260         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 261                 return false;
 262         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 263                 return false;
 264         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 265                 return false;
 266         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 267                 return false;
 268
 269         return true;
 270 }
 271
 272 static void tar_pull_job_on_finished(PullJob *j) {
 273         TarPull *i;
 274         int r;
 275
 276         assert(j);
 277         assert(j->userdata);
 278
 279         i = j->userdata;
 280
 281         if (j == i->settings_job) {
 282                 if (j->error != 0)
 283                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 284         } else if (j->error != 0 && j != i->signature_job) {
 285                 if (j == i->checksum_job)
 286                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 287                 else
 288                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 289
 290                 r = j->error;
 291                 goto finish;
 292         }
 293
 294         /* This is invoked if either the download completed
 295          * successfully, or the download was skipped because we
 296          * already have the etag. */
 297
 298         if (!tar_pull_is_done(i))
 299                 return;
 300
 301         if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) {
 302                 log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 303
 304                 r = i->signature_job->error;
 305                 goto finish;
 306         }
 307
 308         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 309         if (i->settings_job)
 310                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 311
 312         r = tar_pull_determine_path(i, NULL, &i->final_path);
 313         if (r < 0)
 314                 goto finish;
 315
 316         if (i->tar_pid > 0) {
 317                 r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG);
 318                 i->tar_pid = 0;
 319                 if (r < 0)
 320                         goto finish;
 321                 if (r != EXIT_SUCCESS) {
 322                         r = -EIO;
 323                         goto finish;
 324                 }
 325         }
 326
 327         if (!i->tar_job->etag_exists) {
 328                 /* This is a new download, verify it, and move it into place */
 329
 330                 tar_pull_report_progress(i, TAR_VERIFYING);
 331
 332                 r = pull_verify(i->tar_job, NULL, i->settings_job, i->checksum_job, i->signature_job);
 333                 if (r < 0)
 334                         goto finish;
 335
 336                 tar_pull_report_progress(i, TAR_FINALIZING);
 337
 338                 r = import_make_read_only(i->temp_path);
 339                 if (r < 0)
 340                         goto finish;
 341
 342                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 343                 if (r < 0) {
 344                         log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
 345                         goto finish;
 346                 }
 347
 348                 i->temp_path = mfree(i->temp_path);
 349
 350                 if (i->settings_job &&
 351                     i->settings_job->error == 0) {
 352
 353                         /* Also move the settings file into place, if it exists. Note that we do so only if we also
 354                          * moved the tar file in place, to keep things strictly in sync. */
 355                         assert(i->settings_temp_path);
 356
 357                         /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
 358                          * we should incorporate it in the file name if we can */
 359                         i->settings_path = mfree(i->settings_path);
 360
 361                         r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 362                         if (r < 0)
 363                                 goto finish;
 364
 365                         r = import_make_read_only(i->settings_temp_path);
 366                         if (r < 0)
 367                                 goto finish;
 368
 369                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 370                         if (r < 0) {
 371                                 log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
 372                                 goto finish;
 373                         }
 374
 375                         i->settings_temp_path = mfree(i->settings_temp_path);
 376                 }
 377         }
 378
 379         tar_pull_report_progress(i, TAR_COPYING);
 380
 381         r = tar_pull_make_local_copy(i);
 382         if (r < 0)
 383                 goto finish;
 384
 385         r = 0;
 386
 387 finish:
 388         if (i->on_finished)
 389                 i->on_finished(i, r, i->userdata);
 390         else
 391                 sd_event_exit(i->event, r);
 392 }
 393
 394 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 395         TarPull *i;
 396         int r;
 397
 398         assert(j);
 399         assert(j->userdata);
 400
 401         i = j->userdata;
 402         assert(i->tar_job == j);
 403         assert(i->tar_pid <= 0);
 404
 405         if (!i->temp_path) {
 406                 r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
 407                 if (r < 0)
 408                         return log_oom();
 409         }
 410
 411         mkdir_parents_label(i->temp_path, 0700);
 412
 413         r = btrfs_subvol_make(i->temp_path);
 414         if (r == -ENOTTY) {
 415                 if (mkdir(i->temp_path, 0755) < 0)
 416                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 417         } else if (r < 0)
 418                 return log_error_errno(r, "Failed to create subvolume %s: %m", i->temp_path);
 419         else
 420                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 421
 422         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 423         if (j->disk_fd < 0)
 424                 return j->disk_fd;
 425
 426         return 0;
 427 }
 428
 429 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 430         TarPull *i;
 431         int r;
 432
 433         assert(j);
 434         assert(j->userdata);
 435
 436         i = j->userdata;
 437         assert(i->settings_job == j);
 438
 439         if (!i->settings_temp_path) {
 440                 r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path);
 441                 if (r < 0)
 442                         return log_oom();
 443         }
 444
 445         mkdir_parents_label(i->settings_temp_path, 0700);
 446
 447         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 448         if (j->disk_fd < 0)
 449                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 450
 451         return 0;
 452 }
 453
 454 static void tar_pull_job_on_progress(PullJob *j) {
 455         TarPull *i;
 456
 457         assert(j);
 458         assert(j->userdata);
 459
 460         i = j->userdata;
 461
 462         tar_pull_report_progress(i, TAR_DOWNLOADING);
 463 }
 464
 465 int tar_pull_start(
 466                 TarPull *i,
 467                 const char *url,
 468                 const char *local,
 469                 bool force_local,
 470                 ImportVerify verify,
 471                 bool settings) {
 472
 473         int r;
 474
 475         assert(i);
 476         assert(verify < _IMPORT_VERIFY_MAX);
 477         assert(verify >= 0);
 478
 479         if (!http_url_is_valid(url))
 480                 return -EINVAL;
 481
 482         if (local && !machine_name_is_valid(local))
 483                 return -EINVAL;
 484
 485         if (i->tar_job)
 486                 return -EBUSY;
 487
 488         r = free_and_strdup(&i->local, local);
 489         if (r < 0)
 490                 return r;
 491
 492         i->force_local = force_local;
 493         i->verify = verify;
 494         i->settings = settings;
 495
 496         /* Set up download job for TAR file */
 497         r = pull_job_new(&i->tar_job, url, i->glue, i);
 498         if (r < 0)
 499                 return r;
 500
 501         i->tar_job->on_finished = tar_pull_job_on_finished;
 502         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 503         i->tar_job->on_progress = tar_pull_job_on_progress;
 504         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 505         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 506
 507         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 508         if (r < 0)
 509                 return r;
 510
 511         /* Set up download job for the settings file (.nspawn) */
 512         if (settings) {
 513                 r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i);
 514                 if (r < 0)
 515                         return r;
 516
 517                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 518                 i->settings_job->on_progress = tar_pull_job_on_progress;
 519                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 520         }
 521
 522         /* Set up download of checksum/signature files */
 523         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 524         if (r < 0)
 525                 return r;
 526
 527         r = pull_job_begin(i->tar_job);
 528         if (r < 0)
 529                 return r;
 530
 531         if (i->settings_job) {
 532                 r = pull_job_begin(i->settings_job);
 533                 if (r < 0)
 534                         return r;
 535         }
 536
 537         if (i->checksum_job) {
 538                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 539                 i->checksum_job->style = VERIFICATION_PER_FILE;
 540
 541                 r = pull_job_begin(i->checksum_job);
 542                 if (r < 0)
 543                         return r;
 544         }
 545
 546         if (i->signature_job) {
 547                 i->signature_job->on_progress = tar_pull_job_on_progress;
 548
 549                 r = pull_job_begin(i->signature_job);
 550                 if (r < 0)
 551                         return r;
 552         }
 553
 554         return 0;
 555 }