src/import/pull-tar.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2 /***
   3   This file is part of systemd.
   4
   5   Copyright 2015 Lennart Poettering
   6
   7   systemd is free software; you can redistribute it and/or modify it
   8   under the terms of the GNU Lesser General Public License as published by
   9   the Free Software Foundation; either version 2.1 of the License, or
  10   (at your option) any later version.
  11
  12   systemd is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   Lesser General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  19 ***/
  20
  21 #include <curl/curl.h>
  22 #include <sys/prctl.h>
  23
  24 #include "sd-daemon.h"
  25
  26 #include "alloc-util.h"
  27 #include "btrfs-util.h"
  28 #include "copy.h"
  29 #include "curl-util.h"
  30 #include "fd-util.h"
  31 #include "fileio.h"
  32 #include "fs-util.h"
  33 #include "hostname-util.h"
  34 #include "import-common.h"
  35 #include "import-util.h"
  36 #include "macro.h"
  37 #include "mkdir.h"
  38 #include "path-util.h"
  39 #include "process-util.h"
  40 #include "pull-common.h"
  41 #include "pull-job.h"
  42 #include "pull-tar.h"
  43 #include "rm-rf.h"
  44 #include "string-util.h"
  45 #include "strv.h"
  46 #include "utf8.h"
  47 #include "util.h"
  48 #include "web-util.h"
  49
  50 typedef enum TarProgress {
  51         TAR_DOWNLOADING,
  52         TAR_VERIFYING,
  53         TAR_FINALIZING,
  54         TAR_COPYING,
  55 } TarProgress;
  56
  57 struct TarPull {
  58         sd_event *event;
  59         CurlGlue *glue;
  60
  61         char *image_root;
  62
  63         PullJob *tar_job;
  64         PullJob *settings_job;
  65         PullJob *checksum_job;
  66         PullJob *signature_job;
  67
  68         TarPullFinished on_finished;
  69         void *userdata;
  70
  71         char *local;
  72         bool force_local;
  73         bool grow_machine_directory;
  74         bool settings;
  75
  76         pid_t tar_pid;
  77
  78         char *final_path;
  79         char *temp_path;
  80
  81         char *settings_path;
  82         char *settings_temp_path;
  83
  84         ImportVerify verify;
  85 };
  86
  87 TarPull* tar_pull_unref(TarPull *i) {
  88         if (!i)
  89                 return NULL;
  90
  91         if (i->tar_pid > 1) {
  92                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  93                 (void) wait_for_terminate(i->tar_pid, NULL);
  94         }
  95
  96         pull_job_unref(i->tar_job);
  97         pull_job_unref(i->settings_job);
  98         pull_job_unref(i->checksum_job);
  99         pull_job_unref(i->signature_job);
 100
 101         curl_glue_unref(i->glue);
 102         sd_event_unref(i->event);
 103
 104         if (i->temp_path) {
 105                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 106                 free(i->temp_path);
 107         }
 108
 109         if (i->settings_temp_path) {
 110                 (void) unlink(i->settings_temp_path);
 111                 free(i->settings_temp_path);
 112         }
 113
 114         free(i->final_path);
 115         free(i->settings_path);
 116         free(i->image_root);
 117         free(i->local);
 118
 119         return mfree(i);
 120 }
 121
 122 int tar_pull_new(
 123                 TarPull **ret,
 124                 sd_event *event,
 125                 const char *image_root,
 126                 TarPullFinished on_finished,
 127                 void *userdata) {
 128
 129         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 130         int r;
 131
 132         assert(ret);
 133
 134         i = new0(TarPull, 1);
 135         if (!i)
 136                 return -ENOMEM;
 137
 138         i->on_finished = on_finished;
 139         i->userdata = userdata;
 140
 141         i->image_root = strdup(image_root ?: "/var/lib/machines");
 142         if (!i->image_root)
 143                 return -ENOMEM;
 144
 145         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 146
 147         if (event)
 148                 i->event = sd_event_ref(event);
 149         else {
 150                 r = sd_event_default(&i->event);
 151                 if (r < 0)
 152                         return r;
 153         }
 154
 155         r = curl_glue_new(&i->glue, i->event);
 156         if (r < 0)
 157                 return r;
 158
 159         i->glue->on_finished = pull_job_curl_on_finished;
 160         i->glue->userdata = i;
 161
 162         *ret = TAKE_PTR(i);
 163
 164         return 0;
 165 }
 166
 167 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 168         unsigned percent;
 169
 170         assert(i);
 171
 172         switch (p) {
 173
 174         case TAR_DOWNLOADING: {
 175                 unsigned remain = 85;
 176
 177                 percent = 0;
 178
 179                 if (i->settings_job) {
 180                         percent += i->settings_job->progress_percent * 5 / 100;
 181                         remain -= 5;
 182                 }
 183
 184                 if (i->checksum_job) {
 185                         percent += i->checksum_job->progress_percent * 5 / 100;
 186                         remain -= 5;
 187                 }
 188
 189                 if (i->signature_job) {
 190                         percent += i->signature_job->progress_percent * 5 / 100;
 191                         remain -= 5;
 192                 }
 193
 194                 if (i->tar_job)
 195                         percent += i->tar_job->progress_percent * remain / 100;
 196                 break;
 197         }
 198
 199         case TAR_VERIFYING:
 200                 percent = 85;
 201                 break;
 202
 203         case TAR_FINALIZING:
 204                 percent = 90;
 205                 break;
 206
 207         case TAR_COPYING:
 208                 percent = 95;
 209                 break;
 210
 211         default:
 212                 assert_not_reached("Unknown progress state");
 213         }
 214
 215         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 216         log_debug("Combined progress %u%%", percent);
 217 }
 218
 219 static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) {
 220         int r;
 221
 222         assert(i);
 223         assert(field);
 224
 225         if (*field)
 226                 return 0;
 227
 228         assert(i->tar_job);
 229
 230         r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field);
 231         if (r < 0)
 232                 return log_oom();
 233
 234         return 1;
 235 }
 236
 237 static int tar_pull_make_local_copy(TarPull *i) {
 238         int r;
 239
 240         assert(i);
 241         assert(i->tar_job);
 242
 243         if (!i->local)
 244                 return 0;
 245
 246         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 247         if (r < 0)
 248                 return r;
 249
 250         if (i->settings) {
 251                 const char *local_settings;
 252                 assert(i->settings_job);
 253
 254                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 255                 if (r < 0)
 256                         return r;
 257
 258                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 259
 260                 r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0));
 261                 if (r == -EEXIST)
 262                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 263                 else if (r == -ENOENT)
 264                         log_debug_errno(r, "Skipping creation of settings file, since none was found.");
 265                 else if (r < 0)
 266                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 267                 else
 268                         log_info("Created new settings file %s.", local_settings);
 269         }
 270
 271         return 0;
 272 }
 273
 274 static bool tar_pull_is_done(TarPull *i) {
 275         assert(i);
 276         assert(i->tar_job);
 277
 278         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 279                 return false;
 280         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 281                 return false;
 282         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 283                 return false;
 284         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 285                 return false;
 286
 287         return true;
 288 }
 289
 290 static void tar_pull_job_on_finished(PullJob *j) {
 291         TarPull *i;
 292         int r;
 293
 294         assert(j);
 295         assert(j->userdata);
 296
 297         i = j->userdata;
 298
 299         if (j == i->settings_job) {
 300                 if (j->error != 0)
 301                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 302         } else if (j->error != 0 && j != i->signature_job) {
 303                 if (j == i->checksum_job)
 304                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 305                 else
 306                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 307
 308                 r = j->error;
 309                 goto finish;
 310         }
 311
 312         /* This is invoked if either the download completed
 313          * successfully, or the download was skipped because we
 314          * already have the etag. */
 315
 316         if (!tar_pull_is_done(i))
 317                 return;
 318
 319         if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) {
 320                 log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 321
 322                 r = i->signature_job->error;
 323                 goto finish;
 324         }
 325
 326         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 327         if (i->settings_job)
 328                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 329
 330         r = tar_pull_determine_path(i, NULL, &i->final_path);
 331         if (r < 0)
 332                 goto finish;
 333
 334         if (i->tar_pid > 0) {
 335                 r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG);
 336                 i->tar_pid = 0;
 337                 if (r < 0)
 338                         goto finish;
 339                 if (r != EXIT_SUCCESS) {
 340                         r = -EIO;
 341                         goto finish;
 342                 }
 343         }
 344
 345         if (!i->tar_job->etag_exists) {
 346                 /* This is a new download, verify it, and move it into place */
 347
 348                 tar_pull_report_progress(i, TAR_VERIFYING);
 349
 350                 r = pull_verify(i->tar_job, NULL, i->settings_job, i->checksum_job, i->signature_job);
 351                 if (r < 0)
 352                         goto finish;
 353
 354                 tar_pull_report_progress(i, TAR_FINALIZING);
 355
 356                 r = import_make_read_only(i->temp_path);
 357                 if (r < 0)
 358                         goto finish;
 359
 360                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 361                 if (r < 0) {
 362                         log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
 363                         goto finish;
 364                 }
 365
 366                 i->temp_path = mfree(i->temp_path);
 367
 368                 if (i->settings_job &&
 369                     i->settings_job->error == 0) {
 370
 371                         /* Also move the settings file into place, if it exists. Note that we do so only if we also
 372                          * moved the tar file in place, to keep things strictly in sync. */
 373                         assert(i->settings_temp_path);
 374
 375                         /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
 376                          * we should incorporate it in the file name if we can */
 377                         i->settings_path = mfree(i->settings_path);
 378
 379                         r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 380                         if (r < 0)
 381                                 goto finish;
 382
 383                         r = import_make_read_only(i->settings_temp_path);
 384                         if (r < 0)
 385                                 goto finish;
 386
 387                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 388                         if (r < 0) {
 389                                 log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
 390                                 goto finish;
 391                         }
 392
 393                         i->settings_temp_path = mfree(i->settings_temp_path);
 394                 }
 395         }
 396
 397         tar_pull_report_progress(i, TAR_COPYING);
 398
 399         r = tar_pull_make_local_copy(i);
 400         if (r < 0)
 401                 goto finish;
 402
 403         r = 0;
 404
 405 finish:
 406         if (i->on_finished)
 407                 i->on_finished(i, r, i->userdata);
 408         else
 409                 sd_event_exit(i->event, r);
 410 }
 411
 412 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 413         TarPull *i;
 414         int r;
 415
 416         assert(j);
 417         assert(j->userdata);
 418
 419         i = j->userdata;
 420         assert(i->tar_job == j);
 421         assert(i->tar_pid <= 0);
 422
 423         if (!i->temp_path) {
 424                 r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
 425                 if (r < 0)
 426                         return log_oom();
 427         }
 428
 429         mkdir_parents_label(i->temp_path, 0700);
 430
 431         r = btrfs_subvol_make(i->temp_path);
 432         if (r == -ENOTTY) {
 433                 if (mkdir(i->temp_path, 0755) < 0)
 434                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 435         } else if (r < 0)
 436                 return log_error_errno(r, "Failed to create subvolume %s: %m", i->temp_path);
 437         else
 438                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 439
 440         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 441         if (j->disk_fd < 0)
 442                 return j->disk_fd;
 443
 444         return 0;
 445 }
 446
 447 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 448         TarPull *i;
 449         int r;
 450
 451         assert(j);
 452         assert(j->userdata);
 453
 454         i = j->userdata;
 455         assert(i->settings_job == j);
 456
 457         if (!i->settings_temp_path) {
 458                 r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path);
 459                 if (r < 0)
 460                         return log_oom();
 461         }
 462
 463         mkdir_parents_label(i->settings_temp_path, 0700);
 464
 465         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 466         if (j->disk_fd < 0)
 467                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 468
 469         return 0;
 470 }
 471
 472 static void tar_pull_job_on_progress(PullJob *j) {
 473         TarPull *i;
 474
 475         assert(j);
 476         assert(j->userdata);
 477
 478         i = j->userdata;
 479
 480         tar_pull_report_progress(i, TAR_DOWNLOADING);
 481 }
 482
 483 int tar_pull_start(
 484                 TarPull *i,
 485                 const char *url,
 486                 const char *local,
 487                 bool force_local,
 488                 ImportVerify verify,
 489                 bool settings) {
 490
 491         int r;
 492
 493         assert(i);
 494         assert(verify < _IMPORT_VERIFY_MAX);
 495         assert(verify >= 0);
 496
 497         if (!http_url_is_valid(url))
 498                 return -EINVAL;
 499
 500         if (local && !machine_name_is_valid(local))
 501                 return -EINVAL;
 502
 503         if (i->tar_job)
 504                 return -EBUSY;
 505
 506         r = free_and_strdup(&i->local, local);
 507         if (r < 0)
 508                 return r;
 509
 510         i->force_local = force_local;
 511         i->verify = verify;
 512         i->settings = settings;
 513
 514         /* Set up download job for TAR file */
 515         r = pull_job_new(&i->tar_job, url, i->glue, i);
 516         if (r < 0)
 517                 return r;
 518
 519         i->tar_job->on_finished = tar_pull_job_on_finished;
 520         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 521         i->tar_job->on_progress = tar_pull_job_on_progress;
 522         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 523         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 524
 525         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 526         if (r < 0)
 527                 return r;
 528
 529         /* Set up download job for the settings file (.nspawn) */
 530         if (settings) {
 531                 r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i);
 532                 if (r < 0)
 533                         return r;
 534
 535                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 536                 i->settings_job->on_progress = tar_pull_job_on_progress;
 537                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 538         }
 539
 540         /* Set up download of checksum/signature files */
 541         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 542         if (r < 0)
 543                 return r;
 544
 545         r = pull_job_begin(i->tar_job);
 546         if (r < 0)
 547                 return r;
 548
 549         if (i->settings_job) {
 550                 r = pull_job_begin(i->settings_job);
 551                 if (r < 0)
 552                         return r;
 553         }
 554
 555         if (i->checksum_job) {
 556                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 557                 i->checksum_job->style = VERIFICATION_PER_FILE;
 558
 559                 r = pull_job_begin(i->checksum_job);
 560                 if (r < 0)
 561                         return r;
 562         }
 563
 564         if (i->signature_job) {
 565                 i->signature_job->on_progress = tar_pull_job_on_progress;
 566
 567                 r = pull_job_begin(i->signature_job);
 568                 if (r < 0)
 569                         return r;
 570         }
 571
 572         return 0;
 573 }