src/import/pull-tar.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2 /***
   3   This file is part of systemd.
   4
   5   Copyright 2015 Lennart Poettering
   6
   7   systemd is free software; you can redistribute it and/or modify it
   8   under the terms of the GNU Lesser General Public License as published by
   9   the Free Software Foundation; either version 2.1 of the License, or
  10   (at your option) any later version.
  11
  12   systemd is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   Lesser General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  19 ***/
  20
  21 #include <curl/curl.h>
  22 #include <sys/prctl.h>
  23
  24 #include "sd-daemon.h"
  25
  26 #include "alloc-util.h"
  27 #include "btrfs-util.h"
  28 #include "copy.h"
  29 #include "curl-util.h"
  30 #include "fd-util.h"
  31 #include "fileio.h"
  32 #include "fs-util.h"
  33 #include "hostname-util.h"
  34 #include "import-common.h"
  35 #include "import-util.h"
  36 #include "macro.h"
  37 #include "mkdir.h"
  38 #include "path-util.h"
  39 #include "process-util.h"
  40 #include "pull-common.h"
  41 #include "pull-job.h"
  42 #include "pull-tar.h"
  43 #include "rm-rf.h"
  44 #include "string-util.h"
  45 #include "strv.h"
  46 #include "utf8.h"
  47 #include "util.h"
  48 #include "web-util.h"
  49
  50 typedef enum TarProgress {
  51         TAR_DOWNLOADING,
  52         TAR_VERIFYING,
  53         TAR_FINALIZING,
  54         TAR_COPYING,
  55 } TarProgress;
  56
  57 struct TarPull {
  58         sd_event *event;
  59         CurlGlue *glue;
  60
  61         char *image_root;
  62
  63         PullJob *tar_job;
  64         PullJob *settings_job;
  65         PullJob *checksum_job;
  66         PullJob *signature_job;
  67
  68         TarPullFinished on_finished;
  69         void *userdata;
  70
  71         char *local;
  72         bool force_local;
  73         bool grow_machine_directory;
  74         bool settings;
  75
  76         pid_t tar_pid;
  77
  78         char *final_path;
  79         char *temp_path;
  80
  81         char *settings_path;
  82         char *settings_temp_path;
  83
  84         ImportVerify verify;
  85 };
  86
  87 TarPull* tar_pull_unref(TarPull *i) {
  88         if (!i)
  89                 return NULL;
  90
  91         if (i->tar_pid > 1) {
  92                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  93                 (void) wait_for_terminate(i->tar_pid, NULL);
  94         }
  95
  96         pull_job_unref(i->tar_job);
  97         pull_job_unref(i->settings_job);
  98         pull_job_unref(i->checksum_job);
  99         pull_job_unref(i->signature_job);
 100
 101         curl_glue_unref(i->glue);
 102         sd_event_unref(i->event);
 103
 104         if (i->temp_path) {
 105                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 106                 free(i->temp_path);
 107         }
 108
 109         if (i->settings_temp_path) {
 110                 (void) unlink(i->settings_temp_path);
 111                 free(i->settings_temp_path);
 112         }
 113
 114         free(i->final_path);
 115         free(i->settings_path);
 116         free(i->image_root);
 117         free(i->local);
 118
 119         return mfree(i);
 120 }
 121
 122 int tar_pull_new(
 123                 TarPull **ret,
 124                 sd_event *event,
 125                 const char *image_root,
 126                 TarPullFinished on_finished,
 127                 void *userdata) {
 128
 129         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 130         int r;
 131
 132         assert(ret);
 133
 134         i = new0(TarPull, 1);
 135         if (!i)
 136                 return -ENOMEM;
 137
 138         i->on_finished = on_finished;
 139         i->userdata = userdata;
 140
 141         i->image_root = strdup(image_root ?: "/var/lib/machines");
 142         if (!i->image_root)
 143                 return -ENOMEM;
 144
 145         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 146
 147         if (event)
 148                 i->event = sd_event_ref(event);
 149         else {
 150                 r = sd_event_default(&i->event);
 151                 if (r < 0)
 152                         return r;
 153         }
 154
 155         r = curl_glue_new(&i->glue, i->event);
 156         if (r < 0)
 157                 return r;
 158
 159         i->glue->on_finished = pull_job_curl_on_finished;
 160         i->glue->userdata = i;
 161
 162         *ret = i;
 163         i = NULL;
 164
 165         return 0;
 166 }
 167
 168 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 169         unsigned percent;
 170
 171         assert(i);
 172
 173         switch (p) {
 174
 175         case TAR_DOWNLOADING: {
 176                 unsigned remain = 85;
 177
 178                 percent = 0;
 179
 180                 if (i->settings_job) {
 181                         percent += i->settings_job->progress_percent * 5 / 100;
 182                         remain -= 5;
 183                 }
 184
 185                 if (i->checksum_job) {
 186                         percent += i->checksum_job->progress_percent * 5 / 100;
 187                         remain -= 5;
 188                 }
 189
 190                 if (i->signature_job) {
 191                         percent += i->signature_job->progress_percent * 5 / 100;
 192                         remain -= 5;
 193                 }
 194
 195                 if (i->tar_job)
 196                         percent += i->tar_job->progress_percent * remain / 100;
 197                 break;
 198         }
 199
 200         case TAR_VERIFYING:
 201                 percent = 85;
 202                 break;
 203
 204         case TAR_FINALIZING:
 205                 percent = 90;
 206                 break;
 207
 208         case TAR_COPYING:
 209                 percent = 95;
 210                 break;
 211
 212         default:
 213                 assert_not_reached("Unknown progress state");
 214         }
 215
 216         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 217         log_debug("Combined progress %u%%", percent);
 218 }
 219
 220 static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) {
 221         int r;
 222
 223         assert(i);
 224         assert(field);
 225
 226         if (*field)
 227                 return 0;
 228
 229         assert(i->tar_job);
 230
 231         r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field);
 232         if (r < 0)
 233                 return log_oom();
 234
 235         return 1;
 236 }
 237
 238 static int tar_pull_make_local_copy(TarPull *i) {
 239         int r;
 240
 241         assert(i);
 242         assert(i->tar_job);
 243
 244         if (!i->local)
 245                 return 0;
 246
 247         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 248         if (r < 0)
 249                 return r;
 250
 251         if (i->settings) {
 252                 const char *local_settings;
 253                 assert(i->settings_job);
 254
 255                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 256                 if (r < 0)
 257                         return r;
 258
 259                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 260
 261                 r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0));
 262                 if (r == -EEXIST)
 263                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 264                 else if (r == -ENOENT)
 265                         log_debug_errno(r, "Skipping creation of settings file, since none was found.");
 266                 else if (r < 0)
 267                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 268                 else
 269                         log_info("Created new settings file %s.", local_settings);
 270         }
 271
 272         return 0;
 273 }
 274
 275 static bool tar_pull_is_done(TarPull *i) {
 276         assert(i);
 277         assert(i->tar_job);
 278
 279         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 280                 return false;
 281         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 282                 return false;
 283         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 284                 return false;
 285         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 286                 return false;
 287
 288         return true;
 289 }
 290
 291 static void tar_pull_job_on_finished(PullJob *j) {
 292         TarPull *i;
 293         int r;
 294
 295         assert(j);
 296         assert(j->userdata);
 297
 298         i = j->userdata;
 299
 300         if (j == i->settings_job) {
 301                 if (j->error != 0)
 302                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 303         } else if (j->error != 0 && j != i->signature_job) {
 304                 if (j == i->checksum_job)
 305                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 306                 else
 307                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 308
 309                 r = j->error;
 310                 goto finish;
 311         }
 312
 313         /* This is invoked if either the download completed
 314          * successfully, or the download was skipped because we
 315          * already have the etag. */
 316
 317         if (!tar_pull_is_done(i))
 318                 return;
 319
 320         if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) {
 321                 log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 322
 323                 r = i->signature_job->error;
 324                 goto finish;
 325         }
 326
 327         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 328         if (i->settings_job)
 329                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 330
 331         r = tar_pull_determine_path(i, NULL, &i->final_path);
 332         if (r < 0)
 333                 goto finish;
 334
 335         if (i->tar_pid > 0) {
 336                 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
 337                 i->tar_pid = 0;
 338                 if (r < 0)
 339                         goto finish;
 340                 if (r > 0) {
 341                         r = -EIO;
 342                         goto finish;
 343                 }
 344         }
 345
 346         if (!i->tar_job->etag_exists) {
 347                 /* This is a new download, verify it, and move it into place */
 348
 349                 tar_pull_report_progress(i, TAR_VERIFYING);
 350
 351                 r = pull_verify(i->tar_job, NULL, i->settings_job, i->checksum_job, i->signature_job);
 352                 if (r < 0)
 353                         goto finish;
 354
 355                 tar_pull_report_progress(i, TAR_FINALIZING);
 356
 357                 r = import_make_read_only(i->temp_path);
 358                 if (r < 0)
 359                         goto finish;
 360
 361                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 362                 if (r < 0) {
 363                         log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
 364                         goto finish;
 365                 }
 366
 367                 i->temp_path = mfree(i->temp_path);
 368
 369                 if (i->settings_job &&
 370                     i->settings_job->error == 0) {
 371
 372                         /* Also move the settings file into place, if it exists. Note that we do so only if we also
 373                          * moved the tar file in place, to keep things strictly in sync. */
 374                         assert(i->settings_temp_path);
 375
 376                         /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
 377                          * we should incorporate it in the file name if we can */
 378                         i->settings_path = mfree(i->settings_path);
 379
 380                         r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 381                         if (r < 0)
 382                                 goto finish;
 383
 384                         r = import_make_read_only(i->settings_temp_path);
 385                         if (r < 0)
 386                                 goto finish;
 387
 388                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 389                         if (r < 0) {
 390                                 log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
 391                                 goto finish;
 392                         }
 393
 394                         i->settings_temp_path = mfree(i->settings_temp_path);
 395                 }
 396         }
 397
 398         tar_pull_report_progress(i, TAR_COPYING);
 399
 400         r = tar_pull_make_local_copy(i);
 401         if (r < 0)
 402                 goto finish;
 403
 404         r = 0;
 405
 406 finish:
 407         if (i->on_finished)
 408                 i->on_finished(i, r, i->userdata);
 409         else
 410                 sd_event_exit(i->event, r);
 411 }
 412
 413 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 414         TarPull *i;
 415         int r;
 416
 417         assert(j);
 418         assert(j->userdata);
 419
 420         i = j->userdata;
 421         assert(i->tar_job == j);
 422         assert(i->tar_pid <= 0);
 423
 424         if (!i->temp_path) {
 425                 r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
 426                 if (r < 0)
 427                         return log_oom();
 428         }
 429
 430         mkdir_parents_label(i->temp_path, 0700);
 431
 432         r = btrfs_subvol_make(i->temp_path);
 433         if (r == -ENOTTY) {
 434                 if (mkdir(i->temp_path, 0755) < 0)
 435                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 436         } else if (r < 0)
 437                 return log_error_errno(r, "Failed to create subvolume %s: %m", i->temp_path);
 438         else
 439                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 440
 441         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 442         if (j->disk_fd < 0)
 443                 return j->disk_fd;
 444
 445         return 0;
 446 }
 447
 448 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 449         TarPull *i;
 450         int r;
 451
 452         assert(j);
 453         assert(j->userdata);
 454
 455         i = j->userdata;
 456         assert(i->settings_job == j);
 457
 458         if (!i->settings_temp_path) {
 459                 r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path);
 460                 if (r < 0)
 461                         return log_oom();
 462         }
 463
 464         mkdir_parents_label(i->settings_temp_path, 0700);
 465
 466         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 467         if (j->disk_fd < 0)
 468                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 469
 470         return 0;
 471 }
 472
 473 static void tar_pull_job_on_progress(PullJob *j) {
 474         TarPull *i;
 475
 476         assert(j);
 477         assert(j->userdata);
 478
 479         i = j->userdata;
 480
 481         tar_pull_report_progress(i, TAR_DOWNLOADING);
 482 }
 483
 484 int tar_pull_start(
 485                 TarPull *i,
 486                 const char *url,
 487                 const char *local,
 488                 bool force_local,
 489                 ImportVerify verify,
 490                 bool settings) {
 491
 492         int r;
 493
 494         assert(i);
 495         assert(verify < _IMPORT_VERIFY_MAX);
 496         assert(verify >= 0);
 497
 498         if (!http_url_is_valid(url))
 499                 return -EINVAL;
 500
 501         if (local && !machine_name_is_valid(local))
 502                 return -EINVAL;
 503
 504         if (i->tar_job)
 505                 return -EBUSY;
 506
 507         r = free_and_strdup(&i->local, local);
 508         if (r < 0)
 509                 return r;
 510
 511         i->force_local = force_local;
 512         i->verify = verify;
 513         i->settings = settings;
 514
 515         /* Set up download job for TAR file */
 516         r = pull_job_new(&i->tar_job, url, i->glue, i);
 517         if (r < 0)
 518                 return r;
 519
 520         i->tar_job->on_finished = tar_pull_job_on_finished;
 521         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 522         i->tar_job->on_progress = tar_pull_job_on_progress;
 523         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 524         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 525
 526         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 527         if (r < 0)
 528                 return r;
 529
 530         /* Set up download job for the settings file (.nspawn) */
 531         if (settings) {
 532                 r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i);
 533                 if (r < 0)
 534                         return r;
 535
 536                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 537                 i->settings_job->on_progress = tar_pull_job_on_progress;
 538                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 539         }
 540
 541         /* Set up download of checksum/signature files */
 542         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 543         if (r < 0)
 544                 return r;
 545
 546         r = pull_job_begin(i->tar_job);
 547         if (r < 0)
 548                 return r;
 549
 550         if (i->settings_job) {
 551                 r = pull_job_begin(i->settings_job);
 552                 if (r < 0)
 553                         return r;
 554         }
 555
 556         if (i->checksum_job) {
 557                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 558                 i->checksum_job->style = VERIFICATION_PER_FILE;
 559
 560                 r = pull_job_begin(i->checksum_job);
 561                 if (r < 0)
 562                         return r;
 563         }
 564
 565         if (i->signature_job) {
 566                 i->signature_job->on_progress = tar_pull_job_on_progress;
 567
 568                 r = pull_job_begin(i->signature_job);
 569                 if (r < 0)
 570                         return r;
 571         }
 572
 573         return 0;
 574 }