src/import/pull-tar.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2015 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/prctl.h>
  23 #include <curl/curl.h>
  24
  25 #include "sd-daemon.h"
  26
  27 #include "btrfs-util.h"
  28 #include "copy.h"
  29 #include "curl-util.h"
  30 #include "fd-util.h"
  31 #include "hostname-util.h"
  32 #include "import-common.h"
  33 #include "import-util.h"
  34 #include "macro.h"
  35 #include "mkdir.h"
  36 #include "path-util.h"
  37 #include "process-util.h"
  38 #include "pull-common.h"
  39 #include "pull-job.h"
  40 #include "pull-tar.h"
  41 #include "rm-rf.h"
  42 #include "string-util.h"
  43 #include "strv.h"
  44 #include "utf8.h"
  45 #include "util.h"
  46
  47 typedef enum TarProgress {
  48         TAR_DOWNLOADING,
  49         TAR_VERIFYING,
  50         TAR_FINALIZING,
  51         TAR_COPYING,
  52 } TarProgress;
  53
  54 struct TarPull {
  55         sd_event *event;
  56         CurlGlue *glue;
  57
  58         char *image_root;
  59
  60         PullJob *tar_job;
  61         PullJob *settings_job;
  62         PullJob *checksum_job;
  63         PullJob *signature_job;
  64
  65         TarPullFinished on_finished;
  66         void *userdata;
  67
  68         char *local;
  69         bool force_local;
  70         bool grow_machine_directory;
  71         bool settings;
  72
  73         pid_t tar_pid;
  74
  75         char *final_path;
  76         char *temp_path;
  77
  78         char *settings_path;
  79         char *settings_temp_path;
  80
  81         ImportVerify verify;
  82 };
  83
  84 TarPull* tar_pull_unref(TarPull *i) {
  85         if (!i)
  86                 return NULL;
  87
  88         if (i->tar_pid > 1) {
  89                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  90                 (void) wait_for_terminate(i->tar_pid, NULL);
  91         }
  92
  93         pull_job_unref(i->tar_job);
  94         pull_job_unref(i->settings_job);
  95         pull_job_unref(i->checksum_job);
  96         pull_job_unref(i->signature_job);
  97
  98         curl_glue_unref(i->glue);
  99         sd_event_unref(i->event);
 100
 101         if (i->temp_path) {
 102                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 103                 free(i->temp_path);
 104         }
 105
 106         if (i->settings_temp_path) {
 107                 (void) unlink(i->settings_temp_path);
 108                 free(i->settings_temp_path);
 109         }
 110
 111         free(i->final_path);
 112         free(i->settings_path);
 113         free(i->image_root);
 114         free(i->local);
 115         free(i);
 116
 117         return NULL;
 118 }
 119
 120 int tar_pull_new(
 121                 TarPull **ret,
 122                 sd_event *event,
 123                 const char *image_root,
 124                 TarPullFinished on_finished,
 125                 void *userdata) {
 126
 127         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 128         int r;
 129
 130         assert(ret);
 131
 132         i = new0(TarPull, 1);
 133         if (!i)
 134                 return -ENOMEM;
 135
 136         i->on_finished = on_finished;
 137         i->userdata = userdata;
 138
 139         i->image_root = strdup(image_root ?: "/var/lib/machines");
 140         if (!i->image_root)
 141                 return -ENOMEM;
 142
 143         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 144
 145         if (event)
 146                 i->event = sd_event_ref(event);
 147         else {
 148                 r = sd_event_default(&i->event);
 149                 if (r < 0)
 150                         return r;
 151         }
 152
 153         r = curl_glue_new(&i->glue, i->event);
 154         if (r < 0)
 155                 return r;
 156
 157         i->glue->on_finished = pull_job_curl_on_finished;
 158         i->glue->userdata = i;
 159
 160         *ret = i;
 161         i = NULL;
 162
 163         return 0;
 164 }
 165
 166 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 167         unsigned percent;
 168
 169         assert(i);
 170
 171         switch (p) {
 172
 173         case TAR_DOWNLOADING: {
 174                 unsigned remain = 85;
 175
 176                 percent = 0;
 177
 178                 if (i->settings_job) {
 179                         percent += i->settings_job->progress_percent * 5 / 100;
 180                         remain -= 5;
 181                 }
 182
 183                 if (i->checksum_job) {
 184                         percent += i->checksum_job->progress_percent * 5 / 100;
 185                         remain -= 5;
 186                 }
 187
 188                 if (i->signature_job) {
 189                         percent += i->signature_job->progress_percent * 5 / 100;
 190                         remain -= 5;
 191                 }
 192
 193                 if (i->tar_job)
 194                         percent += i->tar_job->progress_percent * remain / 100;
 195                 break;
 196         }
 197
 198         case TAR_VERIFYING:
 199                 percent = 85;
 200                 break;
 201
 202         case TAR_FINALIZING:
 203                 percent = 90;
 204                 break;
 205
 206         case TAR_COPYING:
 207                 percent = 95;
 208                 break;
 209
 210         default:
 211                 assert_not_reached("Unknown progress state");
 212         }
 213
 214         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 215         log_debug("Combined progress %u%%", percent);
 216 }
 217
 218 static int tar_pull_make_local_copy(TarPull *i) {
 219         int r;
 220
 221         assert(i);
 222         assert(i->tar_job);
 223
 224         if (!i->local)
 225                 return 0;
 226
 227         if (!i->final_path) {
 228                 r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", NULL, &i->final_path);
 229                 if (r < 0)
 230                         return log_oom();
 231         }
 232
 233         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 234         if (r < 0)
 235                 return r;
 236
 237         if (i->settings) {
 238                 const char *local_settings;
 239                 assert(i->settings_job);
 240
 241                 if (!i->settings_path) {
 242                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 243                         if (r < 0)
 244                                 return log_oom();
 245                 }
 246
 247                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 248
 249                 r = copy_file_atomic(i->settings_path, local_settings, 0664, i->force_local, 0);
 250                 if (r == -EEXIST)
 251                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 252                 else if (r < 0 && r != -ENOENT)
 253                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 254                 else
 255                         log_info("Created new settings file '%s.nspawn'", i->local);
 256         }
 257
 258         return 0;
 259 }
 260
 261 static bool tar_pull_is_done(TarPull *i) {
 262         assert(i);
 263         assert(i->tar_job);
 264
 265         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 266                 return false;
 267         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 268                 return false;
 269         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 270                 return false;
 271         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 272                 return false;
 273
 274         return true;
 275 }
 276
 277 static void tar_pull_job_on_finished(PullJob *j) {
 278         TarPull *i;
 279         int r;
 280
 281         assert(j);
 282         assert(j->userdata);
 283
 284         i = j->userdata;
 285
 286         if (j == i->settings_job) {
 287                 if (j->error != 0)
 288                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 289         } else if (j->error != 0) {
 290                 if (j == i->checksum_job)
 291                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 292                 else if (j == i->signature_job)
 293                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 294                 else
 295                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 296
 297                 r = j->error;
 298                 goto finish;
 299         }
 300
 301         /* This is invoked if either the download completed
 302          * successfully, or the download was skipped because we
 303          * already have the etag. */
 304
 305         if (!tar_pull_is_done(i))
 306                 return;
 307
 308         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 309         if (i->settings_job)
 310                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 311
 312         if (i->tar_pid > 0) {
 313                 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
 314                 i->tar_pid = 0;
 315                 if (r < 0)
 316                         goto finish;
 317                 if (r > 0) {
 318                         r = -EIO;
 319                         goto finish;
 320                 }
 321         }
 322
 323         if (!i->tar_job->etag_exists) {
 324                 /* This is a new download, verify it, and move it into place */
 325
 326                 tar_pull_report_progress(i, TAR_VERIFYING);
 327
 328                 r = pull_verify(i->tar_job, i->settings_job, i->checksum_job, i->signature_job);
 329                 if (r < 0)
 330                         goto finish;
 331
 332                 tar_pull_report_progress(i, TAR_FINALIZING);
 333
 334                 r = import_make_read_only(i->temp_path);
 335                 if (r < 0)
 336                         goto finish;
 337
 338                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 339                 if (r < 0) {
 340                         log_error_errno(r, "Failed to rename to final image name: %m");
 341                         goto finish;
 342                 }
 343
 344                 i->temp_path = mfree(i->temp_path);
 345
 346                 if (i->settings_job &&
 347                     i->settings_job->error == 0 &&
 348                     !i->settings_job->etag_exists) {
 349
 350                         assert(i->settings_temp_path);
 351                         assert(i->settings_path);
 352
 353                         /* Also move the settings file into place, if
 354                          * it exist. Note that we do so only if we
 355                          * also moved the tar file in place, to keep
 356                          * things strictly in sync. */
 357
 358                         r = import_make_read_only(i->settings_temp_path);
 359                         if (r < 0)
 360                                 goto finish;
 361
 362                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 363                         if (r < 0) {
 364                                 log_error_errno(r, "Failed to rename settings file: %m");
 365                                 goto finish;
 366                         }
 367
 368                         i->settings_temp_path = mfree(i->settings_temp_path);
 369                 }
 370         }
 371
 372         tar_pull_report_progress(i, TAR_COPYING);
 373
 374         r = tar_pull_make_local_copy(i);
 375         if (r < 0)
 376                 goto finish;
 377
 378         r = 0;
 379
 380 finish:
 381         if (i->on_finished)
 382                 i->on_finished(i, r, i->userdata);
 383         else
 384                 sd_event_exit(i->event, r);
 385 }
 386
 387 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 388         TarPull *i;
 389         int r;
 390
 391         assert(j);
 392         assert(j->userdata);
 393
 394         i = j->userdata;
 395         assert(i->tar_job == j);
 396         assert(!i->final_path);
 397         assert(!i->temp_path);
 398         assert(i->tar_pid <= 0);
 399
 400         r = pull_make_path(j->url, j->etag, i->image_root, ".tar-", NULL, &i->final_path);
 401         if (r < 0)
 402                 return log_oom();
 403
 404         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 405         if (r < 0)
 406                 return log_oom();
 407
 408         mkdir_parents_label(i->temp_path, 0700);
 409
 410         r = btrfs_subvol_make(i->temp_path);
 411         if (r == -ENOTTY) {
 412                 if (mkdir(i->temp_path, 0755) < 0)
 413                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 414         } else if (r < 0)
 415                 return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path);
 416         else
 417                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 418
 419         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 420         if (j->disk_fd < 0)
 421                 return j->disk_fd;
 422
 423         return 0;
 424 }
 425
 426 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 427         TarPull *i;
 428         int r;
 429
 430         assert(j);
 431         assert(j->userdata);
 432
 433         i = j->userdata;
 434         assert(i->settings_job == j);
 435         assert(!i->settings_path);
 436         assert(!i->settings_temp_path);
 437
 438         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 439         if (r < 0)
 440                 return log_oom();
 441
 442         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 443         if (r < 0)
 444                 return log_oom();
 445
 446         mkdir_parents_label(i->settings_temp_path, 0700);
 447
 448         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 449         if (j->disk_fd < 0)
 450                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 451
 452         return 0;
 453 }
 454
 455 static void tar_pull_job_on_progress(PullJob *j) {
 456         TarPull *i;
 457
 458         assert(j);
 459         assert(j->userdata);
 460
 461         i = j->userdata;
 462
 463         tar_pull_report_progress(i, TAR_DOWNLOADING);
 464 }
 465
 466 int tar_pull_start(
 467                 TarPull *i,
 468                 const char *url,
 469                 const char *local,
 470                 bool force_local,
 471                 ImportVerify verify,
 472                 bool settings) {
 473
 474         int r;
 475
 476         assert(i);
 477         assert(verify < _IMPORT_VERIFY_MAX);
 478         assert(verify >= 0);
 479
 480         if (!http_url_is_valid(url))
 481                 return -EINVAL;
 482
 483         if (local && !machine_name_is_valid(local))
 484                 return -EINVAL;
 485
 486         if (i->tar_job)
 487                 return -EBUSY;
 488
 489         r = free_and_strdup(&i->local, local);
 490         if (r < 0)
 491                 return r;
 492
 493         i->force_local = force_local;
 494         i->verify = verify;
 495         i->settings = settings;
 496
 497         /* Set up download job for TAR file */
 498         r = pull_job_new(&i->tar_job, url, i->glue, i);
 499         if (r < 0)
 500                 return r;
 501
 502         i->tar_job->on_finished = tar_pull_job_on_finished;
 503         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 504         i->tar_job->on_progress = tar_pull_job_on_progress;
 505         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 506         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 507
 508         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 509         if (r < 0)
 510                 return r;
 511
 512         /* Set up download job for the settings file (.nspawn) */
 513         if (settings) {
 514                 r = pull_make_settings_job(&i->settings_job, url, i->glue, tar_pull_job_on_finished, i);
 515                 if (r < 0)
 516                         return r;
 517
 518                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 519                 i->settings_job->on_progress = tar_pull_job_on_progress;
 520                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 521
 522                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 523                 if (r < 0)
 524                         return r;
 525         }
 526
 527         /* Set up download of checksum/signature files */
 528         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 529         if (r < 0)
 530                 return r;
 531
 532         r = pull_job_begin(i->tar_job);
 533         if (r < 0)
 534                 return r;
 535
 536         if (i->settings_job) {
 537                 r = pull_job_begin(i->settings_job);
 538                 if (r < 0)
 539                         return r;
 540         }
 541
 542         if (i->checksum_job) {
 543                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 544
 545                 r = pull_job_begin(i->checksum_job);
 546                 if (r < 0)
 547                         return r;
 548         }
 549
 550         if (i->signature_job) {
 551                 i->signature_job->on_progress = tar_pull_job_on_progress;
 552
 553                 r = pull_job_begin(i->signature_job);
 554                 if (r < 0)
 555                         return r;
 556         }
 557
 558         return 0;
 559 }