src/import/pull-tar.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2015 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/prctl.h>
  23 #include <curl/curl.h>
  24
  25 #include "sd-daemon.h"
  26
  27 #include "btrfs-util.h"
  28 #include "copy.h"
  29 #include "curl-util.h"
  30 #include "hostname-util.h"
  31 #include "import-common.h"
  32 #include "import-util.h"
  33 #include "macro.h"
  34 #include "mkdir.h"
  35 #include "path-util.h"
  36 #include "process-util.h"
  37 #include "pull-common.h"
  38 #include "pull-job.h"
  39 #include "rm-rf.h"
  40 #include "string-util.h"
  41 #include "strv.h"
  42 #include "utf8.h"
  43 #include "util.h"
  44 #include "pull-tar.h"
  45
  46 typedef enum TarProgress {
  47         TAR_DOWNLOADING,
  48         TAR_VERIFYING,
  49         TAR_FINALIZING,
  50         TAR_COPYING,
  51 } TarProgress;
  52
  53 struct TarPull {
  54         sd_event *event;
  55         CurlGlue *glue;
  56
  57         char *image_root;
  58
  59         PullJob *tar_job;
  60         PullJob *settings_job;
  61         PullJob *checksum_job;
  62         PullJob *signature_job;
  63
  64         TarPullFinished on_finished;
  65         void *userdata;
  66
  67         char *local;
  68         bool force_local;
  69         bool grow_machine_directory;
  70         bool settings;
  71
  72         pid_t tar_pid;
  73
  74         char *final_path;
  75         char *temp_path;
  76
  77         char *settings_path;
  78         char *settings_temp_path;
  79
  80         ImportVerify verify;
  81 };
  82
  83 TarPull* tar_pull_unref(TarPull *i) {
  84         if (!i)
  85                 return NULL;
  86
  87         if (i->tar_pid > 1) {
  88                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  89                 (void) wait_for_terminate(i->tar_pid, NULL);
  90         }
  91
  92         pull_job_unref(i->tar_job);
  93         pull_job_unref(i->settings_job);
  94         pull_job_unref(i->checksum_job);
  95         pull_job_unref(i->signature_job);
  96
  97         curl_glue_unref(i->glue);
  98         sd_event_unref(i->event);
  99
 100         if (i->temp_path) {
 101                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 102                 free(i->temp_path);
 103         }
 104
 105         if (i->settings_temp_path) {
 106                 (void) unlink(i->settings_temp_path);
 107                 free(i->settings_temp_path);
 108         }
 109
 110         free(i->final_path);
 111         free(i->settings_path);
 112         free(i->image_root);
 113         free(i->local);
 114         free(i);
 115
 116         return NULL;
 117 }
 118
 119 int tar_pull_new(
 120                 TarPull **ret,
 121                 sd_event *event,
 122                 const char *image_root,
 123                 TarPullFinished on_finished,
 124                 void *userdata) {
 125
 126         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 127         int r;
 128
 129         assert(ret);
 130
 131         i = new0(TarPull, 1);
 132         if (!i)
 133                 return -ENOMEM;
 134
 135         i->on_finished = on_finished;
 136         i->userdata = userdata;
 137
 138         i->image_root = strdup(image_root ?: "/var/lib/machines");
 139         if (!i->image_root)
 140                 return -ENOMEM;
 141
 142         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 143
 144         if (event)
 145                 i->event = sd_event_ref(event);
 146         else {
 147                 r = sd_event_default(&i->event);
 148                 if (r < 0)
 149                         return r;
 150         }
 151
 152         r = curl_glue_new(&i->glue, i->event);
 153         if (r < 0)
 154                 return r;
 155
 156         i->glue->on_finished = pull_job_curl_on_finished;
 157         i->glue->userdata = i;
 158
 159         *ret = i;
 160         i = NULL;
 161
 162         return 0;
 163 }
 164
 165 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 166         unsigned percent;
 167
 168         assert(i);
 169
 170         switch (p) {
 171
 172         case TAR_DOWNLOADING: {
 173                 unsigned remain = 85;
 174
 175                 percent = 0;
 176
 177                 if (i->settings_job) {
 178                         percent += i->settings_job->progress_percent * 5 / 100;
 179                         remain -= 5;
 180                 }
 181
 182                 if (i->checksum_job) {
 183                         percent += i->checksum_job->progress_percent * 5 / 100;
 184                         remain -= 5;
 185                 }
 186
 187                 if (i->signature_job) {
 188                         percent += i->signature_job->progress_percent * 5 / 100;
 189                         remain -= 5;
 190                 }
 191
 192                 if (i->tar_job)
 193                         percent += i->tar_job->progress_percent * remain / 100;
 194                 break;
 195         }
 196
 197         case TAR_VERIFYING:
 198                 percent = 85;
 199                 break;
 200
 201         case TAR_FINALIZING:
 202                 percent = 90;
 203                 break;
 204
 205         case TAR_COPYING:
 206                 percent = 95;
 207                 break;
 208
 209         default:
 210                 assert_not_reached("Unknown progress state");
 211         }
 212
 213         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 214         log_debug("Combined progress %u%%", percent);
 215 }
 216
 217 static int tar_pull_make_local_copy(TarPull *i) {
 218         int r;
 219
 220         assert(i);
 221         assert(i->tar_job);
 222
 223         if (!i->local)
 224                 return 0;
 225
 226         if (!i->final_path) {
 227                 r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", NULL, &i->final_path);
 228                 if (r < 0)
 229                         return log_oom();
 230         }
 231
 232         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 233         if (r < 0)
 234                 return r;
 235
 236         if (i->settings) {
 237                 const char *local_settings;
 238                 assert(i->settings_job);
 239
 240                 if (!i->settings_path) {
 241                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 242                         if (r < 0)
 243                                 return log_oom();
 244                 }
 245
 246                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 247
 248                 r = copy_file_atomic(i->settings_path, local_settings, 0664, i->force_local, 0);
 249                 if (r == -EEXIST)
 250                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 251                 else if (r < 0 && r != -ENOENT)
 252                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 253                 else
 254                         log_info("Created new settings file '%s.nspawn'", i->local);
 255         }
 256
 257         return 0;
 258 }
 259
 260 static bool tar_pull_is_done(TarPull *i) {
 261         assert(i);
 262         assert(i->tar_job);
 263
 264         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 265                 return false;
 266         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 267                 return false;
 268         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 269                 return false;
 270         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 271                 return false;
 272
 273         return true;
 274 }
 275
 276 static void tar_pull_job_on_finished(PullJob *j) {
 277         TarPull *i;
 278         int r;
 279
 280         assert(j);
 281         assert(j->userdata);
 282
 283         i = j->userdata;
 284
 285         if (j == i->settings_job) {
 286                 if (j->error != 0)
 287                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 288         } else if (j->error != 0) {
 289                 if (j == i->checksum_job)
 290                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 291                 else if (j == i->signature_job)
 292                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 293                 else
 294                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 295
 296                 r = j->error;
 297                 goto finish;
 298         }
 299
 300         /* This is invoked if either the download completed
 301          * successfully, or the download was skipped because we
 302          * already have the etag. */
 303
 304         if (!tar_pull_is_done(i))
 305                 return;
 306
 307         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 308         if (i->settings_job)
 309                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 310
 311         if (i->tar_pid > 0) {
 312                 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
 313                 i->tar_pid = 0;
 314                 if (r < 0)
 315                         goto finish;
 316                 if (r > 0) {
 317                         r = -EIO;
 318                         goto finish;
 319                 }
 320         }
 321
 322         if (!i->tar_job->etag_exists) {
 323                 /* This is a new download, verify it, and move it into place */
 324
 325                 tar_pull_report_progress(i, TAR_VERIFYING);
 326
 327                 r = pull_verify(i->tar_job, i->settings_job, i->checksum_job, i->signature_job);
 328                 if (r < 0)
 329                         goto finish;
 330
 331                 tar_pull_report_progress(i, TAR_FINALIZING);
 332
 333                 r = import_make_read_only(i->temp_path);
 334                 if (r < 0)
 335                         goto finish;
 336
 337                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 338                 if (r < 0) {
 339                         log_error_errno(r, "Failed to rename to final image name: %m");
 340                         goto finish;
 341                 }
 342
 343                 i->temp_path = mfree(i->temp_path);
 344
 345                 if (i->settings_job &&
 346                     i->settings_job->error == 0 &&
 347                     !i->settings_job->etag_exists) {
 348
 349                         assert(i->settings_temp_path);
 350                         assert(i->settings_path);
 351
 352                         /* Also move the settings file into place, if
 353                          * it exist. Note that we do so only if we
 354                          * also moved the tar file in place, to keep
 355                          * things strictly in sync. */
 356
 357                         r = import_make_read_only(i->settings_temp_path);
 358                         if (r < 0)
 359                                 goto finish;
 360
 361                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 362                         if (r < 0) {
 363                                 log_error_errno(r, "Failed to rename settings file: %m");
 364                                 goto finish;
 365                         }
 366
 367                         i->settings_temp_path = mfree(i->settings_temp_path);
 368                 }
 369         }
 370
 371         tar_pull_report_progress(i, TAR_COPYING);
 372
 373         r = tar_pull_make_local_copy(i);
 374         if (r < 0)
 375                 goto finish;
 376
 377         r = 0;
 378
 379 finish:
 380         if (i->on_finished)
 381                 i->on_finished(i, r, i->userdata);
 382         else
 383                 sd_event_exit(i->event, r);
 384 }
 385
 386 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 387         TarPull *i;
 388         int r;
 389
 390         assert(j);
 391         assert(j->userdata);
 392
 393         i = j->userdata;
 394         assert(i->tar_job == j);
 395         assert(!i->final_path);
 396         assert(!i->temp_path);
 397         assert(i->tar_pid <= 0);
 398
 399         r = pull_make_path(j->url, j->etag, i->image_root, ".tar-", NULL, &i->final_path);
 400         if (r < 0)
 401                 return log_oom();
 402
 403         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 404         if (r < 0)
 405                 return log_oom();
 406
 407         mkdir_parents_label(i->temp_path, 0700);
 408
 409         r = btrfs_subvol_make(i->temp_path);
 410         if (r == -ENOTTY) {
 411                 if (mkdir(i->temp_path, 0755) < 0)
 412                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 413         } else if (r < 0)
 414                 return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path);
 415         else
 416                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 417
 418         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 419         if (j->disk_fd < 0)
 420                 return j->disk_fd;
 421
 422         return 0;
 423 }
 424
 425 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 426         TarPull *i;
 427         int r;
 428
 429         assert(j);
 430         assert(j->userdata);
 431
 432         i = j->userdata;
 433         assert(i->settings_job == j);
 434         assert(!i->settings_path);
 435         assert(!i->settings_temp_path);
 436
 437         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 438         if (r < 0)
 439                 return log_oom();
 440
 441         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 442         if (r < 0)
 443                 return log_oom();
 444
 445         mkdir_parents_label(i->settings_temp_path, 0700);
 446
 447         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 448         if (j->disk_fd < 0)
 449                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 450
 451         return 0;
 452 }
 453
 454 static void tar_pull_job_on_progress(PullJob *j) {
 455         TarPull *i;
 456
 457         assert(j);
 458         assert(j->userdata);
 459
 460         i = j->userdata;
 461
 462         tar_pull_report_progress(i, TAR_DOWNLOADING);
 463 }
 464
 465 int tar_pull_start(
 466                 TarPull *i,
 467                 const char *url,
 468                 const char *local,
 469                 bool force_local,
 470                 ImportVerify verify,
 471                 bool settings) {
 472
 473         int r;
 474
 475         assert(i);
 476         assert(verify < _IMPORT_VERIFY_MAX);
 477         assert(verify >= 0);
 478
 479         if (!http_url_is_valid(url))
 480                 return -EINVAL;
 481
 482         if (local && !machine_name_is_valid(local))
 483                 return -EINVAL;
 484
 485         if (i->tar_job)
 486                 return -EBUSY;
 487
 488         r = free_and_strdup(&i->local, local);
 489         if (r < 0)
 490                 return r;
 491
 492         i->force_local = force_local;
 493         i->verify = verify;
 494         i->settings = settings;
 495
 496         /* Set up download job for TAR file */
 497         r = pull_job_new(&i->tar_job, url, i->glue, i);
 498         if (r < 0)
 499                 return r;
 500
 501         i->tar_job->on_finished = tar_pull_job_on_finished;
 502         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 503         i->tar_job->on_progress = tar_pull_job_on_progress;
 504         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 505         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 506
 507         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 508         if (r < 0)
 509                 return r;
 510
 511         /* Set up download job for the settings file (.nspawn) */
 512         if (settings) {
 513                 r = pull_make_settings_job(&i->settings_job, url, i->glue, tar_pull_job_on_finished, i);
 514                 if (r < 0)
 515                         return r;
 516
 517                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 518                 i->settings_job->on_progress = tar_pull_job_on_progress;
 519                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 520
 521                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 522                 if (r < 0)
 523                         return r;
 524         }
 525
 526         /* Set up download of checksum/signature files */
 527         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 528         if (r < 0)
 529                 return r;
 530
 531         r = pull_job_begin(i->tar_job);
 532         if (r < 0)
 533                 return r;
 534
 535         if (i->settings_job) {
 536                 r = pull_job_begin(i->settings_job);
 537                 if (r < 0)
 538                         return r;
 539         }
 540
 541         if (i->checksum_job) {
 542                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 543
 544                 r = pull_job_begin(i->checksum_job);
 545                 if (r < 0)
 546                         return r;
 547         }
 548
 549         if (i->signature_job) {
 550                 i->signature_job->on_progress = tar_pull_job_on_progress;
 551
 552                 r = pull_job_begin(i->signature_job);
 553                 if (r < 0)
 554                         return r;
 555         }
 556
 557         return 0;
 558 }