src/import/pull-tar.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2015 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <curl/curl.h>
  23 #include <sys/prctl.h>
  24
  25 #include "sd-daemon.h"
  26
  27 #include "alloc-util.h"
  28 #include "btrfs-util.h"
  29 #include "copy.h"
  30 #include "curl-util.h"
  31 #include "fd-util.h"
  32 #include "fileio.h"
  33 #include "fs-util.h"
  34 #include "hostname-util.h"
  35 #include "import-common.h"
  36 #include "import-util.h"
  37 #include "macro.h"
  38 #include "mkdir.h"
  39 #include "path-util.h"
  40 #include "process-util.h"
  41 #include "pull-common.h"
  42 #include "pull-job.h"
  43 #include "pull-tar.h"
  44 #include "rm-rf.h"
  45 #include "string-util.h"
  46 #include "strv.h"
  47 #include "utf8.h"
  48 #include "util.h"
  49 #include "web-util.h"
  50
  51 typedef enum TarProgress {
  52         TAR_DOWNLOADING,
  53         TAR_VERIFYING,
  54         TAR_FINALIZING,
  55         TAR_COPYING,
  56 } TarProgress;
  57
  58 struct TarPull {
  59         sd_event *event;
  60         CurlGlue *glue;
  61
  62         char *image_root;
  63
  64         PullJob *tar_job;
  65         PullJob *settings_job;
  66         PullJob *checksum_job;
  67         PullJob *signature_job;
  68
  69         TarPullFinished on_finished;
  70         void *userdata;
  71
  72         char *local;
  73         bool force_local;
  74         bool grow_machine_directory;
  75         bool settings;
  76
  77         pid_t tar_pid;
  78
  79         char *final_path;
  80         char *temp_path;
  81
  82         char *settings_path;
  83         char *settings_temp_path;
  84
  85         ImportVerify verify;
  86 };
  87
  88 TarPull* tar_pull_unref(TarPull *i) {
  89         if (!i)
  90                 return NULL;
  91
  92         if (i->tar_pid > 1) {
  93                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  94                 (void) wait_for_terminate(i->tar_pid, NULL);
  95         }
  96
  97         pull_job_unref(i->tar_job);
  98         pull_job_unref(i->settings_job);
  99         pull_job_unref(i->checksum_job);
 100         pull_job_unref(i->signature_job);
 101
 102         curl_glue_unref(i->glue);
 103         sd_event_unref(i->event);
 104
 105         if (i->temp_path) {
 106                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 107                 free(i->temp_path);
 108         }
 109
 110         if (i->settings_temp_path) {
 111                 (void) unlink(i->settings_temp_path);
 112                 free(i->settings_temp_path);
 113         }
 114
 115         free(i->final_path);
 116         free(i->settings_path);
 117         free(i->image_root);
 118         free(i->local);
 119         free(i);
 120
 121         return NULL;
 122 }
 123
 124 int tar_pull_new(
 125                 TarPull **ret,
 126                 sd_event *event,
 127                 const char *image_root,
 128                 TarPullFinished on_finished,
 129                 void *userdata) {
 130
 131         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 132         int r;
 133
 134         assert(ret);
 135
 136         i = new0(TarPull, 1);
 137         if (!i)
 138                 return -ENOMEM;
 139
 140         i->on_finished = on_finished;
 141         i->userdata = userdata;
 142
 143         i->image_root = strdup(image_root ?: "/var/lib/machines");
 144         if (!i->image_root)
 145                 return -ENOMEM;
 146
 147         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 148
 149         if (event)
 150                 i->event = sd_event_ref(event);
 151         else {
 152                 r = sd_event_default(&i->event);
 153                 if (r < 0)
 154                         return r;
 155         }
 156
 157         r = curl_glue_new(&i->glue, i->event);
 158         if (r < 0)
 159                 return r;
 160
 161         i->glue->on_finished = pull_job_curl_on_finished;
 162         i->glue->userdata = i;
 163
 164         *ret = i;
 165         i = NULL;
 166
 167         return 0;
 168 }
 169
 170 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 171         unsigned percent;
 172
 173         assert(i);
 174
 175         switch (p) {
 176
 177         case TAR_DOWNLOADING: {
 178                 unsigned remain = 85;
 179
 180                 percent = 0;
 181
 182                 if (i->settings_job) {
 183                         percent += i->settings_job->progress_percent * 5 / 100;
 184                         remain -= 5;
 185                 }
 186
 187                 if (i->checksum_job) {
 188                         percent += i->checksum_job->progress_percent * 5 / 100;
 189                         remain -= 5;
 190                 }
 191
 192                 if (i->signature_job) {
 193                         percent += i->signature_job->progress_percent * 5 / 100;
 194                         remain -= 5;
 195                 }
 196
 197                 if (i->tar_job)
 198                         percent += i->tar_job->progress_percent * remain / 100;
 199                 break;
 200         }
 201
 202         case TAR_VERIFYING:
 203                 percent = 85;
 204                 break;
 205
 206         case TAR_FINALIZING:
 207                 percent = 90;
 208                 break;
 209
 210         case TAR_COPYING:
 211                 percent = 95;
 212                 break;
 213
 214         default:
 215                 assert_not_reached("Unknown progress state");
 216         }
 217
 218         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 219         log_debug("Combined progress %u%%", percent);
 220 }
 221
 222 static int tar_pull_make_local_copy(TarPull *i) {
 223         int r;
 224
 225         assert(i);
 226         assert(i->tar_job);
 227
 228         if (!i->local)
 229                 return 0;
 230
 231         if (!i->final_path) {
 232                 r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", NULL, &i->final_path);
 233                 if (r < 0)
 234                         return log_oom();
 235         }
 236
 237         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 238         if (r < 0)
 239                 return r;
 240
 241         if (i->settings) {
 242                 const char *local_settings;
 243                 assert(i->settings_job);
 244
 245                 if (!i->settings_path) {
 246                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 247                         if (r < 0)
 248                                 return log_oom();
 249                 }
 250
 251                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 252
 253                 r = copy_file_atomic(i->settings_path, local_settings, 0664, i->force_local, 0);
 254                 if (r == -EEXIST)
 255                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 256                 else if (r < 0 && r != -ENOENT)
 257                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 258                 else
 259                         log_info("Created new settings file '%s.nspawn'", i->local);
 260         }
 261
 262         return 0;
 263 }
 264
 265 static bool tar_pull_is_done(TarPull *i) {
 266         assert(i);
 267         assert(i->tar_job);
 268
 269         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 270                 return false;
 271         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 272                 return false;
 273         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 274                 return false;
 275         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 276                 return false;
 277
 278         return true;
 279 }
 280
 281 static void tar_pull_job_on_finished(PullJob *j) {
 282         TarPull *i;
 283         int r;
 284
 285         assert(j);
 286         assert(j->userdata);
 287
 288         i = j->userdata;
 289
 290         if (j == i->settings_job) {
 291                 if (j->error != 0)
 292                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 293         } else if (j->error != 0) {
 294                 if (j == i->checksum_job)
 295                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 296                 else if (j == i->signature_job)
 297                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 298                 else
 299                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 300
 301                 r = j->error;
 302                 goto finish;
 303         }
 304
 305         /* This is invoked if either the download completed
 306          * successfully, or the download was skipped because we
 307          * already have the etag. */
 308
 309         if (!tar_pull_is_done(i))
 310                 return;
 311
 312         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 313         if (i->settings_job)
 314                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 315
 316         if (i->tar_pid > 0) {
 317                 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
 318                 i->tar_pid = 0;
 319                 if (r < 0)
 320                         goto finish;
 321                 if (r > 0) {
 322                         r = -EIO;
 323                         goto finish;
 324                 }
 325         }
 326
 327         if (!i->tar_job->etag_exists) {
 328                 /* This is a new download, verify it, and move it into place */
 329
 330                 tar_pull_report_progress(i, TAR_VERIFYING);
 331
 332                 r = pull_verify(i->tar_job, i->settings_job, i->checksum_job, i->signature_job);
 333                 if (r < 0)
 334                         goto finish;
 335
 336                 tar_pull_report_progress(i, TAR_FINALIZING);
 337
 338                 r = import_make_read_only(i->temp_path);
 339                 if (r < 0)
 340                         goto finish;
 341
 342                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 343                 if (r < 0) {
 344                         log_error_errno(r, "Failed to rename to final image name: %m");
 345                         goto finish;
 346                 }
 347
 348                 i->temp_path = mfree(i->temp_path);
 349
 350                 if (i->settings_job &&
 351                     i->settings_job->error == 0 &&
 352                     !i->settings_job->etag_exists) {
 353
 354                         assert(i->settings_temp_path);
 355                         assert(i->settings_path);
 356
 357                         /* Also move the settings file into place, if
 358                          * it exist. Note that we do so only if we
 359                          * also moved the tar file in place, to keep
 360                          * things strictly in sync. */
 361
 362                         r = import_make_read_only(i->settings_temp_path);
 363                         if (r < 0)
 364                                 goto finish;
 365
 366                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 367                         if (r < 0) {
 368                                 log_error_errno(r, "Failed to rename settings file: %m");
 369                                 goto finish;
 370                         }
 371
 372                         i->settings_temp_path = mfree(i->settings_temp_path);
 373                 }
 374         }
 375
 376         tar_pull_report_progress(i, TAR_COPYING);
 377
 378         r = tar_pull_make_local_copy(i);
 379         if (r < 0)
 380                 goto finish;
 381
 382         r = 0;
 383
 384 finish:
 385         if (i->on_finished)
 386                 i->on_finished(i, r, i->userdata);
 387         else
 388                 sd_event_exit(i->event, r);
 389 }
 390
 391 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 392         TarPull *i;
 393         int r;
 394
 395         assert(j);
 396         assert(j->userdata);
 397
 398         i = j->userdata;
 399         assert(i->tar_job == j);
 400         assert(!i->final_path);
 401         assert(!i->temp_path);
 402         assert(i->tar_pid <= 0);
 403
 404         r = pull_make_path(j->url, j->etag, i->image_root, ".tar-", NULL, &i->final_path);
 405         if (r < 0)
 406                 return log_oom();
 407
 408         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 409         if (r < 0)
 410                 return log_oom();
 411
 412         mkdir_parents_label(i->temp_path, 0700);
 413
 414         r = btrfs_subvol_make(i->temp_path);
 415         if (r == -ENOTTY) {
 416                 if (mkdir(i->temp_path, 0755) < 0)
 417                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 418         } else if (r < 0)
 419                 return log_error_errno(r, "Failed to create subvolume %s: %m", i->temp_path);
 420         else
 421                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 422
 423         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 424         if (j->disk_fd < 0)
 425                 return j->disk_fd;
 426
 427         return 0;
 428 }
 429
 430 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 431         TarPull *i;
 432         int r;
 433
 434         assert(j);
 435         assert(j->userdata);
 436
 437         i = j->userdata;
 438         assert(i->settings_job == j);
 439         assert(!i->settings_path);
 440         assert(!i->settings_temp_path);
 441
 442         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 443         if (r < 0)
 444                 return log_oom();
 445
 446         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 447         if (r < 0)
 448                 return log_oom();
 449
 450         mkdir_parents_label(i->settings_temp_path, 0700);
 451
 452         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 453         if (j->disk_fd < 0)
 454                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 455
 456         return 0;
 457 }
 458
 459 static void tar_pull_job_on_progress(PullJob *j) {
 460         TarPull *i;
 461
 462         assert(j);
 463         assert(j->userdata);
 464
 465         i = j->userdata;
 466
 467         tar_pull_report_progress(i, TAR_DOWNLOADING);
 468 }
 469
 470 int tar_pull_start(
 471                 TarPull *i,
 472                 const char *url,
 473                 const char *local,
 474                 bool force_local,
 475                 ImportVerify verify,
 476                 bool settings) {
 477
 478         int r;
 479
 480         assert(i);
 481         assert(verify < _IMPORT_VERIFY_MAX);
 482         assert(verify >= 0);
 483
 484         if (!http_url_is_valid(url))
 485                 return -EINVAL;
 486
 487         if (local && !machine_name_is_valid(local))
 488                 return -EINVAL;
 489
 490         if (i->tar_job)
 491                 return -EBUSY;
 492
 493         r = free_and_strdup(&i->local, local);
 494         if (r < 0)
 495                 return r;
 496
 497         i->force_local = force_local;
 498         i->verify = verify;
 499         i->settings = settings;
 500
 501         /* Set up download job for TAR file */
 502         r = pull_job_new(&i->tar_job, url, i->glue, i);
 503         if (r < 0)
 504                 return r;
 505
 506         i->tar_job->on_finished = tar_pull_job_on_finished;
 507         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 508         i->tar_job->on_progress = tar_pull_job_on_progress;
 509         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 510         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 511
 512         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 513         if (r < 0)
 514                 return r;
 515
 516         /* Set up download job for the settings file (.nspawn) */
 517         if (settings) {
 518                 r = pull_make_settings_job(&i->settings_job, url, i->glue, tar_pull_job_on_finished, i);
 519                 if (r < 0)
 520                         return r;
 521
 522                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 523                 i->settings_job->on_progress = tar_pull_job_on_progress;
 524                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 525
 526                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 527                 if (r < 0)
 528                         return r;
 529         }
 530
 531         /* Set up download of checksum/signature files */
 532         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 533         if (r < 0)
 534                 return r;
 535
 536         r = pull_job_begin(i->tar_job);
 537         if (r < 0)
 538                 return r;
 539
 540         if (i->settings_job) {
 541                 r = pull_job_begin(i->settings_job);
 542                 if (r < 0)
 543                         return r;
 544         }
 545
 546         if (i->checksum_job) {
 547                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 548
 549                 r = pull_job_begin(i->checksum_job);
 550                 if (r < 0)
 551                         return r;
 552         }
 553
 554         if (i->signature_job) {
 555                 i->signature_job->on_progress = tar_pull_job_on_progress;
 556
 557                 r = pull_job_begin(i->signature_job);
 558                 if (r < 0)
 559                         return r;
 560         }
 561
 562         return 0;
 563 }