src/import/pull-tar.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2015 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/prctl.h>
  23 #include <curl/curl.h>
  24
  25 #include "sd-daemon.h"
  26 #include "utf8.h"
  27 #include "strv.h"
  28 #include "copy.h"
  29 #include "btrfs-util.h"
  30 #include "util.h"
  31 #include "macro.h"
  32 #include "mkdir.h"
  33 #include "rm-rf.h"
  34 #include "path-util.h"
  35 #include "process-util.h"
  36 #include "hostname-util.h"
  37 #include "import-util.h"
  38 #include "import-common.h"
  39 #include "curl-util.h"
  40 #include "pull-job.h"
  41 #include "pull-common.h"
  42 #include "pull-tar.h"
  43
  44 typedef enum TarProgress {
  45         TAR_DOWNLOADING,
  46         TAR_VERIFYING,
  47         TAR_FINALIZING,
  48         TAR_COPYING,
  49 } TarProgress;
  50
  51 struct TarPull {
  52         sd_event *event;
  53         CurlGlue *glue;
  54
  55         char *image_root;
  56
  57         PullJob *tar_job;
  58         PullJob *settings_job;
  59         PullJob *checksum_job;
  60         PullJob *signature_job;
  61
  62         TarPullFinished on_finished;
  63         void *userdata;
  64
  65         char *local;
  66         bool force_local;
  67         bool grow_machine_directory;
  68         bool settings;
  69
  70         pid_t tar_pid;
  71
  72         char *final_path;
  73         char *temp_path;
  74
  75         char *settings_path;
  76         char *settings_temp_path;
  77
  78         ImportVerify verify;
  79 };
  80
  81 TarPull* tar_pull_unref(TarPull *i) {
  82         if (!i)
  83                 return NULL;
  84
  85         if (i->tar_pid > 1) {
  86                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  87                 (void) wait_for_terminate(i->tar_pid, NULL);
  88         }
  89
  90         pull_job_unref(i->tar_job);
  91         pull_job_unref(i->settings_job);
  92         pull_job_unref(i->checksum_job);
  93         pull_job_unref(i->signature_job);
  94
  95         curl_glue_unref(i->glue);
  96         sd_event_unref(i->event);
  97
  98         if (i->temp_path) {
  99                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 100                 free(i->temp_path);
 101         }
 102
 103         if (i->settings_temp_path) {
 104                 (void) unlink(i->settings_temp_path);
 105                 free(i->settings_temp_path);
 106         }
 107
 108         free(i->final_path);
 109         free(i->settings_path);
 110         free(i->image_root);
 111         free(i->local);
 112         free(i);
 113
 114         return NULL;
 115 }
 116
 117 int tar_pull_new(
 118                 TarPull **ret,
 119                 sd_event *event,
 120                 const char *image_root,
 121                 TarPullFinished on_finished,
 122                 void *userdata) {
 123
 124         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 125         int r;
 126
 127         assert(ret);
 128
 129         i = new0(TarPull, 1);
 130         if (!i)
 131                 return -ENOMEM;
 132
 133         i->on_finished = on_finished;
 134         i->userdata = userdata;
 135
 136         i->image_root = strdup(image_root ?: "/var/lib/machines");
 137         if (!i->image_root)
 138                 return -ENOMEM;
 139
 140         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 141
 142         if (event)
 143                 i->event = sd_event_ref(event);
 144         else {
 145                 r = sd_event_default(&i->event);
 146                 if (r < 0)
 147                         return r;
 148         }
 149
 150         r = curl_glue_new(&i->glue, i->event);
 151         if (r < 0)
 152                 return r;
 153
 154         i->glue->on_finished = pull_job_curl_on_finished;
 155         i->glue->userdata = i;
 156
 157         *ret = i;
 158         i = NULL;
 159
 160         return 0;
 161 }
 162
 163 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 164         unsigned percent;
 165
 166         assert(i);
 167
 168         switch (p) {
 169
 170         case TAR_DOWNLOADING: {
 171                 unsigned remain = 85;
 172
 173                 percent = 0;
 174
 175                 if (i->settings_job) {
 176                         percent += i->settings_job->progress_percent * 5 / 100;
 177                         remain -= 5;
 178                 }
 179
 180                 if (i->checksum_job) {
 181                         percent += i->checksum_job->progress_percent * 5 / 100;
 182                         remain -= 5;
 183                 }
 184
 185                 if (i->signature_job) {
 186                         percent += i->signature_job->progress_percent * 5 / 100;
 187                         remain -= 5;
 188                 }
 189
 190                 if (i->tar_job)
 191                         percent += i->tar_job->progress_percent * remain / 100;
 192                 break;
 193         }
 194
 195         case TAR_VERIFYING:
 196                 percent = 85;
 197                 break;
 198
 199         case TAR_FINALIZING:
 200                 percent = 90;
 201                 break;
 202
 203         case TAR_COPYING:
 204                 percent = 95;
 205                 break;
 206
 207         default:
 208                 assert_not_reached("Unknown progress state");
 209         }
 210
 211         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 212         log_debug("Combined progress %u%%", percent);
 213 }
 214
 215 static int tar_pull_make_local_copy(TarPull *i) {
 216         int r;
 217
 218         assert(i);
 219         assert(i->tar_job);
 220
 221         if (!i->local)
 222                 return 0;
 223
 224         if (!i->final_path) {
 225                 r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", NULL, &i->final_path);
 226                 if (r < 0)
 227                         return log_oom();
 228         }
 229
 230         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 231         if (r < 0)
 232                 return r;
 233
 234         if (i->settings) {
 235                 const char *local_settings;
 236                 assert(i->settings_job);
 237
 238                 if (!i->settings_path) {
 239                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 240                         if (r < 0)
 241                                 return log_oom();
 242                 }
 243
 244                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 245
 246                 r = copy_file_atomic(i->settings_path, local_settings, 0664, i->force_local, 0);
 247                 if (r == -EEXIST)
 248                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 249                 else if (r < 0 && r != -ENOENT)
 250                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 251                 else
 252                         log_info("Created new settings file '%s.nspawn'", i->local);
 253         }
 254
 255         return 0;
 256 }
 257
 258 static bool tar_pull_is_done(TarPull *i) {
 259         assert(i);
 260         assert(i->tar_job);
 261
 262         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 263                 return false;
 264         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 265                 return false;
 266         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 267                 return false;
 268         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 269                 return false;
 270
 271         return true;
 272 }
 273
 274 static void tar_pull_job_on_finished(PullJob *j) {
 275         TarPull *i;
 276         int r;
 277
 278         assert(j);
 279         assert(j->userdata);
 280
 281         i = j->userdata;
 282
 283         if (j == i->settings_job) {
 284                 if (j->error != 0)
 285                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 286         } else if (j->error != 0) {
 287                 if (j == i->checksum_job)
 288                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 289                 else if (j == i->signature_job)
 290                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 291                 else
 292                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 293
 294                 r = j->error;
 295                 goto finish;
 296         }
 297
 298         /* This is invoked if either the download completed
 299          * successfully, or the download was skipped because we
 300          * already have the etag. */
 301
 302         if (!tar_pull_is_done(i))
 303                 return;
 304
 305         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 306         if (i->settings_job)
 307                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 308
 309         if (i->tar_pid > 0) {
 310                 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
 311                 i->tar_pid = 0;
 312                 if (r < 0)
 313                         goto finish;
 314                 if (r > 0) {
 315                         r = -EIO;
 316                         goto finish;
 317                 }
 318         }
 319
 320         if (!i->tar_job->etag_exists) {
 321                 /* This is a new download, verify it, and move it into place */
 322
 323                 tar_pull_report_progress(i, TAR_VERIFYING);
 324
 325                 r = pull_verify(i->tar_job, i->settings_job, i->checksum_job, i->signature_job);
 326                 if (r < 0)
 327                         goto finish;
 328
 329                 tar_pull_report_progress(i, TAR_FINALIZING);
 330
 331                 r = import_make_read_only(i->temp_path);
 332                 if (r < 0)
 333                         goto finish;
 334
 335                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 336                 if (r < 0) {
 337                         log_error_errno(r, "Failed to rename to final image name: %m");
 338                         goto finish;
 339                 }
 340
 341                 i->temp_path = mfree(i->temp_path);
 342
 343                 if (i->settings_job &&
 344                     i->settings_job->error == 0 &&
 345                     !i->settings_job->etag_exists) {
 346
 347                         assert(i->settings_temp_path);
 348                         assert(i->settings_path);
 349
 350                         /* Also move the settings file into place, if
 351                          * it exist. Note that we do so only if we
 352                          * also moved the tar file in place, to keep
 353                          * things strictly in sync. */
 354
 355                         r = import_make_read_only(i->settings_temp_path);
 356                         if (r < 0)
 357                                 goto finish;
 358
 359                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 360                         if (r < 0) {
 361                                 log_error_errno(r, "Failed to rename settings file: %m");
 362                                 goto finish;
 363                         }
 364
 365                         i->settings_temp_path = mfree(i->settings_temp_path);
 366                 }
 367         }
 368
 369         tar_pull_report_progress(i, TAR_COPYING);
 370
 371         r = tar_pull_make_local_copy(i);
 372         if (r < 0)
 373                 goto finish;
 374
 375         r = 0;
 376
 377 finish:
 378         if (i->on_finished)
 379                 i->on_finished(i, r, i->userdata);
 380         else
 381                 sd_event_exit(i->event, r);
 382 }
 383
 384 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 385         TarPull *i;
 386         int r;
 387
 388         assert(j);
 389         assert(j->userdata);
 390
 391         i = j->userdata;
 392         assert(i->tar_job == j);
 393         assert(!i->final_path);
 394         assert(!i->temp_path);
 395         assert(i->tar_pid <= 0);
 396
 397         r = pull_make_path(j->url, j->etag, i->image_root, ".tar-", NULL, &i->final_path);
 398         if (r < 0)
 399                 return log_oom();
 400
 401         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 402         if (r < 0)
 403                 return log_oom();
 404
 405         mkdir_parents_label(i->temp_path, 0700);
 406
 407         r = btrfs_subvol_make(i->temp_path);
 408         if (r == -ENOTTY) {
 409                 if (mkdir(i->temp_path, 0755) < 0)
 410                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 411         } else if (r < 0)
 412                 return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path);
 413         else
 414                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 415
 416         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 417         if (j->disk_fd < 0)
 418                 return j->disk_fd;
 419
 420         return 0;
 421 }
 422
 423 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 424         TarPull *i;
 425         int r;
 426
 427         assert(j);
 428         assert(j->userdata);
 429
 430         i = j->userdata;
 431         assert(i->settings_job == j);
 432         assert(!i->settings_path);
 433         assert(!i->settings_temp_path);
 434
 435         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 436         if (r < 0)
 437                 return log_oom();
 438
 439         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 440         if (r < 0)
 441                 return log_oom();
 442
 443         mkdir_parents_label(i->settings_temp_path, 0700);
 444
 445         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 446         if (j->disk_fd < 0)
 447                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 448
 449         return 0;
 450 }
 451
 452 static void tar_pull_job_on_progress(PullJob *j) {
 453         TarPull *i;
 454
 455         assert(j);
 456         assert(j->userdata);
 457
 458         i = j->userdata;
 459
 460         tar_pull_report_progress(i, TAR_DOWNLOADING);
 461 }
 462
 463 int tar_pull_start(
 464                 TarPull *i,
 465                 const char *url,
 466                 const char *local,
 467                 bool force_local,
 468                 ImportVerify verify,
 469                 bool settings) {
 470
 471         int r;
 472
 473         assert(i);
 474         assert(verify < _IMPORT_VERIFY_MAX);
 475         assert(verify >= 0);
 476
 477         if (!http_url_is_valid(url))
 478                 return -EINVAL;
 479
 480         if (local && !machine_name_is_valid(local))
 481                 return -EINVAL;
 482
 483         if (i->tar_job)
 484                 return -EBUSY;
 485
 486         r = free_and_strdup(&i->local, local);
 487         if (r < 0)
 488                 return r;
 489
 490         i->force_local = force_local;
 491         i->verify = verify;
 492         i->settings = settings;
 493
 494         /* Set up download job for TAR file */
 495         r = pull_job_new(&i->tar_job, url, i->glue, i);
 496         if (r < 0)
 497                 return r;
 498
 499         i->tar_job->on_finished = tar_pull_job_on_finished;
 500         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 501         i->tar_job->on_progress = tar_pull_job_on_progress;
 502         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 503         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 504
 505         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 506         if (r < 0)
 507                 return r;
 508
 509         /* Set up download job for the settings file (.nspawn) */
 510         if (settings) {
 511                 r = pull_make_settings_job(&i->settings_job, url, i->glue, tar_pull_job_on_finished, i);
 512                 if (r < 0)
 513                         return r;
 514
 515                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 516                 i->settings_job->on_progress = tar_pull_job_on_progress;
 517                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 518
 519                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 520                 if (r < 0)
 521                         return r;
 522         }
 523
 524         /* Set up download of checksum/signature files */
 525         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 526         if (r < 0)
 527                 return r;
 528
 529         r = pull_job_begin(i->tar_job);
 530         if (r < 0)
 531                 return r;
 532
 533         if (i->settings_job) {
 534                 r = pull_job_begin(i->settings_job);
 535                 if (r < 0)
 536                         return r;
 537         }
 538
 539         if (i->checksum_job) {
 540                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 541
 542                 r = pull_job_begin(i->checksum_job);
 543                 if (r < 0)
 544                         return r;
 545         }
 546
 547         if (i->signature_job) {
 548                 i->signature_job->on_progress = tar_pull_job_on_progress;
 549
 550                 r = pull_job_begin(i->signature_job);
 551                 if (r < 0)
 552                         return r;
 553         }
 554
 555         return 0;
 556 }