src/import/pull-tar.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2015 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/prctl.h>
  23 #include <curl/curl.h>
  24
  25 #include "sd-daemon.h"
  26
  27 #include "btrfs-util.h"
  28 #include "copy.h"
  29 #include "curl-util.h"
  30 #include "fd-util.h"
  31 #include "fileio.h"
  32 #include "fs-util.h"
  33 #include "hostname-util.h"
  34 #include "import-common.h"
  35 #include "import-util.h"
  36 #include "macro.h"
  37 #include "mkdir.h"
  38 #include "path-util.h"
  39 #include "process-util.h"
  40 #include "pull-common.h"
  41 #include "pull-job.h"
  42 #include "pull-tar.h"
  43 #include "rm-rf.h"
  44 #include "string-util.h"
  45 #include "strv.h"
  46 #include "utf8.h"
  47 #include "util.h"
  48
  49 typedef enum TarProgress {
  50         TAR_DOWNLOADING,
  51         TAR_VERIFYING,
  52         TAR_FINALIZING,
  53         TAR_COPYING,
  54 } TarProgress;
  55
  56 struct TarPull {
  57         sd_event *event;
  58         CurlGlue *glue;
  59
  60         char *image_root;
  61
  62         PullJob *tar_job;
  63         PullJob *settings_job;
  64         PullJob *checksum_job;
  65         PullJob *signature_job;
  66
  67         TarPullFinished on_finished;
  68         void *userdata;
  69
  70         char *local;
  71         bool force_local;
  72         bool grow_machine_directory;
  73         bool settings;
  74
  75         pid_t tar_pid;
  76
  77         char *final_path;
  78         char *temp_path;
  79
  80         char *settings_path;
  81         char *settings_temp_path;
  82
  83         ImportVerify verify;
  84 };
  85
  86 TarPull* tar_pull_unref(TarPull *i) {
  87         if (!i)
  88                 return NULL;
  89
  90         if (i->tar_pid > 1) {
  91                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  92                 (void) wait_for_terminate(i->tar_pid, NULL);
  93         }
  94
  95         pull_job_unref(i->tar_job);
  96         pull_job_unref(i->settings_job);
  97         pull_job_unref(i->checksum_job);
  98         pull_job_unref(i->signature_job);
  99
 100         curl_glue_unref(i->glue);
 101         sd_event_unref(i->event);
 102
 103         if (i->temp_path) {
 104                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 105                 free(i->temp_path);
 106         }
 107
 108         if (i->settings_temp_path) {
 109                 (void) unlink(i->settings_temp_path);
 110                 free(i->settings_temp_path);
 111         }
 112
 113         free(i->final_path);
 114         free(i->settings_path);
 115         free(i->image_root);
 116         free(i->local);
 117         free(i);
 118
 119         return NULL;
 120 }
 121
 122 int tar_pull_new(
 123                 TarPull **ret,
 124                 sd_event *event,
 125                 const char *image_root,
 126                 TarPullFinished on_finished,
 127                 void *userdata) {
 128
 129         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 130         int r;
 131
 132         assert(ret);
 133
 134         i = new0(TarPull, 1);
 135         if (!i)
 136                 return -ENOMEM;
 137
 138         i->on_finished = on_finished;
 139         i->userdata = userdata;
 140
 141         i->image_root = strdup(image_root ?: "/var/lib/machines");
 142         if (!i->image_root)
 143                 return -ENOMEM;
 144
 145         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 146
 147         if (event)
 148                 i->event = sd_event_ref(event);
 149         else {
 150                 r = sd_event_default(&i->event);
 151                 if (r < 0)
 152                         return r;
 153         }
 154
 155         r = curl_glue_new(&i->glue, i->event);
 156         if (r < 0)
 157                 return r;
 158
 159         i->glue->on_finished = pull_job_curl_on_finished;
 160         i->glue->userdata = i;
 161
 162         *ret = i;
 163         i = NULL;
 164
 165         return 0;
 166 }
 167
 168 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 169         unsigned percent;
 170
 171         assert(i);
 172
 173         switch (p) {
 174
 175         case TAR_DOWNLOADING: {
 176                 unsigned remain = 85;
 177
 178                 percent = 0;
 179
 180                 if (i->settings_job) {
 181                         percent += i->settings_job->progress_percent * 5 / 100;
 182                         remain -= 5;
 183                 }
 184
 185                 if (i->checksum_job) {
 186                         percent += i->checksum_job->progress_percent * 5 / 100;
 187                         remain -= 5;
 188                 }
 189
 190                 if (i->signature_job) {
 191                         percent += i->signature_job->progress_percent * 5 / 100;
 192                         remain -= 5;
 193                 }
 194
 195                 if (i->tar_job)
 196                         percent += i->tar_job->progress_percent * remain / 100;
 197                 break;
 198         }
 199
 200         case TAR_VERIFYING:
 201                 percent = 85;
 202                 break;
 203
 204         case TAR_FINALIZING:
 205                 percent = 90;
 206                 break;
 207
 208         case TAR_COPYING:
 209                 percent = 95;
 210                 break;
 211
 212         default:
 213                 assert_not_reached("Unknown progress state");
 214         }
 215
 216         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 217         log_debug("Combined progress %u%%", percent);
 218 }
 219
 220 static int tar_pull_make_local_copy(TarPull *i) {
 221         int r;
 222
 223         assert(i);
 224         assert(i->tar_job);
 225
 226         if (!i->local)
 227                 return 0;
 228
 229         if (!i->final_path) {
 230                 r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", NULL, &i->final_path);
 231                 if (r < 0)
 232                         return log_oom();
 233         }
 234
 235         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 236         if (r < 0)
 237                 return r;
 238
 239         if (i->settings) {
 240                 const char *local_settings;
 241                 assert(i->settings_job);
 242
 243                 if (!i->settings_path) {
 244                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 245                         if (r < 0)
 246                                 return log_oom();
 247                 }
 248
 249                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 250
 251                 r = copy_file_atomic(i->settings_path, local_settings, 0664, i->force_local, 0);
 252                 if (r == -EEXIST)
 253                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 254                 else if (r < 0 && r != -ENOENT)
 255                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 256                 else
 257                         log_info("Created new settings file '%s.nspawn'", i->local);
 258         }
 259
 260         return 0;
 261 }
 262
 263 static bool tar_pull_is_done(TarPull *i) {
 264         assert(i);
 265         assert(i->tar_job);
 266
 267         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 268                 return false;
 269         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 270                 return false;
 271         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 272                 return false;
 273         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 274                 return false;
 275
 276         return true;
 277 }
 278
 279 static void tar_pull_job_on_finished(PullJob *j) {
 280         TarPull *i;
 281         int r;
 282
 283         assert(j);
 284         assert(j->userdata);
 285
 286         i = j->userdata;
 287
 288         if (j == i->settings_job) {
 289                 if (j->error != 0)
 290                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 291         } else if (j->error != 0) {
 292                 if (j == i->checksum_job)
 293                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 294                 else if (j == i->signature_job)
 295                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 296                 else
 297                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 298
 299                 r = j->error;
 300                 goto finish;
 301         }
 302
 303         /* This is invoked if either the download completed
 304          * successfully, or the download was skipped because we
 305          * already have the etag. */
 306
 307         if (!tar_pull_is_done(i))
 308                 return;
 309
 310         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 311         if (i->settings_job)
 312                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 313
 314         if (i->tar_pid > 0) {
 315                 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
 316                 i->tar_pid = 0;
 317                 if (r < 0)
 318                         goto finish;
 319                 if (r > 0) {
 320                         r = -EIO;
 321                         goto finish;
 322                 }
 323         }
 324
 325         if (!i->tar_job->etag_exists) {
 326                 /* This is a new download, verify it, and move it into place */
 327
 328                 tar_pull_report_progress(i, TAR_VERIFYING);
 329
 330                 r = pull_verify(i->tar_job, i->settings_job, i->checksum_job, i->signature_job);
 331                 if (r < 0)
 332                         goto finish;
 333
 334                 tar_pull_report_progress(i, TAR_FINALIZING);
 335
 336                 r = import_make_read_only(i->temp_path);
 337                 if (r < 0)
 338                         goto finish;
 339
 340                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 341                 if (r < 0) {
 342                         log_error_errno(r, "Failed to rename to final image name: %m");
 343                         goto finish;
 344                 }
 345
 346                 i->temp_path = mfree(i->temp_path);
 347
 348                 if (i->settings_job &&
 349                     i->settings_job->error == 0 &&
 350                     !i->settings_job->etag_exists) {
 351
 352                         assert(i->settings_temp_path);
 353                         assert(i->settings_path);
 354
 355                         /* Also move the settings file into place, if
 356                          * it exist. Note that we do so only if we
 357                          * also moved the tar file in place, to keep
 358                          * things strictly in sync. */
 359
 360                         r = import_make_read_only(i->settings_temp_path);
 361                         if (r < 0)
 362                                 goto finish;
 363
 364                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 365                         if (r < 0) {
 366                                 log_error_errno(r, "Failed to rename settings file: %m");
 367                                 goto finish;
 368                         }
 369
 370                         i->settings_temp_path = mfree(i->settings_temp_path);
 371                 }
 372         }
 373
 374         tar_pull_report_progress(i, TAR_COPYING);
 375
 376         r = tar_pull_make_local_copy(i);
 377         if (r < 0)
 378                 goto finish;
 379
 380         r = 0;
 381
 382 finish:
 383         if (i->on_finished)
 384                 i->on_finished(i, r, i->userdata);
 385         else
 386                 sd_event_exit(i->event, r);
 387 }
 388
 389 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 390         TarPull *i;
 391         int r;
 392
 393         assert(j);
 394         assert(j->userdata);
 395
 396         i = j->userdata;
 397         assert(i->tar_job == j);
 398         assert(!i->final_path);
 399         assert(!i->temp_path);
 400         assert(i->tar_pid <= 0);
 401
 402         r = pull_make_path(j->url, j->etag, i->image_root, ".tar-", NULL, &i->final_path);
 403         if (r < 0)
 404                 return log_oom();
 405
 406         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 407         if (r < 0)
 408                 return log_oom();
 409
 410         mkdir_parents_label(i->temp_path, 0700);
 411
 412         r = btrfs_subvol_make(i->temp_path);
 413         if (r == -ENOTTY) {
 414                 if (mkdir(i->temp_path, 0755) < 0)
 415                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 416         } else if (r < 0)
 417                 return log_error_errno(errno, "Failed to create subvolume %s: %m", i->temp_path);
 418         else
 419                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 420
 421         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 422         if (j->disk_fd < 0)
 423                 return j->disk_fd;
 424
 425         return 0;
 426 }
 427
 428 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 429         TarPull *i;
 430         int r;
 431
 432         assert(j);
 433         assert(j->userdata);
 434
 435         i = j->userdata;
 436         assert(i->settings_job == j);
 437         assert(!i->settings_path);
 438         assert(!i->settings_temp_path);
 439
 440         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 441         if (r < 0)
 442                 return log_oom();
 443
 444         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 445         if (r < 0)
 446                 return log_oom();
 447
 448         mkdir_parents_label(i->settings_temp_path, 0700);
 449
 450         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 451         if (j->disk_fd < 0)
 452                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 453
 454         return 0;
 455 }
 456
 457 static void tar_pull_job_on_progress(PullJob *j) {
 458         TarPull *i;
 459
 460         assert(j);
 461         assert(j->userdata);
 462
 463         i = j->userdata;
 464
 465         tar_pull_report_progress(i, TAR_DOWNLOADING);
 466 }
 467
 468 int tar_pull_start(
 469                 TarPull *i,
 470                 const char *url,
 471                 const char *local,
 472                 bool force_local,
 473                 ImportVerify verify,
 474                 bool settings) {
 475
 476         int r;
 477
 478         assert(i);
 479         assert(verify < _IMPORT_VERIFY_MAX);
 480         assert(verify >= 0);
 481
 482         if (!http_url_is_valid(url))
 483                 return -EINVAL;
 484
 485         if (local && !machine_name_is_valid(local))
 486                 return -EINVAL;
 487
 488         if (i->tar_job)
 489                 return -EBUSY;
 490
 491         r = free_and_strdup(&i->local, local);
 492         if (r < 0)
 493                 return r;
 494
 495         i->force_local = force_local;
 496         i->verify = verify;
 497         i->settings = settings;
 498
 499         /* Set up download job for TAR file */
 500         r = pull_job_new(&i->tar_job, url, i->glue, i);
 501         if (r < 0)
 502                 return r;
 503
 504         i->tar_job->on_finished = tar_pull_job_on_finished;
 505         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 506         i->tar_job->on_progress = tar_pull_job_on_progress;
 507         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 508         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 509
 510         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 511         if (r < 0)
 512                 return r;
 513
 514         /* Set up download job for the settings file (.nspawn) */
 515         if (settings) {
 516                 r = pull_make_settings_job(&i->settings_job, url, i->glue, tar_pull_job_on_finished, i);
 517                 if (r < 0)
 518                         return r;
 519
 520                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 521                 i->settings_job->on_progress = tar_pull_job_on_progress;
 522                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 523
 524                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 525                 if (r < 0)
 526                         return r;
 527         }
 528
 529         /* Set up download of checksum/signature files */
 530         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 531         if (r < 0)
 532                 return r;
 533
 534         r = pull_job_begin(i->tar_job);
 535         if (r < 0)
 536                 return r;
 537
 538         if (i->settings_job) {
 539                 r = pull_job_begin(i->settings_job);
 540                 if (r < 0)
 541                         return r;
 542         }
 543
 544         if (i->checksum_job) {
 545                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 546
 547                 r = pull_job_begin(i->checksum_job);
 548                 if (r < 0)
 549                         return r;
 550         }
 551
 552         if (i->signature_job) {
 553                 i->signature_job->on_progress = tar_pull_job_on_progress;
 554
 555                 r = pull_job_begin(i->signature_job);
 556                 if (r < 0)
 557                         return r;
 558         }
 559
 560         return 0;
 561 }