src/import/pull-tar.c

   1 /***
   2   This file is part of systemd.
   3
   4   Copyright 2015 Lennart Poettering
   5
   6   systemd is free software; you can redistribute it and/or modify it
   7   under the terms of the GNU Lesser General Public License as published by
   8   the Free Software Foundation; either version 2.1 of the License, or
   9   (at your option) any later version.
  10
  11   systemd is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14   Lesser General Public License for more details.
  15
  16   You should have received a copy of the GNU Lesser General Public License
  17   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  18 ***/
  19
  20 #include <curl/curl.h>
  21 #include <sys/prctl.h>
  22
  23 #include "sd-daemon.h"
  24
  25 #include "alloc-util.h"
  26 #include "btrfs-util.h"
  27 #include "copy.h"
  28 #include "curl-util.h"
  29 #include "fd-util.h"
  30 #include "fileio.h"
  31 #include "fs-util.h"
  32 #include "hostname-util.h"
  33 #include "import-common.h"
  34 #include "import-util.h"
  35 #include "macro.h"
  36 #include "mkdir.h"
  37 #include "path-util.h"
  38 #include "process-util.h"
  39 #include "pull-common.h"
  40 #include "pull-job.h"
  41 #include "pull-tar.h"
  42 #include "rm-rf.h"
  43 #include "string-util.h"
  44 #include "strv.h"
  45 #include "utf8.h"
  46 #include "util.h"
  47 #include "web-util.h"
  48
  49 typedef enum TarProgress {
  50         TAR_DOWNLOADING,
  51         TAR_VERIFYING,
  52         TAR_FINALIZING,
  53         TAR_COPYING,
  54 } TarProgress;
  55
  56 struct TarPull {
  57         sd_event *event;
  58         CurlGlue *glue;
  59
  60         char *image_root;
  61
  62         PullJob *tar_job;
  63         PullJob *settings_job;
  64         PullJob *checksum_job;
  65         PullJob *signature_job;
  66
  67         TarPullFinished on_finished;
  68         void *userdata;
  69
  70         char *local;
  71         bool force_local;
  72         bool grow_machine_directory;
  73         bool settings;
  74
  75         pid_t tar_pid;
  76
  77         char *final_path;
  78         char *temp_path;
  79
  80         char *settings_path;
  81         char *settings_temp_path;
  82
  83         ImportVerify verify;
  84 };
  85
  86 TarPull* tar_pull_unref(TarPull *i) {
  87         if (!i)
  88                 return NULL;
  89
  90         if (i->tar_pid > 1) {
  91                 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
  92                 (void) wait_for_terminate(i->tar_pid, NULL);
  93         }
  94
  95         pull_job_unref(i->tar_job);
  96         pull_job_unref(i->settings_job);
  97         pull_job_unref(i->checksum_job);
  98         pull_job_unref(i->signature_job);
  99
 100         curl_glue_unref(i->glue);
 101         sd_event_unref(i->event);
 102
 103         if (i->temp_path) {
 104                 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 105                 free(i->temp_path);
 106         }
 107
 108         if (i->settings_temp_path) {
 109                 (void) unlink(i->settings_temp_path);
 110                 free(i->settings_temp_path);
 111         }
 112
 113         free(i->final_path);
 114         free(i->settings_path);
 115         free(i->image_root);
 116         free(i->local);
 117
 118         return mfree(i);
 119 }
 120
 121 int tar_pull_new(
 122                 TarPull **ret,
 123                 sd_event *event,
 124                 const char *image_root,
 125                 TarPullFinished on_finished,
 126                 void *userdata) {
 127
 128         _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
 129         int r;
 130
 131         assert(ret);
 132
 133         i = new0(TarPull, 1);
 134         if (!i)
 135                 return -ENOMEM;
 136
 137         i->on_finished = on_finished;
 138         i->userdata = userdata;
 139
 140         i->image_root = strdup(image_root ?: "/var/lib/machines");
 141         if (!i->image_root)
 142                 return -ENOMEM;
 143
 144         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 145
 146         if (event)
 147                 i->event = sd_event_ref(event);
 148         else {
 149                 r = sd_event_default(&i->event);
 150                 if (r < 0)
 151                         return r;
 152         }
 153
 154         r = curl_glue_new(&i->glue, i->event);
 155         if (r < 0)
 156                 return r;
 157
 158         i->glue->on_finished = pull_job_curl_on_finished;
 159         i->glue->userdata = i;
 160
 161         *ret = i;
 162         i = NULL;
 163
 164         return 0;
 165 }
 166
 167 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
 168         unsigned percent;
 169
 170         assert(i);
 171
 172         switch (p) {
 173
 174         case TAR_DOWNLOADING: {
 175                 unsigned remain = 85;
 176
 177                 percent = 0;
 178
 179                 if (i->settings_job) {
 180                         percent += i->settings_job->progress_percent * 5 / 100;
 181                         remain -= 5;
 182                 }
 183
 184                 if (i->checksum_job) {
 185                         percent += i->checksum_job->progress_percent * 5 / 100;
 186                         remain -= 5;
 187                 }
 188
 189                 if (i->signature_job) {
 190                         percent += i->signature_job->progress_percent * 5 / 100;
 191                         remain -= 5;
 192                 }
 193
 194                 if (i->tar_job)
 195                         percent += i->tar_job->progress_percent * remain / 100;
 196                 break;
 197         }
 198
 199         case TAR_VERIFYING:
 200                 percent = 85;
 201                 break;
 202
 203         case TAR_FINALIZING:
 204                 percent = 90;
 205                 break;
 206
 207         case TAR_COPYING:
 208                 percent = 95;
 209                 break;
 210
 211         default:
 212                 assert_not_reached("Unknown progress state");
 213         }
 214
 215         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 216         log_debug("Combined progress %u%%", percent);
 217 }
 218
 219 static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) {
 220         int r;
 221
 222         assert(i);
 223         assert(field);
 224
 225         if (*field)
 226                 return 0;
 227
 228         assert(i->tar_job);
 229
 230         r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field);
 231         if (r < 0)
 232                 return log_oom();
 233
 234         return 1;
 235 }
 236
 237 static int tar_pull_make_local_copy(TarPull *i) {
 238         int r;
 239
 240         assert(i);
 241         assert(i->tar_job);
 242
 243         if (!i->local)
 244                 return 0;
 245
 246         r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);
 247         if (r < 0)
 248                 return r;
 249
 250         if (i->settings) {
 251                 const char *local_settings;
 252                 assert(i->settings_job);
 253
 254                 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 255                 if (r < 0)
 256                         return r;
 257
 258                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 259
 260                 r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0));
 261                 if (r == -EEXIST)
 262                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 263                 else if (r == -ENOENT)
 264                         log_debug_errno(r, "Skipping creation of settings file, since none was found.");
 265                 else if (r < 0)
 266                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 267                 else
 268                         log_info("Created new settings file %s.", local_settings);
 269         }
 270
 271         return 0;
 272 }
 273
 274 static bool tar_pull_is_done(TarPull *i) {
 275         assert(i);
 276         assert(i->tar_job);
 277
 278         if (!PULL_JOB_IS_COMPLETE(i->tar_job))
 279                 return false;
 280         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 281                 return false;
 282         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 283                 return false;
 284         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 285                 return false;
 286
 287         return true;
 288 }
 289
 290 static void tar_pull_job_on_finished(PullJob *j) {
 291         TarPull *i;
 292         int r;
 293
 294         assert(j);
 295         assert(j->userdata);
 296
 297         i = j->userdata;
 298
 299         if (j == i->settings_job) {
 300                 if (j->error != 0)
 301                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 302         } else if (j->error != 0 && j != i->signature_job) {
 303                 if (j == i->checksum_job)
 304                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 305                 else
 306                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 307
 308                 r = j->error;
 309                 goto finish;
 310         }
 311
 312         /* This is invoked if either the download completed
 313          * successfully, or the download was skipped because we
 314          * already have the etag. */
 315
 316         if (!tar_pull_is_done(i))
 317                 return;
 318
 319         if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) {
 320                 log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 321
 322                 r = i->signature_job->error;
 323                 goto finish;
 324         }
 325
 326         i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
 327         if (i->settings_job)
 328                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 329
 330         r = tar_pull_determine_path(i, NULL, &i->final_path);
 331         if (r < 0)
 332                 goto finish;
 333
 334         if (i->tar_pid > 0) {
 335                 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
 336                 i->tar_pid = 0;
 337                 if (r < 0)
 338                         goto finish;
 339                 if (r > 0) {
 340                         r = -EIO;
 341                         goto finish;
 342                 }
 343         }
 344
 345         if (!i->tar_job->etag_exists) {
 346                 /* This is a new download, verify it, and move it into place */
 347
 348                 tar_pull_report_progress(i, TAR_VERIFYING);
 349
 350                 r = pull_verify(i->tar_job, NULL, i->settings_job, i->checksum_job, i->signature_job);
 351                 if (r < 0)
 352                         goto finish;
 353
 354                 tar_pull_report_progress(i, TAR_FINALIZING);
 355
 356                 r = import_make_read_only(i->temp_path);
 357                 if (r < 0)
 358                         goto finish;
 359
 360                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 361                 if (r < 0) {
 362                         log_error_errno(r, "Failed to rename to final image name: %m");
 363                         goto finish;
 364                 }
 365
 366                 i->temp_path = mfree(i->temp_path);
 367
 368                 if (i->settings_job &&
 369                     i->settings_job->error == 0) {
 370
 371                         /* Also move the settings file into place, if it exists. Note that we do so only if we also
 372                          * moved the tar file in place, to keep things strictly in sync. */
 373                         assert(i->settings_temp_path);
 374
 375                         /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
 376                          * we should incorporate it in the file name if we can */
 377                         i->settings_path = mfree(i->settings_path);
 378
 379                         r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
 380                         if (r < 0)
 381                                 goto finish;
 382
 383                         r = import_make_read_only(i->settings_temp_path);
 384                         if (r < 0)
 385                                 goto finish;
 386
 387                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 388                         if (r < 0) {
 389                                 log_error_errno(r, "Failed to rename settings file: %m");
 390                                 goto finish;
 391                         }
 392
 393                         i->settings_temp_path = mfree(i->settings_temp_path);
 394                 }
 395         }
 396
 397         tar_pull_report_progress(i, TAR_COPYING);
 398
 399         r = tar_pull_make_local_copy(i);
 400         if (r < 0)
 401                 goto finish;
 402
 403         r = 0;
 404
 405 finish:
 406         if (i->on_finished)
 407                 i->on_finished(i, r, i->userdata);
 408         else
 409                 sd_event_exit(i->event, r);
 410 }
 411
 412 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
 413         TarPull *i;
 414         int r;
 415
 416         assert(j);
 417         assert(j->userdata);
 418
 419         i = j->userdata;
 420         assert(i->tar_job == j);
 421         assert(i->tar_pid <= 0);
 422
 423         if (!i->temp_path) {
 424                 r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
 425                 if (r < 0)
 426                         return log_oom();
 427         }
 428
 429         mkdir_parents_label(i->temp_path, 0700);
 430
 431         r = btrfs_subvol_make(i->temp_path);
 432         if (r == -ENOTTY) {
 433                 if (mkdir(i->temp_path, 0755) < 0)
 434                         return log_error_errno(errno, "Failed to create directory %s: %m", i->temp_path);
 435         } else if (r < 0)
 436                 return log_error_errno(r, "Failed to create subvolume %s: %m", i->temp_path);
 437         else
 438                 (void) import_assign_pool_quota_and_warn(i->temp_path);
 439
 440         j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
 441         if (j->disk_fd < 0)
 442                 return j->disk_fd;
 443
 444         return 0;
 445 }
 446
 447 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
 448         TarPull *i;
 449         int r;
 450
 451         assert(j);
 452         assert(j->userdata);
 453
 454         i = j->userdata;
 455         assert(i->settings_job == j);
 456
 457         if (!i->settings_temp_path) {
 458                 r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path);
 459                 if (r < 0)
 460                         return log_oom();
 461         }
 462
 463         mkdir_parents_label(i->settings_temp_path, 0700);
 464
 465         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 466         if (j->disk_fd < 0)
 467                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 468
 469         return 0;
 470 }
 471
 472 static void tar_pull_job_on_progress(PullJob *j) {
 473         TarPull *i;
 474
 475         assert(j);
 476         assert(j->userdata);
 477
 478         i = j->userdata;
 479
 480         tar_pull_report_progress(i, TAR_DOWNLOADING);
 481 }
 482
 483 int tar_pull_start(
 484                 TarPull *i,
 485                 const char *url,
 486                 const char *local,
 487                 bool force_local,
 488                 ImportVerify verify,
 489                 bool settings) {
 490
 491         int r;
 492
 493         assert(i);
 494         assert(verify < _IMPORT_VERIFY_MAX);
 495         assert(verify >= 0);
 496
 497         if (!http_url_is_valid(url))
 498                 return -EINVAL;
 499
 500         if (local && !machine_name_is_valid(local))
 501                 return -EINVAL;
 502
 503         if (i->tar_job)
 504                 return -EBUSY;
 505
 506         r = free_and_strdup(&i->local, local);
 507         if (r < 0)
 508                 return r;
 509
 510         i->force_local = force_local;
 511         i->verify = verify;
 512         i->settings = settings;
 513
 514         /* Set up download job for TAR file */
 515         r = pull_job_new(&i->tar_job, url, i->glue, i);
 516         if (r < 0)
 517                 return r;
 518
 519         i->tar_job->on_finished = tar_pull_job_on_finished;
 520         i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
 521         i->tar_job->on_progress = tar_pull_job_on_progress;
 522         i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 523         i->tar_job->grow_machine_directory = i->grow_machine_directory;
 524
 525         r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
 526         if (r < 0)
 527                 return r;
 528
 529         /* Set up download job for the settings file (.nspawn) */
 530         if (settings) {
 531                 r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i);
 532                 if (r < 0)
 533                         return r;
 534
 535                 i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
 536                 i->settings_job->on_progress = tar_pull_job_on_progress;
 537                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 538         }
 539
 540         /* Set up download of checksum/signature files */
 541         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
 542         if (r < 0)
 543                 return r;
 544
 545         r = pull_job_begin(i->tar_job);
 546         if (r < 0)
 547                 return r;
 548
 549         if (i->settings_job) {
 550                 r = pull_job_begin(i->settings_job);
 551                 if (r < 0)
 552                         return r;
 553         }
 554
 555         if (i->checksum_job) {
 556                 i->checksum_job->on_progress = tar_pull_job_on_progress;
 557                 i->checksum_job->style = VERIFICATION_PER_FILE;
 558
 559                 r = pull_job_begin(i->checksum_job);
 560                 if (r < 0)
 561                         return r;
 562         }
 563
 564         if (i->signature_job) {
 565                 i->signature_job->on_progress = tar_pull_job_on_progress;
 566
 567                 r = pull_job_begin(i->signature_job);
 568                 if (r < 0)
 569                         return r;
 570         }
 571
 572         return 0;
 573 }