src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <curl/curl.h>
  23 #include <linux/fs.h>
  24 #include <sys/xattr.h>
  25
  26 #include "sd-daemon.h"
  27
  28 #include "alloc-util.h"
  29 #include "btrfs-util.h"
  30 #include "chattr-util.h"
  31 #include "copy.h"
  32 #include "curl-util.h"
  33 #include "fd-util.h"
  34 #include "fileio.h"
  35 #include "fs-util.h"
  36 #include "hostname-util.h"
  37 #include "import-common.h"
  38 #include "import-util.h"
  39 #include "macro.h"
  40 #include "mkdir.h"
  41 #include "path-util.h"
  42 #include "pull-common.h"
  43 #include "pull-job.h"
  44 #include "pull-raw.h"
  45 #include "qcow2-util.h"
  46 #include "rm-rf.h"
  47 #include "string-util.h"
  48 #include "strv.h"
  49 #include "utf8.h"
  50 #include "util.h"
  51 #include "web-util.h"
  52
  53 typedef enum RawProgress {
  54         RAW_DOWNLOADING,
  55         RAW_VERIFYING,
  56         RAW_UNPACKING,
  57         RAW_FINALIZING,
  58         RAW_COPYING,
  59 } RawProgress;
  60
  61 struct RawPull {
  62         sd_event *event;
  63         CurlGlue *glue;
  64
  65         char *image_root;
  66
  67         PullJob *raw_job;
  68         PullJob *settings_job;
  69         PullJob *checksum_job;
  70         PullJob *signature_job;
  71
  72         RawPullFinished on_finished;
  73         void *userdata;
  74
  75         char *local;
  76         bool force_local;
  77         bool grow_machine_directory;
  78         bool settings;
  79
  80         char *final_path;
  81         char *temp_path;
  82
  83         char *settings_path;
  84         char *settings_temp_path;
  85
  86         ImportVerify verify;
  87 };
  88
  89 RawPull* raw_pull_unref(RawPull *i) {
  90         if (!i)
  91                 return NULL;
  92
  93         pull_job_unref(i->raw_job);
  94         pull_job_unref(i->settings_job);
  95         pull_job_unref(i->checksum_job);
  96         pull_job_unref(i->signature_job);
  97
  98         curl_glue_unref(i->glue);
  99         sd_event_unref(i->event);
 100
 101         if (i->temp_path) {
 102                 (void) unlink(i->temp_path);
 103                 free(i->temp_path);
 104         }
 105
 106         if (i->settings_temp_path) {
 107                 (void) unlink(i->settings_temp_path);
 108                 free(i->settings_temp_path);
 109         }
 110
 111         free(i->final_path);
 112         free(i->settings_path);
 113         free(i->image_root);
 114         free(i->local);
 115         free(i);
 116
 117         return NULL;
 118 }
 119
 120 int raw_pull_new(
 121                 RawPull **ret,
 122                 sd_event *event,
 123                 const char *image_root,
 124                 RawPullFinished on_finished,
 125                 void *userdata) {
 126
 127         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 128         int r;
 129
 130         assert(ret);
 131
 132         i = new0(RawPull, 1);
 133         if (!i)
 134                 return -ENOMEM;
 135
 136         i->on_finished = on_finished;
 137         i->userdata = userdata;
 138
 139         i->image_root = strdup(image_root ?: "/var/lib/machines");
 140         if (!i->image_root)
 141                 return -ENOMEM;
 142
 143         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 144
 145         if (event)
 146                 i->event = sd_event_ref(event);
 147         else {
 148                 r = sd_event_default(&i->event);
 149                 if (r < 0)
 150                         return r;
 151         }
 152
 153         r = curl_glue_new(&i->glue, i->event);
 154         if (r < 0)
 155                 return r;
 156
 157         i->glue->on_finished = pull_job_curl_on_finished;
 158         i->glue->userdata = i;
 159
 160         *ret = i;
 161         i = NULL;
 162
 163         return 0;
 164 }
 165
 166 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 167         unsigned percent;
 168
 169         assert(i);
 170
 171         switch (p) {
 172
 173         case RAW_DOWNLOADING: {
 174                 unsigned remain = 80;
 175
 176                 percent = 0;
 177
 178                 if (i->settings_job) {
 179                         percent += i->settings_job->progress_percent * 5 / 100;
 180                         remain -= 5;
 181                 }
 182
 183                 if (i->checksum_job) {
 184                         percent += i->checksum_job->progress_percent * 5 / 100;
 185                         remain -= 5;
 186                 }
 187
 188                 if (i->signature_job) {
 189                         percent += i->signature_job->progress_percent * 5 / 100;
 190                         remain -= 5;
 191                 }
 192
 193                 if (i->raw_job)
 194                         percent += i->raw_job->progress_percent * remain / 100;
 195                 break;
 196         }
 197
 198         case RAW_VERIFYING:
 199                 percent = 80;
 200                 break;
 201
 202         case RAW_UNPACKING:
 203                 percent = 85;
 204                 break;
 205
 206         case RAW_FINALIZING:
 207                 percent = 90;
 208                 break;
 209
 210         case RAW_COPYING:
 211                 percent = 95;
 212                 break;
 213
 214         default:
 215                 assert_not_reached("Unknown progress state");
 216         }
 217
 218         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 219         log_debug("Combined progress %u%%", percent);
 220 }
 221
 222 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 223         _cleanup_close_ int converted_fd = -1;
 224         _cleanup_free_ char *t = NULL;
 225         int r;
 226
 227         assert(i);
 228         assert(i->raw_job);
 229
 230         r = qcow2_detect(i->raw_job->disk_fd);
 231         if (r < 0)
 232                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 233         if (r == 0)
 234                 return 0;
 235
 236         /* This is a QCOW2 image, let's convert it */
 237         r = tempfn_random(i->final_path, NULL, &t);
 238         if (r < 0)
 239                 return log_oom();
 240
 241         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 242         if (converted_fd < 0)
 243                 return log_error_errno(errno, "Failed to create %s: %m", t);
 244
 245         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 246         if (r < 0)
 247                 log_warning_errno(r, "Failed to set file attributes on %s: %m", t);
 248
 249         log_info("Unpacking QCOW2 file.");
 250
 251         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 252         if (r < 0) {
 253                 unlink(t);
 254                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 255         }
 256
 257         (void) unlink(i->temp_path);
 258         free(i->temp_path);
 259         i->temp_path = t;
 260         t = NULL;
 261
 262         safe_close(i->raw_job->disk_fd);
 263         i->raw_job->disk_fd = converted_fd;
 264         converted_fd = -1;
 265
 266         return 1;
 267 }
 268
 269 static int raw_pull_make_local_copy(RawPull *i) {
 270         _cleanup_free_ char *tp = NULL;
 271         _cleanup_close_ int dfd = -1;
 272         const char *p;
 273         int r;
 274
 275         assert(i);
 276         assert(i->raw_job);
 277
 278         if (!i->local)
 279                 return 0;
 280
 281         if (!i->final_path) {
 282                 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 283                 if (r < 0)
 284                         return log_oom();
 285         }
 286
 287         if (i->raw_job->etag_exists) {
 288                 /* We have downloaded this one previously, reopen it */
 289
 290                 assert(i->raw_job->disk_fd < 0);
 291
 292                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 293                 if (i->raw_job->disk_fd < 0)
 294                         return log_error_errno(errno, "Failed to open vendor image: %m");
 295         } else {
 296                 /* We freshly downloaded the image, use it */
 297
 298                 assert(i->raw_job->disk_fd >= 0);
 299
 300                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 301                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 302         }
 303
 304         p = strjoina(i->image_root, "/", i->local, ".raw");
 305
 306         if (i->force_local)
 307                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 308
 309         r = tempfn_random(p, NULL, &tp);
 310         if (r < 0)
 311                 return log_oom();
 312
 313         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 314         if (dfd < 0)
 315                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 316
 317         /* Turn off COW writing. This should greatly improve
 318          * performance on COW file systems like btrfs, since it
 319          * reduces fragmentation caused by not allowing in-place
 320          * writes. */
 321         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 322         if (r < 0)
 323                 log_warning_errno(r, "Failed to set file attributes on %s: %m", tp);
 324
 325         r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, true);
 326         if (r < 0) {
 327                 unlink(tp);
 328                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 329         }
 330
 331         (void) copy_times(i->raw_job->disk_fd, dfd);
 332         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 333
 334         dfd = safe_close(dfd);
 335
 336         r = rename(tp, p);
 337         if (r < 0)  {
 338                 r = log_error_errno(errno, "Failed to move writable image into place: %m");
 339                 unlink(tp);
 340                 return r;
 341         }
 342
 343         log_info("Created new local image '%s'.", i->local);
 344
 345         if (i->settings) {
 346                 const char *local_settings;
 347                 assert(i->settings_job);
 348
 349                 if (!i->settings_path) {
 350                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 351                         if (r < 0)
 352                                 return log_oom();
 353                 }
 354
 355                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 356
 357                 r = copy_file_atomic(i->settings_path, local_settings, 0644, i->force_local, 0);
 358                 if (r == -EEXIST)
 359                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 360                 else if (r < 0 && r != -ENOENT)
 361                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 362                 else
 363                         log_info("Created new settings file '%s.nspawn'", i->local);
 364         }
 365
 366         return 0;
 367 }
 368
 369 static bool raw_pull_is_done(RawPull *i) {
 370         assert(i);
 371         assert(i->raw_job);
 372
 373         if (!PULL_JOB_IS_COMPLETE(i->raw_job))
 374                 return false;
 375         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 376                 return false;
 377         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 378                 return false;
 379         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 380                 return false;
 381
 382         return true;
 383 }
 384
 385 static void raw_pull_job_on_finished(PullJob *j) {
 386         RawPull *i;
 387         int r;
 388
 389         assert(j);
 390         assert(j->userdata);
 391
 392         i = j->userdata;
 393         if (j == i->settings_job) {
 394                 if (j->error != 0)
 395                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 396         } else if (j->error != 0) {
 397                 if (j == i->checksum_job)
 398                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 399                 else if (j == i->signature_job)
 400                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 401                 else
 402                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 403
 404                 r = j->error;
 405                 goto finish;
 406         }
 407
 408         /* This is invoked if either the download completed
 409          * successfully, or the download was skipped because we
 410          * already have the etag. In this case ->etag_exists is
 411          * true.
 412          *
 413          * We only do something when we got all three files */
 414
 415         if (!raw_pull_is_done(i))
 416                 return;
 417
 418         if (i->settings_job)
 419                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 420
 421         if (!i->raw_job->etag_exists) {
 422                 /* This is a new download, verify it, and move it into place */
 423                 assert(i->raw_job->disk_fd >= 0);
 424
 425                 raw_pull_report_progress(i, RAW_VERIFYING);
 426
 427                 r = pull_verify(i->raw_job, i->settings_job, i->checksum_job, i->signature_job);
 428                 if (r < 0)
 429                         goto finish;
 430
 431                 raw_pull_report_progress(i, RAW_UNPACKING);
 432
 433                 r = raw_pull_maybe_convert_qcow2(i);
 434                 if (r < 0)
 435                         goto finish;
 436
 437                 raw_pull_report_progress(i, RAW_FINALIZING);
 438
 439                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 440                 if (r < 0)
 441                         goto finish;
 442
 443                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 444                 if (r < 0) {
 445                         log_error_errno(r, "Failed to move RAW file into place: %m");
 446                         goto finish;
 447                 }
 448
 449                 i->temp_path = mfree(i->temp_path);
 450
 451                 if (i->settings_job &&
 452                     i->settings_job->error == 0 &&
 453                     !i->settings_job->etag_exists) {
 454
 455                         assert(i->settings_temp_path);
 456                         assert(i->settings_path);
 457
 458                         r = import_make_read_only(i->settings_temp_path);
 459                         if (r < 0)
 460                                 goto finish;
 461
 462                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 463                         if (r < 0) {
 464                                 log_error_errno(r, "Failed to rename settings file: %m");
 465                                 goto finish;
 466                         }
 467
 468                         i->settings_temp_path = mfree(i->settings_temp_path);
 469                 }
 470         }
 471
 472         raw_pull_report_progress(i, RAW_COPYING);
 473
 474         r = raw_pull_make_local_copy(i);
 475         if (r < 0)
 476                 goto finish;
 477
 478         r = 0;
 479
 480 finish:
 481         if (i->on_finished)
 482                 i->on_finished(i, r, i->userdata);
 483         else
 484                 sd_event_exit(i->event, r);
 485 }
 486
 487 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 488         RawPull *i;
 489         int r;
 490
 491         assert(j);
 492         assert(j->userdata);
 493
 494         i = j->userdata;
 495         assert(i->raw_job == j);
 496         assert(!i->final_path);
 497         assert(!i->temp_path);
 498
 499         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 500         if (r < 0)
 501                 return log_oom();
 502
 503         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 504         if (r < 0)
 505                 return log_oom();
 506
 507         (void) mkdir_parents_label(i->temp_path, 0700);
 508
 509         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 510         if (j->disk_fd < 0)
 511                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 512
 513         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 514         if (r < 0)
 515                 log_warning_errno(r, "Failed to set file attributes on %s: %m", i->temp_path);
 516
 517         return 0;
 518 }
 519
 520 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
 521         RawPull *i;
 522         int r;
 523
 524         assert(j);
 525         assert(j->userdata);
 526
 527         i = j->userdata;
 528         assert(i->settings_job == j);
 529         assert(!i->settings_path);
 530         assert(!i->settings_temp_path);
 531
 532         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 533         if (r < 0)
 534                 return log_oom();
 535
 536         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 537         if (r < 0)
 538                 return log_oom();
 539
 540         mkdir_parents_label(i->settings_temp_path, 0700);
 541
 542         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 543         if (j->disk_fd < 0)
 544                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 545
 546         return 0;
 547 }
 548
 549 static void raw_pull_job_on_progress(PullJob *j) {
 550         RawPull *i;
 551
 552         assert(j);
 553         assert(j->userdata);
 554
 555         i = j->userdata;
 556
 557         raw_pull_report_progress(i, RAW_DOWNLOADING);
 558 }
 559
 560 int raw_pull_start(
 561                 RawPull *i,
 562                 const char *url,
 563                 const char *local,
 564                 bool force_local,
 565                 ImportVerify verify,
 566                 bool settings) {
 567
 568         int r;
 569
 570         assert(i);
 571         assert(verify < _IMPORT_VERIFY_MAX);
 572         assert(verify >= 0);
 573
 574         if (!http_url_is_valid(url))
 575                 return -EINVAL;
 576
 577         if (local && !machine_name_is_valid(local))
 578                 return -EINVAL;
 579
 580         if (i->raw_job)
 581                 return -EBUSY;
 582
 583         r = free_and_strdup(&i->local, local);
 584         if (r < 0)
 585                 return r;
 586
 587         i->force_local = force_local;
 588         i->verify = verify;
 589         i->settings = settings;
 590
 591         /* Queue job for the image itself */
 592         r = pull_job_new(&i->raw_job, url, i->glue, i);
 593         if (r < 0)
 594                 return r;
 595
 596         i->raw_job->on_finished = raw_pull_job_on_finished;
 597         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 598         i->raw_job->on_progress = raw_pull_job_on_progress;
 599         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 600         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 601
 602         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 603         if (r < 0)
 604                 return r;
 605
 606         if (settings) {
 607                 r = pull_make_settings_job(&i->settings_job, url, i->glue, raw_pull_job_on_finished, i);
 608                 if (r < 0)
 609                         return r;
 610
 611                 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
 612                 i->settings_job->on_progress = raw_pull_job_on_progress;
 613                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 614
 615                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 616                 if (r < 0)
 617                         return r;
 618         }
 619
 620         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 621         if (r < 0)
 622                 return r;
 623
 624         r = pull_job_begin(i->raw_job);
 625         if (r < 0)
 626                 return r;
 627
 628         if (i->settings_job) {
 629                 r = pull_job_begin(i->settings_job);
 630                 if (r < 0)
 631                         return r;
 632         }
 633
 634         if (i->checksum_job) {
 635                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 636
 637                 r = pull_job_begin(i->checksum_job);
 638                 if (r < 0)
 639                         return r;
 640         }
 641
 642         if (i->signature_job) {
 643                 i->signature_job->on_progress = raw_pull_job_on_progress;
 644
 645                 r = pull_job_begin(i->signature_job);
 646                 if (r < 0)
 647                         return r;
 648         }
 649
 650         return 0;
 651 }