src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25
  26 #include "sd-daemon.h"
  27 #include "utf8.h"
  28 #include "strv.h"
  29 #include "copy.h"
  30 #include "btrfs-util.h"
  31 #include "util.h"
  32 #include "macro.h"
  33 #include "mkdir.h"
  34 #include "rm-rf.h"
  35 #include "path-util.h"
  36 #include "hostname-util.h"
  37 #include "import-util.h"
  38 #include "import-common.h"
  39 #include "curl-util.h"
  40 #include "qcow2-util.h"
  41 #include "pull-job.h"
  42 #include "pull-common.h"
  43 #include "pull-raw.h"
  44
  45 typedef enum RawProgress {
  46         RAW_DOWNLOADING,
  47         RAW_VERIFYING,
  48         RAW_UNPACKING,
  49         RAW_FINALIZING,
  50         RAW_COPYING,
  51 } RawProgress;
  52
  53 struct RawPull {
  54         sd_event *event;
  55         CurlGlue *glue;
  56
  57         char *image_root;
  58
  59         PullJob *raw_job;
  60         PullJob *settings_job;
  61         PullJob *checksum_job;
  62         PullJob *signature_job;
  63
  64         RawPullFinished on_finished;
  65         void *userdata;
  66
  67         char *local;
  68         bool force_local;
  69         bool grow_machine_directory;
  70         bool settings;
  71
  72         char *final_path;
  73         char *temp_path;
  74
  75         char *settings_path;
  76         char *settings_temp_path;
  77
  78         ImportVerify verify;
  79 };
  80
  81 RawPull* raw_pull_unref(RawPull *i) {
  82         if (!i)
  83                 return NULL;
  84
  85         pull_job_unref(i->raw_job);
  86         pull_job_unref(i->settings_job);
  87         pull_job_unref(i->checksum_job);
  88         pull_job_unref(i->signature_job);
  89
  90         curl_glue_unref(i->glue);
  91         sd_event_unref(i->event);
  92
  93         if (i->temp_path) {
  94                 (void) unlink(i->temp_path);
  95                 free(i->temp_path);
  96         }
  97
  98         if (i->settings_temp_path) {
  99                 (void) unlink(i->settings_temp_path);
 100                 free(i->settings_temp_path);
 101         }
 102
 103         free(i->final_path);
 104         free(i->settings_path);
 105         free(i->image_root);
 106         free(i->local);
 107         free(i);
 108
 109         return NULL;
 110 }
 111
 112 int raw_pull_new(
 113                 RawPull **ret,
 114                 sd_event *event,
 115                 const char *image_root,
 116                 RawPullFinished on_finished,
 117                 void *userdata) {
 118
 119         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 120         int r;
 121
 122         assert(ret);
 123
 124         i = new0(RawPull, 1);
 125         if (!i)
 126                 return -ENOMEM;
 127
 128         i->on_finished = on_finished;
 129         i->userdata = userdata;
 130
 131         i->image_root = strdup(image_root ?: "/var/lib/machines");
 132         if (!i->image_root)
 133                 return -ENOMEM;
 134
 135         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 136
 137         if (event)
 138                 i->event = sd_event_ref(event);
 139         else {
 140                 r = sd_event_default(&i->event);
 141                 if (r < 0)
 142                         return r;
 143         }
 144
 145         r = curl_glue_new(&i->glue, i->event);
 146         if (r < 0)
 147                 return r;
 148
 149         i->glue->on_finished = pull_job_curl_on_finished;
 150         i->glue->userdata = i;
 151
 152         *ret = i;
 153         i = NULL;
 154
 155         return 0;
 156 }
 157
 158 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 159         unsigned percent;
 160
 161         assert(i);
 162
 163         switch (p) {
 164
 165         case RAW_DOWNLOADING: {
 166                 unsigned remain = 80;
 167
 168                 percent = 0;
 169
 170                 if (i->settings_job) {
 171                         percent += i->settings_job->progress_percent * 5 / 100;
 172                         remain -= 5;
 173                 }
 174
 175                 if (i->checksum_job) {
 176                         percent += i->checksum_job->progress_percent * 5 / 100;
 177                         remain -= 5;
 178                 }
 179
 180                 if (i->signature_job) {
 181                         percent += i->signature_job->progress_percent * 5 / 100;
 182                         remain -= 5;
 183                 }
 184
 185                 if (i->raw_job)
 186                         percent += i->raw_job->progress_percent * remain / 100;
 187                 break;
 188         }
 189
 190         case RAW_VERIFYING:
 191                 percent = 80;
 192                 break;
 193
 194         case RAW_UNPACKING:
 195                 percent = 85;
 196                 break;
 197
 198         case RAW_FINALIZING:
 199                 percent = 90;
 200                 break;
 201
 202         case RAW_COPYING:
 203                 percent = 95;
 204                 break;
 205
 206         default:
 207                 assert_not_reached("Unknown progress state");
 208         }
 209
 210         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 211         log_debug("Combined progress %u%%", percent);
 212 }
 213
 214 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 215         _cleanup_close_ int converted_fd = -1;
 216         _cleanup_free_ char *t = NULL;
 217         int r;
 218
 219         assert(i);
 220         assert(i->raw_job);
 221
 222         r = qcow2_detect(i->raw_job->disk_fd);
 223         if (r < 0)
 224                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 225         if (r == 0)
 226                 return 0;
 227
 228         /* This is a QCOW2 image, let's convert it */
 229         r = tempfn_random(i->final_path, NULL, &t);
 230         if (r < 0)
 231                 return log_oom();
 232
 233         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 234         if (converted_fd < 0)
 235                 return log_error_errno(errno, "Failed to create %s: %m", t);
 236
 237         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 238         if (r < 0)
 239                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 240
 241         log_info("Unpacking QCOW2 file.");
 242
 243         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 244         if (r < 0) {
 245                 unlink(t);
 246                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 247         }
 248
 249         (void) unlink(i->temp_path);
 250         free(i->temp_path);
 251         i->temp_path = t;
 252         t = NULL;
 253
 254         safe_close(i->raw_job->disk_fd);
 255         i->raw_job->disk_fd = converted_fd;
 256         converted_fd = -1;
 257
 258         return 1;
 259 }
 260
 261 static int raw_pull_make_local_copy(RawPull *i) {
 262         _cleanup_free_ char *tp = NULL;
 263         _cleanup_close_ int dfd = -1;
 264         const char *p;
 265         int r;
 266
 267         assert(i);
 268         assert(i->raw_job);
 269
 270         if (!i->local)
 271                 return 0;
 272
 273         if (!i->final_path) {
 274                 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 275                 if (r < 0)
 276                         return log_oom();
 277         }
 278
 279         if (i->raw_job->etag_exists) {
 280                 /* We have downloaded this one previously, reopen it */
 281
 282                 assert(i->raw_job->disk_fd < 0);
 283
 284                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 285                 if (i->raw_job->disk_fd < 0)
 286                         return log_error_errno(errno, "Failed to open vendor image: %m");
 287         } else {
 288                 /* We freshly downloaded the image, use it */
 289
 290                 assert(i->raw_job->disk_fd >= 0);
 291
 292                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 293                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 294         }
 295
 296         p = strjoina(i->image_root, "/", i->local, ".raw");
 297
 298         if (i->force_local)
 299                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 300
 301         r = tempfn_random(p, NULL, &tp);
 302         if (r < 0)
 303                 return log_oom();
 304
 305         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 306         if (dfd < 0)
 307                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 308
 309         /* Turn off COW writing. This should greatly improve
 310          * performance on COW file systems like btrfs, since it
 311          * reduces fragmentation caused by not allowing in-place
 312          * writes. */
 313         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 314         if (r < 0)
 315                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 316
 317         r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, true);
 318         if (r < 0) {
 319                 unlink(tp);
 320                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 321         }
 322
 323         (void) copy_times(i->raw_job->disk_fd, dfd);
 324         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 325
 326         dfd = safe_close(dfd);
 327
 328         r = rename(tp, p);
 329         if (r < 0)  {
 330                 unlink(tp);
 331                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 332         }
 333
 334         log_info("Created new local image '%s'.", i->local);
 335
 336         if (i->settings) {
 337                 const char *local_settings;
 338                 assert(i->settings_job);
 339
 340                 if (!i->settings_path) {
 341                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 342                         if (r < 0)
 343                                 return log_oom();
 344                 }
 345
 346                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 347
 348                 r = copy_file_atomic(i->settings_path, local_settings, 0644, i->force_local, 0);
 349                 if (r == -EEXIST)
 350                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 351                 else if (r < 0 && r != -ENOENT)
 352                         log_warning_errno(r, "Failed to copy settings files %s: %m", local_settings);
 353
 354                 log_info("Create new settings file '%s.nspawn'", i->local);
 355         }
 356
 357         return 0;
 358 }
 359
 360 static bool raw_pull_is_done(RawPull *i) {
 361         assert(i);
 362         assert(i->raw_job);
 363
 364         if (!PULL_JOB_IS_COMPLETE(i->raw_job))
 365                 return false;
 366         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 367                 return false;
 368         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 369                 return false;
 370         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 371                 return false;
 372
 373         return true;
 374 }
 375
 376 static void raw_pull_job_on_finished(PullJob *j) {
 377         RawPull *i;
 378         int r;
 379
 380         assert(j);
 381         assert(j->userdata);
 382
 383         i = j->userdata;
 384         if (j == i->settings_job) {
 385                 if (j->error != 0)
 386                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 387         } else if (j->error != 0) {
 388                 if (j == i->checksum_job)
 389                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 390                 else if (j == i->signature_job)
 391                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 392                 else
 393                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 394
 395                 r = j->error;
 396                 goto finish;
 397         }
 398
 399         /* This is invoked if either the download completed
 400          * successfully, or the download was skipped because we
 401          * already have the etag. In this case ->etag_exists is
 402          * true.
 403          *
 404          * We only do something when we got all three files */
 405
 406         if (!raw_pull_is_done(i))
 407                 return;
 408
 409         if (i->settings_job)
 410                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 411
 412         if (!i->raw_job->etag_exists) {
 413                 /* This is a new download, verify it, and move it into place */
 414                 assert(i->raw_job->disk_fd >= 0);
 415
 416                 raw_pull_report_progress(i, RAW_VERIFYING);
 417
 418                 r = pull_verify(i->raw_job, i->settings_job, i->checksum_job, i->signature_job);
 419                 if (r < 0)
 420                         goto finish;
 421
 422                 raw_pull_report_progress(i, RAW_UNPACKING);
 423
 424                 r = raw_pull_maybe_convert_qcow2(i);
 425                 if (r < 0)
 426                         goto finish;
 427
 428                 raw_pull_report_progress(i, RAW_FINALIZING);
 429
 430                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 431                 if (r < 0)
 432                         goto finish;
 433
 434                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 435                 if (r < 0) {
 436                         log_error_errno(r, "Failed to move RAW file into place: %m");
 437                         goto finish;
 438                 }
 439
 440                 i->temp_path = mfree(i->temp_path);
 441
 442                 if (i->settings_job &&
 443                     i->settings_job->error == 0 &&
 444                     !i->settings_job->etag_exists) {
 445
 446                         assert(i->settings_temp_path);
 447                         assert(i->settings_path);
 448
 449                         r = import_make_read_only(i->settings_temp_path);
 450                         if (r < 0)
 451                                 goto finish;
 452
 453                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 454                         if (r < 0) {
 455                                 log_error_errno(r, "Failed to rename settings file: %m");
 456                                 goto finish;
 457                         }
 458
 459                         i->settings_temp_path = mfree(i->settings_temp_path);
 460                 }
 461         }
 462
 463         raw_pull_report_progress(i, RAW_COPYING);
 464
 465         r = raw_pull_make_local_copy(i);
 466         if (r < 0)
 467                 goto finish;
 468
 469         r = 0;
 470
 471 finish:
 472         if (i->on_finished)
 473                 i->on_finished(i, r, i->userdata);
 474         else
 475                 sd_event_exit(i->event, r);
 476 }
 477
 478 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 479         RawPull *i;
 480         int r;
 481
 482         assert(j);
 483         assert(j->userdata);
 484
 485         i = j->userdata;
 486         assert(i->raw_job == j);
 487         assert(!i->final_path);
 488         assert(!i->temp_path);
 489
 490         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 491         if (r < 0)
 492                 return log_oom();
 493
 494         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 495         if (r < 0)
 496                 return log_oom();
 497
 498         (void) mkdir_parents_label(i->temp_path, 0700);
 499
 500         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 501         if (j->disk_fd < 0)
 502                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 503
 504         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 505         if (r < 0)
 506                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 507
 508         return 0;
 509 }
 510
 511 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
 512         RawPull *i;
 513         int r;
 514
 515         assert(j);
 516         assert(j->userdata);
 517
 518         i = j->userdata;
 519         assert(i->settings_job == j);
 520         assert(!i->settings_path);
 521         assert(!i->settings_temp_path);
 522
 523         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 524         if (r < 0)
 525                 return log_oom();
 526
 527         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 528         if (r < 0)
 529                 return log_oom();
 530
 531         mkdir_parents_label(i->settings_temp_path, 0700);
 532
 533         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 534         if (j->disk_fd < 0)
 535                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 536
 537         return 0;
 538 }
 539
 540 static void raw_pull_job_on_progress(PullJob *j) {
 541         RawPull *i;
 542
 543         assert(j);
 544         assert(j->userdata);
 545
 546         i = j->userdata;
 547
 548         raw_pull_report_progress(i, RAW_DOWNLOADING);
 549 }
 550
 551 int raw_pull_start(
 552                 RawPull *i,
 553                 const char *url,
 554                 const char *local,
 555                 bool force_local,
 556                 ImportVerify verify,
 557                 bool settings) {
 558
 559         int r;
 560
 561         assert(i);
 562         assert(verify < _IMPORT_VERIFY_MAX);
 563         assert(verify >= 0);
 564
 565         if (!http_url_is_valid(url))
 566                 return -EINVAL;
 567
 568         if (local && !machine_name_is_valid(local))
 569                 return -EINVAL;
 570
 571         if (i->raw_job)
 572                 return -EBUSY;
 573
 574         r = free_and_strdup(&i->local, local);
 575         if (r < 0)
 576                 return r;
 577
 578         i->force_local = force_local;
 579         i->verify = verify;
 580         i->settings = settings;
 581
 582         /* Queue job for the image itself */
 583         r = pull_job_new(&i->raw_job, url, i->glue, i);
 584         if (r < 0)
 585                 return r;
 586
 587         i->raw_job->on_finished = raw_pull_job_on_finished;
 588         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 589         i->raw_job->on_progress = raw_pull_job_on_progress;
 590         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 591         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 592
 593         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 594         if (r < 0)
 595                 return r;
 596
 597         if (settings) {
 598                 r = pull_make_settings_job(&i->settings_job, url, i->glue, raw_pull_job_on_finished, i);
 599                 if (r < 0)
 600                         return r;
 601
 602                 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
 603                 i->settings_job->on_progress = raw_pull_job_on_progress;
 604                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 605
 606                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 607                 if (r < 0)
 608                         return r;
 609         }
 610
 611         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 612         if (r < 0)
 613                 return r;
 614
 615         r = pull_job_begin(i->raw_job);
 616         if (r < 0)
 617                 return r;
 618
 619         if (i->settings_job) {
 620                 r = pull_job_begin(i->settings_job);
 621                 if (r < 0)
 622                         return r;
 623         }
 624
 625         if (i->checksum_job) {
 626                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 627
 628                 r = pull_job_begin(i->checksum_job);
 629                 if (r < 0)
 630                         return r;
 631         }
 632
 633         if (i->signature_job) {
 634                 i->signature_job->on_progress = raw_pull_job_on_progress;
 635
 636                 r = pull_job_begin(i->signature_job);
 637                 if (r < 0)
 638                         return r;
 639         }
 640
 641         return 0;
 642 }