src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <curl/curl.h>
  23 #include <linux/fs.h>
  24 #include <sys/xattr.h>
  25
  26 #include "sd-daemon.h"
  27
  28 #include "alloc-util.h"
  29 #include "btrfs-util.h"
  30 #include "chattr-util.h"
  31 #include "copy.h"
  32 #include "curl-util.h"
  33 #include "fd-util.h"
  34 #include "fileio.h"
  35 #include "fs-util.h"
  36 #include "hostname-util.h"
  37 #include "import-common.h"
  38 #include "import-util.h"
  39 #include "macro.h"
  40 #include "mkdir.h"
  41 #include "path-util.h"
  42 #include "pull-common.h"
  43 #include "pull-job.h"
  44 #include "pull-raw.h"
  45 #include "qcow2-util.h"
  46 #include "rm-rf.h"
  47 #include "string-util.h"
  48 #include "strv.h"
  49 #include "utf8.h"
  50 #include "util.h"
  51 #include "web-util.h"
  52
  53 typedef enum RawProgress {
  54         RAW_DOWNLOADING,
  55         RAW_VERIFYING,
  56         RAW_UNPACKING,
  57         RAW_FINALIZING,
  58         RAW_COPYING,
  59 } RawProgress;
  60
  61 struct RawPull {
  62         sd_event *event;
  63         CurlGlue *glue;
  64
  65         char *image_root;
  66
  67         PullJob *raw_job;
  68         PullJob *settings_job;
  69         PullJob *checksum_job;
  70         PullJob *signature_job;
  71
  72         RawPullFinished on_finished;
  73         void *userdata;
  74
  75         char *local;
  76         bool force_local;
  77         bool grow_machine_directory;
  78         bool settings;
  79
  80         char *final_path;
  81         char *temp_path;
  82
  83         char *settings_path;
  84         char *settings_temp_path;
  85
  86         ImportVerify verify;
  87 };
  88
  89 RawPull* raw_pull_unref(RawPull *i) {
  90         if (!i)
  91                 return NULL;
  92
  93         pull_job_unref(i->raw_job);
  94         pull_job_unref(i->settings_job);
  95         pull_job_unref(i->checksum_job);
  96         pull_job_unref(i->signature_job);
  97
  98         curl_glue_unref(i->glue);
  99         sd_event_unref(i->event);
 100
 101         if (i->temp_path) {
 102                 (void) unlink(i->temp_path);
 103                 free(i->temp_path);
 104         }
 105
 106         if (i->settings_temp_path) {
 107                 (void) unlink(i->settings_temp_path);
 108                 free(i->settings_temp_path);
 109         }
 110
 111         free(i->final_path);
 112         free(i->settings_path);
 113         free(i->image_root);
 114         free(i->local);
 115         free(i);
 116
 117         return NULL;
 118 }
 119
 120 int raw_pull_new(
 121                 RawPull **ret,
 122                 sd_event *event,
 123                 const char *image_root,
 124                 RawPullFinished on_finished,
 125                 void *userdata) {
 126
 127         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 128         int r;
 129
 130         assert(ret);
 131
 132         i = new0(RawPull, 1);
 133         if (!i)
 134                 return -ENOMEM;
 135
 136         i->on_finished = on_finished;
 137         i->userdata = userdata;
 138
 139         i->image_root = strdup(image_root ?: "/var/lib/machines");
 140         if (!i->image_root)
 141                 return -ENOMEM;
 142
 143         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 144
 145         if (event)
 146                 i->event = sd_event_ref(event);
 147         else {
 148                 r = sd_event_default(&i->event);
 149                 if (r < 0)
 150                         return r;
 151         }
 152
 153         r = curl_glue_new(&i->glue, i->event);
 154         if (r < 0)
 155                 return r;
 156
 157         i->glue->on_finished = pull_job_curl_on_finished;
 158         i->glue->userdata = i;
 159
 160         *ret = i;
 161         i = NULL;
 162
 163         return 0;
 164 }
 165
 166 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 167         unsigned percent;
 168
 169         assert(i);
 170
 171         switch (p) {
 172
 173         case RAW_DOWNLOADING: {
 174                 unsigned remain = 80;
 175
 176                 percent = 0;
 177
 178                 if (i->settings_job) {
 179                         percent += i->settings_job->progress_percent * 5 / 100;
 180                         remain -= 5;
 181                 }
 182
 183                 if (i->checksum_job) {
 184                         percent += i->checksum_job->progress_percent * 5 / 100;
 185                         remain -= 5;
 186                 }
 187
 188                 if (i->signature_job) {
 189                         percent += i->signature_job->progress_percent * 5 / 100;
 190                         remain -= 5;
 191                 }
 192
 193                 if (i->raw_job)
 194                         percent += i->raw_job->progress_percent * remain / 100;
 195                 break;
 196         }
 197
 198         case RAW_VERIFYING:
 199                 percent = 80;
 200                 break;
 201
 202         case RAW_UNPACKING:
 203                 percent = 85;
 204                 break;
 205
 206         case RAW_FINALIZING:
 207                 percent = 90;
 208                 break;
 209
 210         case RAW_COPYING:
 211                 percent = 95;
 212                 break;
 213
 214         default:
 215                 assert_not_reached("Unknown progress state");
 216         }
 217
 218         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 219         log_debug("Combined progress %u%%", percent);
 220 }
 221
 222 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 223         _cleanup_close_ int converted_fd = -1;
 224         _cleanup_free_ char *t = NULL;
 225         int r;
 226
 227         assert(i);
 228         assert(i->raw_job);
 229
 230         r = qcow2_detect(i->raw_job->disk_fd);
 231         if (r < 0)
 232                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 233         if (r == 0)
 234                 return 0;
 235
 236         /* This is a QCOW2 image, let's convert it */
 237         r = tempfn_random(i->final_path, NULL, &t);
 238         if (r < 0)
 239                 return log_oom();
 240
 241         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 242         if (converted_fd < 0)
 243                 return log_error_errno(errno, "Failed to create %s: %m", t);
 244
 245         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 246         if (r < 0)
 247                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 248
 249         log_info("Unpacking QCOW2 file.");
 250
 251         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 252         if (r < 0) {
 253                 unlink(t);
 254                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 255         }
 256
 257         (void) unlink(i->temp_path);
 258         free(i->temp_path);
 259         i->temp_path = t;
 260         t = NULL;
 261
 262         safe_close(i->raw_job->disk_fd);
 263         i->raw_job->disk_fd = converted_fd;
 264         converted_fd = -1;
 265
 266         return 1;
 267 }
 268
 269 static int raw_pull_make_local_copy(RawPull *i) {
 270         _cleanup_free_ char *tp = NULL;
 271         _cleanup_close_ int dfd = -1;
 272         const char *p;
 273         int r;
 274
 275         assert(i);
 276         assert(i->raw_job);
 277
 278         if (!i->local)
 279                 return 0;
 280
 281         if (!i->final_path) {
 282                 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 283                 if (r < 0)
 284                         return log_oom();
 285         }
 286
 287         if (i->raw_job->etag_exists) {
 288                 /* We have downloaded this one previously, reopen it */
 289
 290                 assert(i->raw_job->disk_fd < 0);
 291
 292                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 293                 if (i->raw_job->disk_fd < 0)
 294                         return log_error_errno(errno, "Failed to open vendor image: %m");
 295         } else {
 296                 /* We freshly downloaded the image, use it */
 297
 298                 assert(i->raw_job->disk_fd >= 0);
 299
 300                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 301                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 302         }
 303
 304         p = strjoina(i->image_root, "/", i->local, ".raw");
 305
 306         if (i->force_local)
 307                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 308
 309         r = tempfn_random(p, NULL, &tp);
 310         if (r < 0)
 311                 return log_oom();
 312
 313         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 314         if (dfd < 0)
 315                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 316
 317         /* Turn off COW writing. This should greatly improve
 318          * performance on COW file systems like btrfs, since it
 319          * reduces fragmentation caused by not allowing in-place
 320          * writes. */
 321         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 322         if (r < 0)
 323                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 324
 325         r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, true);
 326         if (r < 0) {
 327                 unlink(tp);
 328                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 329         }
 330
 331         (void) copy_times(i->raw_job->disk_fd, dfd);
 332         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 333
 334         dfd = safe_close(dfd);
 335
 336         r = rename(tp, p);
 337         if (r < 0)  {
 338                 unlink(tp);
 339                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 340         }
 341
 342         log_info("Created new local image '%s'.", i->local);
 343
 344         if (i->settings) {
 345                 const char *local_settings;
 346                 assert(i->settings_job);
 347
 348                 if (!i->settings_path) {
 349                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 350                         if (r < 0)
 351                                 return log_oom();
 352                 }
 353
 354                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 355
 356                 r = copy_file_atomic(i->settings_path, local_settings, 0644, i->force_local, 0);
 357                 if (r == -EEXIST)
 358                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 359                 else if (r < 0 && r != -ENOENT)
 360                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 361                 else
 362                         log_info("Created new settings file '%s.nspawn'", i->local);
 363         }
 364
 365         return 0;
 366 }
 367
 368 static bool raw_pull_is_done(RawPull *i) {
 369         assert(i);
 370         assert(i->raw_job);
 371
 372         if (!PULL_JOB_IS_COMPLETE(i->raw_job))
 373                 return false;
 374         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 375                 return false;
 376         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 377                 return false;
 378         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 379                 return false;
 380
 381         return true;
 382 }
 383
 384 static void raw_pull_job_on_finished(PullJob *j) {
 385         RawPull *i;
 386         int r;
 387
 388         assert(j);
 389         assert(j->userdata);
 390
 391         i = j->userdata;
 392         if (j == i->settings_job) {
 393                 if (j->error != 0)
 394                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 395         } else if (j->error != 0) {
 396                 if (j == i->checksum_job)
 397                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 398                 else if (j == i->signature_job)
 399                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 400                 else
 401                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 402
 403                 r = j->error;
 404                 goto finish;
 405         }
 406
 407         /* This is invoked if either the download completed
 408          * successfully, or the download was skipped because we
 409          * already have the etag. In this case ->etag_exists is
 410          * true.
 411          *
 412          * We only do something when we got all three files */
 413
 414         if (!raw_pull_is_done(i))
 415                 return;
 416
 417         if (i->settings_job)
 418                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 419
 420         if (!i->raw_job->etag_exists) {
 421                 /* This is a new download, verify it, and move it into place */
 422                 assert(i->raw_job->disk_fd >= 0);
 423
 424                 raw_pull_report_progress(i, RAW_VERIFYING);
 425
 426                 r = pull_verify(i->raw_job, i->settings_job, i->checksum_job, i->signature_job);
 427                 if (r < 0)
 428                         goto finish;
 429
 430                 raw_pull_report_progress(i, RAW_UNPACKING);
 431
 432                 r = raw_pull_maybe_convert_qcow2(i);
 433                 if (r < 0)
 434                         goto finish;
 435
 436                 raw_pull_report_progress(i, RAW_FINALIZING);
 437
 438                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 439                 if (r < 0)
 440                         goto finish;
 441
 442                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 443                 if (r < 0) {
 444                         log_error_errno(r, "Failed to move RAW file into place: %m");
 445                         goto finish;
 446                 }
 447
 448                 i->temp_path = mfree(i->temp_path);
 449
 450                 if (i->settings_job &&
 451                     i->settings_job->error == 0 &&
 452                     !i->settings_job->etag_exists) {
 453
 454                         assert(i->settings_temp_path);
 455                         assert(i->settings_path);
 456
 457                         r = import_make_read_only(i->settings_temp_path);
 458                         if (r < 0)
 459                                 goto finish;
 460
 461                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 462                         if (r < 0) {
 463                                 log_error_errno(r, "Failed to rename settings file: %m");
 464                                 goto finish;
 465                         }
 466
 467                         i->settings_temp_path = mfree(i->settings_temp_path);
 468                 }
 469         }
 470
 471         raw_pull_report_progress(i, RAW_COPYING);
 472
 473         r = raw_pull_make_local_copy(i);
 474         if (r < 0)
 475                 goto finish;
 476
 477         r = 0;
 478
 479 finish:
 480         if (i->on_finished)
 481                 i->on_finished(i, r, i->userdata);
 482         else
 483                 sd_event_exit(i->event, r);
 484 }
 485
 486 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 487         RawPull *i;
 488         int r;
 489
 490         assert(j);
 491         assert(j->userdata);
 492
 493         i = j->userdata;
 494         assert(i->raw_job == j);
 495         assert(!i->final_path);
 496         assert(!i->temp_path);
 497
 498         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 499         if (r < 0)
 500                 return log_oom();
 501
 502         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 503         if (r < 0)
 504                 return log_oom();
 505
 506         (void) mkdir_parents_label(i->temp_path, 0700);
 507
 508         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 509         if (j->disk_fd < 0)
 510                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 511
 512         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 513         if (r < 0)
 514                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 515
 516         return 0;
 517 }
 518
 519 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
 520         RawPull *i;
 521         int r;
 522
 523         assert(j);
 524         assert(j->userdata);
 525
 526         i = j->userdata;
 527         assert(i->settings_job == j);
 528         assert(!i->settings_path);
 529         assert(!i->settings_temp_path);
 530
 531         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 532         if (r < 0)
 533                 return log_oom();
 534
 535         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 536         if (r < 0)
 537                 return log_oom();
 538
 539         mkdir_parents_label(i->settings_temp_path, 0700);
 540
 541         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 542         if (j->disk_fd < 0)
 543                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 544
 545         return 0;
 546 }
 547
 548 static void raw_pull_job_on_progress(PullJob *j) {
 549         RawPull *i;
 550
 551         assert(j);
 552         assert(j->userdata);
 553
 554         i = j->userdata;
 555
 556         raw_pull_report_progress(i, RAW_DOWNLOADING);
 557 }
 558
 559 int raw_pull_start(
 560                 RawPull *i,
 561                 const char *url,
 562                 const char *local,
 563                 bool force_local,
 564                 ImportVerify verify,
 565                 bool settings) {
 566
 567         int r;
 568
 569         assert(i);
 570         assert(verify < _IMPORT_VERIFY_MAX);
 571         assert(verify >= 0);
 572
 573         if (!http_url_is_valid(url))
 574                 return -EINVAL;
 575
 576         if (local && !machine_name_is_valid(local))
 577                 return -EINVAL;
 578
 579         if (i->raw_job)
 580                 return -EBUSY;
 581
 582         r = free_and_strdup(&i->local, local);
 583         if (r < 0)
 584                 return r;
 585
 586         i->force_local = force_local;
 587         i->verify = verify;
 588         i->settings = settings;
 589
 590         /* Queue job for the image itself */
 591         r = pull_job_new(&i->raw_job, url, i->glue, i);
 592         if (r < 0)
 593                 return r;
 594
 595         i->raw_job->on_finished = raw_pull_job_on_finished;
 596         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 597         i->raw_job->on_progress = raw_pull_job_on_progress;
 598         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 599         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 600
 601         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 602         if (r < 0)
 603                 return r;
 604
 605         if (settings) {
 606                 r = pull_make_settings_job(&i->settings_job, url, i->glue, raw_pull_job_on_finished, i);
 607                 if (r < 0)
 608                         return r;
 609
 610                 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
 611                 i->settings_job->on_progress = raw_pull_job_on_progress;
 612                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 613
 614                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 615                 if (r < 0)
 616                         return r;
 617         }
 618
 619         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 620         if (r < 0)
 621                 return r;
 622
 623         r = pull_job_begin(i->raw_job);
 624         if (r < 0)
 625                 return r;
 626
 627         if (i->settings_job) {
 628                 r = pull_job_begin(i->settings_job);
 629                 if (r < 0)
 630                         return r;
 631         }
 632
 633         if (i->checksum_job) {
 634                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 635
 636                 r = pull_job_begin(i->checksum_job);
 637                 if (r < 0)
 638                         return r;
 639         }
 640
 641         if (i->signature_job) {
 642                 i->signature_job->on_progress = raw_pull_job_on_progress;
 643
 644                 r = pull_job_begin(i->signature_job);
 645                 if (r < 0)
 646                         return r;
 647         }
 648
 649         return 0;
 650 }