src/import/pull-raw.c

   1 /***
   2   This file is part of systemd.
   3
   4   Copyright 2014 Lennart Poettering
   5
   6   systemd is free software; you can redistribute it and/or modify it
   7   under the terms of the GNU Lesser General Public License as published by
   8   the Free Software Foundation; either version 2.1 of the License, or
   9   (at your option) any later version.
  10
  11   systemd is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14   Lesser General Public License for more details.
  15
  16   You should have received a copy of the GNU Lesser General Public License
  17   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  18 ***/
  19
  20 #include <curl/curl.h>
  21 #include <linux/fs.h>
  22 #include <sys/xattr.h>
  23
  24 #include "sd-daemon.h"
  25
  26 #include "alloc-util.h"
  27 #include "btrfs-util.h"
  28 #include "chattr-util.h"
  29 #include "copy.h"
  30 #include "curl-util.h"
  31 #include "fd-util.h"
  32 #include "fileio.h"
  33 #include "fs-util.h"
  34 #include "hostname-util.h"
  35 #include "import-common.h"
  36 #include "import-util.h"
  37 #include "macro.h"
  38 #include "mkdir.h"
  39 #include "path-util.h"
  40 #include "pull-common.h"
  41 #include "pull-job.h"
  42 #include "pull-raw.h"
  43 #include "qcow2-util.h"
  44 #include "rm-rf.h"
  45 #include "string-util.h"
  46 #include "strv.h"
  47 #include "utf8.h"
  48 #include "util.h"
  49 #include "web-util.h"
  50
  51 typedef enum RawProgress {
  52         RAW_DOWNLOADING,
  53         RAW_VERIFYING,
  54         RAW_UNPACKING,
  55         RAW_FINALIZING,
  56         RAW_COPYING,
  57 } RawProgress;
  58
  59 struct RawPull {
  60         sd_event *event;
  61         CurlGlue *glue;
  62
  63         char *image_root;
  64
  65         PullJob *raw_job;
  66         PullJob *settings_job;
  67         PullJob *checksum_job;
  68         PullJob *signature_job;
  69
  70         RawPullFinished on_finished;
  71         void *userdata;
  72
  73         char *local;
  74         bool force_local;
  75         bool grow_machine_directory;
  76         bool settings;
  77
  78         char *final_path;
  79         char *temp_path;
  80
  81         char *settings_path;
  82         char *settings_temp_path;
  83
  84         ImportVerify verify;
  85 };
  86
  87 RawPull* raw_pull_unref(RawPull *i) {
  88         if (!i)
  89                 return NULL;
  90
  91         pull_job_unref(i->raw_job);
  92         pull_job_unref(i->settings_job);
  93         pull_job_unref(i->checksum_job);
  94         pull_job_unref(i->signature_job);
  95
  96         curl_glue_unref(i->glue);
  97         sd_event_unref(i->event);
  98
  99         if (i->temp_path) {
 100                 (void) unlink(i->temp_path);
 101                 free(i->temp_path);
 102         }
 103
 104         if (i->settings_temp_path) {
 105                 (void) unlink(i->settings_temp_path);
 106                 free(i->settings_temp_path);
 107         }
 108
 109         free(i->final_path);
 110         free(i->settings_path);
 111         free(i->image_root);
 112         free(i->local);
 113         return mfree(i);
 114 }
 115
 116 int raw_pull_new(
 117                 RawPull **ret,
 118                 sd_event *event,
 119                 const char *image_root,
 120                 RawPullFinished on_finished,
 121                 void *userdata) {
 122
 123         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 124         int r;
 125
 126         assert(ret);
 127
 128         i = new0(RawPull, 1);
 129         if (!i)
 130                 return -ENOMEM;
 131
 132         i->on_finished = on_finished;
 133         i->userdata = userdata;
 134
 135         i->image_root = strdup(image_root ?: "/var/lib/machines");
 136         if (!i->image_root)
 137                 return -ENOMEM;
 138
 139         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 140
 141         if (event)
 142                 i->event = sd_event_ref(event);
 143         else {
 144                 r = sd_event_default(&i->event);
 145                 if (r < 0)
 146                         return r;
 147         }
 148
 149         r = curl_glue_new(&i->glue, i->event);
 150         if (r < 0)
 151                 return r;
 152
 153         i->glue->on_finished = pull_job_curl_on_finished;
 154         i->glue->userdata = i;
 155
 156         *ret = i;
 157         i = NULL;
 158
 159         return 0;
 160 }
 161
 162 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 163         unsigned percent;
 164
 165         assert(i);
 166
 167         switch (p) {
 168
 169         case RAW_DOWNLOADING: {
 170                 unsigned remain = 80;
 171
 172                 percent = 0;
 173
 174                 if (i->settings_job) {
 175                         percent += i->settings_job->progress_percent * 5 / 100;
 176                         remain -= 5;
 177                 }
 178
 179                 if (i->checksum_job) {
 180                         percent += i->checksum_job->progress_percent * 5 / 100;
 181                         remain -= 5;
 182                 }
 183
 184                 if (i->signature_job) {
 185                         percent += i->signature_job->progress_percent * 5 / 100;
 186                         remain -= 5;
 187                 }
 188
 189                 if (i->raw_job)
 190                         percent += i->raw_job->progress_percent * remain / 100;
 191                 break;
 192         }
 193
 194         case RAW_VERIFYING:
 195                 percent = 80;
 196                 break;
 197
 198         case RAW_UNPACKING:
 199                 percent = 85;
 200                 break;
 201
 202         case RAW_FINALIZING:
 203                 percent = 90;
 204                 break;
 205
 206         case RAW_COPYING:
 207                 percent = 95;
 208                 break;
 209
 210         default:
 211                 assert_not_reached("Unknown progress state");
 212         }
 213
 214         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 215         log_debug("Combined progress %u%%", percent);
 216 }
 217
 218 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 219         _cleanup_close_ int converted_fd = -1;
 220         _cleanup_free_ char *t = NULL;
 221         int r;
 222
 223         assert(i);
 224         assert(i->raw_job);
 225
 226         r = qcow2_detect(i->raw_job->disk_fd);
 227         if (r < 0)
 228                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 229         if (r == 0)
 230                 return 0;
 231
 232         /* This is a QCOW2 image, let's convert it */
 233         r = tempfn_random(i->final_path, NULL, &t);
 234         if (r < 0)
 235                 return log_oom();
 236
 237         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 238         if (converted_fd < 0)
 239                 return log_error_errno(errno, "Failed to create %s: %m", t);
 240
 241         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 242         if (r < 0)
 243                 log_warning_errno(r, "Failed to set file attributes on %s: %m", t);
 244
 245         log_info("Unpacking QCOW2 file.");
 246
 247         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 248         if (r < 0) {
 249                 unlink(t);
 250                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 251         }
 252
 253         (void) unlink(i->temp_path);
 254         free(i->temp_path);
 255         i->temp_path = t;
 256         t = NULL;
 257
 258         safe_close(i->raw_job->disk_fd);
 259         i->raw_job->disk_fd = converted_fd;
 260         converted_fd = -1;
 261
 262         return 1;
 263 }
 264
 265 static int raw_pull_make_local_copy(RawPull *i) {
 266         _cleanup_free_ char *tp = NULL;
 267         _cleanup_close_ int dfd = -1;
 268         const char *p;
 269         int r;
 270
 271         assert(i);
 272         assert(i->raw_job);
 273
 274         if (!i->local)
 275                 return 0;
 276
 277         if (!i->final_path) {
 278                 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 279                 if (r < 0)
 280                         return log_oom();
 281         }
 282
 283         if (i->raw_job->etag_exists) {
 284                 /* We have downloaded this one previously, reopen it */
 285
 286                 assert(i->raw_job->disk_fd < 0);
 287
 288                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 289                 if (i->raw_job->disk_fd < 0)
 290                         return log_error_errno(errno, "Failed to open vendor image: %m");
 291         } else {
 292                 /* We freshly downloaded the image, use it */
 293
 294                 assert(i->raw_job->disk_fd >= 0);
 295
 296                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 297                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 298         }
 299
 300         p = strjoina(i->image_root, "/", i->local, ".raw");
 301
 302         if (i->force_local)
 303                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 304
 305         r = tempfn_random(p, NULL, &tp);
 306         if (r < 0)
 307                 return log_oom();
 308
 309         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 310         if (dfd < 0)
 311                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 312
 313         /* Turn off COW writing. This should greatly improve
 314          * performance on COW file systems like btrfs, since it
 315          * reduces fragmentation caused by not allowing in-place
 316          * writes. */
 317         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 318         if (r < 0)
 319                 log_warning_errno(r, "Failed to set file attributes on %s: %m", tp);
 320
 321         r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, true);
 322         if (r < 0) {
 323                 unlink(tp);
 324                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 325         }
 326
 327         (void) copy_times(i->raw_job->disk_fd, dfd);
 328         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 329
 330         dfd = safe_close(dfd);
 331
 332         r = rename(tp, p);
 333         if (r < 0)  {
 334                 r = log_error_errno(errno, "Failed to move writable image into place: %m");
 335                 unlink(tp);
 336                 return r;
 337         }
 338
 339         log_info("Created new local image '%s'.", i->local);
 340
 341         if (i->settings) {
 342                 const char *local_settings;
 343                 assert(i->settings_job);
 344
 345                 if (!i->settings_path) {
 346                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 347                         if (r < 0)
 348                                 return log_oom();
 349                 }
 350
 351                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 352
 353                 r = copy_file_atomic(i->settings_path, local_settings, 0644, i->force_local, 0);
 354                 if (r == -EEXIST)
 355                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 356                 else if (r == -ENOENT)
 357                         log_debug_errno(r, "Skipping creation of settings file, since none was found.");
 358                 else if (r < 0)
 359                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 360                 else
 361                         log_info("Created new settings file %s.", local_settings);
 362         }
 363
 364         return 0;
 365 }
 366
 367 static bool raw_pull_is_done(RawPull *i) {
 368         assert(i);
 369         assert(i->raw_job);
 370
 371         if (!PULL_JOB_IS_COMPLETE(i->raw_job))
 372                 return false;
 373         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 374                 return false;
 375         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 376                 return false;
 377         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 378                 return false;
 379
 380         return true;
 381 }
 382
 383 static void raw_pull_job_on_finished(PullJob *j) {
 384         RawPull *i;
 385         int r;
 386
 387         assert(j);
 388         assert(j->userdata);
 389
 390         i = j->userdata;
 391         if (j == i->settings_job) {
 392                 if (j->error != 0)
 393                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 394         } else if (j->error != 0) {
 395                 if (j == i->checksum_job)
 396                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 397                 else if (j == i->signature_job)
 398                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 399                 else
 400                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 401
 402                 r = j->error;
 403                 goto finish;
 404         }
 405
 406         /* This is invoked if either the download completed
 407          * successfully, or the download was skipped because we
 408          * already have the etag. In this case ->etag_exists is
 409          * true.
 410          *
 411          * We only do something when we got all three files */
 412
 413         if (!raw_pull_is_done(i))
 414                 return;
 415
 416         if (i->settings_job)
 417                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 418
 419         if (!i->raw_job->etag_exists) {
 420                 /* This is a new download, verify it, and move it into place */
 421                 assert(i->raw_job->disk_fd >= 0);
 422
 423                 raw_pull_report_progress(i, RAW_VERIFYING);
 424
 425                 r = pull_verify(i->raw_job, i->settings_job, i->checksum_job, i->signature_job);
 426                 if (r < 0)
 427                         goto finish;
 428
 429                 raw_pull_report_progress(i, RAW_UNPACKING);
 430
 431                 r = raw_pull_maybe_convert_qcow2(i);
 432                 if (r < 0)
 433                         goto finish;
 434
 435                 raw_pull_report_progress(i, RAW_FINALIZING);
 436
 437                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 438                 if (r < 0)
 439                         goto finish;
 440
 441                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 442                 if (r < 0) {
 443                         log_error_errno(r, "Failed to move RAW file into place: %m");
 444                         goto finish;
 445                 }
 446
 447                 i->temp_path = mfree(i->temp_path);
 448
 449                 if (i->settings_job &&
 450                     i->settings_job->error == 0 &&
 451                     !i->settings_job->etag_exists) {
 452
 453                         assert(i->settings_temp_path);
 454                         assert(i->settings_path);
 455
 456                         r = import_make_read_only(i->settings_temp_path);
 457                         if (r < 0)
 458                                 goto finish;
 459
 460                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 461                         if (r < 0) {
 462                                 log_error_errno(r, "Failed to rename settings file: %m");
 463                                 goto finish;
 464                         }
 465
 466                         i->settings_temp_path = mfree(i->settings_temp_path);
 467                 }
 468         }
 469
 470         raw_pull_report_progress(i, RAW_COPYING);
 471
 472         r = raw_pull_make_local_copy(i);
 473         if (r < 0)
 474                 goto finish;
 475
 476         r = 0;
 477
 478 finish:
 479         if (i->on_finished)
 480                 i->on_finished(i, r, i->userdata);
 481         else
 482                 sd_event_exit(i->event, r);
 483 }
 484
 485 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 486         RawPull *i;
 487         int r;
 488
 489         assert(j);
 490         assert(j->userdata);
 491
 492         i = j->userdata;
 493         assert(i->raw_job == j);
 494         assert(!i->final_path);
 495         assert(!i->temp_path);
 496
 497         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 498         if (r < 0)
 499                 return log_oom();
 500
 501         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 502         if (r < 0)
 503                 return log_oom();
 504
 505         (void) mkdir_parents_label(i->temp_path, 0700);
 506
 507         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 508         if (j->disk_fd < 0)
 509                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 510
 511         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 512         if (r < 0)
 513                 log_warning_errno(r, "Failed to set file attributes on %s: %m", i->temp_path);
 514
 515         return 0;
 516 }
 517
 518 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
 519         RawPull *i;
 520         int r;
 521
 522         assert(j);
 523         assert(j->userdata);
 524
 525         i = j->userdata;
 526         assert(i->settings_job == j);
 527         assert(!i->settings_path);
 528         assert(!i->settings_temp_path);
 529
 530         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 531         if (r < 0)
 532                 return log_oom();
 533
 534         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 535         if (r < 0)
 536                 return log_oom();
 537
 538         mkdir_parents_label(i->settings_temp_path, 0700);
 539
 540         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 541         if (j->disk_fd < 0)
 542                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 543
 544         return 0;
 545 }
 546
 547 static void raw_pull_job_on_progress(PullJob *j) {
 548         RawPull *i;
 549
 550         assert(j);
 551         assert(j->userdata);
 552
 553         i = j->userdata;
 554
 555         raw_pull_report_progress(i, RAW_DOWNLOADING);
 556 }
 557
 558 int raw_pull_start(
 559                 RawPull *i,
 560                 const char *url,
 561                 const char *local,
 562                 bool force_local,
 563                 ImportVerify verify,
 564                 bool settings) {
 565
 566         int r;
 567
 568         assert(i);
 569         assert(verify < _IMPORT_VERIFY_MAX);
 570         assert(verify >= 0);
 571
 572         if (!http_url_is_valid(url))
 573                 return -EINVAL;
 574
 575         if (local && !machine_name_is_valid(local))
 576                 return -EINVAL;
 577
 578         if (i->raw_job)
 579                 return -EBUSY;
 580
 581         r = free_and_strdup(&i->local, local);
 582         if (r < 0)
 583                 return r;
 584
 585         i->force_local = force_local;
 586         i->verify = verify;
 587         i->settings = settings;
 588
 589         /* Queue job for the image itself */
 590         r = pull_job_new(&i->raw_job, url, i->glue, i);
 591         if (r < 0)
 592                 return r;
 593
 594         i->raw_job->on_finished = raw_pull_job_on_finished;
 595         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 596         i->raw_job->on_progress = raw_pull_job_on_progress;
 597         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 598         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 599
 600         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 601         if (r < 0)
 602                 return r;
 603
 604         if (settings) {
 605                 r = pull_make_settings_job(&i->settings_job, url, i->glue, raw_pull_job_on_finished, i);
 606                 if (r < 0)
 607                         return r;
 608
 609                 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
 610                 i->settings_job->on_progress = raw_pull_job_on_progress;
 611                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 612
 613                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 614                 if (r < 0)
 615                         return r;
 616         }
 617
 618         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 619         if (r < 0)
 620                 return r;
 621
 622         r = pull_job_begin(i->raw_job);
 623         if (r < 0)
 624                 return r;
 625
 626         if (i->settings_job) {
 627                 r = pull_job_begin(i->settings_job);
 628                 if (r < 0)
 629                         return r;
 630         }
 631
 632         if (i->checksum_job) {
 633                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 634
 635                 r = pull_job_begin(i->checksum_job);
 636                 if (r < 0)
 637                         return r;
 638         }
 639
 640         if (i->signature_job) {
 641                 i->signature_job->on_progress = raw_pull_job_on_progress;
 642
 643                 r = pull_job_begin(i->signature_job);
 644                 if (r < 0)
 645                         return r;
 646         }
 647
 648         return 0;
 649 }