src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <curl/curl.h>
  23 #include <linux/fs.h>
  24 #include <sys/xattr.h>
  25
  26 #include "sd-daemon.h"
  27
  28 #include "btrfs-util.h"
  29 #include "copy.h"
  30 #include "curl-util.h"
  31 #include "hostname-util.h"
  32 #include "import-common.h"
  33 #include "import-util.h"
  34 #include "macro.h"
  35 #include "mkdir.h"
  36 #include "path-util.h"
  37 #include "pull-common.h"
  38 #include "pull-job.h"
  39 #include "qcow2-util.h"
  40 #include "rm-rf.h"
  41 #include "string-util.h"
  42 #include "strv.h"
  43 #include "utf8.h"
  44 #include "util.h"
  45 #include "pull-raw.h"
  46
  47 typedef enum RawProgress {
  48         RAW_DOWNLOADING,
  49         RAW_VERIFYING,
  50         RAW_UNPACKING,
  51         RAW_FINALIZING,
  52         RAW_COPYING,
  53 } RawProgress;
  54
  55 struct RawPull {
  56         sd_event *event;
  57         CurlGlue *glue;
  58
  59         char *image_root;
  60
  61         PullJob *raw_job;
  62         PullJob *settings_job;
  63         PullJob *checksum_job;
  64         PullJob *signature_job;
  65
  66         RawPullFinished on_finished;
  67         void *userdata;
  68
  69         char *local;
  70         bool force_local;
  71         bool grow_machine_directory;
  72         bool settings;
  73
  74         char *final_path;
  75         char *temp_path;
  76
  77         char *settings_path;
  78         char *settings_temp_path;
  79
  80         ImportVerify verify;
  81 };
  82
  83 RawPull* raw_pull_unref(RawPull *i) {
  84         if (!i)
  85                 return NULL;
  86
  87         pull_job_unref(i->raw_job);
  88         pull_job_unref(i->settings_job);
  89         pull_job_unref(i->checksum_job);
  90         pull_job_unref(i->signature_job);
  91
  92         curl_glue_unref(i->glue);
  93         sd_event_unref(i->event);
  94
  95         if (i->temp_path) {
  96                 (void) unlink(i->temp_path);
  97                 free(i->temp_path);
  98         }
  99
 100         if (i->settings_temp_path) {
 101                 (void) unlink(i->settings_temp_path);
 102                 free(i->settings_temp_path);
 103         }
 104
 105         free(i->final_path);
 106         free(i->settings_path);
 107         free(i->image_root);
 108         free(i->local);
 109         free(i);
 110
 111         return NULL;
 112 }
 113
 114 int raw_pull_new(
 115                 RawPull **ret,
 116                 sd_event *event,
 117                 const char *image_root,
 118                 RawPullFinished on_finished,
 119                 void *userdata) {
 120
 121         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 122         int r;
 123
 124         assert(ret);
 125
 126         i = new0(RawPull, 1);
 127         if (!i)
 128                 return -ENOMEM;
 129
 130         i->on_finished = on_finished;
 131         i->userdata = userdata;
 132
 133         i->image_root = strdup(image_root ?: "/var/lib/machines");
 134         if (!i->image_root)
 135                 return -ENOMEM;
 136
 137         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 138
 139         if (event)
 140                 i->event = sd_event_ref(event);
 141         else {
 142                 r = sd_event_default(&i->event);
 143                 if (r < 0)
 144                         return r;
 145         }
 146
 147         r = curl_glue_new(&i->glue, i->event);
 148         if (r < 0)
 149                 return r;
 150
 151         i->glue->on_finished = pull_job_curl_on_finished;
 152         i->glue->userdata = i;
 153
 154         *ret = i;
 155         i = NULL;
 156
 157         return 0;
 158 }
 159
 160 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 161         unsigned percent;
 162
 163         assert(i);
 164
 165         switch (p) {
 166
 167         case RAW_DOWNLOADING: {
 168                 unsigned remain = 80;
 169
 170                 percent = 0;
 171
 172                 if (i->settings_job) {
 173                         percent += i->settings_job->progress_percent * 5 / 100;
 174                         remain -= 5;
 175                 }
 176
 177                 if (i->checksum_job) {
 178                         percent += i->checksum_job->progress_percent * 5 / 100;
 179                         remain -= 5;
 180                 }
 181
 182                 if (i->signature_job) {
 183                         percent += i->signature_job->progress_percent * 5 / 100;
 184                         remain -= 5;
 185                 }
 186
 187                 if (i->raw_job)
 188                         percent += i->raw_job->progress_percent * remain / 100;
 189                 break;
 190         }
 191
 192         case RAW_VERIFYING:
 193                 percent = 80;
 194                 break;
 195
 196         case RAW_UNPACKING:
 197                 percent = 85;
 198                 break;
 199
 200         case RAW_FINALIZING:
 201                 percent = 90;
 202                 break;
 203
 204         case RAW_COPYING:
 205                 percent = 95;
 206                 break;
 207
 208         default:
 209                 assert_not_reached("Unknown progress state");
 210         }
 211
 212         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 213         log_debug("Combined progress %u%%", percent);
 214 }
 215
 216 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 217         _cleanup_close_ int converted_fd = -1;
 218         _cleanup_free_ char *t = NULL;
 219         int r;
 220
 221         assert(i);
 222         assert(i->raw_job);
 223
 224         r = qcow2_detect(i->raw_job->disk_fd);
 225         if (r < 0)
 226                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 227         if (r == 0)
 228                 return 0;
 229
 230         /* This is a QCOW2 image, let's convert it */
 231         r = tempfn_random(i->final_path, NULL, &t);
 232         if (r < 0)
 233                 return log_oom();
 234
 235         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 236         if (converted_fd < 0)
 237                 return log_error_errno(errno, "Failed to create %s: %m", t);
 238
 239         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 240         if (r < 0)
 241                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 242
 243         log_info("Unpacking QCOW2 file.");
 244
 245         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 246         if (r < 0) {
 247                 unlink(t);
 248                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 249         }
 250
 251         (void) unlink(i->temp_path);
 252         free(i->temp_path);
 253         i->temp_path = t;
 254         t = NULL;
 255
 256         safe_close(i->raw_job->disk_fd);
 257         i->raw_job->disk_fd = converted_fd;
 258         converted_fd = -1;
 259
 260         return 1;
 261 }
 262
 263 static int raw_pull_make_local_copy(RawPull *i) {
 264         _cleanup_free_ char *tp = NULL;
 265         _cleanup_close_ int dfd = -1;
 266         const char *p;
 267         int r;
 268
 269         assert(i);
 270         assert(i->raw_job);
 271
 272         if (!i->local)
 273                 return 0;
 274
 275         if (!i->final_path) {
 276                 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 277                 if (r < 0)
 278                         return log_oom();
 279         }
 280
 281         if (i->raw_job->etag_exists) {
 282                 /* We have downloaded this one previously, reopen it */
 283
 284                 assert(i->raw_job->disk_fd < 0);
 285
 286                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 287                 if (i->raw_job->disk_fd < 0)
 288                         return log_error_errno(errno, "Failed to open vendor image: %m");
 289         } else {
 290                 /* We freshly downloaded the image, use it */
 291
 292                 assert(i->raw_job->disk_fd >= 0);
 293
 294                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 295                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 296         }
 297
 298         p = strjoina(i->image_root, "/", i->local, ".raw");
 299
 300         if (i->force_local)
 301                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 302
 303         r = tempfn_random(p, NULL, &tp);
 304         if (r < 0)
 305                 return log_oom();
 306
 307         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 308         if (dfd < 0)
 309                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 310
 311         /* Turn off COW writing. This should greatly improve
 312          * performance on COW file systems like btrfs, since it
 313          * reduces fragmentation caused by not allowing in-place
 314          * writes. */
 315         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 316         if (r < 0)
 317                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 318
 319         r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, true);
 320         if (r < 0) {
 321                 unlink(tp);
 322                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 323         }
 324
 325         (void) copy_times(i->raw_job->disk_fd, dfd);
 326         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 327
 328         dfd = safe_close(dfd);
 329
 330         r = rename(tp, p);
 331         if (r < 0)  {
 332                 unlink(tp);
 333                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 334         }
 335
 336         log_info("Created new local image '%s'.", i->local);
 337
 338         if (i->settings) {
 339                 const char *local_settings;
 340                 assert(i->settings_job);
 341
 342                 if (!i->settings_path) {
 343                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 344                         if (r < 0)
 345                                 return log_oom();
 346                 }
 347
 348                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 349
 350                 r = copy_file_atomic(i->settings_path, local_settings, 0644, i->force_local, 0);
 351                 if (r == -EEXIST)
 352                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 353                 else if (r < 0 && r != -ENOENT)
 354                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 355                 else
 356                         log_info("Created new settings file '%s.nspawn'", i->local);
 357         }
 358
 359         return 0;
 360 }
 361
 362 static bool raw_pull_is_done(RawPull *i) {
 363         assert(i);
 364         assert(i->raw_job);
 365
 366         if (!PULL_JOB_IS_COMPLETE(i->raw_job))
 367                 return false;
 368         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 369                 return false;
 370         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 371                 return false;
 372         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 373                 return false;
 374
 375         return true;
 376 }
 377
 378 static void raw_pull_job_on_finished(PullJob *j) {
 379         RawPull *i;
 380         int r;
 381
 382         assert(j);
 383         assert(j->userdata);
 384
 385         i = j->userdata;
 386         if (j == i->settings_job) {
 387                 if (j->error != 0)
 388                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 389         } else if (j->error != 0) {
 390                 if (j == i->checksum_job)
 391                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 392                 else if (j == i->signature_job)
 393                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 394                 else
 395                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 396
 397                 r = j->error;
 398                 goto finish;
 399         }
 400
 401         /* This is invoked if either the download completed
 402          * successfully, or the download was skipped because we
 403          * already have the etag. In this case ->etag_exists is
 404          * true.
 405          *
 406          * We only do something when we got all three files */
 407
 408         if (!raw_pull_is_done(i))
 409                 return;
 410
 411         if (i->settings_job)
 412                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 413
 414         if (!i->raw_job->etag_exists) {
 415                 /* This is a new download, verify it, and move it into place */
 416                 assert(i->raw_job->disk_fd >= 0);
 417
 418                 raw_pull_report_progress(i, RAW_VERIFYING);
 419
 420                 r = pull_verify(i->raw_job, i->settings_job, i->checksum_job, i->signature_job);
 421                 if (r < 0)
 422                         goto finish;
 423
 424                 raw_pull_report_progress(i, RAW_UNPACKING);
 425
 426                 r = raw_pull_maybe_convert_qcow2(i);
 427                 if (r < 0)
 428                         goto finish;
 429
 430                 raw_pull_report_progress(i, RAW_FINALIZING);
 431
 432                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 433                 if (r < 0)
 434                         goto finish;
 435
 436                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 437                 if (r < 0) {
 438                         log_error_errno(r, "Failed to move RAW file into place: %m");
 439                         goto finish;
 440                 }
 441
 442                 i->temp_path = mfree(i->temp_path);
 443
 444                 if (i->settings_job &&
 445                     i->settings_job->error == 0 &&
 446                     !i->settings_job->etag_exists) {
 447
 448                         assert(i->settings_temp_path);
 449                         assert(i->settings_path);
 450
 451                         r = import_make_read_only(i->settings_temp_path);
 452                         if (r < 0)
 453                                 goto finish;
 454
 455                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 456                         if (r < 0) {
 457                                 log_error_errno(r, "Failed to rename settings file: %m");
 458                                 goto finish;
 459                         }
 460
 461                         i->settings_temp_path = mfree(i->settings_temp_path);
 462                 }
 463         }
 464
 465         raw_pull_report_progress(i, RAW_COPYING);
 466
 467         r = raw_pull_make_local_copy(i);
 468         if (r < 0)
 469                 goto finish;
 470
 471         r = 0;
 472
 473 finish:
 474         if (i->on_finished)
 475                 i->on_finished(i, r, i->userdata);
 476         else
 477                 sd_event_exit(i->event, r);
 478 }
 479
 480 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 481         RawPull *i;
 482         int r;
 483
 484         assert(j);
 485         assert(j->userdata);
 486
 487         i = j->userdata;
 488         assert(i->raw_job == j);
 489         assert(!i->final_path);
 490         assert(!i->temp_path);
 491
 492         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 493         if (r < 0)
 494                 return log_oom();
 495
 496         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 497         if (r < 0)
 498                 return log_oom();
 499
 500         (void) mkdir_parents_label(i->temp_path, 0700);
 501
 502         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 503         if (j->disk_fd < 0)
 504                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 505
 506         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 507         if (r < 0)
 508                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 509
 510         return 0;
 511 }
 512
 513 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
 514         RawPull *i;
 515         int r;
 516
 517         assert(j);
 518         assert(j->userdata);
 519
 520         i = j->userdata;
 521         assert(i->settings_job == j);
 522         assert(!i->settings_path);
 523         assert(!i->settings_temp_path);
 524
 525         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 526         if (r < 0)
 527                 return log_oom();
 528
 529         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 530         if (r < 0)
 531                 return log_oom();
 532
 533         mkdir_parents_label(i->settings_temp_path, 0700);
 534
 535         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 536         if (j->disk_fd < 0)
 537                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 538
 539         return 0;
 540 }
 541
 542 static void raw_pull_job_on_progress(PullJob *j) {
 543         RawPull *i;
 544
 545         assert(j);
 546         assert(j->userdata);
 547
 548         i = j->userdata;
 549
 550         raw_pull_report_progress(i, RAW_DOWNLOADING);
 551 }
 552
 553 int raw_pull_start(
 554                 RawPull *i,
 555                 const char *url,
 556                 const char *local,
 557                 bool force_local,
 558                 ImportVerify verify,
 559                 bool settings) {
 560
 561         int r;
 562
 563         assert(i);
 564         assert(verify < _IMPORT_VERIFY_MAX);
 565         assert(verify >= 0);
 566
 567         if (!http_url_is_valid(url))
 568                 return -EINVAL;
 569
 570         if (local && !machine_name_is_valid(local))
 571                 return -EINVAL;
 572
 573         if (i->raw_job)
 574                 return -EBUSY;
 575
 576         r = free_and_strdup(&i->local, local);
 577         if (r < 0)
 578                 return r;
 579
 580         i->force_local = force_local;
 581         i->verify = verify;
 582         i->settings = settings;
 583
 584         /* Queue job for the image itself */
 585         r = pull_job_new(&i->raw_job, url, i->glue, i);
 586         if (r < 0)
 587                 return r;
 588
 589         i->raw_job->on_finished = raw_pull_job_on_finished;
 590         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 591         i->raw_job->on_progress = raw_pull_job_on_progress;
 592         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 593         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 594
 595         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 596         if (r < 0)
 597                 return r;
 598
 599         if (settings) {
 600                 r = pull_make_settings_job(&i->settings_job, url, i->glue, raw_pull_job_on_finished, i);
 601                 if (r < 0)
 602                         return r;
 603
 604                 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
 605                 i->settings_job->on_progress = raw_pull_job_on_progress;
 606                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 607
 608                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 609                 if (r < 0)
 610                         return r;
 611         }
 612
 613         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 614         if (r < 0)
 615                 return r;
 616
 617         r = pull_job_begin(i->raw_job);
 618         if (r < 0)
 619                 return r;
 620
 621         if (i->settings_job) {
 622                 r = pull_job_begin(i->settings_job);
 623                 if (r < 0)
 624                         return r;
 625         }
 626
 627         if (i->checksum_job) {
 628                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 629
 630                 r = pull_job_begin(i->checksum_job);
 631                 if (r < 0)
 632                         return r;
 633         }
 634
 635         if (i->signature_job) {
 636                 i->signature_job->on_progress = raw_pull_job_on_progress;
 637
 638                 r = pull_job_begin(i->signature_job);
 639                 if (r < 0)
 640                         return r;
 641         }
 642
 643         return 0;
 644 }