src/import/pull-raw.c

   1 /***
   2   This file is part of systemd.
   3
   4   Copyright 2014 Lennart Poettering
   5
   6   systemd is free software; you can redistribute it and/or modify it
   7   under the terms of the GNU Lesser General Public License as published by
   8   the Free Software Foundation; either version 2.1 of the License, or
   9   (at your option) any later version.
  10
  11   systemd is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14   Lesser General Public License for more details.
  15
  16   You should have received a copy of the GNU Lesser General Public License
  17   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  18 ***/
  19
  20 #include <curl/curl.h>
  21 #include <linux/fs.h>
  22 #include <sys/xattr.h>
  23
  24 #include "sd-daemon.h"
  25
  26 #include "alloc-util.h"
  27 #include "btrfs-util.h"
  28 #include "chattr-util.h"
  29 #include "copy.h"
  30 #include "curl-util.h"
  31 #include "fd-util.h"
  32 #include "fileio.h"
  33 #include "fs-util.h"
  34 #include "hostname-util.h"
  35 #include "import-common.h"
  36 #include "import-util.h"
  37 #include "macro.h"
  38 #include "mkdir.h"
  39 #include "path-util.h"
  40 #include "pull-common.h"
  41 #include "pull-job.h"
  42 #include "pull-raw.h"
  43 #include "qcow2-util.h"
  44 #include "rm-rf.h"
  45 #include "string-util.h"
  46 #include "strv.h"
  47 #include "utf8.h"
  48 #include "util.h"
  49 #include "web-util.h"
  50
  51 typedef enum RawProgress {
  52         RAW_DOWNLOADING,
  53         RAW_VERIFYING,
  54         RAW_UNPACKING,
  55         RAW_FINALIZING,
  56         RAW_COPYING,
  57 } RawProgress;
  58
  59 struct RawPull {
  60         sd_event *event;
  61         CurlGlue *glue;
  62
  63         char *image_root;
  64
  65         PullJob *raw_job;
  66         PullJob *settings_job;
  67         PullJob *checksum_job;
  68         PullJob *signature_job;
  69
  70         RawPullFinished on_finished;
  71         void *userdata;
  72
  73         char *local;
  74         bool force_local;
  75         bool grow_machine_directory;
  76         bool settings;
  77
  78         char *final_path;
  79         char *temp_path;
  80
  81         char *settings_path;
  82         char *settings_temp_path;
  83
  84         ImportVerify verify;
  85 };
  86
  87 RawPull* raw_pull_unref(RawPull *i) {
  88         if (!i)
  89                 return NULL;
  90
  91         pull_job_unref(i->raw_job);
  92         pull_job_unref(i->settings_job);
  93         pull_job_unref(i->checksum_job);
  94         pull_job_unref(i->signature_job);
  95
  96         curl_glue_unref(i->glue);
  97         sd_event_unref(i->event);
  98
  99         if (i->temp_path) {
 100                 (void) unlink(i->temp_path);
 101                 free(i->temp_path);
 102         }
 103
 104         if (i->settings_temp_path) {
 105                 (void) unlink(i->settings_temp_path);
 106                 free(i->settings_temp_path);
 107         }
 108
 109         free(i->final_path);
 110         free(i->settings_path);
 111         free(i->image_root);
 112         free(i->local);
 113         free(i);
 114
 115         return NULL;
 116 }
 117
 118 int raw_pull_new(
 119                 RawPull **ret,
 120                 sd_event *event,
 121                 const char *image_root,
 122                 RawPullFinished on_finished,
 123                 void *userdata) {
 124
 125         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 126         int r;
 127
 128         assert(ret);
 129
 130         i = new0(RawPull, 1);
 131         if (!i)
 132                 return -ENOMEM;
 133
 134         i->on_finished = on_finished;
 135         i->userdata = userdata;
 136
 137         i->image_root = strdup(image_root ?: "/var/lib/machines");
 138         if (!i->image_root)
 139                 return -ENOMEM;
 140
 141         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 142
 143         if (event)
 144                 i->event = sd_event_ref(event);
 145         else {
 146                 r = sd_event_default(&i->event);
 147                 if (r < 0)
 148                         return r;
 149         }
 150
 151         r = curl_glue_new(&i->glue, i->event);
 152         if (r < 0)
 153                 return r;
 154
 155         i->glue->on_finished = pull_job_curl_on_finished;
 156         i->glue->userdata = i;
 157
 158         *ret = i;
 159         i = NULL;
 160
 161         return 0;
 162 }
 163
 164 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 165         unsigned percent;
 166
 167         assert(i);
 168
 169         switch (p) {
 170
 171         case RAW_DOWNLOADING: {
 172                 unsigned remain = 80;
 173
 174                 percent = 0;
 175
 176                 if (i->settings_job) {
 177                         percent += i->settings_job->progress_percent * 5 / 100;
 178                         remain -= 5;
 179                 }
 180
 181                 if (i->checksum_job) {
 182                         percent += i->checksum_job->progress_percent * 5 / 100;
 183                         remain -= 5;
 184                 }
 185
 186                 if (i->signature_job) {
 187                         percent += i->signature_job->progress_percent * 5 / 100;
 188                         remain -= 5;
 189                 }
 190
 191                 if (i->raw_job)
 192                         percent += i->raw_job->progress_percent * remain / 100;
 193                 break;
 194         }
 195
 196         case RAW_VERIFYING:
 197                 percent = 80;
 198                 break;
 199
 200         case RAW_UNPACKING:
 201                 percent = 85;
 202                 break;
 203
 204         case RAW_FINALIZING:
 205                 percent = 90;
 206                 break;
 207
 208         case RAW_COPYING:
 209                 percent = 95;
 210                 break;
 211
 212         default:
 213                 assert_not_reached("Unknown progress state");
 214         }
 215
 216         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 217         log_debug("Combined progress %u%%", percent);
 218 }
 219
 220 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 221         _cleanup_close_ int converted_fd = -1;
 222         _cleanup_free_ char *t = NULL;
 223         int r;
 224
 225         assert(i);
 226         assert(i->raw_job);
 227
 228         r = qcow2_detect(i->raw_job->disk_fd);
 229         if (r < 0)
 230                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 231         if (r == 0)
 232                 return 0;
 233
 234         /* This is a QCOW2 image, let's convert it */
 235         r = tempfn_random(i->final_path, NULL, &t);
 236         if (r < 0)
 237                 return log_oom();
 238
 239         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 240         if (converted_fd < 0)
 241                 return log_error_errno(errno, "Failed to create %s: %m", t);
 242
 243         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 244         if (r < 0)
 245                 log_warning_errno(r, "Failed to set file attributes on %s: %m", t);
 246
 247         log_info("Unpacking QCOW2 file.");
 248
 249         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 250         if (r < 0) {
 251                 unlink(t);
 252                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 253         }
 254
 255         (void) unlink(i->temp_path);
 256         free(i->temp_path);
 257         i->temp_path = t;
 258         t = NULL;
 259
 260         safe_close(i->raw_job->disk_fd);
 261         i->raw_job->disk_fd = converted_fd;
 262         converted_fd = -1;
 263
 264         return 1;
 265 }
 266
 267 static int raw_pull_make_local_copy(RawPull *i) {
 268         _cleanup_free_ char *tp = NULL;
 269         _cleanup_close_ int dfd = -1;
 270         const char *p;
 271         int r;
 272
 273         assert(i);
 274         assert(i->raw_job);
 275
 276         if (!i->local)
 277                 return 0;
 278
 279         if (!i->final_path) {
 280                 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 281                 if (r < 0)
 282                         return log_oom();
 283         }
 284
 285         if (i->raw_job->etag_exists) {
 286                 /* We have downloaded this one previously, reopen it */
 287
 288                 assert(i->raw_job->disk_fd < 0);
 289
 290                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 291                 if (i->raw_job->disk_fd < 0)
 292                         return log_error_errno(errno, "Failed to open vendor image: %m");
 293         } else {
 294                 /* We freshly downloaded the image, use it */
 295
 296                 assert(i->raw_job->disk_fd >= 0);
 297
 298                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 299                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 300         }
 301
 302         p = strjoina(i->image_root, "/", i->local, ".raw");
 303
 304         if (i->force_local)
 305                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 306
 307         r = tempfn_random(p, NULL, &tp);
 308         if (r < 0)
 309                 return log_oom();
 310
 311         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 312         if (dfd < 0)
 313                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 314
 315         /* Turn off COW writing. This should greatly improve
 316          * performance on COW file systems like btrfs, since it
 317          * reduces fragmentation caused by not allowing in-place
 318          * writes. */
 319         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 320         if (r < 0)
 321                 log_warning_errno(r, "Failed to set file attributes on %s: %m", tp);
 322
 323         r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, true);
 324         if (r < 0) {
 325                 unlink(tp);
 326                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 327         }
 328
 329         (void) copy_times(i->raw_job->disk_fd, dfd);
 330         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 331
 332         dfd = safe_close(dfd);
 333
 334         r = rename(tp, p);
 335         if (r < 0)  {
 336                 r = log_error_errno(errno, "Failed to move writable image into place: %m");
 337                 unlink(tp);
 338                 return r;
 339         }
 340
 341         log_info("Created new local image '%s'.", i->local);
 342
 343         if (i->settings) {
 344                 const char *local_settings;
 345                 assert(i->settings_job);
 346
 347                 if (!i->settings_path) {
 348                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 349                         if (r < 0)
 350                                 return log_oom();
 351                 }
 352
 353                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 354
 355                 r = copy_file_atomic(i->settings_path, local_settings, 0644, i->force_local, 0);
 356                 if (r == -EEXIST)
 357                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 358                 else if (r < 0 && r != -ENOENT)
 359                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 360                 else
 361                         log_info("Created new settings file '%s.nspawn'", i->local);
 362         }
 363
 364         return 0;
 365 }
 366
 367 static bool raw_pull_is_done(RawPull *i) {
 368         assert(i);
 369         assert(i->raw_job);
 370
 371         if (!PULL_JOB_IS_COMPLETE(i->raw_job))
 372                 return false;
 373         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 374                 return false;
 375         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 376                 return false;
 377         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 378                 return false;
 379
 380         return true;
 381 }
 382
 383 static void raw_pull_job_on_finished(PullJob *j) {
 384         RawPull *i;
 385         int r;
 386
 387         assert(j);
 388         assert(j->userdata);
 389
 390         i = j->userdata;
 391         if (j == i->settings_job) {
 392                 if (j->error != 0)
 393                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 394         } else if (j->error != 0) {
 395                 if (j == i->checksum_job)
 396                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 397                 else if (j == i->signature_job)
 398                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 399                 else
 400                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 401
 402                 r = j->error;
 403                 goto finish;
 404         }
 405
 406         /* This is invoked if either the download completed
 407          * successfully, or the download was skipped because we
 408          * already have the etag. In this case ->etag_exists is
 409          * true.
 410          *
 411          * We only do something when we got all three files */
 412
 413         if (!raw_pull_is_done(i))
 414                 return;
 415
 416         if (i->settings_job)
 417                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 418
 419         if (!i->raw_job->etag_exists) {
 420                 /* This is a new download, verify it, and move it into place */
 421                 assert(i->raw_job->disk_fd >= 0);
 422
 423                 raw_pull_report_progress(i, RAW_VERIFYING);
 424
 425                 r = pull_verify(i->raw_job, i->settings_job, i->checksum_job, i->signature_job);
 426                 if (r < 0)
 427                         goto finish;
 428
 429                 raw_pull_report_progress(i, RAW_UNPACKING);
 430
 431                 r = raw_pull_maybe_convert_qcow2(i);
 432                 if (r < 0)
 433                         goto finish;
 434
 435                 raw_pull_report_progress(i, RAW_FINALIZING);
 436
 437                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 438                 if (r < 0)
 439                         goto finish;
 440
 441                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 442                 if (r < 0) {
 443                         log_error_errno(r, "Failed to move RAW file into place: %m");
 444                         goto finish;
 445                 }
 446
 447                 i->temp_path = mfree(i->temp_path);
 448
 449                 if (i->settings_job &&
 450                     i->settings_job->error == 0 &&
 451                     !i->settings_job->etag_exists) {
 452
 453                         assert(i->settings_temp_path);
 454                         assert(i->settings_path);
 455
 456                         r = import_make_read_only(i->settings_temp_path);
 457                         if (r < 0)
 458                                 goto finish;
 459
 460                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 461                         if (r < 0) {
 462                                 log_error_errno(r, "Failed to rename settings file: %m");
 463                                 goto finish;
 464                         }
 465
 466                         i->settings_temp_path = mfree(i->settings_temp_path);
 467                 }
 468         }
 469
 470         raw_pull_report_progress(i, RAW_COPYING);
 471
 472         r = raw_pull_make_local_copy(i);
 473         if (r < 0)
 474                 goto finish;
 475
 476         r = 0;
 477
 478 finish:
 479         if (i->on_finished)
 480                 i->on_finished(i, r, i->userdata);
 481         else
 482                 sd_event_exit(i->event, r);
 483 }
 484
 485 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 486         RawPull *i;
 487         int r;
 488
 489         assert(j);
 490         assert(j->userdata);
 491
 492         i = j->userdata;
 493         assert(i->raw_job == j);
 494         assert(!i->final_path);
 495         assert(!i->temp_path);
 496
 497         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 498         if (r < 0)
 499                 return log_oom();
 500
 501         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 502         if (r < 0)
 503                 return log_oom();
 504
 505         (void) mkdir_parents_label(i->temp_path, 0700);
 506
 507         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 508         if (j->disk_fd < 0)
 509                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 510
 511         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 512         if (r < 0)
 513                 log_warning_errno(r, "Failed to set file attributes on %s: %m", i->temp_path);
 514
 515         return 0;
 516 }
 517
 518 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
 519         RawPull *i;
 520         int r;
 521
 522         assert(j);
 523         assert(j->userdata);
 524
 525         i = j->userdata;
 526         assert(i->settings_job == j);
 527         assert(!i->settings_path);
 528         assert(!i->settings_temp_path);
 529
 530         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 531         if (r < 0)
 532                 return log_oom();
 533
 534         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 535         if (r < 0)
 536                 return log_oom();
 537
 538         mkdir_parents_label(i->settings_temp_path, 0700);
 539
 540         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 541         if (j->disk_fd < 0)
 542                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 543
 544         return 0;
 545 }
 546
 547 static void raw_pull_job_on_progress(PullJob *j) {
 548         RawPull *i;
 549
 550         assert(j);
 551         assert(j->userdata);
 552
 553         i = j->userdata;
 554
 555         raw_pull_report_progress(i, RAW_DOWNLOADING);
 556 }
 557
 558 int raw_pull_start(
 559                 RawPull *i,
 560                 const char *url,
 561                 const char *local,
 562                 bool force_local,
 563                 ImportVerify verify,
 564                 bool settings) {
 565
 566         int r;
 567
 568         assert(i);
 569         assert(verify < _IMPORT_VERIFY_MAX);
 570         assert(verify >= 0);
 571
 572         if (!http_url_is_valid(url))
 573                 return -EINVAL;
 574
 575         if (local && !machine_name_is_valid(local))
 576                 return -EINVAL;
 577
 578         if (i->raw_job)
 579                 return -EBUSY;
 580
 581         r = free_and_strdup(&i->local, local);
 582         if (r < 0)
 583                 return r;
 584
 585         i->force_local = force_local;
 586         i->verify = verify;
 587         i->settings = settings;
 588
 589         /* Queue job for the image itself */
 590         r = pull_job_new(&i->raw_job, url, i->glue, i);
 591         if (r < 0)
 592                 return r;
 593
 594         i->raw_job->on_finished = raw_pull_job_on_finished;
 595         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 596         i->raw_job->on_progress = raw_pull_job_on_progress;
 597         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 598         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 599
 600         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 601         if (r < 0)
 602                 return r;
 603
 604         if (settings) {
 605                 r = pull_make_settings_job(&i->settings_job, url, i->glue, raw_pull_job_on_finished, i);
 606                 if (r < 0)
 607                         return r;
 608
 609                 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
 610                 i->settings_job->on_progress = raw_pull_job_on_progress;
 611                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 612
 613                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 614                 if (r < 0)
 615                         return r;
 616         }
 617
 618         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 619         if (r < 0)
 620                 return r;
 621
 622         r = pull_job_begin(i->raw_job);
 623         if (r < 0)
 624                 return r;
 625
 626         if (i->settings_job) {
 627                 r = pull_job_begin(i->settings_job);
 628                 if (r < 0)
 629                         return r;
 630         }
 631
 632         if (i->checksum_job) {
 633                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 634
 635                 r = pull_job_begin(i->checksum_job);
 636                 if (r < 0)
 637                         return r;
 638         }
 639
 640         if (i->signature_job) {
 641                 i->signature_job->on_progress = raw_pull_job_on_progress;
 642
 643                 r = pull_job_begin(i->signature_job);
 644                 if (r < 0)
 645                         return r;
 646         }
 647
 648         return 0;
 649 }