src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <curl/curl.h>
  23 #include <linux/fs.h>
  24 #include <sys/xattr.h>
  25
  26 #include "sd-daemon.h"
  27
  28 #include "btrfs-util.h"
  29 #include "copy.h"
  30 #include "curl-util.h"
  31 #include "fd-util.h"
  32 #include "hostname-util.h"
  33 #include "import-common.h"
  34 #include "import-util.h"
  35 #include "macro.h"
  36 #include "mkdir.h"
  37 #include "path-util.h"
  38 #include "pull-common.h"
  39 #include "pull-job.h"
  40 #include "pull-raw.h"
  41 #include "qcow2-util.h"
  42 #include "rm-rf.h"
  43 #include "string-util.h"
  44 #include "strv.h"
  45 #include "utf8.h"
  46 #include "util.h"
  47
  48 typedef enum RawProgress {
  49         RAW_DOWNLOADING,
  50         RAW_VERIFYING,
  51         RAW_UNPACKING,
  52         RAW_FINALIZING,
  53         RAW_COPYING,
  54 } RawProgress;
  55
  56 struct RawPull {
  57         sd_event *event;
  58         CurlGlue *glue;
  59
  60         char *image_root;
  61
  62         PullJob *raw_job;
  63         PullJob *settings_job;
  64         PullJob *checksum_job;
  65         PullJob *signature_job;
  66
  67         RawPullFinished on_finished;
  68         void *userdata;
  69
  70         char *local;
  71         bool force_local;
  72         bool grow_machine_directory;
  73         bool settings;
  74
  75         char *final_path;
  76         char *temp_path;
  77
  78         char *settings_path;
  79         char *settings_temp_path;
  80
  81         ImportVerify verify;
  82 };
  83
  84 RawPull* raw_pull_unref(RawPull *i) {
  85         if (!i)
  86                 return NULL;
  87
  88         pull_job_unref(i->raw_job);
  89         pull_job_unref(i->settings_job);
  90         pull_job_unref(i->checksum_job);
  91         pull_job_unref(i->signature_job);
  92
  93         curl_glue_unref(i->glue);
  94         sd_event_unref(i->event);
  95
  96         if (i->temp_path) {
  97                 (void) unlink(i->temp_path);
  98                 free(i->temp_path);
  99         }
 100
 101         if (i->settings_temp_path) {
 102                 (void) unlink(i->settings_temp_path);
 103                 free(i->settings_temp_path);
 104         }
 105
 106         free(i->final_path);
 107         free(i->settings_path);
 108         free(i->image_root);
 109         free(i->local);
 110         free(i);
 111
 112         return NULL;
 113 }
 114
 115 int raw_pull_new(
 116                 RawPull **ret,
 117                 sd_event *event,
 118                 const char *image_root,
 119                 RawPullFinished on_finished,
 120                 void *userdata) {
 121
 122         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 123         int r;
 124
 125         assert(ret);
 126
 127         i = new0(RawPull, 1);
 128         if (!i)
 129                 return -ENOMEM;
 130
 131         i->on_finished = on_finished;
 132         i->userdata = userdata;
 133
 134         i->image_root = strdup(image_root ?: "/var/lib/machines");
 135         if (!i->image_root)
 136                 return -ENOMEM;
 137
 138         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 139
 140         if (event)
 141                 i->event = sd_event_ref(event);
 142         else {
 143                 r = sd_event_default(&i->event);
 144                 if (r < 0)
 145                         return r;
 146         }
 147
 148         r = curl_glue_new(&i->glue, i->event);
 149         if (r < 0)
 150                 return r;
 151
 152         i->glue->on_finished = pull_job_curl_on_finished;
 153         i->glue->userdata = i;
 154
 155         *ret = i;
 156         i = NULL;
 157
 158         return 0;
 159 }
 160
 161 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 162         unsigned percent;
 163
 164         assert(i);
 165
 166         switch (p) {
 167
 168         case RAW_DOWNLOADING: {
 169                 unsigned remain = 80;
 170
 171                 percent = 0;
 172
 173                 if (i->settings_job) {
 174                         percent += i->settings_job->progress_percent * 5 / 100;
 175                         remain -= 5;
 176                 }
 177
 178                 if (i->checksum_job) {
 179                         percent += i->checksum_job->progress_percent * 5 / 100;
 180                         remain -= 5;
 181                 }
 182
 183                 if (i->signature_job) {
 184                         percent += i->signature_job->progress_percent * 5 / 100;
 185                         remain -= 5;
 186                 }
 187
 188                 if (i->raw_job)
 189                         percent += i->raw_job->progress_percent * remain / 100;
 190                 break;
 191         }
 192
 193         case RAW_VERIFYING:
 194                 percent = 80;
 195                 break;
 196
 197         case RAW_UNPACKING:
 198                 percent = 85;
 199                 break;
 200
 201         case RAW_FINALIZING:
 202                 percent = 90;
 203                 break;
 204
 205         case RAW_COPYING:
 206                 percent = 95;
 207                 break;
 208
 209         default:
 210                 assert_not_reached("Unknown progress state");
 211         }
 212
 213         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 214         log_debug("Combined progress %u%%", percent);
 215 }
 216
 217 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 218         _cleanup_close_ int converted_fd = -1;
 219         _cleanup_free_ char *t = NULL;
 220         int r;
 221
 222         assert(i);
 223         assert(i->raw_job);
 224
 225         r = qcow2_detect(i->raw_job->disk_fd);
 226         if (r < 0)
 227                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 228         if (r == 0)
 229                 return 0;
 230
 231         /* This is a QCOW2 image, let's convert it */
 232         r = tempfn_random(i->final_path, NULL, &t);
 233         if (r < 0)
 234                 return log_oom();
 235
 236         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 237         if (converted_fd < 0)
 238                 return log_error_errno(errno, "Failed to create %s: %m", t);
 239
 240         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 241         if (r < 0)
 242                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 243
 244         log_info("Unpacking QCOW2 file.");
 245
 246         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 247         if (r < 0) {
 248                 unlink(t);
 249                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 250         }
 251
 252         (void) unlink(i->temp_path);
 253         free(i->temp_path);
 254         i->temp_path = t;
 255         t = NULL;
 256
 257         safe_close(i->raw_job->disk_fd);
 258         i->raw_job->disk_fd = converted_fd;
 259         converted_fd = -1;
 260
 261         return 1;
 262 }
 263
 264 static int raw_pull_make_local_copy(RawPull *i) {
 265         _cleanup_free_ char *tp = NULL;
 266         _cleanup_close_ int dfd = -1;
 267         const char *p;
 268         int r;
 269
 270         assert(i);
 271         assert(i->raw_job);
 272
 273         if (!i->local)
 274                 return 0;
 275
 276         if (!i->final_path) {
 277                 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 278                 if (r < 0)
 279                         return log_oom();
 280         }
 281
 282         if (i->raw_job->etag_exists) {
 283                 /* We have downloaded this one previously, reopen it */
 284
 285                 assert(i->raw_job->disk_fd < 0);
 286
 287                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 288                 if (i->raw_job->disk_fd < 0)
 289                         return log_error_errno(errno, "Failed to open vendor image: %m");
 290         } else {
 291                 /* We freshly downloaded the image, use it */
 292
 293                 assert(i->raw_job->disk_fd >= 0);
 294
 295                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 296                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 297         }
 298
 299         p = strjoina(i->image_root, "/", i->local, ".raw");
 300
 301         if (i->force_local)
 302                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 303
 304         r = tempfn_random(p, NULL, &tp);
 305         if (r < 0)
 306                 return log_oom();
 307
 308         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 309         if (dfd < 0)
 310                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 311
 312         /* Turn off COW writing. This should greatly improve
 313          * performance on COW file systems like btrfs, since it
 314          * reduces fragmentation caused by not allowing in-place
 315          * writes. */
 316         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 317         if (r < 0)
 318                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 319
 320         r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, true);
 321         if (r < 0) {
 322                 unlink(tp);
 323                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 324         }
 325
 326         (void) copy_times(i->raw_job->disk_fd, dfd);
 327         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 328
 329         dfd = safe_close(dfd);
 330
 331         r = rename(tp, p);
 332         if (r < 0)  {
 333                 unlink(tp);
 334                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 335         }
 336
 337         log_info("Created new local image '%s'.", i->local);
 338
 339         if (i->settings) {
 340                 const char *local_settings;
 341                 assert(i->settings_job);
 342
 343                 if (!i->settings_path) {
 344                         r = pull_make_path(i->settings_job->url, i->settings_job->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 345                         if (r < 0)
 346                                 return log_oom();
 347                 }
 348
 349                 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
 350
 351                 r = copy_file_atomic(i->settings_path, local_settings, 0644, i->force_local, 0);
 352                 if (r == -EEXIST)
 353                         log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
 354                 else if (r < 0 && r != -ENOENT)
 355                         log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
 356                 else
 357                         log_info("Created new settings file '%s.nspawn'", i->local);
 358         }
 359
 360         return 0;
 361 }
 362
 363 static bool raw_pull_is_done(RawPull *i) {
 364         assert(i);
 365         assert(i->raw_job);
 366
 367         if (!PULL_JOB_IS_COMPLETE(i->raw_job))
 368                 return false;
 369         if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
 370                 return false;
 371         if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
 372                 return false;
 373         if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
 374                 return false;
 375
 376         return true;
 377 }
 378
 379 static void raw_pull_job_on_finished(PullJob *j) {
 380         RawPull *i;
 381         int r;
 382
 383         assert(j);
 384         assert(j->userdata);
 385
 386         i = j->userdata;
 387         if (j == i->settings_job) {
 388                 if (j->error != 0)
 389                         log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
 390         } else if (j->error != 0) {
 391                 if (j == i->checksum_job)
 392                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 393                 else if (j == i->signature_job)
 394                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 395                 else
 396                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 397
 398                 r = j->error;
 399                 goto finish;
 400         }
 401
 402         /* This is invoked if either the download completed
 403          * successfully, or the download was skipped because we
 404          * already have the etag. In this case ->etag_exists is
 405          * true.
 406          *
 407          * We only do something when we got all three files */
 408
 409         if (!raw_pull_is_done(i))
 410                 return;
 411
 412         if (i->settings_job)
 413                 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
 414
 415         if (!i->raw_job->etag_exists) {
 416                 /* This is a new download, verify it, and move it into place */
 417                 assert(i->raw_job->disk_fd >= 0);
 418
 419                 raw_pull_report_progress(i, RAW_VERIFYING);
 420
 421                 r = pull_verify(i->raw_job, i->settings_job, i->checksum_job, i->signature_job);
 422                 if (r < 0)
 423                         goto finish;
 424
 425                 raw_pull_report_progress(i, RAW_UNPACKING);
 426
 427                 r = raw_pull_maybe_convert_qcow2(i);
 428                 if (r < 0)
 429                         goto finish;
 430
 431                 raw_pull_report_progress(i, RAW_FINALIZING);
 432
 433                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 434                 if (r < 0)
 435                         goto finish;
 436
 437                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 438                 if (r < 0) {
 439                         log_error_errno(r, "Failed to move RAW file into place: %m");
 440                         goto finish;
 441                 }
 442
 443                 i->temp_path = mfree(i->temp_path);
 444
 445                 if (i->settings_job &&
 446                     i->settings_job->error == 0 &&
 447                     !i->settings_job->etag_exists) {
 448
 449                         assert(i->settings_temp_path);
 450                         assert(i->settings_path);
 451
 452                         r = import_make_read_only(i->settings_temp_path);
 453                         if (r < 0)
 454                                 goto finish;
 455
 456                         r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
 457                         if (r < 0) {
 458                                 log_error_errno(r, "Failed to rename settings file: %m");
 459                                 goto finish;
 460                         }
 461
 462                         i->settings_temp_path = mfree(i->settings_temp_path);
 463                 }
 464         }
 465
 466         raw_pull_report_progress(i, RAW_COPYING);
 467
 468         r = raw_pull_make_local_copy(i);
 469         if (r < 0)
 470                 goto finish;
 471
 472         r = 0;
 473
 474 finish:
 475         if (i->on_finished)
 476                 i->on_finished(i, r, i->userdata);
 477         else
 478                 sd_event_exit(i->event, r);
 479 }
 480
 481 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
 482         RawPull *i;
 483         int r;
 484
 485         assert(j);
 486         assert(j->userdata);
 487
 488         i = j->userdata;
 489         assert(i->raw_job == j);
 490         assert(!i->final_path);
 491         assert(!i->temp_path);
 492
 493         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 494         if (r < 0)
 495                 return log_oom();
 496
 497         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 498         if (r < 0)
 499                 return log_oom();
 500
 501         (void) mkdir_parents_label(i->temp_path, 0700);
 502
 503         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 504         if (j->disk_fd < 0)
 505                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 506
 507         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 508         if (r < 0)
 509                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 510
 511         return 0;
 512 }
 513
 514 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
 515         RawPull *i;
 516         int r;
 517
 518         assert(j);
 519         assert(j->userdata);
 520
 521         i = j->userdata;
 522         assert(i->settings_job == j);
 523         assert(!i->settings_path);
 524         assert(!i->settings_temp_path);
 525
 526         r = pull_make_path(j->url, j->etag, i->image_root, ".settings-", NULL, &i->settings_path);
 527         if (r < 0)
 528                 return log_oom();
 529
 530         r = tempfn_random(i->settings_path, NULL, &i->settings_temp_path);
 531         if (r < 0)
 532                 return log_oom();
 533
 534         mkdir_parents_label(i->settings_temp_path, 0700);
 535
 536         j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 537         if (j->disk_fd < 0)
 538                 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
 539
 540         return 0;
 541 }
 542
 543 static void raw_pull_job_on_progress(PullJob *j) {
 544         RawPull *i;
 545
 546         assert(j);
 547         assert(j->userdata);
 548
 549         i = j->userdata;
 550
 551         raw_pull_report_progress(i, RAW_DOWNLOADING);
 552 }
 553
 554 int raw_pull_start(
 555                 RawPull *i,
 556                 const char *url,
 557                 const char *local,
 558                 bool force_local,
 559                 ImportVerify verify,
 560                 bool settings) {
 561
 562         int r;
 563
 564         assert(i);
 565         assert(verify < _IMPORT_VERIFY_MAX);
 566         assert(verify >= 0);
 567
 568         if (!http_url_is_valid(url))
 569                 return -EINVAL;
 570
 571         if (local && !machine_name_is_valid(local))
 572                 return -EINVAL;
 573
 574         if (i->raw_job)
 575                 return -EBUSY;
 576
 577         r = free_and_strdup(&i->local, local);
 578         if (r < 0)
 579                 return r;
 580
 581         i->force_local = force_local;
 582         i->verify = verify;
 583         i->settings = settings;
 584
 585         /* Queue job for the image itself */
 586         r = pull_job_new(&i->raw_job, url, i->glue, i);
 587         if (r < 0)
 588                 return r;
 589
 590         i->raw_job->on_finished = raw_pull_job_on_finished;
 591         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 592         i->raw_job->on_progress = raw_pull_job_on_progress;
 593         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 594         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 595
 596         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 597         if (r < 0)
 598                 return r;
 599
 600         if (settings) {
 601                 r = pull_make_settings_job(&i->settings_job, url, i->glue, raw_pull_job_on_finished, i);
 602                 if (r < 0)
 603                         return r;
 604
 605                 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
 606                 i->settings_job->on_progress = raw_pull_job_on_progress;
 607                 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 608
 609                 r = pull_find_old_etags(i->settings_job->url, i->image_root, DT_REG, ".settings-", NULL, &i->settings_job->old_etags);
 610                 if (r < 0)
 611                         return r;
 612         }
 613
 614         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 615         if (r < 0)
 616                 return r;
 617
 618         r = pull_job_begin(i->raw_job);
 619         if (r < 0)
 620                 return r;
 621
 622         if (i->settings_job) {
 623                 r = pull_job_begin(i->settings_job);
 624                 if (r < 0)
 625                         return r;
 626         }
 627
 628         if (i->checksum_job) {
 629                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 630
 631                 r = pull_job_begin(i->checksum_job);
 632                 if (r < 0)
 633                         return r;
 634         }
 635
 636         if (i->signature_job) {
 637                 i->signature_job->on_progress = raw_pull_job_on_progress;
 638
 639                 r = pull_job_begin(i->signature_job);
 640                 if (r < 0)
 641                         return r;
 642         }
 643
 644         return 0;
 645 }