src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25
  26 #include "sd-daemon.h"
  27 #include "utf8.h"
  28 #include "strv.h"
  29 #include "copy.h"
  30 #include "btrfs-util.h"
  31 #include "util.h"
  32 #include "macro.h"
  33 #include "mkdir.h"
  34 #include "rm-rf.h"
  35 #include "path-util.h"
  36 #include "hostname-util.h"
  37 #include "import-util.h"
  38 #include "import-common.h"
  39 #include "curl-util.h"
  40 #include "qcow2-util.h"
  41 #include "pull-job.h"
  42 #include "pull-common.h"
  43 #include "pull-raw.h"
  44
  45 typedef enum RawProgress {
  46         RAW_DOWNLOADING,
  47         RAW_VERIFYING,
  48         RAW_UNPACKING,
  49         RAW_FINALIZING,
  50         RAW_COPYING,
  51 } RawProgress;
  52
  53 struct RawPull {
  54         sd_event *event;
  55         CurlGlue *glue;
  56
  57         char *image_root;
  58
  59         PullJob *raw_job;
  60         PullJob *checksum_job;
  61         PullJob *signature_job;
  62
  63         RawPullFinished on_finished;
  64         void *userdata;
  65
  66         char *local;
  67         bool force_local;
  68         bool grow_machine_directory;
  69
  70         char *temp_path;
  71         char *final_path;
  72
  73         ImportVerify verify;
  74 };
  75
  76 RawPull* raw_pull_unref(RawPull *i) {
  77         if (!i)
  78                 return NULL;
  79
  80         pull_job_unref(i->raw_job);
  81         pull_job_unref(i->checksum_job);
  82         pull_job_unref(i->signature_job);
  83
  84         curl_glue_unref(i->glue);
  85         sd_event_unref(i->event);
  86
  87         if (i->temp_path) {
  88                 (void) unlink(i->temp_path);
  89                 free(i->temp_path);
  90         }
  91
  92         free(i->final_path);
  93         free(i->image_root);
  94         free(i->local);
  95         free(i);
  96
  97         return NULL;
  98 }
  99
 100 int raw_pull_new(
 101                 RawPull **ret,
 102                 sd_event *event,
 103                 const char *image_root,
 104                 RawPullFinished on_finished,
 105                 void *userdata) {
 106
 107         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 108         int r;
 109
 110         assert(ret);
 111
 112         i = new0(RawPull, 1);
 113         if (!i)
 114                 return -ENOMEM;
 115
 116         i->on_finished = on_finished;
 117         i->userdata = userdata;
 118
 119         i->image_root = strdup(image_root ?: "/var/lib/machines");
 120         if (!i->image_root)
 121                 return -ENOMEM;
 122
 123         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 124
 125         if (event)
 126                 i->event = sd_event_ref(event);
 127         else {
 128                 r = sd_event_default(&i->event);
 129                 if (r < 0)
 130                         return r;
 131         }
 132
 133         r = curl_glue_new(&i->glue, i->event);
 134         if (r < 0)
 135                 return r;
 136
 137         i->glue->on_finished = pull_job_curl_on_finished;
 138         i->glue->userdata = i;
 139
 140         *ret = i;
 141         i = NULL;
 142
 143         return 0;
 144 }
 145
 146 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 147         unsigned percent;
 148
 149         assert(i);
 150
 151         switch (p) {
 152
 153         case RAW_DOWNLOADING: {
 154                 unsigned remain = 80;
 155
 156                 percent = 0;
 157
 158                 if (i->checksum_job) {
 159                         percent += i->checksum_job->progress_percent * 5 / 100;
 160                         remain -= 5;
 161                 }
 162
 163                 if (i->signature_job) {
 164                         percent += i->signature_job->progress_percent * 5 / 100;
 165                         remain -= 5;
 166                 }
 167
 168                 if (i->raw_job)
 169                         percent += i->raw_job->progress_percent * remain / 100;
 170                 break;
 171         }
 172
 173         case RAW_VERIFYING:
 174                 percent = 80;
 175                 break;
 176
 177         case RAW_UNPACKING:
 178                 percent = 85;
 179                 break;
 180
 181         case RAW_FINALIZING:
 182                 percent = 90;
 183                 break;
 184
 185         case RAW_COPYING:
 186                 percent = 95;
 187                 break;
 188
 189         default:
 190                 assert_not_reached("Unknown progress state");
 191         }
 192
 193         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 194         log_debug("Combined progress %u%%", percent);
 195 }
 196
 197 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 198         _cleanup_close_ int converted_fd = -1;
 199         _cleanup_free_ char *t = NULL;
 200         int r;
 201
 202         assert(i);
 203         assert(i->raw_job);
 204
 205         r = qcow2_detect(i->raw_job->disk_fd);
 206         if (r < 0)
 207                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 208         if (r == 0)
 209                 return 0;
 210
 211         /* This is a QCOW2 image, let's convert it */
 212         r = tempfn_random(i->final_path, NULL, &t);
 213         if (r < 0)
 214                 return log_oom();
 215
 216         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 217         if (converted_fd < 0)
 218                 return log_error_errno(errno, "Failed to create %s: %m", t);
 219
 220         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 221         if (r < 0)
 222                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 223
 224         log_info("Unpacking QCOW2 file.");
 225
 226         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 227         if (r < 0) {
 228                 unlink(t);
 229                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 230         }
 231
 232         (void) unlink(i->temp_path);
 233         free(i->temp_path);
 234         i->temp_path = t;
 235         t = NULL;
 236
 237         safe_close(i->raw_job->disk_fd);
 238         i->raw_job->disk_fd = converted_fd;
 239         converted_fd = -1;
 240
 241         return 1;
 242 }
 243
 244 static int raw_pull_make_local_copy(RawPull *i) {
 245         _cleanup_free_ char *tp = NULL;
 246         _cleanup_close_ int dfd = -1;
 247         const char *p;
 248         int r;
 249
 250         assert(i);
 251         assert(i->raw_job);
 252
 253         if (!i->local)
 254                 return 0;
 255
 256         if (i->raw_job->etag_exists) {
 257                 /* We have downloaded this one previously, reopen it */
 258
 259                 assert(i->raw_job->disk_fd < 0);
 260
 261                 if (!i->final_path) {
 262                         r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 263                         if (r < 0)
 264                                 return log_oom();
 265                 }
 266
 267                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 268                 if (i->raw_job->disk_fd < 0)
 269                         return log_error_errno(errno, "Failed to open vendor image: %m");
 270         } else {
 271                 /* We freshly downloaded the image, use it */
 272
 273                 assert(i->raw_job->disk_fd >= 0);
 274
 275                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 276                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 277         }
 278
 279         p = strjoina(i->image_root, "/", i->local, ".raw");
 280
 281         if (i->force_local)
 282                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 283
 284         r = tempfn_random(p, NULL, &tp);
 285         if (r < 0)
 286                 return log_oom();
 287
 288         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 289         if (dfd < 0)
 290                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 291
 292         /* Turn off COW writing. This should greatly improve
 293          * performance on COW file systems like btrfs, since it
 294          * reduces fragmentation caused by not allowing in-place
 295          * writes. */
 296         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 297         if (r < 0)
 298                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 299
 300         r = copy_bytes(i->raw_job->disk_fd, dfd, (off_t) -1, true);
 301         if (r < 0) {
 302                 unlink(tp);
 303                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 304         }
 305
 306         (void) copy_times(i->raw_job->disk_fd, dfd);
 307         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 308
 309         dfd = safe_close(dfd);
 310
 311         r = rename(tp, p);
 312         if (r < 0)  {
 313                 unlink(tp);
 314                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 315         }
 316
 317         log_info("Created new local image '%s'.", i->local);
 318         return 0;
 319 }
 320
 321 static bool raw_pull_is_done(RawPull *i) {
 322         assert(i);
 323         assert(i->raw_job);
 324
 325         if (i->raw_job->state != PULL_JOB_DONE)
 326                 return false;
 327         if (i->checksum_job && i->checksum_job->state != PULL_JOB_DONE)
 328                 return false;
 329         if (i->signature_job && i->signature_job->state != PULL_JOB_DONE)
 330                 return false;
 331
 332         return true;
 333 }
 334
 335 static void raw_pull_job_on_finished(PullJob *j) {
 336         RawPull *i;
 337         int r;
 338
 339         assert(j);
 340         assert(j->userdata);
 341
 342         i = j->userdata;
 343         if (j->error != 0) {
 344                 if (j == i->checksum_job)
 345                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 346                 else if (j == i->signature_job)
 347                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 348                 else
 349                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 350
 351                 r = j->error;
 352                 goto finish;
 353         }
 354
 355         /* This is invoked if either the download completed
 356          * successfully, or the download was skipped because we
 357          * already have the etag. In this case ->etag_exists is
 358          * true.
 359          *
 360          * We only do something when we got all three files */
 361
 362         if (!raw_pull_is_done(i))
 363                 return;
 364
 365         if (!i->raw_job->etag_exists) {
 366                 /* This is a new download, verify it, and move it into place */
 367                 assert(i->raw_job->disk_fd >= 0);
 368
 369                 raw_pull_report_progress(i, RAW_VERIFYING);
 370
 371                 r = pull_verify(i->raw_job, i->checksum_job, i->signature_job);
 372                 if (r < 0)
 373                         goto finish;
 374
 375                 raw_pull_report_progress(i, RAW_UNPACKING);
 376
 377                 r = raw_pull_maybe_convert_qcow2(i);
 378                 if (r < 0)
 379                         goto finish;
 380
 381                 raw_pull_report_progress(i, RAW_FINALIZING);
 382
 383                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 384                 if (r < 0)
 385                         goto finish;
 386
 387                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 388                 if (r < 0) {
 389                         log_error_errno(r, "Failed to move RAW file into place: %m");
 390                         goto finish;
 391                 }
 392
 393                 free(i->temp_path);
 394                 i->temp_path = NULL;
 395         }
 396
 397         raw_pull_report_progress(i, RAW_COPYING);
 398
 399         r = raw_pull_make_local_copy(i);
 400         if (r < 0)
 401                 goto finish;
 402
 403         r = 0;
 404
 405 finish:
 406         if (i->on_finished)
 407                 i->on_finished(i, r, i->userdata);
 408         else
 409                 sd_event_exit(i->event, r);
 410 }
 411
 412 static int raw_pull_job_on_open_disk(PullJob *j) {
 413         RawPull *i;
 414         int r;
 415
 416         assert(j);
 417         assert(j->userdata);
 418
 419         i = j->userdata;
 420         assert(i->raw_job == j);
 421         assert(!i->final_path);
 422         assert(!i->temp_path);
 423
 424         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 425         if (r < 0)
 426                 return log_oom();
 427
 428         r = tempfn_random(i->final_path, NULL, &i->temp_path);
 429         if (r < 0)
 430                 return log_oom();
 431
 432         (void) mkdir_parents_label(i->temp_path, 0700);
 433
 434         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 435         if (j->disk_fd < 0)
 436                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 437
 438         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 439         if (r < 0)
 440                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 441
 442         return 0;
 443 }
 444
 445 static void raw_pull_job_on_progress(PullJob *j) {
 446         RawPull *i;
 447
 448         assert(j);
 449         assert(j->userdata);
 450
 451         i = j->userdata;
 452
 453         raw_pull_report_progress(i, RAW_DOWNLOADING);
 454 }
 455
 456 int raw_pull_start(RawPull *i, const char *url, const char *local, bool force_local, ImportVerify verify) {
 457         int r;
 458
 459         assert(i);
 460         assert(verify < _IMPORT_VERIFY_MAX);
 461         assert(verify >= 0);
 462
 463         if (!http_url_is_valid(url))
 464                 return -EINVAL;
 465
 466         if (local && !machine_name_is_valid(local))
 467                 return -EINVAL;
 468
 469         if (i->raw_job)
 470                 return -EBUSY;
 471
 472         r = free_and_strdup(&i->local, local);
 473         if (r < 0)
 474                 return r;
 475         i->force_local = force_local;
 476         i->verify = verify;
 477
 478         /* Queue job for the image itself */
 479         r = pull_job_new(&i->raw_job, url, i->glue, i);
 480         if (r < 0)
 481                 return r;
 482
 483         i->raw_job->on_finished = raw_pull_job_on_finished;
 484         i->raw_job->on_open_disk = raw_pull_job_on_open_disk;
 485         i->raw_job->on_progress = raw_pull_job_on_progress;
 486         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 487         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 488
 489         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 490         if (r < 0)
 491                 return r;
 492
 493         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 494         if (r < 0)
 495                 return r;
 496
 497         r = pull_job_begin(i->raw_job);
 498         if (r < 0)
 499                 return r;
 500
 501         if (i->checksum_job) {
 502                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 503
 504                 r = pull_job_begin(i->checksum_job);
 505                 if (r < 0)
 506                         return r;
 507         }
 508
 509         if (i->signature_job) {
 510                 i->signature_job->on_progress = raw_pull_job_on_progress;
 511
 512                 r = pull_job_begin(i->signature_job);
 513                 if (r < 0)
 514                         return r;
 515         }
 516
 517         return 0;
 518 }