src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25
  26 #include "sd-daemon.h"
  27 #include "utf8.h"
  28 #include "strv.h"
  29 #include "copy.h"
  30 #include "btrfs-util.h"
  31 #include "util.h"
  32 #include "macro.h"
  33 #include "mkdir.h"
  34 #include "rm-rf.h"
  35 #include "path-util.h"
  36 #include "import-util.h"
  37 #include "import-common.h"
  38 #include "curl-util.h"
  39 #include "qcow2-util.h"
  40 #include "pull-job.h"
  41 #include "pull-common.h"
  42 #include "pull-raw.h"
  43
  44 typedef enum RawProgress {
  45         RAW_DOWNLOADING,
  46         RAW_VERIFYING,
  47         RAW_UNPACKING,
  48         RAW_FINALIZING,
  49         RAW_COPYING,
  50 } RawProgress;
  51
  52 struct RawPull {
  53         sd_event *event;
  54         CurlGlue *glue;
  55
  56         char *image_root;
  57
  58         PullJob *raw_job;
  59         PullJob *checksum_job;
  60         PullJob *signature_job;
  61
  62         RawPullFinished on_finished;
  63         void *userdata;
  64
  65         char *local;
  66         bool force_local;
  67         bool grow_machine_directory;
  68
  69         char *temp_path;
  70         char *final_path;
  71
  72         ImportVerify verify;
  73 };
  74
  75 RawPull* raw_pull_unref(RawPull *i) {
  76         if (!i)
  77                 return NULL;
  78
  79         pull_job_unref(i->raw_job);
  80         pull_job_unref(i->checksum_job);
  81         pull_job_unref(i->signature_job);
  82
  83         curl_glue_unref(i->glue);
  84         sd_event_unref(i->event);
  85
  86         if (i->temp_path) {
  87                 (void) unlink(i->temp_path);
  88                 free(i->temp_path);
  89         }
  90
  91         free(i->final_path);
  92         free(i->image_root);
  93         free(i->local);
  94         free(i);
  95
  96         return NULL;
  97 }
  98
  99 int raw_pull_new(
 100                 RawPull **ret,
 101                 sd_event *event,
 102                 const char *image_root,
 103                 RawPullFinished on_finished,
 104                 void *userdata) {
 105
 106         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 107         int r;
 108
 109         assert(ret);
 110
 111         i = new0(RawPull, 1);
 112         if (!i)
 113                 return -ENOMEM;
 114
 115         i->on_finished = on_finished;
 116         i->userdata = userdata;
 117
 118         i->image_root = strdup(image_root ?: "/var/lib/machines");
 119         if (!i->image_root)
 120                 return -ENOMEM;
 121
 122         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 123
 124         if (event)
 125                 i->event = sd_event_ref(event);
 126         else {
 127                 r = sd_event_default(&i->event);
 128                 if (r < 0)
 129                         return r;
 130         }
 131
 132         r = curl_glue_new(&i->glue, i->event);
 133         if (r < 0)
 134                 return r;
 135
 136         i->glue->on_finished = pull_job_curl_on_finished;
 137         i->glue->userdata = i;
 138
 139         *ret = i;
 140         i = NULL;
 141
 142         return 0;
 143 }
 144
 145 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 146         unsigned percent;
 147
 148         assert(i);
 149
 150         switch (p) {
 151
 152         case RAW_DOWNLOADING: {
 153                 unsigned remain = 80;
 154
 155                 percent = 0;
 156
 157                 if (i->checksum_job) {
 158                         percent += i->checksum_job->progress_percent * 5 / 100;
 159                         remain -= 5;
 160                 }
 161
 162                 if (i->signature_job) {
 163                         percent += i->signature_job->progress_percent * 5 / 100;
 164                         remain -= 5;
 165                 }
 166
 167                 if (i->raw_job)
 168                         percent += i->raw_job->progress_percent * remain / 100;
 169                 break;
 170         }
 171
 172         case RAW_VERIFYING:
 173                 percent = 80;
 174                 break;
 175
 176         case RAW_UNPACKING:
 177                 percent = 85;
 178                 break;
 179
 180         case RAW_FINALIZING:
 181                 percent = 90;
 182                 break;
 183
 184         case RAW_COPYING:
 185                 percent = 95;
 186                 break;
 187
 188         default:
 189                 assert_not_reached("Unknown progress state");
 190         }
 191
 192         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 193         log_debug("Combined progress %u%%", percent);
 194 }
 195
 196 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 197         _cleanup_close_ int converted_fd = -1;
 198         _cleanup_free_ char *t = NULL;
 199         int r;
 200
 201         assert(i);
 202         assert(i->raw_job);
 203
 204         r = qcow2_detect(i->raw_job->disk_fd);
 205         if (r < 0)
 206                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 207         if (r == 0)
 208                 return 0;
 209
 210         /* This is a QCOW2 image, let's convert it */
 211         r = tempfn_random(i->final_path, &t);
 212         if (r < 0)
 213                 return log_oom();
 214
 215         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 216         if (converted_fd < 0)
 217                 return log_error_errno(errno, "Failed to create %s: %m", t);
 218
 219         r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 220         if (r < 0)
 221                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 222
 223         log_info("Unpacking QCOW2 file.");
 224
 225         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 226         if (r < 0) {
 227                 unlink(t);
 228                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 229         }
 230
 231         (void) unlink(i->temp_path);
 232         free(i->temp_path);
 233         i->temp_path = t;
 234         t = NULL;
 235
 236         safe_close(i->raw_job->disk_fd);
 237         i->raw_job->disk_fd = converted_fd;
 238         converted_fd = -1;
 239
 240         return 1;
 241 }
 242
 243 static int raw_pull_make_local_copy(RawPull *i) {
 244         _cleanup_free_ char *tp = NULL;
 245         _cleanup_close_ int dfd = -1;
 246         const char *p;
 247         int r;
 248
 249         assert(i);
 250         assert(i->raw_job);
 251
 252         if (!i->local)
 253                 return 0;
 254
 255         if (i->raw_job->etag_exists) {
 256                 /* We have downloaded this one previously, reopen it */
 257
 258                 assert(i->raw_job->disk_fd < 0);
 259
 260                 if (!i->final_path) {
 261                         r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 262                         if (r < 0)
 263                                 return log_oom();
 264                 }
 265
 266                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 267                 if (i->raw_job->disk_fd < 0)
 268                         return log_error_errno(errno, "Failed to open vendor image: %m");
 269         } else {
 270                 /* We freshly downloaded the image, use it */
 271
 272                 assert(i->raw_job->disk_fd >= 0);
 273
 274                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 275                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 276         }
 277
 278         p = strjoina(i->image_root, "/", i->local, ".raw");
 279
 280         if (i->force_local)
 281                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
 282
 283         r = tempfn_random(p, &tp);
 284         if (r < 0)
 285                 return log_oom();
 286
 287         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 288         if (dfd < 0)
 289                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 290
 291         /* Turn off COW writing. This should greatly improve
 292          * performance on COW file systems like btrfs, since it
 293          * reduces fragmentation caused by not allowing in-place
 294          * writes. */
 295         r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
 296         if (r < 0)
 297                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 298
 299         r = copy_bytes(i->raw_job->disk_fd, dfd, (off_t) -1, true);
 300         if (r < 0) {
 301                 unlink(tp);
 302                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 303         }
 304
 305         (void) copy_times(i->raw_job->disk_fd, dfd);
 306         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 307
 308         dfd = safe_close(dfd);
 309
 310         r = rename(tp, p);
 311         if (r < 0)  {
 312                 unlink(tp);
 313                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 314         }
 315
 316         log_info("Created new local image '%s'.", i->local);
 317         return 0;
 318 }
 319
 320 static bool raw_pull_is_done(RawPull *i) {
 321         assert(i);
 322         assert(i->raw_job);
 323
 324         if (i->raw_job->state != PULL_JOB_DONE)
 325                 return false;
 326         if (i->checksum_job && i->checksum_job->state != PULL_JOB_DONE)
 327                 return false;
 328         if (i->signature_job && i->signature_job->state != PULL_JOB_DONE)
 329                 return false;
 330
 331         return true;
 332 }
 333
 334 static void raw_pull_job_on_finished(PullJob *j) {
 335         RawPull *i;
 336         int r;
 337
 338         assert(j);
 339         assert(j->userdata);
 340
 341         i = j->userdata;
 342         if (j->error != 0) {
 343                 if (j == i->checksum_job)
 344                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 345                 else if (j == i->signature_job)
 346                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 347                 else
 348                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 349
 350                 r = j->error;
 351                 goto finish;
 352         }
 353
 354         /* This is invoked if either the download completed
 355          * successfully, or the download was skipped because we
 356          * already have the etag. In this case ->etag_exists is
 357          * true.
 358          *
 359          * We only do something when we got all three files */
 360
 361         if (!raw_pull_is_done(i))
 362                 return;
 363
 364         if (!i->raw_job->etag_exists) {
 365                 /* This is a new download, verify it, and move it into place */
 366                 assert(i->raw_job->disk_fd >= 0);
 367
 368                 raw_pull_report_progress(i, RAW_VERIFYING);
 369
 370                 r = pull_verify(i->raw_job, i->checksum_job, i->signature_job);
 371                 if (r < 0)
 372                         goto finish;
 373
 374                 raw_pull_report_progress(i, RAW_UNPACKING);
 375
 376                 r = raw_pull_maybe_convert_qcow2(i);
 377                 if (r < 0)
 378                         goto finish;
 379
 380                 raw_pull_report_progress(i, RAW_FINALIZING);
 381
 382                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 383                 if (r < 0)
 384                         goto finish;
 385
 386                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 387                 if (r < 0) {
 388                         log_error_errno(r, "Failed to move RAW file into place: %m");
 389                         goto finish;
 390                 }
 391
 392                 free(i->temp_path);
 393                 i->temp_path = NULL;
 394         }
 395
 396         raw_pull_report_progress(i, RAW_COPYING);
 397
 398         r = raw_pull_make_local_copy(i);
 399         if (r < 0)
 400                 goto finish;
 401
 402         r = 0;
 403
 404 finish:
 405         if (i->on_finished)
 406                 i->on_finished(i, r, i->userdata);
 407         else
 408                 sd_event_exit(i->event, r);
 409 }
 410
 411 static int raw_pull_job_on_open_disk(PullJob *j) {
 412         RawPull *i;
 413         int r;
 414
 415         assert(j);
 416         assert(j->userdata);
 417
 418         i = j->userdata;
 419         assert(i->raw_job == j);
 420         assert(!i->final_path);
 421         assert(!i->temp_path);
 422
 423         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 424         if (r < 0)
 425                 return log_oom();
 426
 427         r = tempfn_random(i->final_path, &i->temp_path);
 428         if (r < 0)
 429                 return log_oom();
 430
 431         (void) mkdir_parents_label(i->temp_path, 0700);
 432
 433         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 434         if (j->disk_fd < 0)
 435                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 436
 437         r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
 438         if (r < 0)
 439                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 440
 441         return 0;
 442 }
 443
 444 static void raw_pull_job_on_progress(PullJob *j) {
 445         RawPull *i;
 446
 447         assert(j);
 448         assert(j->userdata);
 449
 450         i = j->userdata;
 451
 452         raw_pull_report_progress(i, RAW_DOWNLOADING);
 453 }
 454
 455 int raw_pull_start(RawPull *i, const char *url, const char *local, bool force_local, ImportVerify verify) {
 456         int r;
 457
 458         assert(i);
 459         assert(verify < _IMPORT_VERIFY_MAX);
 460         assert(verify >= 0);
 461
 462         if (!http_url_is_valid(url))
 463                 return -EINVAL;
 464
 465         if (local && !machine_name_is_valid(local))
 466                 return -EINVAL;
 467
 468         if (i->raw_job)
 469                 return -EBUSY;
 470
 471         r = free_and_strdup(&i->local, local);
 472         if (r < 0)
 473                 return r;
 474         i->force_local = force_local;
 475         i->verify = verify;
 476
 477         /* Queue job for the image itself */
 478         r = pull_job_new(&i->raw_job, url, i->glue, i);
 479         if (r < 0)
 480                 return r;
 481
 482         i->raw_job->on_finished = raw_pull_job_on_finished;
 483         i->raw_job->on_open_disk = raw_pull_job_on_open_disk;
 484         i->raw_job->on_progress = raw_pull_job_on_progress;
 485         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 486         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 487
 488         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 489         if (r < 0)
 490                 return r;
 491
 492         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 493         if (r < 0)
 494                 return r;
 495
 496         r = pull_job_begin(i->raw_job);
 497         if (r < 0)
 498                 return r;
 499
 500         if (i->checksum_job) {
 501                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 502
 503                 r = pull_job_begin(i->checksum_job);
 504                 if (r < 0)
 505                         return r;
 506         }
 507
 508         if (i->signature_job) {
 509                 i->signature_job->on_progress = raw_pull_job_on_progress;
 510
 511                 r = pull_job_begin(i->signature_job);
 512                 if (r < 0)
 513                         return r;
 514         }
 515
 516         return 0;
 517 }