src/import/pull-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25
  26 #include "sd-daemon.h"
  27 #include "utf8.h"
  28 #include "strv.h"
  29 #include "copy.h"
  30 #include "btrfs-util.h"
  31 #include "util.h"
  32 #include "macro.h"
  33 #include "mkdir.h"
  34 #include "rm-rf.h"
  35 #include "path-util.h"
  36 #include "import-util.h"
  37 #include "import-common.h"
  38 #include "curl-util.h"
  39 #include "qcow2-util.h"
  40 #include "pull-job.h"
  41 #include "pull-common.h"
  42 #include "pull-raw.h"
  43
  44 typedef enum RawProgress {
  45         RAW_DOWNLOADING,
  46         RAW_VERIFYING,
  47         RAW_UNPACKING,
  48         RAW_FINALIZING,
  49         RAW_COPYING,
  50 } RawProgress;
  51
  52 struct RawPull {
  53         sd_event *event;
  54         CurlGlue *glue;
  55
  56         char *image_root;
  57
  58         PullJob *raw_job;
  59         PullJob *checksum_job;
  60         PullJob *signature_job;
  61
  62         RawPullFinished on_finished;
  63         void *userdata;
  64
  65         char *local;
  66         bool force_local;
  67         bool grow_machine_directory;
  68
  69         char *temp_path;
  70         char *final_path;
  71
  72         ImportVerify verify;
  73 };
  74
  75 RawPull* raw_pull_unref(RawPull *i) {
  76         if (!i)
  77                 return NULL;
  78
  79         pull_job_unref(i->raw_job);
  80         pull_job_unref(i->checksum_job);
  81         pull_job_unref(i->signature_job);
  82
  83         curl_glue_unref(i->glue);
  84         sd_event_unref(i->event);
  85
  86         if (i->temp_path) {
  87                 (void) unlink(i->temp_path);
  88                 free(i->temp_path);
  89         }
  90
  91         free(i->final_path);
  92         free(i->image_root);
  93         free(i->local);
  94         free(i);
  95
  96         return NULL;
  97 }
  98
  99 int raw_pull_new(
 100                 RawPull **ret,
 101                 sd_event *event,
 102                 const char *image_root,
 103                 RawPullFinished on_finished,
 104                 void *userdata) {
 105
 106         _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
 107         int r;
 108
 109         assert(ret);
 110
 111         i = new0(RawPull, 1);
 112         if (!i)
 113                 return -ENOMEM;
 114
 115         i->on_finished = on_finished;
 116         i->userdata = userdata;
 117
 118         i->image_root = strdup(image_root ?: "/var/lib/machines");
 119         if (!i->image_root)
 120                 return -ENOMEM;
 121
 122         i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
 123
 124         if (event)
 125                 i->event = sd_event_ref(event);
 126         else {
 127                 r = sd_event_default(&i->event);
 128                 if (r < 0)
 129                         return r;
 130         }
 131
 132         r = curl_glue_new(&i->glue, i->event);
 133         if (r < 0)
 134                 return r;
 135
 136         i->glue->on_finished = pull_job_curl_on_finished;
 137         i->glue->userdata = i;
 138
 139         *ret = i;
 140         i = NULL;
 141
 142         return 0;
 143 }
 144
 145 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
 146         unsigned percent;
 147
 148         assert(i);
 149
 150         switch (p) {
 151
 152         case RAW_DOWNLOADING: {
 153                 unsigned remain = 80;
 154
 155                 percent = 0;
 156
 157                 if (i->checksum_job) {
 158                         percent += i->checksum_job->progress_percent * 5 / 100;
 159                         remain -= 5;
 160                 }
 161
 162                 if (i->signature_job) {
 163                         percent += i->signature_job->progress_percent * 5 / 100;
 164                         remain -= 5;
 165                 }
 166
 167                 if (i->raw_job)
 168                         percent += i->raw_job->progress_percent * remain / 100;
 169                 break;
 170         }
 171
 172         case RAW_VERIFYING:
 173                 percent = 80;
 174                 break;
 175
 176         case RAW_UNPACKING:
 177                 percent = 85;
 178                 break;
 179
 180         case RAW_FINALIZING:
 181                 percent = 90;
 182                 break;
 183
 184         case RAW_COPYING:
 185                 percent = 95;
 186                 break;
 187
 188         default:
 189                 assert_not_reached("Unknown progress state");
 190         }
 191
 192         sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
 193         log_debug("Combined progress %u%%", percent);
 194 }
 195
 196 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
 197         _cleanup_close_ int converted_fd = -1;
 198         _cleanup_free_ char *t = NULL;
 199         int r;
 200
 201         assert(i);
 202         assert(i->raw_job);
 203
 204         r = qcow2_detect(i->raw_job->disk_fd);
 205         if (r < 0)
 206                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 207         if (r == 0)
 208                 return 0;
 209
 210         /* This is a QCOW2 image, let's convert it */
 211         r = tempfn_random(i->final_path, &t);
 212         if (r < 0)
 213                 return log_oom();
 214
 215         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 216         if (converted_fd < 0)
 217                 return log_error_errno(errno, "Failed to create %s: %m", t);
 218
 219         r = chattr_fd(converted_fd, true, FS_NOCOW_FL);
 220         if (r < 0)
 221                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", t);
 222
 223         log_info("Unpacking QCOW2 file.");
 224
 225         r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
 226         if (r < 0) {
 227                 unlink(t);
 228                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 229         }
 230
 231         (void) unlink(i->temp_path);
 232         free(i->temp_path);
 233         i->temp_path = t;
 234         t = NULL;
 235
 236         safe_close(i->raw_job->disk_fd);
 237         i->raw_job->disk_fd = converted_fd;
 238         converted_fd = -1;
 239
 240         return 1;
 241 }
 242
 243 static int raw_pull_make_local_copy(RawPull *i) {
 244         _cleanup_free_ char *tp = NULL;
 245         _cleanup_close_ int dfd = -1;
 246         const char *p;
 247         int r;
 248
 249         assert(i);
 250         assert(i->raw_job);
 251
 252         if (!i->local)
 253                 return 0;
 254
 255         if (i->raw_job->etag_exists) {
 256                 /* We have downloaded this one previously, reopen it */
 257
 258                 assert(i->raw_job->disk_fd < 0);
 259
 260                 if (!i->final_path) {
 261                         r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 262                         if (r < 0)
 263                                 return log_oom();
 264                 }
 265
 266                 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 267                 if (i->raw_job->disk_fd < 0)
 268                         return log_error_errno(errno, "Failed to open vendor image: %m");
 269         } else {
 270                 /* We freshly downloaded the image, use it */
 271
 272                 assert(i->raw_job->disk_fd >= 0);
 273
 274                 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
 275                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 276         }
 277
 278         p = strjoina(i->image_root, "/", i->local, ".raw");
 279
 280         if (i->force_local) {
 281                 (void) btrfs_subvol_remove(p);
 282                 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL);
 283         }
 284
 285         r = tempfn_random(p, &tp);
 286         if (r < 0)
 287                 return log_oom();
 288
 289         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 290         if (dfd < 0)
 291                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 292
 293         /* Turn off COW writing. This should greatly improve
 294          * performance on COW file systems like btrfs, since it
 295          * reduces fragmentation caused by not allowing in-place
 296          * writes. */
 297         r = chattr_fd(dfd, true, FS_NOCOW_FL);
 298         if (r < 0)
 299                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 300
 301         r = copy_bytes(i->raw_job->disk_fd, dfd, (off_t) -1, true);
 302         if (r < 0) {
 303                 unlink(tp);
 304                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 305         }
 306
 307         (void) copy_times(i->raw_job->disk_fd, dfd);
 308         (void) copy_xattr(i->raw_job->disk_fd, dfd);
 309
 310         dfd = safe_close(dfd);
 311
 312         r = rename(tp, p);
 313         if (r < 0)  {
 314                 unlink(tp);
 315                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 316         }
 317
 318         log_info("Created new local image '%s'.", i->local);
 319         return 0;
 320 }
 321
 322 static bool raw_pull_is_done(RawPull *i) {
 323         assert(i);
 324         assert(i->raw_job);
 325
 326         if (i->raw_job->state != PULL_JOB_DONE)
 327                 return false;
 328         if (i->checksum_job && i->checksum_job->state != PULL_JOB_DONE)
 329                 return false;
 330         if (i->signature_job && i->signature_job->state != PULL_JOB_DONE)
 331                 return false;
 332
 333         return true;
 334 }
 335
 336 static void raw_pull_job_on_finished(PullJob *j) {
 337         RawPull *i;
 338         int r;
 339
 340         assert(j);
 341         assert(j->userdata);
 342
 343         i = j->userdata;
 344         if (j->error != 0) {
 345                 if (j == i->checksum_job)
 346                         log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
 347                 else if (j == i->signature_job)
 348                         log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
 349                 else
 350                         log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
 351
 352                 r = j->error;
 353                 goto finish;
 354         }
 355
 356         /* This is invoked if either the download completed
 357          * successfully, or the download was skipped because we
 358          * already have the etag. In this case ->etag_exists is
 359          * true.
 360          *
 361          * We only do something when we got all three files */
 362
 363         if (!raw_pull_is_done(i))
 364                 return;
 365
 366         if (!i->raw_job->etag_exists) {
 367                 /* This is a new download, verify it, and move it into place */
 368                 assert(i->raw_job->disk_fd >= 0);
 369
 370                 raw_pull_report_progress(i, RAW_VERIFYING);
 371
 372                 r = pull_verify(i->raw_job, i->checksum_job, i->signature_job);
 373                 if (r < 0)
 374                         goto finish;
 375
 376                 raw_pull_report_progress(i, RAW_UNPACKING);
 377
 378                 r = raw_pull_maybe_convert_qcow2(i);
 379                 if (r < 0)
 380                         goto finish;
 381
 382                 raw_pull_report_progress(i, RAW_FINALIZING);
 383
 384                 r = import_make_read_only_fd(i->raw_job->disk_fd);
 385                 if (r < 0)
 386                         goto finish;
 387
 388                 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
 389                 if (r < 0) {
 390                         log_error_errno(r, "Failed to move RAW file into place: %m");
 391                         goto finish;
 392                 }
 393
 394                 free(i->temp_path);
 395                 i->temp_path = NULL;
 396         }
 397
 398         raw_pull_report_progress(i, RAW_COPYING);
 399
 400         r = raw_pull_make_local_copy(i);
 401         if (r < 0)
 402                 goto finish;
 403
 404         r = 0;
 405
 406 finish:
 407         if (i->on_finished)
 408                 i->on_finished(i, r, i->userdata);
 409         else
 410                 sd_event_exit(i->event, r);
 411 }
 412
 413 static int raw_pull_job_on_open_disk(PullJob *j) {
 414         RawPull *i;
 415         int r;
 416
 417         assert(j);
 418         assert(j->userdata);
 419
 420         i = j->userdata;
 421         assert(i->raw_job == j);
 422         assert(!i->final_path);
 423         assert(!i->temp_path);
 424
 425         r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path);
 426         if (r < 0)
 427                 return log_oom();
 428
 429         r = tempfn_random(i->final_path, &i->temp_path);
 430         if (r < 0)
 431                 return log_oom();
 432
 433         (void) mkdir_parents_label(i->temp_path, 0700);
 434
 435         j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 436         if (j->disk_fd < 0)
 437                 return log_error_errno(errno, "Failed to create %s: %m", i->temp_path);
 438
 439         r = chattr_fd(j->disk_fd, true, FS_NOCOW_FL);
 440         if (r < 0)
 441                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path);
 442
 443         return 0;
 444 }
 445
 446 static void raw_pull_job_on_progress(PullJob *j) {
 447         RawPull *i;
 448
 449         assert(j);
 450         assert(j->userdata);
 451
 452         i = j->userdata;
 453
 454         raw_pull_report_progress(i, RAW_DOWNLOADING);
 455 }
 456
 457 int raw_pull_start(RawPull *i, const char *url, const char *local, bool force_local, ImportVerify verify) {
 458         int r;
 459
 460         assert(i);
 461         assert(verify < _IMPORT_VERIFY_MAX);
 462         assert(verify >= 0);
 463
 464         if (!http_url_is_valid(url))
 465                 return -EINVAL;
 466
 467         if (local && !machine_name_is_valid(local))
 468                 return -EINVAL;
 469
 470         if (i->raw_job)
 471                 return -EBUSY;
 472
 473         r = free_and_strdup(&i->local, local);
 474         if (r < 0)
 475                 return r;
 476         i->force_local = force_local;
 477         i->verify = verify;
 478
 479         /* Queue job for the image itself */
 480         r = pull_job_new(&i->raw_job, url, i->glue, i);
 481         if (r < 0)
 482                 return r;
 483
 484         i->raw_job->on_finished = raw_pull_job_on_finished;
 485         i->raw_job->on_open_disk = raw_pull_job_on_open_disk;
 486         i->raw_job->on_progress = raw_pull_job_on_progress;
 487         i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
 488         i->raw_job->grow_machine_directory = i->grow_machine_directory;
 489
 490         r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
 491         if (r < 0)
 492                 return r;
 493
 494         r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
 495         if (r < 0)
 496                 return r;
 497
 498         r = pull_job_begin(i->raw_job);
 499         if (r < 0)
 500                 return r;
 501
 502         if (i->checksum_job) {
 503                 i->checksum_job->on_progress = raw_pull_job_on_progress;
 504
 505                 r = pull_job_begin(i->checksum_job);
 506                 if (r < 0)
 507                         return r;
 508         }
 509
 510         if (i->signature_job) {
 511                 i->signature_job->on_progress = raw_pull_job_on_progress;
 512
 513                 r = pull_job_begin(i->signature_job);
 514                 if (r < 0)
 515                         return r;
 516         }
 517
 518         return 0;
 519 }