src/import/import-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25 #include <lzma.h>
  26
  27 #include "hashmap.h"
  28 #include "utf8.h"
  29 #include "curl-util.h"
  30 #include "qcow2-util.h"
  31 #include "import-raw.h"
  32 #include "strv.h"
  33 #include "copy.h"
  34
  35 typedef struct RawImportFile RawImportFile;
  36
  37 struct RawImportFile {
  38         RawImport *import;
  39
  40         char *url;
  41         char *local;
  42
  43         CURL *curl;
  44         struct curl_slist *request_header;
  45
  46         char *temp_path;
  47         char *final_path;
  48         char *etag;
  49         char **old_etags;
  50
  51         uint64_t content_length;
  52         uint64_t written_compressed;
  53         uint64_t written_uncompressed;
  54
  55         void *payload;
  56         size_t payload_size;
  57
  58         usec_t mtime;
  59
  60         bool force_local;
  61         bool done;
  62
  63         int disk_fd;
  64
  65         lzma_stream lzma;
  66         bool compressed;
  67
  68         unsigned progress_percent;
  69         usec_t start_usec;
  70         usec_t last_status_usec;
  71 };
  72
  73 struct RawImport {
  74         sd_event *event;
  75         CurlGlue *glue;
  76
  77         char *image_root;
  78         Hashmap *files;
  79
  80         raw_import_on_finished on_finished;
  81         void *userdata;
  82
  83         bool finished;
  84 };
  85
  86 #define FILENAME_ESCAPE "/.#\"\'"
  87
  88 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
  89
  90 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
  91         if (!f)
  92                 return NULL;
  93
  94         if (f->import)
  95                 curl_glue_remove_and_free(f->import->glue, f->curl);
  96         curl_slist_free_all(f->request_header);
  97
  98         safe_close(f->disk_fd);
  99
 100         free(f->final_path);
 101
 102         if (f->temp_path) {
 103                 unlink(f->temp_path);
 104                 free(f->temp_path);
 105         }
 106
 107         lzma_end(&f->lzma);
 108         free(f->url);
 109         free(f->local);
 110         free(f->etag);
 111         strv_free(f->old_etags);
 112         free(f->payload);
 113         free(f);
 114
 115         return NULL;
 116 }
 117
 118 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
 119
 120 static void raw_import_finish(RawImport *import, int error) {
 121         assert(import);
 122
 123         if (import->finished)
 124                 return;
 125
 126         import->finished = true;
 127
 128         if (import->on_finished)
 129                 import->on_finished(import, error, import->userdata);
 130         else
 131                 sd_event_exit(import->event, error);
 132 }
 133
 134 static int raw_import_file_make_final_path(RawImportFile *f) {
 135         _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
 136
 137         assert(f);
 138
 139         if (f->final_path)
 140                 return 0;
 141
 142         escaped_url = xescape(f->url, FILENAME_ESCAPE);
 143         if (!escaped_url)
 144                 return -ENOMEM;
 145
 146         if (f->etag) {
 147                 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
 148                 if (!escaped_etag)
 149                         return -ENOMEM;
 150
 151                 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
 152         } else
 153                 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
 154         if (!f->final_path)
 155                 return -ENOMEM;
 156
 157         return 0;
 158 }
 159
 160 static int raw_import_file_make_local_copy(RawImportFile *f) {
 161         _cleanup_free_ char *tp = NULL;
 162         _cleanup_close_ int dfd = -1;
 163         const char *p;
 164         int r;
 165
 166         assert(f);
 167
 168         if (!f->local)
 169                 return 0;
 170
 171         if (f->disk_fd >= 0) {
 172                 if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1)
 173                         return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 174         } else {
 175                 r = raw_import_file_make_final_path(f);
 176                 if (r < 0)
 177                         return log_oom();
 178
 179                 f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 180                 if (f->disk_fd < 0)
 181                         return log_error_errno(errno, "Failed to open vendor image: %m");
 182         }
 183
 184         p = strappenda(f->import->image_root, "/", f->local, ".raw");
 185         if (f->force_local)
 186                 (void) rm_rf_dangerous(p, false, true, false);
 187
 188         r = tempfn_random(p, &tp);
 189         if (r < 0)
 190                 return log_oom();
 191
 192         dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 193         if (dfd < 0)
 194                 return log_error_errno(errno, "Failed to create writable copy of image: %m");
 195
 196         /* Turn off COW writing. This should greatly improve
 197          * performance on COW file systems like btrfs, since it
 198          * reduces fragmentation caused by not allowing in-place
 199          * writes. */
 200         r = chattr_fd(dfd, true, FS_NOCOW_FL);
 201         if (r < 0)
 202                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
 203
 204         r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
 205         if (r < 0) {
 206                 unlink(tp);
 207                 return log_error_errno(r, "Failed to make writable copy of image: %m");
 208         }
 209
 210         (void) copy_times(f->disk_fd, dfd);
 211         (void) copy_xattr(f->disk_fd, dfd);
 212
 213         dfd = safe_close(dfd);
 214
 215         r = rename(tp, p);
 216         if (r < 0)  {
 217                 unlink(tp);
 218                 return log_error_errno(errno, "Failed to move writable image into place: %m");
 219         }
 220
 221         log_info("Created new local image %s.", p);
 222         return 0;
 223 }
 224
 225 static void raw_import_file_success(RawImportFile *f) {
 226         int r;
 227
 228         assert(f);
 229
 230         f->done = true;
 231
 232         r = raw_import_file_make_local_copy(f);
 233         if (r < 0)
 234                 goto finish;
 235
 236         f->disk_fd = safe_close(f->disk_fd);
 237         r = 0;
 238
 239 finish:
 240         raw_import_finish(f->import, r);
 241 }
 242
 243 static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
 244         _cleanup_close_ int converted_fd = -1;
 245         _cleanup_free_ char *t = NULL;
 246         int r;
 247
 248         assert(f);
 249         assert(f->disk_fd);
 250         assert(f->temp_path);
 251
 252         r = qcow2_detect(f->disk_fd);
 253         if (r < 0)
 254                 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
 255         if (r == 0)
 256                 return 0;
 257
 258         /* This is a QCOW2 image, let's convert it */
 259         r = tempfn_random(f->final_path, &t);
 260         if (r < 0)
 261                 return log_oom();
 262
 263         converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
 264         if (converted_fd < 0)
 265                 return log_error_errno(errno, "Failed to create %s: %m", t);
 266
 267         log_info("Unpacking QCOW2 file.");
 268
 269         r = qcow2_convert(f->disk_fd, converted_fd);
 270         if (r < 0) {
 271                 unlink(t);
 272                 return log_error_errno(r, "Failed to convert qcow2 image: %m");
 273         }
 274
 275         unlink(f->temp_path);
 276         free(f->temp_path);
 277
 278         f->temp_path = t;
 279         t = NULL;
 280
 281         safe_close(f->disk_fd);
 282         f->disk_fd = converted_fd;
 283         converted_fd = -1;
 284
 285         return 1;
 286 }
 287
 288 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 289         RawImportFile *f = NULL;
 290         struct stat st;
 291         CURLcode code;
 292         long status;
 293         int r;
 294
 295         if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
 296                 return;
 297
 298         if (!f || f->done)
 299                 return;
 300
 301         f->done = true;
 302
 303         if (result != CURLE_OK) {
 304                 log_error("Transfer failed: %s", curl_easy_strerror(result));
 305                 r = -EIO;
 306                 goto fail;
 307         }
 308
 309         code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
 310         if (code != CURLE_OK) {
 311                 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
 312                 r = -EIO;
 313                 goto fail;
 314         } else if (status == 304) {
 315                 log_info("Image already downloaded. Skipping download.");
 316                 raw_import_file_success(f);
 317                 return;
 318         } else if (status >= 300) {
 319                 log_error("HTTP request to %s failed with code %li.", f->url, status);
 320                 r = -EIO;
 321                 goto fail;
 322         } else if (status < 200) {
 323                 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
 324                 r = -EIO;
 325                 goto fail;
 326         }
 327
 328         if (f->disk_fd < 0) {
 329                 log_error("No data received.");
 330                 r = -EIO;
 331                 goto fail;
 332         }
 333
 334         if (f->content_length != (uint64_t) -1 &&
 335             f->content_length != f->written_compressed) {
 336                 log_error("Download truncated.");
 337                 r = -EIO;
 338                 goto fail;
 339         }
 340
 341         /* Make sure the file size is right, in case the file was
 342          * sparse and we just seeked for the last part */
 343         if (ftruncate(f->disk_fd, f->written_uncompressed) < 0) {
 344                 log_error_errno(errno, "Failed to truncate file: %m");
 345                 r = -errno;
 346                 goto fail;
 347         }
 348
 349         r = raw_import_maybe_convert_qcow2(f);
 350         if (r < 0)
 351                 goto fail;
 352
 353         if (f->etag)
 354                 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
 355         if (f->url)
 356                 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
 357
 358         if (f->mtime != 0) {
 359                 struct timespec ut[2];
 360
 361                 timespec_store(&ut[0], f->mtime);
 362                 ut[1] = ut[0];
 363                 (void) futimens(f->disk_fd, ut);
 364
 365                 fd_setcrtime(f->disk_fd, f->mtime);
 366         }
 367
 368         if (fstat(f->disk_fd, &st) < 0) {
 369                 r = log_error_errno(errno, "Failed to stat file: %m");
 370                 goto fail;
 371         }
 372
 373         /* Mark read-only */
 374         (void) fchmod(f->disk_fd, st.st_mode & 07444);
 375
 376         assert(f->temp_path);
 377         assert(f->final_path);
 378
 379         r = rename(f->temp_path, f->final_path);
 380         if (r < 0) {
 381                 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
 382                 goto fail;
 383         }
 384
 385         free(f->temp_path);
 386         f->temp_path = NULL;
 387
 388         log_info("Completed writing vendor image %s.", f->final_path);
 389
 390         raw_import_file_success(f);
 391         return;
 392
 393 fail:
 394         raw_import_finish(f->import, r);
 395 }
 396
 397 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
 398         int r;
 399
 400         assert(f);
 401
 402         if (f->disk_fd >= 0)
 403                 return 0;
 404
 405         r = raw_import_file_make_final_path(f);
 406         if (r < 0)
 407                 return log_oom();
 408
 409         if (!f->temp_path) {
 410                 r = tempfn_random(f->final_path, &f->temp_path);
 411                 if (r < 0)
 412                         return log_oom();
 413         }
 414
 415         f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
 416         if (f->disk_fd < 0)
 417                 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
 418
 419         r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
 420         if (r < 0)
 421                 log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
 422
 423         return 0;
 424 }
 425
 426 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
 427         ssize_t n;
 428
 429         assert(f);
 430         assert(p);
 431         assert(sz > 0);
 432         assert(f->disk_fd >= 0);
 433
 434         if (f->written_uncompressed + sz < f->written_uncompressed) {
 435                 log_error("File too large, overflow");
 436                 return -EOVERFLOW;
 437         }
 438
 439         if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
 440                 log_error("File overly large, refusing");
 441                 return -EFBIG;
 442         }
 443
 444         n = sparse_write(f->disk_fd, p, sz, 64);
 445         if (n < 0) {
 446                 log_error_errno(errno, "Failed to write file: %m");
 447                 return -errno;
 448         }
 449         if ((size_t) n < sz) {
 450                 log_error("Short write");
 451                 return -EIO;
 452         }
 453
 454         f->written_uncompressed += sz;
 455
 456         return 0;
 457 }
 458
 459 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
 460         int r;
 461
 462         assert(f);
 463         assert(p);
 464         assert(sz > 0);
 465         assert(f->disk_fd >= 0);
 466
 467         if (f->written_compressed + sz < f->written_compressed) {
 468                 log_error("File too large, overflow");
 469                 return -EOVERFLOW;
 470         }
 471
 472         if (f->content_length != (uint64_t) -1 &&
 473             f->written_compressed + sz > f->content_length) {
 474                 log_error("Content length incorrect.");
 475                 return -EFBIG;
 476         }
 477
 478         if (!f->compressed) {
 479                 r = raw_import_file_write_uncompressed(f, p, sz);
 480                 if (r < 0)
 481                         return r;
 482         } else {
 483                 f->lzma.next_in = p;
 484                 f->lzma.avail_in = sz;
 485
 486                 while (f->lzma.avail_in > 0) {
 487                         uint8_t buffer[16 * 1024];
 488                         lzma_ret lzr;
 489
 490                         f->lzma.next_out = buffer;
 491                         f->lzma.avail_out = sizeof(buffer);
 492
 493                         lzr = lzma_code(&f->lzma, LZMA_RUN);
 494                         if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
 495                                 log_error("Decompression error.");
 496                                 return -EIO;
 497                         }
 498
 499                         r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
 500                         if (r < 0)
 501                                 return r;
 502                 }
 503         }
 504
 505         f->written_compressed += sz;
 506
 507         return 0;
 508 }
 509
 510 static int raw_import_file_detect_xz(RawImportFile *f) {
 511         static const uint8_t xz_signature[] = {
 512                 '\xfd', '7', 'z', 'X', 'Z', '\x00'
 513         };
 514         lzma_ret lzr;
 515         int r;
 516
 517         assert(f);
 518
 519         if (f->payload_size < sizeof(xz_signature))
 520                 return 0;
 521
 522         f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
 523         log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
 524
 525         if (f->compressed) {
 526                 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
 527                 if (lzr != LZMA_OK) {
 528                         log_error("Failed to initialize LZMA decoder.");
 529                         return -EIO;
 530                 }
 531         }
 532
 533         r = raw_import_file_open_disk_for_write(f);
 534         if (r < 0)
 535                 return r;
 536
 537         r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
 538         if (r < 0)
 539                 return r;
 540
 541         free(f->payload);
 542         f->payload = NULL;
 543         f->payload_size = 0;
 544
 545         return 0;
 546 }
 547
 548 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 549         RawImportFile *f = userdata;
 550         size_t sz = size * nmemb;
 551         int r;
 552
 553         assert(contents);
 554         assert(f);
 555
 556         if (f->done) {
 557                 r = -ESTALE;
 558                 goto fail;
 559         }
 560
 561         if (f->disk_fd < 0) {
 562                 uint8_t *p;
 563
 564                 /* We haven't opened the file yet, let's first check what it actually is */
 565
 566                 p = realloc(f->payload, f->payload_size + sz);
 567                 if (!p) {
 568                         r = log_oom();
 569                         goto fail;
 570                 }
 571
 572                 memcpy(p + f->payload_size, contents, sz);
 573                 f->payload_size = sz;
 574                 f->payload = p;
 575
 576                 r = raw_import_file_detect_xz(f);
 577                 if (r < 0)
 578                         goto fail;
 579
 580                 return sz;
 581         }
 582
 583         r = raw_import_file_write_compressed(f, contents, sz);
 584         if (r < 0)
 585                 goto fail;
 586
 587         return sz;
 588
 589 fail:
 590         raw_import_finish(f->import, r);
 591         return 0;
 592 }
 593
 594 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 595         RawImportFile *f = userdata;
 596         size_t sz = size * nmemb;
 597         _cleanup_free_ char *length = NULL, *last_modified = NULL;
 598         char *etag;
 599         int r;
 600
 601         assert(contents);
 602         assert(f);
 603
 604         if (f->done) {
 605                 r = -ESTALE;
 606                 goto fail;
 607         }
 608
 609         r = curl_header_strdup(contents, sz, "ETag:", &etag);
 610         if (r < 0) {
 611                 log_oom();
 612                 goto fail;
 613         }
 614         if (r > 0) {
 615                 free(f->etag);
 616                 f->etag = etag;
 617
 618                 if (strv_contains(f->old_etags, f->etag)) {
 619                         log_info("Image already downloaded. Skipping download.");
 620                         raw_import_file_success(f);
 621                         return sz;
 622                 }
 623
 624                 return sz;
 625         }
 626
 627         r = curl_header_strdup(contents, sz, "Content-Length:", &length);
 628         if (r < 0) {
 629                 log_oom();
 630                 goto fail;
 631         }
 632         if (r > 0) {
 633                 (void) safe_atou64(length, &f->content_length);
 634
 635                 if (f->content_length != (uint64_t) -1) {
 636                         char bytes[FORMAT_BYTES_MAX];
 637                         log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
 638                 }
 639
 640                 return sz;
 641         }
 642
 643         r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
 644         if (r < 0) {
 645                 log_oom();
 646                 goto fail;
 647         }
 648         if (r > 0) {
 649                 (void) curl_parse_http_time(last_modified, &f->mtime);
 650                 return sz;
 651         }
 652
 653         return sz;
 654
 655 fail:
 656         raw_import_finish(f->import, r);
 657         return 0;
 658 }
 659
 660 static int raw_import_file_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
 661         RawImportFile *f = userdata;
 662         unsigned percent;
 663         usec_t n;
 664
 665         assert(f);
 666
 667         if (dltotal <= 0)
 668                 return 0;
 669
 670         percent = ((100 * dlnow) / dltotal);
 671         n = now(CLOCK_MONOTONIC);
 672
 673         if (n > f->last_status_usec + USEC_PER_SEC &&
 674             percent != f->progress_percent) {
 675                 char buf[FORMAT_TIMESPAN_MAX];
 676
 677                 if (n - f->start_usec > USEC_PER_SEC && dlnow > 0) {
 678                         usec_t left, done;
 679
 680                         done = n - f->start_usec;
 681                         left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
 682
 683                         log_info("Got %u%%. %s left.", percent, format_timespan(buf, sizeof(buf), left, USEC_PER_SEC));
 684                 } else
 685                         log_info("Got %u%%.", percent);
 686
 687                 f->progress_percent = percent;
 688                 f->last_status_usec = n;
 689         }
 690
 691         return 0;
 692 }
 693
 694 static bool etag_is_valid(const char *etag) {
 695
 696         if (!endswith(etag, "\""))
 697                 return false;
 698
 699         if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
 700                 return false;
 701
 702         return true;
 703 }
 704
 705 static int raw_import_file_find_old_etags(RawImportFile *f) {
 706         _cleanup_free_ char *escaped_url = NULL;
 707         _cleanup_closedir_ DIR *d = NULL;
 708         struct dirent *de;
 709         int r;
 710
 711         escaped_url = xescape(f->url, FILENAME_ESCAPE);
 712         if (!escaped_url)
 713                 return -ENOMEM;
 714
 715         d = opendir(f->import->image_root);
 716         if (!d) {
 717                 if (errno == ENOENT)
 718                         return 0;
 719
 720                 return -errno;
 721         }
 722
 723         FOREACH_DIRENT_ALL(de, d, return -errno) {
 724                 const char *a, *b;
 725                 char *u;
 726
 727                 if (de->d_type != DT_UNKNOWN &&
 728                     de->d_type != DT_REG)
 729                         continue;
 730
 731                 a = startswith(de->d_name, ".raw-");
 732                 if (!a)
 733                         continue;
 734
 735                 a = startswith(a, escaped_url);
 736                 if (!a)
 737                         continue;
 738
 739                 a = startswith(a, ".");
 740                 if (!a)
 741                         continue;
 742
 743                 b = endswith(de->d_name, ".raw");
 744                 if (!b)
 745                         continue;
 746
 747                 if (a >= b)
 748                         continue;
 749
 750                 u = cunescape_length(a, b - a);
 751                 if (!u)
 752                         return -ENOMEM;
 753
 754                 if (!etag_is_valid(u)) {
 755                         free(u);
 756                         continue;
 757                 }
 758
 759                 r = strv_consume(&f->old_etags, u);
 760                 if (r < 0)
 761                         return r;
 762         }
 763
 764         return 0;
 765 }
 766
 767 static int raw_import_file_begin(RawImportFile *f) {
 768         int r;
 769
 770         assert(f);
 771         assert(!f->curl);
 772
 773         log_info("Getting %s.", f->url);
 774
 775         r = raw_import_file_find_old_etags(f);
 776         if (r < 0)
 777                 return r;
 778
 779         r = curl_glue_make(&f->curl, f->url, f);
 780         if (r < 0)
 781                 return r;
 782
 783         if (!strv_isempty(f->old_etags)) {
 784                 _cleanup_free_ char *cc = NULL, *hdr = NULL;
 785
 786                 cc = strv_join(f->old_etags, ", ");
 787                 if (!cc)
 788                         return -ENOMEM;
 789
 790                 hdr = strappend("If-None-Match: ", cc);
 791                 if (!hdr)
 792                         return -ENOMEM;
 793
 794                 f->request_header = curl_slist_new(hdr, NULL);
 795                 if (!f->request_header)
 796                         return -ENOMEM;
 797
 798                 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
 799                         return -EIO;
 800         }
 801
 802         if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
 803                 return -EIO;
 804
 805         if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
 806                 return -EIO;
 807
 808         if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
 809                 return -EIO;
 810
 811         if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
 812                 return -EIO;
 813
 814         if (curl_easy_setopt(f->curl, CURLOPT_XFERINFOFUNCTION, raw_import_file_progress_callback) != CURLE_OK)
 815                 return -EIO;
 816
 817         if (curl_easy_setopt(f->curl, CURLOPT_XFERINFODATA, f) != CURLE_OK)
 818                 return -EIO;
 819
 820         if (curl_easy_setopt(f->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
 821                 return -EIO;
 822
 823         r = curl_glue_add(f->import->glue, f->curl);
 824         if (r < 0)
 825                 return r;
 826
 827         return 0;
 828 }
 829
 830 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
 831         _cleanup_(raw_import_unrefp) RawImport *i = NULL;
 832         int r;
 833
 834         assert(import);
 835         assert(image_root);
 836
 837         i = new0(RawImport, 1);
 838         if (!i)
 839                 return -ENOMEM;
 840
 841         i->on_finished = on_finished;
 842         i->userdata = userdata;
 843
 844         i->image_root = strdup(image_root);
 845         if (!i->image_root)
 846                 return -ENOMEM;
 847
 848         if (event)
 849                 i->event = sd_event_ref(event);
 850         else {
 851                 r = sd_event_default(&i->event);
 852                 if (r < 0)
 853                         return r;
 854         }
 855
 856         r = curl_glue_new(&i->glue, i->event);
 857         if (r < 0)
 858                 return r;
 859
 860         i->glue->on_finished = raw_import_curl_on_finished;
 861         i->glue->userdata = i;
 862
 863         *import = i;
 864         i = NULL;
 865
 866         return 0;
 867 }
 868
 869 RawImport* raw_import_unref(RawImport *import) {
 870         RawImportFile *f;
 871
 872         if (!import)
 873                 return NULL;
 874
 875         while ((f = hashmap_steal_first(import->files)))
 876                 raw_import_file_unref(f);
 877         hashmap_free(import->files);
 878
 879         curl_glue_unref(import->glue);
 880         sd_event_unref(import->event);
 881
 882         free(import->image_root);
 883         free(import);
 884
 885         return NULL;
 886 }
 887
 888 int raw_import_cancel(RawImport *import, const char *url) {
 889         RawImportFile *f;
 890
 891         assert(import);
 892         assert(url);
 893
 894         f = hashmap_remove(import->files, url);
 895         if (!f)
 896                 return 0;
 897
 898         raw_import_file_unref(f);
 899         return 1;
 900 }
 901
 902 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
 903         _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
 904         int r;
 905
 906         assert(import);
 907         assert(raw_url_is_valid(url));
 908         assert(!local || machine_name_is_valid(local));
 909
 910         if (hashmap_get(import->files, url))
 911                 return -EEXIST;
 912
 913         r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
 914         if (r < 0)
 915                 return r;
 916
 917         f = new0(RawImportFile, 1);
 918         if (!f)
 919                 return -ENOMEM;
 920
 921         f->import = import;
 922         f->disk_fd = -1;
 923         f->content_length = (uint64_t) -1;
 924         f->start_usec = now(CLOCK_MONOTONIC);
 925
 926         f->url = strdup(url);
 927         if (!f->url)
 928                 return -ENOMEM;
 929
 930         if (local) {
 931                 f->local = strdup(local);
 932                 if (!f->local)
 933                         return -ENOMEM;
 934
 935                 f->force_local = force_local;
 936         }
 937
 938         r = hashmap_put(import->files, f->url, f);
 939         if (r < 0)
 940                 return r;
 941
 942         r = raw_import_file_begin(f);
 943         if (r < 0) {
 944                 raw_import_cancel(import, f->url);
 945                 f = NULL;
 946                 return r;
 947         }
 948
 949         f = NULL;
 950         return 0;
 951 }
 952
 953 bool raw_url_is_valid(const char *url) {
 954         if (isempty(url))
 955                 return false;
 956
 957         if (!startswith(url, "http://") &&
 958             !startswith(url, "https://"))
 959                 return false;
 960
 961         return ascii_is_valid(url);
 962 }