src/import/import-raw.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2014 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <sys/xattr.h>
  23 #include <linux/fs.h>
  24 #include <curl/curl.h>
  25 #include <lzma.h>
  26
  27 #include "hashmap.h"
  28 #include "utf8.h"
  29 #include "curl-util.h"
  30 #include "import-raw.h"
  31 #include "strv.h"
  32 #include "copy.h"
  33
  34 typedef struct RawImportFile RawImportFile;
  35
  36 struct RawImportFile {
  37         RawImport *import;
  38
  39         char *url;
  40         char *local;
  41
  42         CURL *curl;
  43         struct curl_slist *request_header;
  44
  45         char *temp_path;
  46         char *final_path;
  47         char *etag;
  48         char **old_etags;
  49
  50         uint64_t content_length;
  51         uint64_t written_compressed;
  52         uint64_t written_uncompressed;
  53
  54         void *payload;
  55         size_t payload_size;
  56
  57         usec_t mtime;
  58
  59         bool force_local;
  60         bool done;
  61
  62         int disk_fd;
  63
  64         lzma_stream lzma;
  65         bool compressed;
  66 };
  67
  68 struct RawImport {
  69         sd_event *event;
  70         CurlGlue *glue;
  71
  72         char *image_root;
  73         Hashmap *files;
  74
  75         raw_import_on_finished on_finished;
  76         void *userdata;
  77
  78         bool finished;
  79 };
  80
  81 #define FILENAME_ESCAPE "/.#\"\'"
  82
  83 #define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
  84
  85 static RawImportFile *raw_import_file_unref(RawImportFile *f) {
  86         if (!f)
  87                 return NULL;
  88
  89         if (f->import)
  90                 curl_glue_remove_and_free(f->import->glue, f->curl);
  91         curl_slist_free_all(f->request_header);
  92
  93         safe_close(f->disk_fd);
  94
  95         free(f->final_path);
  96
  97         if (f->temp_path) {
  98                 unlink(f->temp_path);
  99                 free(f->temp_path);
 100         }
 101
 102         free(f->url);
 103         free(f->local);
 104         free(f->etag);
 105         strv_free(f->old_etags);
 106         free(f->payload);
 107         free(f);
 108
 109         return NULL;
 110 }
 111
 112 DEFINE_TRIVIAL_CLEANUP_FUNC(RawImportFile*, raw_import_file_unref);
 113
 114 static void raw_import_finish(RawImport *import, int error) {
 115         assert(import);
 116
 117         if (import->finished)
 118                 return;
 119
 120         import->finished = true;
 121
 122         if (import->on_finished)
 123                 import->on_finished(import, error, import->userdata);
 124         else
 125                 sd_event_exit(import->event, error);
 126 }
 127
 128 static int raw_import_file_make_final_path(RawImportFile *f) {
 129         _cleanup_free_ char *escaped_url = NULL, *escaped_etag = NULL;
 130
 131         assert(f);
 132
 133         if (f->final_path)
 134                 return 0;
 135
 136         escaped_url = xescape(f->url, FILENAME_ESCAPE);
 137         if (!escaped_url)
 138                 return -ENOMEM;
 139
 140         if (f->etag) {
 141                 escaped_etag = xescape(f->etag, FILENAME_ESCAPE);
 142                 if (!escaped_etag)
 143                         return -ENOMEM;
 144
 145                 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".", escaped_etag, ".raw", NULL);
 146         } else
 147                 f->final_path = strjoin(f->import->image_root, "/.raw-", escaped_url, ".raw", NULL);
 148         if (!f->final_path)
 149                 return -ENOMEM;
 150
 151         return 0;
 152 }
 153
 154 static void raw_import_file_success(RawImportFile *f) {
 155         int r;
 156
 157         assert(f);
 158
 159         f->done = true;
 160
 161         if (f->local) {
 162                 _cleanup_free_ char *tp = NULL;
 163                 _cleanup_close_ int dfd = -1;
 164                 const char *p;
 165
 166                 if (f->disk_fd >= 0) {
 167                         if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1) {
 168                                 r = log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
 169                                 goto finish;
 170                         }
 171                 } else {
 172                         r = raw_import_file_make_final_path(f);
 173                         if (r < 0) {
 174                                 log_oom();
 175                                 goto finish;
 176                         }
 177
 178                         f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
 179                         if (f->disk_fd < 0) {
 180                                 r = log_error_errno(errno, "Failed to open vendor image: %m");
 181                                 goto finish;
 182                         }
 183                 }
 184
 185                 p = strappenda(f->import->image_root, "/", f->local, ".raw");
 186                 if (f->force_local)
 187                         (void) rm_rf_dangerous(p, false, true, false);
 188
 189                 r = tempfn_random(p, &tp);
 190                 if (r < 0) {
 191                         log_oom();
 192                         goto finish;
 193                 }
 194
 195                 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
 196                 if (dfd < 0) {
 197                         r = log_error_errno(errno, "Failed to create writable copy of image: %m");
 198                         goto finish;
 199                 }
 200
 201                 /* Turn off COW writing. This should greatly improve
 202                  * performance on COW file systems like btrfs, since it
 203                  * reduces fragmentation caused by not allowing in-place
 204                  * writes. */
 205                 r = chattr_fd(dfd, true, FS_NOCOW_FL);
 206                 if (r < 0)
 207                         log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
 208
 209                 r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
 210                 if (r < 0) {
 211                         log_error_errno(r, "Failed to make writable copy of image: %m");
 212                         unlink(tp);
 213                         goto finish;
 214                 }
 215
 216                 (void) copy_times(f->disk_fd, dfd);
 217                 (void) copy_xattr(f->disk_fd, dfd);
 218
 219                 dfd = safe_close(dfd);
 220
 221                 r = rename(tp, p);
 222                 if (r < 0) {
 223                         r = log_error_errno(errno, "Failed to move writable image into place: %m");
 224                         unlink(tp);
 225                         goto finish;
 226                 }
 227
 228                 log_info("Created new local image %s.", p);
 229         }
 230
 231         f->disk_fd = safe_close(f->disk_fd);
 232         r = 0;
 233
 234 finish:
 235         raw_import_finish(f->import, r);
 236 }
 237
 238 static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 239         RawImportFile *f = NULL;
 240         struct stat st;
 241         CURLcode code;
 242         long status;
 243         int r;
 244
 245         if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, &f) != CURLE_OK)
 246                 return;
 247
 248         if (!f || f->done)
 249                 return;
 250
 251         f->done = true;
 252
 253         if (result != CURLE_OK) {
 254                 log_error("Transfer failed: %s", curl_easy_strerror(result));
 255                 r = -EIO;
 256                 goto fail;
 257         }
 258
 259         code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
 260         if (code != CURLE_OK) {
 261                 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
 262                 r = -EIO;
 263                 goto fail;
 264         } else if (status == 304) {
 265                 log_info("Image already downloaded. Skipping download.");
 266                 raw_import_file_success(f);
 267                 return;
 268         } else if (status >= 300) {
 269                 log_error("HTTP request to %s failed with code %li.", f->url, status);
 270                 r = -EIO;
 271                 goto fail;
 272         } else if (status < 200) {
 273                 log_error("HTTP request to %s finished with unexpected code %li.", f->url, status);
 274                 r = -EIO;
 275                 goto fail;
 276         }
 277
 278         if (f->disk_fd < 0) {
 279                 log_error("No data received.");
 280                 r = -EIO;
 281                 goto fail;
 282         }
 283
 284         if (f->content_length != (uint64_t) -1 &&
 285             f->content_length != f->written_compressed) {
 286                 log_error("Download truncated.");
 287                 r = -EIO;
 288                 goto fail;
 289         }
 290
 291         if (f->etag)
 292                 (void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
 293         if (f->url)
 294                 (void) fsetxattr(f->disk_fd, "user.source_url", f->url, strlen(f->url), 0);
 295
 296         if (f->mtime != 0) {
 297                 struct timespec ut[2];
 298
 299                 timespec_store(&ut[0], f->mtime);
 300                 ut[1] = ut[0];
 301                 (void) futimens(f->disk_fd, ut);
 302
 303                 fd_setcrtime(f->disk_fd, f->mtime);
 304         }
 305
 306         if (fstat(f->disk_fd, &st) < 0) {
 307                 r = log_error_errno(errno, "Failed to stat file: %m");
 308                 goto fail;
 309         }
 310
 311         /* Mark read-only */
 312         (void) fchmod(f->disk_fd, st.st_mode & 07444);
 313
 314         assert(f->temp_path);
 315         assert(f->final_path);
 316
 317         r = rename(f->temp_path, f->final_path);
 318         if (r < 0) {
 319                 r = log_error_errno(errno, "Failed to move RAW file into place: %m");
 320                 goto fail;
 321         }
 322
 323         free(f->temp_path);
 324         f->temp_path = NULL;
 325
 326         log_info("Completed writing vendor image %s.", f->final_path);
 327
 328         raw_import_file_success(f);
 329         return;
 330
 331 fail:
 332         raw_import_finish(f->import, r);
 333 }
 334
 335 static int raw_import_file_open_disk_for_write(RawImportFile *f) {
 336         int r;
 337
 338         assert(f);
 339
 340         if (f->disk_fd >= 0)
 341                 return 0;
 342
 343         r = raw_import_file_make_final_path(f);
 344         if (r < 0)
 345                 return log_oom();
 346
 347         if (!f->temp_path) {
 348                 r = tempfn_random(f->final_path, &f->temp_path);
 349                 if (r < 0)
 350                         return log_oom();
 351         }
 352
 353         f->disk_fd = open(f->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
 354         if (f->disk_fd < 0)
 355                 return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
 356
 357         return 0;
 358 }
 359
 360 static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
 361         ssize_t n;
 362
 363         assert(f);
 364         assert(p);
 365         assert(sz > 0);
 366         assert(f->disk_fd >= 0);
 367
 368         if (f->written_uncompressed + sz < f->written_uncompressed) {
 369                 log_error("File too large, overflow");
 370                 return -EOVERFLOW;
 371         }
 372
 373         if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
 374                 log_error("File overly large, refusing");
 375                 return -EFBIG;
 376         }
 377
 378         n = write(f->disk_fd, p, sz);
 379         if (n < 0) {
 380                 log_error_errno(errno, "Failed to write file: %m");
 381                 return -errno;
 382         }
 383         if ((size_t) n < sz) {
 384                 log_error("Short write");
 385                 return -EIO;
 386         }
 387
 388         f->written_uncompressed += sz;
 389
 390         return 0;
 391 }
 392
 393 static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
 394         int r;
 395
 396         assert(f);
 397         assert(p);
 398         assert(sz > 0);
 399         assert(f->disk_fd >= 0);
 400
 401         if (f->written_compressed + sz < f->written_compressed) {
 402                 log_error("File too large, overflow");
 403                 return -EOVERFLOW;
 404         }
 405
 406         if (f->content_length != (uint64_t) -1 &&
 407             f->written_compressed + sz > f->content_length) {
 408                 log_error("Content length incorrect.");
 409                 return -EFBIG;
 410         }
 411
 412         if (!f->compressed) {
 413                 r = raw_import_file_write_uncompressed(f, p, sz);
 414                 if (r < 0)
 415                         return r;
 416         } else {
 417                 f->lzma.next_in = p;
 418                 f->lzma.avail_in = sz;
 419
 420                 while (f->lzma.avail_in > 0) {
 421                         uint8_t buffer[16 * 1024];
 422                         lzma_ret lzr;
 423
 424                         f->lzma.next_out = buffer;
 425                         f->lzma.avail_out = sizeof(buffer);
 426
 427                         lzr = lzma_code(&f->lzma, LZMA_RUN);
 428                         if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
 429                                 log_error("Decompression error.");
 430                                 return -EIO;
 431                         }
 432
 433                         r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
 434                         if (r < 0)
 435                                 return r;
 436                 }
 437         }
 438
 439         f->written_compressed += sz;
 440
 441         return 0;
 442 }
 443
 444 static int raw_import_file_detect_xz(RawImportFile *f) {
 445         static const uint8_t xz_signature[] = {
 446                 '\xfd', '7', 'z', 'X', 'Z', '\x00'
 447         };
 448         lzma_ret lzr;
 449         int r;
 450
 451         assert(f);
 452
 453         if (f->payload_size < sizeof(xz_signature))
 454                 return 0;
 455
 456         f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
 457         log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
 458
 459         if (f->compressed) {
 460                 lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
 461                 if (lzr != LZMA_OK) {
 462                         log_error("Failed to initialize LZMA decoder.");
 463                         return -EIO;
 464                 }
 465         }
 466
 467         r = raw_import_file_open_disk_for_write(f);
 468         if (r < 0)
 469                 return r;
 470
 471         r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
 472         if (r < 0)
 473                 return r;
 474
 475         free(f->payload);
 476         f->payload = NULL;
 477         f->payload_size = 0;
 478
 479         return 0;
 480 }
 481
 482 static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 483         RawImportFile *f = userdata;
 484         size_t sz = size * nmemb;
 485         int r;
 486
 487         assert(contents);
 488         assert(f);
 489
 490         if (f->done) {
 491                 r = -ESTALE;
 492                 goto fail;
 493         }
 494
 495         if (f->disk_fd < 0) {
 496                 uint8_t *p;
 497
 498                 /* We haven't opened the file yet, let's first check what it actually is */
 499
 500                 p = realloc(f->payload, f->payload_size + sz);
 501                 if (!p) {
 502                         r = log_oom();
 503                         goto fail;
 504                 }
 505
 506                 memcpy(p + f->payload_size, contents, sz);
 507                 f->payload_size = sz;
 508                 f->payload = p;
 509
 510                 r = raw_import_file_detect_xz(f);
 511                 if (r < 0)
 512                         goto fail;
 513
 514                 return sz;
 515         }
 516
 517         r = raw_import_file_write_compressed(f, contents, sz);
 518         if (r < 0)
 519                 goto fail;
 520
 521         return sz;
 522
 523 fail:
 524         raw_import_finish(f->import, r);
 525         return 0;
 526 }
 527
 528 static size_t raw_import_file_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
 529         RawImportFile *f = userdata;
 530         size_t sz = size * nmemb;
 531         _cleanup_free_ char *length = NULL, *last_modified = NULL;
 532         char *etag;
 533         int r;
 534
 535         assert(contents);
 536         assert(f);
 537
 538         if (f->done) {
 539                 r = -ESTALE;
 540                 goto fail;
 541         }
 542
 543         r = curl_header_strdup(contents, sz, "ETag:", &etag);
 544         if (r < 0) {
 545                 log_oom();
 546                 goto fail;
 547         }
 548         if (r > 0) {
 549                 free(f->etag);
 550                 f->etag = etag;
 551
 552                 if (strv_contains(f->old_etags, f->etag)) {
 553                         log_info("Image already downloaded. Skipping download.");
 554                         raw_import_file_success(f);
 555                         return sz;
 556                 }
 557
 558                 return sz;
 559         }
 560
 561         r = curl_header_strdup(contents, sz, "Content-Length:", &length);
 562         if (r < 0) {
 563                 log_oom();
 564                 goto fail;
 565         }
 566         if (r > 0) {
 567                 (void) safe_atou64(length, &f->content_length);
 568
 569                 if (f->content_length != (uint64_t) -1) {
 570                         char bytes[FORMAT_BYTES_MAX];
 571                         log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
 572                 }
 573
 574                 return sz;
 575         }
 576
 577         r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
 578         if (r < 0) {
 579                 log_oom();
 580                 goto fail;
 581         }
 582         if (r > 0) {
 583                 (void) curl_parse_http_time(last_modified, &f->mtime);
 584                 return sz;
 585         }
 586
 587         return sz;
 588
 589 fail:
 590         raw_import_finish(f->import, r);
 591         return 0;
 592 }
 593
 594 static bool etag_is_valid(const char *etag) {
 595
 596         if (!endswith(etag, "\""))
 597                 return false;
 598
 599         if (!startswith(etag, "\"") && !startswith(etag, "W/\""))
 600                 return false;
 601
 602         return true;
 603 }
 604
 605 static int raw_import_file_find_old_etags(RawImportFile *f) {
 606         _cleanup_free_ char *escaped_url = NULL;
 607         _cleanup_closedir_ DIR *d = NULL;
 608         struct dirent *de;
 609         int r;
 610
 611         escaped_url = xescape(f->url, FILENAME_ESCAPE);
 612         if (!escaped_url)
 613                 return -ENOMEM;
 614
 615         d = opendir(f->import->image_root);
 616         if (!d) {
 617                 if (errno == ENOENT)
 618                         return 0;
 619
 620                 return -errno;
 621         }
 622
 623         FOREACH_DIRENT_ALL(de, d, return -errno) {
 624                 const char *a, *b;
 625                 char *u;
 626
 627                 if (de->d_type != DT_UNKNOWN &&
 628                     de->d_type != DT_REG)
 629                         continue;
 630
 631                 a = startswith(de->d_name, ".raw-");
 632                 if (!a)
 633                         continue;
 634
 635                 a = startswith(a, escaped_url);
 636                 if (!a)
 637                         continue;
 638
 639                 a = startswith(a, ".");
 640                 if (!a)
 641                         continue;
 642
 643                 b = endswith(de->d_name, ".raw");
 644                 if (!b)
 645                         continue;
 646
 647                 if (a >= b)
 648                         continue;
 649
 650                 u = cunescape_length(a, b - a);
 651                 if (!u)
 652                         return -ENOMEM;
 653
 654                 if (!etag_is_valid(u)) {
 655                         free(u);
 656                         continue;
 657                 }
 658
 659                 r = strv_consume(&f->old_etags, u);
 660                 if (r < 0)
 661                         return r;
 662         }
 663
 664         return 0;
 665 }
 666
 667 static int raw_import_file_begin(RawImportFile *f) {
 668         int r;
 669
 670         assert(f);
 671         assert(!f->curl);
 672
 673         log_info("Getting %s.", f->url);
 674
 675         r = raw_import_file_find_old_etags(f);
 676         if (r < 0)
 677                 return r;
 678
 679         r = curl_glue_make(&f->curl, f->url, f);
 680         if (r < 0)
 681                 return r;
 682
 683         if (!strv_isempty(f->old_etags)) {
 684                 _cleanup_free_ char *cc = NULL, *hdr = NULL;
 685
 686                 cc = strv_join(f->old_etags, ", ");
 687                 if (!cc)
 688                         return -ENOMEM;
 689
 690                 hdr = strappend("If-None-Match: ", cc);
 691                 if (!hdr)
 692                         return -ENOMEM;
 693
 694                 f->request_header = curl_slist_new(hdr, NULL);
 695                 if (!f->request_header)
 696                         return -ENOMEM;
 697
 698                 if (curl_easy_setopt(f->curl, CURLOPT_HTTPHEADER, f->request_header) != CURLE_OK)
 699                         return -EIO;
 700         }
 701
 702         if (curl_easy_setopt(f->curl, CURLOPT_WRITEFUNCTION, raw_import_file_write_callback) != CURLE_OK)
 703                 return -EIO;
 704
 705         if (curl_easy_setopt(f->curl, CURLOPT_WRITEDATA, f) != CURLE_OK)
 706                 return -EIO;
 707
 708         if (curl_easy_setopt(f->curl, CURLOPT_HEADERFUNCTION, raw_import_file_header_callback) != CURLE_OK)
 709                 return -EIO;
 710
 711         if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
 712                 return -EIO;
 713
 714         r = curl_glue_add(f->import->glue, f->curl);
 715         if (r < 0)
 716                 return r;
 717
 718         return 0;
 719 }
 720
 721 int raw_import_new(RawImport **import, sd_event *event, const char *image_root, raw_import_on_finished on_finished, void *userdata) {
 722         _cleanup_(raw_import_unrefp) RawImport *i = NULL;
 723         int r;
 724
 725         assert(import);
 726         assert(image_root);
 727
 728         i = new0(RawImport, 1);
 729         if (!i)
 730                 return -ENOMEM;
 731
 732         i->on_finished = on_finished;
 733         i->userdata = userdata;
 734
 735         i->image_root = strdup(image_root);
 736         if (!i->image_root)
 737                 return -ENOMEM;
 738
 739         if (event)
 740                 i->event = sd_event_ref(event);
 741         else {
 742                 r = sd_event_default(&i->event);
 743                 if (r < 0)
 744                         return r;
 745         }
 746
 747         r = curl_glue_new(&i->glue, i->event);
 748         if (r < 0)
 749                 return r;
 750
 751         i->glue->on_finished = raw_import_curl_on_finished;
 752         i->glue->userdata = i;
 753
 754         *import = i;
 755         i = NULL;
 756
 757         return 0;
 758 }
 759
 760 RawImport* raw_import_unref(RawImport *import) {
 761         RawImportFile *f;
 762
 763         if (!import)
 764                 return NULL;
 765
 766         while ((f = hashmap_steal_first(import->files)))
 767                 raw_import_file_unref(f);
 768         hashmap_free(import->files);
 769
 770         curl_glue_unref(import->glue);
 771         sd_event_unref(import->event);
 772
 773         free(import->image_root);
 774         free(import);
 775
 776         return NULL;
 777 }
 778
 779 int raw_import_cancel(RawImport *import, const char *url) {
 780         RawImportFile *f;
 781
 782         assert(import);
 783         assert(url);
 784
 785         f = hashmap_remove(import->files, url);
 786         if (!f)
 787                 return 0;
 788
 789         raw_import_file_unref(f);
 790         return 1;
 791 }
 792
 793 int raw_import_pull(RawImport *import, const char *url, const char *local, bool force_local) {
 794         _cleanup_(raw_import_file_unrefp) RawImportFile *f = NULL;
 795         int r;
 796
 797         assert(import);
 798         assert(raw_url_is_valid(url));
 799         assert(!local || machine_name_is_valid(local));
 800
 801         if (hashmap_get(import->files, url))
 802                 return -EEXIST;
 803
 804         r = hashmap_ensure_allocated(&import->files, &string_hash_ops);
 805         if (r < 0)
 806                 return r;
 807
 808         f = new0(RawImportFile, 1);
 809         if (!f)
 810                 return -ENOMEM;
 811
 812         f->import = import;
 813         f->disk_fd = -1;
 814         f->content_length = (uint64_t) -1;
 815
 816         f->url = strdup(url);
 817         if (!f->url)
 818                 return -ENOMEM;
 819
 820         if (local) {
 821                 f->local = strdup(local);
 822                 if (!f->local)
 823                         return -ENOMEM;
 824
 825                 f->force_local = force_local;
 826         }
 827
 828         r = hashmap_put(import->files, f->url, f);
 829         if (r < 0)
 830                 return r;
 831
 832         r = raw_import_file_begin(f);
 833         if (r < 0) {
 834                 raw_import_cancel(import, f->url);
 835                 f = NULL;
 836                 return r;
 837         }
 838
 839         f = NULL;
 840         return 0;
 841 }
 842
 843 bool raw_url_is_valid(const char *url) {
 844         if (isempty(url))
 845                 return false;
 846
 847         if (!startswith(url, "http://") &&
 848             !startswith(url, "https://"))
 849                 return false;
 850
 851         return ascii_is_valid(url);
 852 }