2 This file is part of systemd.
4 Copyright 2014 Lennart Poettering
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
20 #include <curl/curl.h>
22 #include <sys/xattr.h>
24 #include "sd-daemon.h"
26 #include "alloc-util.h"
27 #include "btrfs-util.h"
28 #include "chattr-util.h"
30 #include "curl-util.h"
34 #include "hostname-util.h"
35 #include "import-common.h"
36 #include "import-util.h"
39 #include "path-util.h"
40 #include "pull-common.h"
43 #include "qcow2-util.h"
45 #include "string-util.h"
51 typedef enum RawProgress
{
66 PullJob
*roothash_job
;
67 PullJob
*settings_job
;
68 PullJob
*checksum_job
;
69 PullJob
*signature_job
;
71 RawPullFinished on_finished
;
76 bool grow_machine_directory
;
84 char *settings_temp_path
;
87 char *roothash_temp_path
;
92 RawPull
* raw_pull_unref(RawPull
*i
) {
96 pull_job_unref(i
->raw_job
);
97 pull_job_unref(i
->settings_job
);
98 pull_job_unref(i
->roothash_job
);
99 pull_job_unref(i
->checksum_job
);
100 pull_job_unref(i
->signature_job
);
102 curl_glue_unref(i
->glue
);
103 sd_event_unref(i
->event
);
106 (void) unlink(i
->temp_path
);
110 if (i
->roothash_temp_path
) {
111 (void) unlink(i
->roothash_temp_path
);
112 free(i
->roothash_temp_path
);
115 if (i
->settings_temp_path
) {
116 (void) unlink(i
->settings_temp_path
);
117 free(i
->settings_temp_path
);
121 free(i
->roothash_path
);
122 free(i
->settings_path
);
131 const char *image_root
,
132 RawPullFinished on_finished
,
135 _cleanup_(raw_pull_unrefp
) RawPull
*i
= NULL
;
140 i
= new0(RawPull
, 1);
144 i
->on_finished
= on_finished
;
145 i
->userdata
= userdata
;
147 i
->image_root
= strdup(image_root
?: "/var/lib/machines");
151 i
->grow_machine_directory
= path_startswith(i
->image_root
, "/var/lib/machines");
154 i
->event
= sd_event_ref(event
);
156 r
= sd_event_default(&i
->event
);
161 r
= curl_glue_new(&i
->glue
, i
->event
);
165 i
->glue
->on_finished
= pull_job_curl_on_finished
;
166 i
->glue
->userdata
= i
;
174 static void raw_pull_report_progress(RawPull
*i
, RawProgress p
) {
181 case RAW_DOWNLOADING
: {
182 unsigned remain
= 80;
186 if (i
->settings_job
) {
187 percent
+= i
->settings_job
->progress_percent
* 5 / 100;
191 if (i
->roothash_job
) {
192 percent
+= i
->roothash_job
->progress_percent
* 5 / 100;
196 if (i
->checksum_job
) {
197 percent
+= i
->checksum_job
->progress_percent
* 5 / 100;
201 if (i
->signature_job
) {
202 percent
+= i
->signature_job
->progress_percent
* 5 / 100;
207 percent
+= i
->raw_job
->progress_percent
* remain
/ 100;
228 assert_not_reached("Unknown progress state");
231 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent
);
232 log_debug("Combined progress %u%%", percent
);
235 static int raw_pull_maybe_convert_qcow2(RawPull
*i
) {
236 _cleanup_close_
int converted_fd
= -1;
237 _cleanup_free_
char *t
= NULL
;
243 r
= qcow2_detect(i
->raw_job
->disk_fd
);
245 return log_error_errno(r
, "Failed to detect whether this is a QCOW2 image: %m");
249 /* This is a QCOW2 image, let's convert it */
250 r
= tempfn_random(i
->final_path
, NULL
, &t
);
254 converted_fd
= open(t
, O_RDWR
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
255 if (converted_fd
< 0)
256 return log_error_errno(errno
, "Failed to create %s: %m", t
);
258 r
= chattr_fd(converted_fd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
260 log_warning_errno(r
, "Failed to set file attributes on %s: %m", t
);
262 log_info("Unpacking QCOW2 file.");
264 r
= qcow2_convert(i
->raw_job
->disk_fd
, converted_fd
);
267 return log_error_errno(r
, "Failed to convert qcow2 image: %m");
270 (void) unlink(i
->temp_path
);
275 safe_close(i
->raw_job
->disk_fd
);
276 i
->raw_job
->disk_fd
= converted_fd
;
282 static int raw_pull_determine_path(RawPull
*i
, const char *suffix
, char **field
) {
293 r
= pull_make_path(i
->raw_job
->url
, i
->raw_job
->etag
, i
->image_root
, ".raw-", suffix
, field
);
300 static int raw_pull_copy_auxiliary_file(
312 r
= raw_pull_determine_path(i
, suffix
, path
);
316 local
= strjoina(i
->image_root
, "/", i
->local
, suffix
);
318 r
= copy_file_atomic(*path
, local
, 0644, 0, COPY_REFLINK
| (i
->force_local
? COPY_REPLACE
: 0));
320 log_warning_errno(r
, "File %s already exists, not replacing.", local
);
321 else if (r
== -ENOENT
)
322 log_debug_errno(r
, "Skipping creation of auxiliary file, since none was found.");
324 log_warning_errno(r
, "Failed to copy file %s, ignoring: %m", local
);
326 log_info("Created new file %s.", local
);
331 static int raw_pull_make_local_copy(RawPull
*i
) {
332 _cleanup_free_
char *tp
= NULL
;
333 _cleanup_close_
int dfd
= -1;
343 if (i
->raw_job
->etag_exists
) {
344 /* We have downloaded this one previously, reopen it */
346 assert(i
->raw_job
->disk_fd
< 0);
348 i
->raw_job
->disk_fd
= open(i
->final_path
, O_RDONLY
|O_NOCTTY
|O_CLOEXEC
);
349 if (i
->raw_job
->disk_fd
< 0)
350 return log_error_errno(errno
, "Failed to open vendor image: %m");
352 /* We freshly downloaded the image, use it */
354 assert(i
->raw_job
->disk_fd
>= 0);
356 if (lseek(i
->raw_job
->disk_fd
, SEEK_SET
, 0) == (off_t
) -1)
357 return log_error_errno(errno
, "Failed to seek to beginning of vendor image: %m");
360 p
= strjoina(i
->image_root
, "/", i
->local
, ".raw");
363 (void) rm_rf(p
, REMOVE_ROOT
|REMOVE_PHYSICAL
|REMOVE_SUBVOLUME
);
365 r
= tempfn_random(p
, NULL
, &tp
);
369 dfd
= open(tp
, O_WRONLY
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
371 return log_error_errno(errno
, "Failed to create writable copy of image: %m");
373 /* Turn off COW writing. This should greatly improve
374 * performance on COW file systems like btrfs, since it
375 * reduces fragmentation caused by not allowing in-place
377 r
= chattr_fd(dfd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
379 log_warning_errno(r
, "Failed to set file attributes on %s: %m", tp
);
381 r
= copy_bytes(i
->raw_job
->disk_fd
, dfd
, (uint64_t) -1, COPY_REFLINK
);
384 return log_error_errno(r
, "Failed to make writable copy of image: %m");
387 (void) copy_times(i
->raw_job
->disk_fd
, dfd
);
388 (void) copy_xattr(i
->raw_job
->disk_fd
, dfd
);
390 dfd
= safe_close(dfd
);
394 r
= log_error_errno(errno
, "Failed to move writable image into place: %m");
399 log_info("Created new local image '%s'.", i
->local
);
402 r
= raw_pull_copy_auxiliary_file(i
, ".roothash", &i
->roothash_path
);
408 r
= raw_pull_copy_auxiliary_file(i
, ".nspawn", &i
->settings_path
);
416 static bool raw_pull_is_done(RawPull
*i
) {
420 if (!PULL_JOB_IS_COMPLETE(i
->raw_job
))
422 if (i
->roothash_job
&& !PULL_JOB_IS_COMPLETE(i
->roothash_job
))
424 if (i
->settings_job
&& !PULL_JOB_IS_COMPLETE(i
->settings_job
))
426 if (i
->checksum_job
&& !PULL_JOB_IS_COMPLETE(i
->checksum_job
))
428 if (i
->signature_job
&& !PULL_JOB_IS_COMPLETE(i
->signature_job
))
434 static int raw_pull_rename_auxiliary_file(
447 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
448 * incorporate it in the file name if we can */
449 *path
= mfree(*path
);
450 r
= raw_pull_determine_path(i
, suffix
, path
);
454 r
= import_make_read_only(*temp_path
);
458 r
= rename_noreplace(AT_FDCWD
, *temp_path
, AT_FDCWD
, *path
);
460 return log_error_errno(r
, "Failed to rename file %s to %s: %m", *temp_path
, *path
);
462 *temp_path
= mfree(*temp_path
);
467 static void raw_pull_job_on_finished(PullJob
*j
) {
475 if (j
== i
->roothash_job
) {
477 log_info_errno(j
->error
, "Root hash file could not be retrieved, proceeding without.");
478 } else if (j
== i
->settings_job
) {
480 log_info_errno(j
->error
, "Settings file could not be retrieved, proceeding without.");
481 } else if (j
->error
!= 0 && j
!= i
->signature_job
) {
482 if (j
== i
->checksum_job
)
483 log_error_errno(j
->error
, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
485 log_error_errno(j
->error
, "Failed to retrieve image file. (Wrong URL?)");
491 /* This is invoked if either the download completed
492 * successfully, or the download was skipped because we
493 * already have the etag. In this case ->etag_exists is
496 * We only do something when we got all three files */
498 if (!raw_pull_is_done(i
))
501 if (i
->signature_job
&& i
->checksum_job
->style
== VERIFICATION_PER_DIRECTORY
&& i
->signature_job
->error
!= 0) {
502 log_error_errno(j
->error
, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
504 r
= i
->signature_job
->error
;
509 i
->roothash_job
->disk_fd
= safe_close(i
->roothash_job
->disk_fd
);
511 i
->settings_job
->disk_fd
= safe_close(i
->settings_job
->disk_fd
);
513 r
= raw_pull_determine_path(i
, ".raw", &i
->final_path
);
517 if (!i
->raw_job
->etag_exists
) {
518 /* This is a new download, verify it, and move it into place */
519 assert(i
->raw_job
->disk_fd
>= 0);
521 raw_pull_report_progress(i
, RAW_VERIFYING
);
523 r
= pull_verify(i
->raw_job
, i
->roothash_job
, i
->settings_job
, i
->checksum_job
, i
->signature_job
);
527 raw_pull_report_progress(i
, RAW_UNPACKING
);
529 r
= raw_pull_maybe_convert_qcow2(i
);
533 raw_pull_report_progress(i
, RAW_FINALIZING
);
535 r
= import_make_read_only_fd(i
->raw_job
->disk_fd
);
539 r
= rename_noreplace(AT_FDCWD
, i
->temp_path
, AT_FDCWD
, i
->final_path
);
541 log_error_errno(r
, "Failed to rename raw file to %s: %m", i
->final_path
);
545 i
->temp_path
= mfree(i
->temp_path
);
547 if (i
->roothash_job
&&
548 i
->roothash_job
->error
== 0) {
549 r
= raw_pull_rename_auxiliary_file(i
, ".roothash", &i
->roothash_temp_path
, &i
->roothash_path
);
554 if (i
->settings_job
&&
555 i
->settings_job
->error
== 0) {
556 r
= raw_pull_rename_auxiliary_file(i
, ".nspawn", &i
->settings_temp_path
, &i
->settings_path
);
562 raw_pull_report_progress(i
, RAW_COPYING
);
564 r
= raw_pull_make_local_copy(i
);
572 i
->on_finished(i
, r
, i
->userdata
);
574 sd_event_exit(i
->event
, r
);
577 static int raw_pull_job_on_open_disk_generic(
591 r
= tempfn_random_child(i
->image_root
, extra
, temp_path
);
596 (void) mkdir_parents_label(*temp_path
, 0700);
598 j
->disk_fd
= open(*temp_path
, O_RDWR
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
600 return log_error_errno(errno
, "Failed to create %s: %m", *temp_path
);
605 static int raw_pull_job_on_open_disk_raw(PullJob
*j
) {
613 assert(i
->raw_job
== j
);
615 r
= raw_pull_job_on_open_disk_generic(i
, j
, "raw", &i
->temp_path
);
619 r
= chattr_fd(j
->disk_fd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
621 log_warning_errno(r
, "Failed to set file attributes on %s, ignoring: %m", i
->temp_path
);
626 static int raw_pull_job_on_open_disk_roothash(PullJob
*j
) {
633 assert(i
->roothash_job
== j
);
635 return raw_pull_job_on_open_disk_generic(i
, j
, "roothash", &i
->roothash_temp_path
);
638 static int raw_pull_job_on_open_disk_settings(PullJob
*j
) {
645 assert(i
->settings_job
== j
);
647 return raw_pull_job_on_open_disk_generic(i
, j
, "settings", &i
->settings_temp_path
);
650 static void raw_pull_job_on_progress(PullJob
*j
) {
658 raw_pull_report_progress(i
, RAW_DOWNLOADING
);
673 assert(verify
< _IMPORT_VERIFY_MAX
);
676 if (!http_url_is_valid(url
))
679 if (local
&& !machine_name_is_valid(local
))
685 r
= free_and_strdup(&i
->local
, local
);
689 i
->force_local
= force_local
;
691 i
->settings
= settings
;
692 i
->roothash
= roothash
;
694 /* Queue job for the image itself */
695 r
= pull_job_new(&i
->raw_job
, url
, i
->glue
, i
);
699 i
->raw_job
->on_finished
= raw_pull_job_on_finished
;
700 i
->raw_job
->on_open_disk
= raw_pull_job_on_open_disk_raw
;
701 i
->raw_job
->on_progress
= raw_pull_job_on_progress
;
702 i
->raw_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
703 i
->raw_job
->grow_machine_directory
= i
->grow_machine_directory
;
705 r
= pull_find_old_etags(url
, i
->image_root
, DT_REG
, ".raw-", ".raw", &i
->raw_job
->old_etags
);
710 r
= pull_make_auxiliary_job(&i
->roothash_job
, url
, raw_strip_suffixes
, ".roothash", i
->glue
, raw_pull_job_on_finished
, i
);
714 i
->roothash_job
->on_open_disk
= raw_pull_job_on_open_disk_roothash
;
715 i
->roothash_job
->on_progress
= raw_pull_job_on_progress
;
716 i
->roothash_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
720 r
= pull_make_auxiliary_job(&i
->settings_job
, url
, raw_strip_suffixes
, ".nspawn", i
->glue
, raw_pull_job_on_finished
, i
);
724 i
->settings_job
->on_open_disk
= raw_pull_job_on_open_disk_settings
;
725 i
->settings_job
->on_progress
= raw_pull_job_on_progress
;
726 i
->settings_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
729 r
= pull_make_verification_jobs(&i
->checksum_job
, &i
->signature_job
, verify
, url
, i
->glue
, raw_pull_job_on_finished
, i
);
733 r
= pull_job_begin(i
->raw_job
);
737 if (i
->roothash_job
) {
738 r
= pull_job_begin(i
->roothash_job
);
743 if (i
->settings_job
) {
744 r
= pull_job_begin(i
->settings_job
);
749 if (i
->checksum_job
) {
750 i
->checksum_job
->on_progress
= raw_pull_job_on_progress
;
751 i
->checksum_job
->style
= VERIFICATION_PER_FILE
;
753 r
= pull_job_begin(i
->checksum_job
);
758 if (i
->signature_job
) {
759 i
->signature_job
->on_progress
= raw_pull_job_on_progress
;
761 r
= pull_job_begin(i
->signature_job
);