1 /* SPDX-License-Identifier: LGPL-2.1+ */
3 This file is part of systemd.
5 Copyright 2014 Lennart Poettering
7 systemd is free software; you can redistribute it and/or modify it
8 under the terms of the GNU Lesser General Public License as published by
9 the Free Software Foundation; either version 2.1 of the License, or
10 (at your option) any later version.
12 systemd is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
17 You should have received a copy of the GNU Lesser General Public License
18 along with systemd; If not, see <http://www.gnu.org/licenses/>.
21 #include <curl/curl.h>
23 #include <sys/xattr.h>
25 #include "sd-daemon.h"
27 #include "alloc-util.h"
28 #include "btrfs-util.h"
29 #include "chattr-util.h"
31 #include "curl-util.h"
35 #include "hostname-util.h"
36 #include "import-common.h"
37 #include "import-util.h"
40 #include "path-util.h"
41 #include "pull-common.h"
44 #include "qcow2-util.h"
46 #include "string-util.h"
52 typedef enum RawProgress
{
67 PullJob
*roothash_job
;
68 PullJob
*settings_job
;
69 PullJob
*checksum_job
;
70 PullJob
*signature_job
;
72 RawPullFinished on_finished
;
77 bool grow_machine_directory
;
85 char *settings_temp_path
;
88 char *roothash_temp_path
;
93 RawPull
* raw_pull_unref(RawPull
*i
) {
97 pull_job_unref(i
->raw_job
);
98 pull_job_unref(i
->settings_job
);
99 pull_job_unref(i
->roothash_job
);
100 pull_job_unref(i
->checksum_job
);
101 pull_job_unref(i
->signature_job
);
103 curl_glue_unref(i
->glue
);
104 sd_event_unref(i
->event
);
107 (void) unlink(i
->temp_path
);
111 if (i
->roothash_temp_path
) {
112 (void) unlink(i
->roothash_temp_path
);
113 free(i
->roothash_temp_path
);
116 if (i
->settings_temp_path
) {
117 (void) unlink(i
->settings_temp_path
);
118 free(i
->settings_temp_path
);
122 free(i
->roothash_path
);
123 free(i
->settings_path
);
132 const char *image_root
,
133 RawPullFinished on_finished
,
136 _cleanup_(raw_pull_unrefp
) RawPull
*i
= NULL
;
141 i
= new0(RawPull
, 1);
145 i
->on_finished
= on_finished
;
146 i
->userdata
= userdata
;
148 i
->image_root
= strdup(image_root
?: "/var/lib/machines");
152 i
->grow_machine_directory
= path_startswith(i
->image_root
, "/var/lib/machines");
155 i
->event
= sd_event_ref(event
);
157 r
= sd_event_default(&i
->event
);
162 r
= curl_glue_new(&i
->glue
, i
->event
);
166 i
->glue
->on_finished
= pull_job_curl_on_finished
;
167 i
->glue
->userdata
= i
;
175 static void raw_pull_report_progress(RawPull
*i
, RawProgress p
) {
182 case RAW_DOWNLOADING
: {
183 unsigned remain
= 80;
187 if (i
->settings_job
) {
188 percent
+= i
->settings_job
->progress_percent
* 5 / 100;
192 if (i
->roothash_job
) {
193 percent
+= i
->roothash_job
->progress_percent
* 5 / 100;
197 if (i
->checksum_job
) {
198 percent
+= i
->checksum_job
->progress_percent
* 5 / 100;
202 if (i
->signature_job
) {
203 percent
+= i
->signature_job
->progress_percent
* 5 / 100;
208 percent
+= i
->raw_job
->progress_percent
* remain
/ 100;
229 assert_not_reached("Unknown progress state");
232 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent
);
233 log_debug("Combined progress %u%%", percent
);
236 static int raw_pull_maybe_convert_qcow2(RawPull
*i
) {
237 _cleanup_close_
int converted_fd
= -1;
238 _cleanup_free_
char *t
= NULL
;
244 r
= qcow2_detect(i
->raw_job
->disk_fd
);
246 return log_error_errno(r
, "Failed to detect whether this is a QCOW2 image: %m");
250 /* This is a QCOW2 image, let's convert it */
251 r
= tempfn_random(i
->final_path
, NULL
, &t
);
255 converted_fd
= open(t
, O_RDWR
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
256 if (converted_fd
< 0)
257 return log_error_errno(errno
, "Failed to create %s: %m", t
);
259 r
= chattr_fd(converted_fd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
261 log_warning_errno(r
, "Failed to set file attributes on %s: %m", t
);
263 log_info("Unpacking QCOW2 file.");
265 r
= qcow2_convert(i
->raw_job
->disk_fd
, converted_fd
);
268 return log_error_errno(r
, "Failed to convert qcow2 image: %m");
271 (void) unlink(i
->temp_path
);
272 free_and_replace(i
->temp_path
, t
);
274 safe_close(i
->raw_job
->disk_fd
);
275 i
->raw_job
->disk_fd
= converted_fd
;
281 static int raw_pull_determine_path(RawPull
*i
, const char *suffix
, char **field
) {
292 r
= pull_make_path(i
->raw_job
->url
, i
->raw_job
->etag
, i
->image_root
, ".raw-", suffix
, field
);
299 static int raw_pull_copy_auxiliary_file(
311 r
= raw_pull_determine_path(i
, suffix
, path
);
315 local
= strjoina(i
->image_root
, "/", i
->local
, suffix
);
317 r
= copy_file_atomic(*path
, local
, 0644, 0, COPY_REFLINK
| (i
->force_local
? COPY_REPLACE
: 0));
319 log_warning_errno(r
, "File %s already exists, not replacing.", local
);
320 else if (r
== -ENOENT
)
321 log_debug_errno(r
, "Skipping creation of auxiliary file, since none was found.");
323 log_warning_errno(r
, "Failed to copy file %s, ignoring: %m", local
);
325 log_info("Created new file %s.", local
);
330 static int raw_pull_make_local_copy(RawPull
*i
) {
331 _cleanup_free_
char *tp
= NULL
;
332 _cleanup_close_
int dfd
= -1;
342 if (i
->raw_job
->etag_exists
) {
343 /* We have downloaded this one previously, reopen it */
345 assert(i
->raw_job
->disk_fd
< 0);
347 i
->raw_job
->disk_fd
= open(i
->final_path
, O_RDONLY
|O_NOCTTY
|O_CLOEXEC
);
348 if (i
->raw_job
->disk_fd
< 0)
349 return log_error_errno(errno
, "Failed to open vendor image: %m");
351 /* We freshly downloaded the image, use it */
353 assert(i
->raw_job
->disk_fd
>= 0);
355 if (lseek(i
->raw_job
->disk_fd
, SEEK_SET
, 0) == (off_t
) -1)
356 return log_error_errno(errno
, "Failed to seek to beginning of vendor image: %m");
359 p
= strjoina(i
->image_root
, "/", i
->local
, ".raw");
362 (void) rm_rf(p
, REMOVE_ROOT
|REMOVE_PHYSICAL
|REMOVE_SUBVOLUME
);
364 r
= tempfn_random(p
, NULL
, &tp
);
368 dfd
= open(tp
, O_WRONLY
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
370 return log_error_errno(errno
, "Failed to create writable copy of image: %m");
372 /* Turn off COW writing. This should greatly improve
373 * performance on COW file systems like btrfs, since it
374 * reduces fragmentation caused by not allowing in-place
376 r
= chattr_fd(dfd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
378 log_warning_errno(r
, "Failed to set file attributes on %s: %m", tp
);
380 r
= copy_bytes(i
->raw_job
->disk_fd
, dfd
, (uint64_t) -1, COPY_REFLINK
);
383 return log_error_errno(r
, "Failed to make writable copy of image: %m");
386 (void) copy_times(i
->raw_job
->disk_fd
, dfd
);
387 (void) copy_xattr(i
->raw_job
->disk_fd
, dfd
);
389 dfd
= safe_close(dfd
);
393 r
= log_error_errno(errno
, "Failed to move writable image into place: %m");
398 log_info("Created new local image '%s'.", i
->local
);
401 r
= raw_pull_copy_auxiliary_file(i
, ".roothash", &i
->roothash_path
);
407 r
= raw_pull_copy_auxiliary_file(i
, ".nspawn", &i
->settings_path
);
415 static bool raw_pull_is_done(RawPull
*i
) {
419 if (!PULL_JOB_IS_COMPLETE(i
->raw_job
))
421 if (i
->roothash_job
&& !PULL_JOB_IS_COMPLETE(i
->roothash_job
))
423 if (i
->settings_job
&& !PULL_JOB_IS_COMPLETE(i
->settings_job
))
425 if (i
->checksum_job
&& !PULL_JOB_IS_COMPLETE(i
->checksum_job
))
427 if (i
->signature_job
&& !PULL_JOB_IS_COMPLETE(i
->signature_job
))
433 static int raw_pull_rename_auxiliary_file(
446 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
447 * incorporate it in the file name if we can */
448 *path
= mfree(*path
);
449 r
= raw_pull_determine_path(i
, suffix
, path
);
453 r
= import_make_read_only(*temp_path
);
457 r
= rename_noreplace(AT_FDCWD
, *temp_path
, AT_FDCWD
, *path
);
459 return log_error_errno(r
, "Failed to rename file %s to %s: %m", *temp_path
, *path
);
461 *temp_path
= mfree(*temp_path
);
466 static void raw_pull_job_on_finished(PullJob
*j
) {
474 if (j
== i
->roothash_job
) {
476 log_info_errno(j
->error
, "Root hash file could not be retrieved, proceeding without.");
477 } else if (j
== i
->settings_job
) {
479 log_info_errno(j
->error
, "Settings file could not be retrieved, proceeding without.");
480 } else if (j
->error
!= 0 && j
!= i
->signature_job
) {
481 if (j
== i
->checksum_job
)
482 log_error_errno(j
->error
, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
484 log_error_errno(j
->error
, "Failed to retrieve image file. (Wrong URL?)");
490 /* This is invoked if either the download completed
491 * successfully, or the download was skipped because we
492 * already have the etag. In this case ->etag_exists is
495 * We only do something when we got all three files */
497 if (!raw_pull_is_done(i
))
500 if (i
->signature_job
&& i
->checksum_job
->style
== VERIFICATION_PER_DIRECTORY
&& i
->signature_job
->error
!= 0) {
501 log_error_errno(j
->error
, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
503 r
= i
->signature_job
->error
;
508 i
->roothash_job
->disk_fd
= safe_close(i
->roothash_job
->disk_fd
);
510 i
->settings_job
->disk_fd
= safe_close(i
->settings_job
->disk_fd
);
512 r
= raw_pull_determine_path(i
, ".raw", &i
->final_path
);
516 if (!i
->raw_job
->etag_exists
) {
517 /* This is a new download, verify it, and move it into place */
518 assert(i
->raw_job
->disk_fd
>= 0);
520 raw_pull_report_progress(i
, RAW_VERIFYING
);
522 r
= pull_verify(i
->raw_job
, i
->roothash_job
, i
->settings_job
, i
->checksum_job
, i
->signature_job
);
526 raw_pull_report_progress(i
, RAW_UNPACKING
);
528 r
= raw_pull_maybe_convert_qcow2(i
);
532 raw_pull_report_progress(i
, RAW_FINALIZING
);
534 r
= import_make_read_only_fd(i
->raw_job
->disk_fd
);
538 r
= rename_noreplace(AT_FDCWD
, i
->temp_path
, AT_FDCWD
, i
->final_path
);
540 log_error_errno(r
, "Failed to rename raw file to %s: %m", i
->final_path
);
544 i
->temp_path
= mfree(i
->temp_path
);
546 if (i
->roothash_job
&&
547 i
->roothash_job
->error
== 0) {
548 r
= raw_pull_rename_auxiliary_file(i
, ".roothash", &i
->roothash_temp_path
, &i
->roothash_path
);
553 if (i
->settings_job
&&
554 i
->settings_job
->error
== 0) {
555 r
= raw_pull_rename_auxiliary_file(i
, ".nspawn", &i
->settings_temp_path
, &i
->settings_path
);
561 raw_pull_report_progress(i
, RAW_COPYING
);
563 r
= raw_pull_make_local_copy(i
);
571 i
->on_finished(i
, r
, i
->userdata
);
573 sd_event_exit(i
->event
, r
);
576 static int raw_pull_job_on_open_disk_generic(
590 r
= tempfn_random_child(i
->image_root
, extra
, temp_path
);
595 (void) mkdir_parents_label(*temp_path
, 0700);
597 j
->disk_fd
= open(*temp_path
, O_RDWR
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
599 return log_error_errno(errno
, "Failed to create %s: %m", *temp_path
);
604 static int raw_pull_job_on_open_disk_raw(PullJob
*j
) {
612 assert(i
->raw_job
== j
);
614 r
= raw_pull_job_on_open_disk_generic(i
, j
, "raw", &i
->temp_path
);
618 r
= chattr_fd(j
->disk_fd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
620 log_warning_errno(r
, "Failed to set file attributes on %s, ignoring: %m", i
->temp_path
);
625 static int raw_pull_job_on_open_disk_roothash(PullJob
*j
) {
632 assert(i
->roothash_job
== j
);
634 return raw_pull_job_on_open_disk_generic(i
, j
, "roothash", &i
->roothash_temp_path
);
637 static int raw_pull_job_on_open_disk_settings(PullJob
*j
) {
644 assert(i
->settings_job
== j
);
646 return raw_pull_job_on_open_disk_generic(i
, j
, "settings", &i
->settings_temp_path
);
649 static void raw_pull_job_on_progress(PullJob
*j
) {
657 raw_pull_report_progress(i
, RAW_DOWNLOADING
);
672 assert(verify
< _IMPORT_VERIFY_MAX
);
675 if (!http_url_is_valid(url
))
678 if (local
&& !machine_name_is_valid(local
))
684 r
= free_and_strdup(&i
->local
, local
);
688 i
->force_local
= force_local
;
690 i
->settings
= settings
;
691 i
->roothash
= roothash
;
693 /* Queue job for the image itself */
694 r
= pull_job_new(&i
->raw_job
, url
, i
->glue
, i
);
698 i
->raw_job
->on_finished
= raw_pull_job_on_finished
;
699 i
->raw_job
->on_open_disk
= raw_pull_job_on_open_disk_raw
;
700 i
->raw_job
->on_progress
= raw_pull_job_on_progress
;
701 i
->raw_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
702 i
->raw_job
->grow_machine_directory
= i
->grow_machine_directory
;
704 r
= pull_find_old_etags(url
, i
->image_root
, DT_REG
, ".raw-", ".raw", &i
->raw_job
->old_etags
);
709 r
= pull_make_auxiliary_job(&i
->roothash_job
, url
, raw_strip_suffixes
, ".roothash", i
->glue
, raw_pull_job_on_finished
, i
);
713 i
->roothash_job
->on_open_disk
= raw_pull_job_on_open_disk_roothash
;
714 i
->roothash_job
->on_progress
= raw_pull_job_on_progress
;
715 i
->roothash_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
719 r
= pull_make_auxiliary_job(&i
->settings_job
, url
, raw_strip_suffixes
, ".nspawn", i
->glue
, raw_pull_job_on_finished
, i
);
723 i
->settings_job
->on_open_disk
= raw_pull_job_on_open_disk_settings
;
724 i
->settings_job
->on_progress
= raw_pull_job_on_progress
;
725 i
->settings_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
728 r
= pull_make_verification_jobs(&i
->checksum_job
, &i
->signature_job
, verify
, url
, i
->glue
, raw_pull_job_on_finished
, i
);
732 r
= pull_job_begin(i
->raw_job
);
736 if (i
->roothash_job
) {
737 r
= pull_job_begin(i
->roothash_job
);
742 if (i
->settings_job
) {
743 r
= pull_job_begin(i
->settings_job
);
748 if (i
->checksum_job
) {
749 i
->checksum_job
->on_progress
= raw_pull_job_on_progress
;
750 i
->checksum_job
->style
= VERIFICATION_PER_FILE
;
752 r
= pull_job_begin(i
->checksum_job
);
757 if (i
->signature_job
) {
758 i
->signature_job
->on_progress
= raw_pull_job_on_progress
;
760 r
= pull_job_begin(i
->signature_job
);