1 /* SPDX-License-Identifier: LGPL-2.1+ */
9 #include "alloc-util.h"
10 #include "btrfs-util.h"
11 #include "chattr-util.h"
13 #include "curl-util.h"
17 #include "hostname-util.h"
18 #include "import-common.h"
19 #include "import-util.h"
22 #include "path-util.h"
23 #include "pull-common.h"
26 #include "qcow2-util.h"
28 #include "string-util.h"
34 typedef enum RawProgress
{
49 PullJob
*roothash_job
;
50 PullJob
*settings_job
;
51 PullJob
*checksum_job
;
52 PullJob
*signature_job
;
54 RawPullFinished on_finished
;
59 bool grow_machine_directory
;
67 char *settings_temp_path
;
70 char *roothash_temp_path
;
75 RawPull
* raw_pull_unref(RawPull
*i
) {
79 pull_job_unref(i
->raw_job
);
80 pull_job_unref(i
->settings_job
);
81 pull_job_unref(i
->roothash_job
);
82 pull_job_unref(i
->checksum_job
);
83 pull_job_unref(i
->signature_job
);
85 curl_glue_unref(i
->glue
);
86 sd_event_unref(i
->event
);
89 (void) unlink(i
->temp_path
);
93 if (i
->roothash_temp_path
) {
94 (void) unlink(i
->roothash_temp_path
);
95 free(i
->roothash_temp_path
);
98 if (i
->settings_temp_path
) {
99 (void) unlink(i
->settings_temp_path
);
100 free(i
->settings_temp_path
);
104 free(i
->roothash_path
);
105 free(i
->settings_path
);
114 const char *image_root
,
115 RawPullFinished on_finished
,
118 _cleanup_(raw_pull_unrefp
) RawPull
*i
= NULL
;
123 i
= new0(RawPull
, 1);
127 i
->on_finished
= on_finished
;
128 i
->userdata
= userdata
;
130 i
->image_root
= strdup(image_root
?: "/var/lib/machines");
134 i
->grow_machine_directory
= path_startswith(i
->image_root
, "/var/lib/machines");
137 i
->event
= sd_event_ref(event
);
139 r
= sd_event_default(&i
->event
);
144 r
= curl_glue_new(&i
->glue
, i
->event
);
148 i
->glue
->on_finished
= pull_job_curl_on_finished
;
149 i
->glue
->userdata
= i
;
156 static void raw_pull_report_progress(RawPull
*i
, RawProgress p
) {
163 case RAW_DOWNLOADING
: {
164 unsigned remain
= 80;
168 if (i
->settings_job
) {
169 percent
+= i
->settings_job
->progress_percent
* 5 / 100;
173 if (i
->roothash_job
) {
174 percent
+= i
->roothash_job
->progress_percent
* 5 / 100;
178 if (i
->checksum_job
) {
179 percent
+= i
->checksum_job
->progress_percent
* 5 / 100;
183 if (i
->signature_job
) {
184 percent
+= i
->signature_job
->progress_percent
* 5 / 100;
189 percent
+= i
->raw_job
->progress_percent
* remain
/ 100;
210 assert_not_reached("Unknown progress state");
213 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent
);
214 log_debug("Combined progress %u%%", percent
);
217 static int raw_pull_maybe_convert_qcow2(RawPull
*i
) {
218 _cleanup_close_
int converted_fd
= -1;
219 _cleanup_free_
char *t
= NULL
;
225 r
= qcow2_detect(i
->raw_job
->disk_fd
);
227 return log_error_errno(r
, "Failed to detect whether this is a QCOW2 image: %m");
231 /* This is a QCOW2 image, let's convert it */
232 r
= tempfn_random(i
->final_path
, NULL
, &t
);
236 converted_fd
= open(t
, O_RDWR
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
237 if (converted_fd
< 0)
238 return log_error_errno(errno
, "Failed to create %s: %m", t
);
240 r
= chattr_fd(converted_fd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
242 log_warning_errno(r
, "Failed to set file attributes on %s: %m", t
);
244 log_info("Unpacking QCOW2 file.");
246 r
= qcow2_convert(i
->raw_job
->disk_fd
, converted_fd
);
249 return log_error_errno(r
, "Failed to convert qcow2 image: %m");
252 (void) unlink(i
->temp_path
);
253 free_and_replace(i
->temp_path
, t
);
255 safe_close(i
->raw_job
->disk_fd
);
256 i
->raw_job
->disk_fd
= TAKE_FD(converted_fd
);
261 static int raw_pull_determine_path(RawPull
*i
, const char *suffix
, char **field
) {
272 r
= pull_make_path(i
->raw_job
->url
, i
->raw_job
->etag
, i
->image_root
, ".raw-", suffix
, field
);
279 static int raw_pull_copy_auxiliary_file(
291 r
= raw_pull_determine_path(i
, suffix
, path
);
295 local
= strjoina(i
->image_root
, "/", i
->local
, suffix
);
297 r
= copy_file_atomic(*path
, local
, 0644, 0, COPY_REFLINK
| (i
->force_local
? COPY_REPLACE
: 0));
299 log_warning_errno(r
, "File %s already exists, not replacing.", local
);
300 else if (r
== -ENOENT
)
301 log_debug_errno(r
, "Skipping creation of auxiliary file, since none was found.");
303 log_warning_errno(r
, "Failed to copy file %s, ignoring: %m", local
);
305 log_info("Created new file %s.", local
);
310 static int raw_pull_make_local_copy(RawPull
*i
) {
311 _cleanup_free_
char *tp
= NULL
;
312 _cleanup_close_
int dfd
= -1;
322 if (i
->raw_job
->etag_exists
) {
323 /* We have downloaded this one previously, reopen it */
325 assert(i
->raw_job
->disk_fd
< 0);
327 i
->raw_job
->disk_fd
= open(i
->final_path
, O_RDONLY
|O_NOCTTY
|O_CLOEXEC
);
328 if (i
->raw_job
->disk_fd
< 0)
329 return log_error_errno(errno
, "Failed to open vendor image: %m");
331 /* We freshly downloaded the image, use it */
333 assert(i
->raw_job
->disk_fd
>= 0);
335 if (lseek(i
->raw_job
->disk_fd
, SEEK_SET
, 0) == (off_t
) -1)
336 return log_error_errno(errno
, "Failed to seek to beginning of vendor image: %m");
339 p
= strjoina(i
->image_root
, "/", i
->local
, ".raw");
342 (void) rm_rf(p
, REMOVE_ROOT
|REMOVE_PHYSICAL
|REMOVE_SUBVOLUME
);
344 r
= tempfn_random(p
, NULL
, &tp
);
348 dfd
= open(tp
, O_WRONLY
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
350 return log_error_errno(errno
, "Failed to create writable copy of image: %m");
352 /* Turn off COW writing. This should greatly improve
353 * performance on COW file systems like btrfs, since it
354 * reduces fragmentation caused by not allowing in-place
356 r
= chattr_fd(dfd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
358 log_warning_errno(r
, "Failed to set file attributes on %s: %m", tp
);
360 r
= copy_bytes(i
->raw_job
->disk_fd
, dfd
, (uint64_t) -1, COPY_REFLINK
);
363 return log_error_errno(r
, "Failed to make writable copy of image: %m");
366 (void) copy_times(i
->raw_job
->disk_fd
, dfd
);
367 (void) copy_xattr(i
->raw_job
->disk_fd
, dfd
);
369 dfd
= safe_close(dfd
);
373 r
= log_error_errno(errno
, "Failed to move writable image into place: %m");
378 log_info("Created new local image '%s'.", i
->local
);
381 r
= raw_pull_copy_auxiliary_file(i
, ".roothash", &i
->roothash_path
);
387 r
= raw_pull_copy_auxiliary_file(i
, ".nspawn", &i
->settings_path
);
395 static bool raw_pull_is_done(RawPull
*i
) {
399 if (!PULL_JOB_IS_COMPLETE(i
->raw_job
))
401 if (i
->roothash_job
&& !PULL_JOB_IS_COMPLETE(i
->roothash_job
))
403 if (i
->settings_job
&& !PULL_JOB_IS_COMPLETE(i
->settings_job
))
405 if (i
->checksum_job
&& !PULL_JOB_IS_COMPLETE(i
->checksum_job
))
407 if (i
->signature_job
&& !PULL_JOB_IS_COMPLETE(i
->signature_job
))
413 static int raw_pull_rename_auxiliary_file(
426 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
427 * incorporate it in the file name if we can */
428 *path
= mfree(*path
);
429 r
= raw_pull_determine_path(i
, suffix
, path
);
433 r
= import_make_read_only(*temp_path
);
437 r
= rename_noreplace(AT_FDCWD
, *temp_path
, AT_FDCWD
, *path
);
439 return log_error_errno(r
, "Failed to rename file %s to %s: %m", *temp_path
, *path
);
441 *temp_path
= mfree(*temp_path
);
446 static void raw_pull_job_on_finished(PullJob
*j
) {
454 if (j
== i
->roothash_job
) {
456 log_info_errno(j
->error
, "Root hash file could not be retrieved, proceeding without.");
457 } else if (j
== i
->settings_job
) {
459 log_info_errno(j
->error
, "Settings file could not be retrieved, proceeding without.");
460 } else if (j
->error
!= 0 && j
!= i
->signature_job
) {
461 if (j
== i
->checksum_job
)
462 log_error_errno(j
->error
, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
464 log_error_errno(j
->error
, "Failed to retrieve image file. (Wrong URL?)");
470 /* This is invoked if either the download completed
471 * successfully, or the download was skipped because we
472 * already have the etag. In this case ->etag_exists is
475 * We only do something when we got all three files */
477 if (!raw_pull_is_done(i
))
480 if (i
->signature_job
&& i
->checksum_job
->style
== VERIFICATION_PER_DIRECTORY
&& i
->signature_job
->error
!= 0) {
481 log_error_errno(j
->error
, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
483 r
= i
->signature_job
->error
;
488 i
->roothash_job
->disk_fd
= safe_close(i
->roothash_job
->disk_fd
);
490 i
->settings_job
->disk_fd
= safe_close(i
->settings_job
->disk_fd
);
492 r
= raw_pull_determine_path(i
, ".raw", &i
->final_path
);
496 if (!i
->raw_job
->etag_exists
) {
497 /* This is a new download, verify it, and move it into place */
498 assert(i
->raw_job
->disk_fd
>= 0);
500 raw_pull_report_progress(i
, RAW_VERIFYING
);
502 r
= pull_verify(i
->raw_job
, i
->roothash_job
, i
->settings_job
, i
->checksum_job
, i
->signature_job
);
506 raw_pull_report_progress(i
, RAW_UNPACKING
);
508 r
= raw_pull_maybe_convert_qcow2(i
);
512 raw_pull_report_progress(i
, RAW_FINALIZING
);
514 r
= import_make_read_only_fd(i
->raw_job
->disk_fd
);
518 r
= rename_noreplace(AT_FDCWD
, i
->temp_path
, AT_FDCWD
, i
->final_path
);
520 log_error_errno(r
, "Failed to rename raw file to %s: %m", i
->final_path
);
524 i
->temp_path
= mfree(i
->temp_path
);
526 if (i
->roothash_job
&&
527 i
->roothash_job
->error
== 0) {
528 r
= raw_pull_rename_auxiliary_file(i
, ".roothash", &i
->roothash_temp_path
, &i
->roothash_path
);
533 if (i
->settings_job
&&
534 i
->settings_job
->error
== 0) {
535 r
= raw_pull_rename_auxiliary_file(i
, ".nspawn", &i
->settings_temp_path
, &i
->settings_path
);
541 raw_pull_report_progress(i
, RAW_COPYING
);
543 r
= raw_pull_make_local_copy(i
);
551 i
->on_finished(i
, r
, i
->userdata
);
553 sd_event_exit(i
->event
, r
);
556 static int raw_pull_job_on_open_disk_generic(
570 r
= tempfn_random_child(i
->image_root
, extra
, temp_path
);
575 (void) mkdir_parents_label(*temp_path
, 0700);
577 j
->disk_fd
= open(*temp_path
, O_RDWR
|O_CREAT
|O_EXCL
|O_NOCTTY
|O_CLOEXEC
, 0664);
579 return log_error_errno(errno
, "Failed to create %s: %m", *temp_path
);
584 static int raw_pull_job_on_open_disk_raw(PullJob
*j
) {
592 assert(i
->raw_job
== j
);
594 r
= raw_pull_job_on_open_disk_generic(i
, j
, "raw", &i
->temp_path
);
598 r
= chattr_fd(j
->disk_fd
, FS_NOCOW_FL
, FS_NOCOW_FL
);
600 log_warning_errno(r
, "Failed to set file attributes on %s, ignoring: %m", i
->temp_path
);
605 static int raw_pull_job_on_open_disk_roothash(PullJob
*j
) {
612 assert(i
->roothash_job
== j
);
614 return raw_pull_job_on_open_disk_generic(i
, j
, "roothash", &i
->roothash_temp_path
);
617 static int raw_pull_job_on_open_disk_settings(PullJob
*j
) {
624 assert(i
->settings_job
== j
);
626 return raw_pull_job_on_open_disk_generic(i
, j
, "settings", &i
->settings_temp_path
);
629 static void raw_pull_job_on_progress(PullJob
*j
) {
637 raw_pull_report_progress(i
, RAW_DOWNLOADING
);
652 assert(verify
< _IMPORT_VERIFY_MAX
);
655 if (!http_url_is_valid(url
))
658 if (local
&& !machine_name_is_valid(local
))
664 r
= free_and_strdup(&i
->local
, local
);
668 i
->force_local
= force_local
;
670 i
->settings
= settings
;
671 i
->roothash
= roothash
;
673 /* Queue job for the image itself */
674 r
= pull_job_new(&i
->raw_job
, url
, i
->glue
, i
);
678 i
->raw_job
->on_finished
= raw_pull_job_on_finished
;
679 i
->raw_job
->on_open_disk
= raw_pull_job_on_open_disk_raw
;
680 i
->raw_job
->on_progress
= raw_pull_job_on_progress
;
681 i
->raw_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
682 i
->raw_job
->grow_machine_directory
= i
->grow_machine_directory
;
684 r
= pull_find_old_etags(url
, i
->image_root
, DT_REG
, ".raw-", ".raw", &i
->raw_job
->old_etags
);
689 r
= pull_make_auxiliary_job(&i
->roothash_job
, url
, raw_strip_suffixes
, ".roothash", i
->glue
, raw_pull_job_on_finished
, i
);
693 i
->roothash_job
->on_open_disk
= raw_pull_job_on_open_disk_roothash
;
694 i
->roothash_job
->on_progress
= raw_pull_job_on_progress
;
695 i
->roothash_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
699 r
= pull_make_auxiliary_job(&i
->settings_job
, url
, raw_strip_suffixes
, ".nspawn", i
->glue
, raw_pull_job_on_finished
, i
);
703 i
->settings_job
->on_open_disk
= raw_pull_job_on_open_disk_settings
;
704 i
->settings_job
->on_progress
= raw_pull_job_on_progress
;
705 i
->settings_job
->calc_checksum
= verify
!= IMPORT_VERIFY_NO
;
708 r
= pull_make_verification_jobs(&i
->checksum_job
, &i
->signature_job
, verify
, url
, i
->glue
, raw_pull_job_on_finished
, i
);
712 r
= pull_job_begin(i
->raw_job
);
716 if (i
->roothash_job
) {
717 r
= pull_job_begin(i
->roothash_job
);
722 if (i
->settings_job
) {
723 r
= pull_job_begin(i
->settings_job
);
728 if (i
->checksum_job
) {
729 i
->checksum_job
->on_progress
= raw_pull_job_on_progress
;
730 i
->checksum_job
->style
= VERIFICATION_PER_FILE
;
732 r
= pull_job_begin(i
->checksum_job
);
737 if (i
->signature_job
) {
738 i
->signature_job
->on_progress
= raw_pull_job_on_progress
;
740 r
= pull_job_begin(i
->signature_job
);