]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/import/pull-raw.c
util-lib: split out all temporary file related calls into tmpfiles-util.c
[thirdparty/systemd.git] / src / import / pull-raw.c
1 /* SPDX-License-Identifier: LGPL-2.1+ */
2
3 #include <curl/curl.h>
4 #include <linux/fs.h>
5 #include <sys/xattr.h>
6
7 #include "sd-daemon.h"
8
9 #include "alloc-util.h"
10 #include "btrfs-util.h"
11 #include "chattr-util.h"
12 #include "copy.h"
13 #include "curl-util.h"
14 #include "fd-util.h"
15 #include "fs-util.h"
16 #include "hostname-util.h"
17 #include "import-common.h"
18 #include "import-util.h"
19 #include "macro.h"
20 #include "mkdir.h"
21 #include "path-util.h"
22 #include "pull-common.h"
23 #include "pull-job.h"
24 #include "pull-raw.h"
25 #include "qcow2-util.h"
26 #include "rm-rf.h"
27 #include "string-util.h"
28 #include "strv.h"
29 #include "tmpfile-util.h"
30 #include "utf8.h"
31 #include "util.h"
32 #include "web-util.h"
33
34 typedef enum RawProgress {
35 RAW_DOWNLOADING,
36 RAW_VERIFYING,
37 RAW_UNPACKING,
38 RAW_FINALIZING,
39 RAW_COPYING,
40 } RawProgress;
41
42 struct RawPull {
43 sd_event *event;
44 CurlGlue *glue;
45
46 char *image_root;
47
48 PullJob *raw_job;
49 PullJob *roothash_job;
50 PullJob *settings_job;
51 PullJob *checksum_job;
52 PullJob *signature_job;
53
54 RawPullFinished on_finished;
55 void *userdata;
56
57 char *local;
58 bool force_local;
59 bool settings;
60 bool roothash;
61
62 char *final_path;
63 char *temp_path;
64
65 char *settings_path;
66 char *settings_temp_path;
67
68 char *roothash_path;
69 char *roothash_temp_path;
70
71 ImportVerify verify;
72 };
73
74 RawPull* raw_pull_unref(RawPull *i) {
75 if (!i)
76 return NULL;
77
78 pull_job_unref(i->raw_job);
79 pull_job_unref(i->settings_job);
80 pull_job_unref(i->roothash_job);
81 pull_job_unref(i->checksum_job);
82 pull_job_unref(i->signature_job);
83
84 curl_glue_unref(i->glue);
85 sd_event_unref(i->event);
86
87 if (i->temp_path) {
88 (void) unlink(i->temp_path);
89 free(i->temp_path);
90 }
91
92 if (i->roothash_temp_path) {
93 (void) unlink(i->roothash_temp_path);
94 free(i->roothash_temp_path);
95 }
96
97 if (i->settings_temp_path) {
98 (void) unlink(i->settings_temp_path);
99 free(i->settings_temp_path);
100 }
101
102 free(i->final_path);
103 free(i->roothash_path);
104 free(i->settings_path);
105 free(i->image_root);
106 free(i->local);
107 return mfree(i);
108 }
109
110 int raw_pull_new(
111 RawPull **ret,
112 sd_event *event,
113 const char *image_root,
114 RawPullFinished on_finished,
115 void *userdata) {
116
117 _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL;
118 _cleanup_(sd_event_unrefp) sd_event *e = NULL;
119 _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
120 _cleanup_free_ char *root = NULL;
121 int r;
122
123 assert(ret);
124
125 root = strdup(image_root ?: "/var/lib/machines");
126 if (!root)
127 return -ENOMEM;
128
129 if (event)
130 e = sd_event_ref(event);
131 else {
132 r = sd_event_default(&e);
133 if (r < 0)
134 return r;
135 }
136
137 r = curl_glue_new(&g, e);
138 if (r < 0)
139 return r;
140
141 i = new(RawPull, 1);
142 if (!i)
143 return -ENOMEM;
144
145 *i = (RawPull) {
146 .on_finished = on_finished,
147 .userdata = userdata,
148 .image_root = TAKE_PTR(root),
149 .event = TAKE_PTR(e),
150 .glue = TAKE_PTR(g),
151 };
152
153 i->glue->on_finished = pull_job_curl_on_finished;
154 i->glue->userdata = i;
155
156 *ret = TAKE_PTR(i);
157
158 return 0;
159 }
160
161 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
162 unsigned percent;
163
164 assert(i);
165
166 switch (p) {
167
168 case RAW_DOWNLOADING: {
169 unsigned remain = 80;
170
171 percent = 0;
172
173 if (i->settings_job) {
174 percent += i->settings_job->progress_percent * 5 / 100;
175 remain -= 5;
176 }
177
178 if (i->roothash_job) {
179 percent += i->roothash_job->progress_percent * 5 / 100;
180 remain -= 5;
181 }
182
183 if (i->checksum_job) {
184 percent += i->checksum_job->progress_percent * 5 / 100;
185 remain -= 5;
186 }
187
188 if (i->signature_job) {
189 percent += i->signature_job->progress_percent * 5 / 100;
190 remain -= 5;
191 }
192
193 if (i->raw_job)
194 percent += i->raw_job->progress_percent * remain / 100;
195 break;
196 }
197
198 case RAW_VERIFYING:
199 percent = 80;
200 break;
201
202 case RAW_UNPACKING:
203 percent = 85;
204 break;
205
206 case RAW_FINALIZING:
207 percent = 90;
208 break;
209
210 case RAW_COPYING:
211 percent = 95;
212 break;
213
214 default:
215 assert_not_reached("Unknown progress state");
216 }
217
218 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
219 log_debug("Combined progress %u%%", percent);
220 }
221
222 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
223 _cleanup_close_ int converted_fd = -1;
224 _cleanup_free_ char *t = NULL;
225 int r;
226
227 assert(i);
228 assert(i->raw_job);
229
230 r = qcow2_detect(i->raw_job->disk_fd);
231 if (r < 0)
232 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
233 if (r == 0)
234 return 0;
235
236 /* This is a QCOW2 image, let's convert it */
237 r = tempfn_random(i->final_path, NULL, &t);
238 if (r < 0)
239 return log_oom();
240
241 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
242 if (converted_fd < 0)
243 return log_error_errno(errno, "Failed to create %s: %m", t);
244
245 r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
246 if (r < 0)
247 log_warning_errno(r, "Failed to set file attributes on %s: %m", t);
248
249 log_info("Unpacking QCOW2 file.");
250
251 r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
252 if (r < 0) {
253 unlink(t);
254 return log_error_errno(r, "Failed to convert qcow2 image: %m");
255 }
256
257 (void) unlink(i->temp_path);
258 free_and_replace(i->temp_path, t);
259
260 safe_close(i->raw_job->disk_fd);
261 i->raw_job->disk_fd = TAKE_FD(converted_fd);
262
263 return 1;
264 }
265
266 static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) {
267 int r;
268
269 assert(i);
270 assert(field);
271
272 if (*field)
273 return 0;
274
275 assert(i->raw_job);
276
277 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", suffix, field);
278 if (r < 0)
279 return log_oom();
280
281 return 1;
282 }
283
284 static int raw_pull_copy_auxiliary_file(
285 RawPull *i,
286 const char *suffix,
287 char **path) {
288
289 const char *local;
290 int r;
291
292 assert(i);
293 assert(suffix);
294 assert(path);
295
296 r = raw_pull_determine_path(i, suffix, path);
297 if (r < 0)
298 return r;
299
300 local = strjoina(i->image_root, "/", i->local, suffix);
301
302 r = copy_file_atomic(*path, local, 0644, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0));
303 if (r == -EEXIST)
304 log_warning_errno(r, "File %s already exists, not replacing.", local);
305 else if (r == -ENOENT)
306 log_debug_errno(r, "Skipping creation of auxiliary file, since none was found.");
307 else if (r < 0)
308 log_warning_errno(r, "Failed to copy file %s, ignoring: %m", local);
309 else
310 log_info("Created new file %s.", local);
311
312 return 0;
313 }
314
315 static int raw_pull_make_local_copy(RawPull *i) {
316 _cleanup_free_ char *tp = NULL;
317 _cleanup_close_ int dfd = -1;
318 const char *p;
319 int r;
320
321 assert(i);
322 assert(i->raw_job);
323
324 if (!i->local)
325 return 0;
326
327 if (i->raw_job->etag_exists) {
328 /* We have downloaded this one previously, reopen it */
329
330 assert(i->raw_job->disk_fd < 0);
331
332 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
333 if (i->raw_job->disk_fd < 0)
334 return log_error_errno(errno, "Failed to open vendor image: %m");
335 } else {
336 /* We freshly downloaded the image, use it */
337
338 assert(i->raw_job->disk_fd >= 0);
339
340 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
341 return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
342 }
343
344 p = strjoina(i->image_root, "/", i->local, ".raw");
345
346 if (i->force_local)
347 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
348
349 r = tempfn_random(p, NULL, &tp);
350 if (r < 0)
351 return log_oom();
352
353 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
354 if (dfd < 0)
355 return log_error_errno(errno, "Failed to create writable copy of image: %m");
356
357 /* Turn off COW writing. This should greatly improve
358 * performance on COW file systems like btrfs, since it
359 * reduces fragmentation caused by not allowing in-place
360 * writes. */
361 r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
362 if (r < 0)
363 log_warning_errno(r, "Failed to set file attributes on %s: %m", tp);
364
365 r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, COPY_REFLINK);
366 if (r < 0) {
367 unlink(tp);
368 return log_error_errno(r, "Failed to make writable copy of image: %m");
369 }
370
371 (void) copy_times(i->raw_job->disk_fd, dfd);
372 (void) copy_xattr(i->raw_job->disk_fd, dfd);
373
374 dfd = safe_close(dfd);
375
376 r = rename(tp, p);
377 if (r < 0) {
378 r = log_error_errno(errno, "Failed to move writable image into place: %m");
379 unlink(tp);
380 return r;
381 }
382
383 log_info("Created new local image '%s'.", i->local);
384
385 if (i->roothash) {
386 r = raw_pull_copy_auxiliary_file(i, ".roothash", &i->roothash_path);
387 if (r < 0)
388 return r;
389 }
390
391 if (i->settings) {
392 r = raw_pull_copy_auxiliary_file(i, ".nspawn", &i->settings_path);
393 if (r < 0)
394 return r;
395 }
396
397 return 0;
398 }
399
400 static bool raw_pull_is_done(RawPull *i) {
401 assert(i);
402 assert(i->raw_job);
403
404 if (!PULL_JOB_IS_COMPLETE(i->raw_job))
405 return false;
406 if (i->roothash_job && !PULL_JOB_IS_COMPLETE(i->roothash_job))
407 return false;
408 if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
409 return false;
410 if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
411 return false;
412 if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
413 return false;
414
415 return true;
416 }
417
418 static int raw_pull_rename_auxiliary_file(
419 RawPull *i,
420 const char *suffix,
421 char **temp_path,
422 char **path) {
423
424 int r;
425
426 assert(i);
427 assert(temp_path);
428 assert(suffix);
429 assert(path);
430
431 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
432 * incorporate it in the file name if we can */
433 *path = mfree(*path);
434 r = raw_pull_determine_path(i, suffix, path);
435 if (r < 0)
436 return r;
437
438 r = import_make_read_only(*temp_path);
439 if (r < 0)
440 return r;
441
442 r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path);
443 if (r < 0)
444 return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path);
445
446 *temp_path = mfree(*temp_path);
447
448 return 1;
449 }
450
451 static void raw_pull_job_on_finished(PullJob *j) {
452 RawPull *i;
453 int r;
454
455 assert(j);
456 assert(j->userdata);
457
458 i = j->userdata;
459 if (j == i->roothash_job) {
460 if (j->error != 0)
461 log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
462 } else if (j == i->settings_job) {
463 if (j->error != 0)
464 log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
465 } else if (j->error != 0 && j != i->signature_job) {
466 if (j == i->checksum_job)
467 log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
468 else
469 log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
470
471 r = j->error;
472 goto finish;
473 }
474
475 /* This is invoked if either the download completed
476 * successfully, or the download was skipped because we
477 * already have the etag. In this case ->etag_exists is
478 * true.
479 *
480 * We only do something when we got all three files */
481
482 if (!raw_pull_is_done(i))
483 return;
484
485 if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) {
486 log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
487
488 r = i->signature_job->error;
489 goto finish;
490 }
491
492 if (i->roothash_job)
493 i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd);
494 if (i->settings_job)
495 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
496
497 r = raw_pull_determine_path(i, ".raw", &i->final_path);
498 if (r < 0)
499 goto finish;
500
501 if (!i->raw_job->etag_exists) {
502 /* This is a new download, verify it, and move it into place */
503 assert(i->raw_job->disk_fd >= 0);
504
505 raw_pull_report_progress(i, RAW_VERIFYING);
506
507 r = pull_verify(i->raw_job, i->roothash_job, i->settings_job, i->checksum_job, i->signature_job);
508 if (r < 0)
509 goto finish;
510
511 raw_pull_report_progress(i, RAW_UNPACKING);
512
513 r = raw_pull_maybe_convert_qcow2(i);
514 if (r < 0)
515 goto finish;
516
517 raw_pull_report_progress(i, RAW_FINALIZING);
518
519 r = import_make_read_only_fd(i->raw_job->disk_fd);
520 if (r < 0)
521 goto finish;
522
523 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
524 if (r < 0) {
525 log_error_errno(r, "Failed to rename raw file to %s: %m", i->final_path);
526 goto finish;
527 }
528
529 i->temp_path = mfree(i->temp_path);
530
531 if (i->roothash_job &&
532 i->roothash_job->error == 0) {
533 r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
534 if (r < 0)
535 goto finish;
536 }
537
538 if (i->settings_job &&
539 i->settings_job->error == 0) {
540 r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
541 if (r < 0)
542 goto finish;
543 }
544 }
545
546 raw_pull_report_progress(i, RAW_COPYING);
547
548 r = raw_pull_make_local_copy(i);
549 if (r < 0)
550 goto finish;
551
552 r = 0;
553
554 finish:
555 if (i->on_finished)
556 i->on_finished(i, r, i->userdata);
557 else
558 sd_event_exit(i->event, r);
559 }
560
561 static int raw_pull_job_on_open_disk_generic(
562 RawPull *i,
563 PullJob *j,
564 const char *extra,
565 char **temp_path) {
566
567 int r;
568
569 assert(i);
570 assert(j);
571 assert(extra);
572 assert(temp_path);
573
574 if (!*temp_path) {
575 r = tempfn_random_child(i->image_root, extra, temp_path);
576 if (r < 0)
577 return log_oom();
578 }
579
580 (void) mkdir_parents_label(*temp_path, 0700);
581
582 j->disk_fd = open(*temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
583 if (j->disk_fd < 0)
584 return log_error_errno(errno, "Failed to create %s: %m", *temp_path);
585
586 return 0;
587 }
588
589 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
590 RawPull *i;
591 int r;
592
593 assert(j);
594 assert(j->userdata);
595
596 i = j->userdata;
597 assert(i->raw_job == j);
598
599 r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
600 if (r < 0)
601 return r;
602
603 r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
604 if (r < 0)
605 log_warning_errno(r, "Failed to set file attributes on %s, ignoring: %m", i->temp_path);
606
607 return 0;
608 }
609
610 static int raw_pull_job_on_open_disk_roothash(PullJob *j) {
611 RawPull *i;
612
613 assert(j);
614 assert(j->userdata);
615
616 i = j->userdata;
617 assert(i->roothash_job == j);
618
619 return raw_pull_job_on_open_disk_generic(i, j, "roothash", &i->roothash_temp_path);
620 }
621
622 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
623 RawPull *i;
624
625 assert(j);
626 assert(j->userdata);
627
628 i = j->userdata;
629 assert(i->settings_job == j);
630
631 return raw_pull_job_on_open_disk_generic(i, j, "settings", &i->settings_temp_path);
632 }
633
634 static void raw_pull_job_on_progress(PullJob *j) {
635 RawPull *i;
636
637 assert(j);
638 assert(j->userdata);
639
640 i = j->userdata;
641
642 raw_pull_report_progress(i, RAW_DOWNLOADING);
643 }
644
645 int raw_pull_start(
646 RawPull *i,
647 const char *url,
648 const char *local,
649 bool force_local,
650 ImportVerify verify,
651 bool settings,
652 bool roothash) {
653
654 int r;
655
656 assert(i);
657 assert(verify < _IMPORT_VERIFY_MAX);
658 assert(verify >= 0);
659
660 if (!http_url_is_valid(url))
661 return -EINVAL;
662
663 if (local && !machine_name_is_valid(local))
664 return -EINVAL;
665
666 if (i->raw_job)
667 return -EBUSY;
668
669 r = free_and_strdup(&i->local, local);
670 if (r < 0)
671 return r;
672
673 i->force_local = force_local;
674 i->verify = verify;
675 i->settings = settings;
676 i->roothash = roothash;
677
678 /* Queue job for the image itself */
679 r = pull_job_new(&i->raw_job, url, i->glue, i);
680 if (r < 0)
681 return r;
682
683 i->raw_job->on_finished = raw_pull_job_on_finished;
684 i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
685 i->raw_job->on_progress = raw_pull_job_on_progress;
686 i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
687
688 r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
689 if (r < 0)
690 return r;
691
692 if (roothash) {
693 r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i);
694 if (r < 0)
695 return r;
696
697 i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash;
698 i->roothash_job->on_progress = raw_pull_job_on_progress;
699 i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO;
700 }
701
702 if (settings) {
703 r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i);
704 if (r < 0)
705 return r;
706
707 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
708 i->settings_job->on_progress = raw_pull_job_on_progress;
709 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
710 }
711
712 r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
713 if (r < 0)
714 return r;
715
716 r = pull_job_begin(i->raw_job);
717 if (r < 0)
718 return r;
719
720 if (i->roothash_job) {
721 r = pull_job_begin(i->roothash_job);
722 if (r < 0)
723 return r;
724 }
725
726 if (i->settings_job) {
727 r = pull_job_begin(i->settings_job);
728 if (r < 0)
729 return r;
730 }
731
732 if (i->checksum_job) {
733 i->checksum_job->on_progress = raw_pull_job_on_progress;
734 i->checksum_job->style = VERIFICATION_PER_FILE;
735
736 r = pull_job_begin(i->checksum_job);
737 if (r < 0)
738 return r;
739 }
740
741 if (i->signature_job) {
742 i->signature_job->on_progress = raw_pull_job_on_progress;
743
744 r = pull_job_begin(i->signature_job);
745 if (r < 0)
746 return r;
747 }
748
749 return 0;
750 }