]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/import/pull-raw.c
only check signature job error if signature job exists (#6118)
[thirdparty/systemd.git] / src / import / pull-raw.c
1 /***
2 This file is part of systemd.
3
4 Copyright 2014 Lennart Poettering
5
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
10
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
18 ***/
19
20 #include <curl/curl.h>
21 #include <linux/fs.h>
22 #include <sys/xattr.h>
23
24 #include "sd-daemon.h"
25
26 #include "alloc-util.h"
27 #include "btrfs-util.h"
28 #include "chattr-util.h"
29 #include "copy.h"
30 #include "curl-util.h"
31 #include "fd-util.h"
32 #include "fileio.h"
33 #include "fs-util.h"
34 #include "hostname-util.h"
35 #include "import-common.h"
36 #include "import-util.h"
37 #include "macro.h"
38 #include "mkdir.h"
39 #include "path-util.h"
40 #include "pull-common.h"
41 #include "pull-job.h"
42 #include "pull-raw.h"
43 #include "qcow2-util.h"
44 #include "rm-rf.h"
45 #include "string-util.h"
46 #include "strv.h"
47 #include "utf8.h"
48 #include "util.h"
49 #include "web-util.h"
50
51 typedef enum RawProgress {
52 RAW_DOWNLOADING,
53 RAW_VERIFYING,
54 RAW_UNPACKING,
55 RAW_FINALIZING,
56 RAW_COPYING,
57 } RawProgress;
58
59 struct RawPull {
60 sd_event *event;
61 CurlGlue *glue;
62
63 char *image_root;
64
65 PullJob *raw_job;
66 PullJob *roothash_job;
67 PullJob *settings_job;
68 PullJob *checksum_job;
69 PullJob *signature_job;
70
71 RawPullFinished on_finished;
72 void *userdata;
73
74 char *local;
75 bool force_local;
76 bool grow_machine_directory;
77 bool settings;
78 bool roothash;
79
80 char *final_path;
81 char *temp_path;
82
83 char *settings_path;
84 char *settings_temp_path;
85
86 char *roothash_path;
87 char *roothash_temp_path;
88
89 ImportVerify verify;
90 };
91
92 RawPull* raw_pull_unref(RawPull *i) {
93 if (!i)
94 return NULL;
95
96 pull_job_unref(i->raw_job);
97 pull_job_unref(i->settings_job);
98 pull_job_unref(i->roothash_job);
99 pull_job_unref(i->checksum_job);
100 pull_job_unref(i->signature_job);
101
102 curl_glue_unref(i->glue);
103 sd_event_unref(i->event);
104
105 if (i->temp_path) {
106 (void) unlink(i->temp_path);
107 free(i->temp_path);
108 }
109
110 if (i->roothash_temp_path) {
111 (void) unlink(i->roothash_temp_path);
112 free(i->roothash_temp_path);
113 }
114
115 if (i->settings_temp_path) {
116 (void) unlink(i->settings_temp_path);
117 free(i->settings_temp_path);
118 }
119
120 free(i->final_path);
121 free(i->roothash_path);
122 free(i->settings_path);
123 free(i->image_root);
124 free(i->local);
125 return mfree(i);
126 }
127
128 int raw_pull_new(
129 RawPull **ret,
130 sd_event *event,
131 const char *image_root,
132 RawPullFinished on_finished,
133 void *userdata) {
134
135 _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
136 int r;
137
138 assert(ret);
139
140 i = new0(RawPull, 1);
141 if (!i)
142 return -ENOMEM;
143
144 i->on_finished = on_finished;
145 i->userdata = userdata;
146
147 i->image_root = strdup(image_root ?: "/var/lib/machines");
148 if (!i->image_root)
149 return -ENOMEM;
150
151 i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
152
153 if (event)
154 i->event = sd_event_ref(event);
155 else {
156 r = sd_event_default(&i->event);
157 if (r < 0)
158 return r;
159 }
160
161 r = curl_glue_new(&i->glue, i->event);
162 if (r < 0)
163 return r;
164
165 i->glue->on_finished = pull_job_curl_on_finished;
166 i->glue->userdata = i;
167
168 *ret = i;
169 i = NULL;
170
171 return 0;
172 }
173
174 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
175 unsigned percent;
176
177 assert(i);
178
179 switch (p) {
180
181 case RAW_DOWNLOADING: {
182 unsigned remain = 80;
183
184 percent = 0;
185
186 if (i->settings_job) {
187 percent += i->settings_job->progress_percent * 5 / 100;
188 remain -= 5;
189 }
190
191 if (i->roothash_job) {
192 percent += i->roothash_job->progress_percent * 5 / 100;
193 remain -= 5;
194 }
195
196 if (i->checksum_job) {
197 percent += i->checksum_job->progress_percent * 5 / 100;
198 remain -= 5;
199 }
200
201 if (i->signature_job) {
202 percent += i->signature_job->progress_percent * 5 / 100;
203 remain -= 5;
204 }
205
206 if (i->raw_job)
207 percent += i->raw_job->progress_percent * remain / 100;
208 break;
209 }
210
211 case RAW_VERIFYING:
212 percent = 80;
213 break;
214
215 case RAW_UNPACKING:
216 percent = 85;
217 break;
218
219 case RAW_FINALIZING:
220 percent = 90;
221 break;
222
223 case RAW_COPYING:
224 percent = 95;
225 break;
226
227 default:
228 assert_not_reached("Unknown progress state");
229 }
230
231 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
232 log_debug("Combined progress %u%%", percent);
233 }
234
235 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
236 _cleanup_close_ int converted_fd = -1;
237 _cleanup_free_ char *t = NULL;
238 int r;
239
240 assert(i);
241 assert(i->raw_job);
242
243 r = qcow2_detect(i->raw_job->disk_fd);
244 if (r < 0)
245 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
246 if (r == 0)
247 return 0;
248
249 /* This is a QCOW2 image, let's convert it */
250 r = tempfn_random(i->final_path, NULL, &t);
251 if (r < 0)
252 return log_oom();
253
254 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
255 if (converted_fd < 0)
256 return log_error_errno(errno, "Failed to create %s: %m", t);
257
258 r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
259 if (r < 0)
260 log_warning_errno(r, "Failed to set file attributes on %s: %m", t);
261
262 log_info("Unpacking QCOW2 file.");
263
264 r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
265 if (r < 0) {
266 unlink(t);
267 return log_error_errno(r, "Failed to convert qcow2 image: %m");
268 }
269
270 (void) unlink(i->temp_path);
271 free(i->temp_path);
272 i->temp_path = t;
273 t = NULL;
274
275 safe_close(i->raw_job->disk_fd);
276 i->raw_job->disk_fd = converted_fd;
277 converted_fd = -1;
278
279 return 1;
280 }
281
282 static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) {
283 int r;
284
285 assert(i);
286 assert(field);
287
288 if (*field)
289 return 0;
290
291 assert(i->raw_job);
292
293 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", suffix, field);
294 if (r < 0)
295 return log_oom();
296
297 return 1;
298 }
299
300 static int raw_pull_copy_auxiliary_file(
301 RawPull *i,
302 const char *suffix,
303 char **path) {
304
305 const char *local;
306 int r;
307
308 assert(i);
309 assert(suffix);
310 assert(path);
311
312 r = raw_pull_determine_path(i, suffix, path);
313 if (r < 0)
314 return r;
315
316 local = strjoina(i->image_root, "/", i->local, suffix);
317
318 r = copy_file_atomic(*path, local, 0644, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0));
319 if (r == -EEXIST)
320 log_warning_errno(r, "File %s already exists, not replacing.", local);
321 else if (r == -ENOENT)
322 log_debug_errno(r, "Skipping creation of auxiliary file, since none was found.");
323 else if (r < 0)
324 log_warning_errno(r, "Failed to copy file %s, ignoring: %m", local);
325 else
326 log_info("Created new file %s.", local);
327
328 return 0;
329 }
330
331 static int raw_pull_make_local_copy(RawPull *i) {
332 _cleanup_free_ char *tp = NULL;
333 _cleanup_close_ int dfd = -1;
334 const char *p;
335 int r;
336
337 assert(i);
338 assert(i->raw_job);
339
340 if (!i->local)
341 return 0;
342
343 if (i->raw_job->etag_exists) {
344 /* We have downloaded this one previously, reopen it */
345
346 assert(i->raw_job->disk_fd < 0);
347
348 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
349 if (i->raw_job->disk_fd < 0)
350 return log_error_errno(errno, "Failed to open vendor image: %m");
351 } else {
352 /* We freshly downloaded the image, use it */
353
354 assert(i->raw_job->disk_fd >= 0);
355
356 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
357 return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
358 }
359
360 p = strjoina(i->image_root, "/", i->local, ".raw");
361
362 if (i->force_local)
363 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
364
365 r = tempfn_random(p, NULL, &tp);
366 if (r < 0)
367 return log_oom();
368
369 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
370 if (dfd < 0)
371 return log_error_errno(errno, "Failed to create writable copy of image: %m");
372
373 /* Turn off COW writing. This should greatly improve
374 * performance on COW file systems like btrfs, since it
375 * reduces fragmentation caused by not allowing in-place
376 * writes. */
377 r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
378 if (r < 0)
379 log_warning_errno(r, "Failed to set file attributes on %s: %m", tp);
380
381 r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, COPY_REFLINK);
382 if (r < 0) {
383 unlink(tp);
384 return log_error_errno(r, "Failed to make writable copy of image: %m");
385 }
386
387 (void) copy_times(i->raw_job->disk_fd, dfd);
388 (void) copy_xattr(i->raw_job->disk_fd, dfd);
389
390 dfd = safe_close(dfd);
391
392 r = rename(tp, p);
393 if (r < 0) {
394 r = log_error_errno(errno, "Failed to move writable image into place: %m");
395 unlink(tp);
396 return r;
397 }
398
399 log_info("Created new local image '%s'.", i->local);
400
401 if (i->roothash) {
402 r = raw_pull_copy_auxiliary_file(i, ".roothash", &i->roothash_path);
403 if (r < 0)
404 return r;
405 }
406
407 if (i->settings) {
408 r = raw_pull_copy_auxiliary_file(i, ".nspawn", &i->settings_path);
409 if (r < 0)
410 return r;
411 }
412
413 return 0;
414 }
415
416 static bool raw_pull_is_done(RawPull *i) {
417 assert(i);
418 assert(i->raw_job);
419
420 if (!PULL_JOB_IS_COMPLETE(i->raw_job))
421 return false;
422 if (i->roothash_job && !PULL_JOB_IS_COMPLETE(i->roothash_job))
423 return false;
424 if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
425 return false;
426 if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
427 return false;
428 if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
429 return false;
430
431 return true;
432 }
433
434 static int raw_pull_rename_auxiliary_file(
435 RawPull *i,
436 const char *suffix,
437 char **temp_path,
438 char **path) {
439
440 int r;
441
442 assert(i);
443 assert(temp_path);
444 assert(suffix);
445 assert(path);
446
447 /* Regenerate final name for this auxiliary file, we might know the etag of the raw file now, and we shoud
448 * incorporate it in the file name if we can */
449 *path = mfree(*path);
450 r = raw_pull_determine_path(i, suffix, path);
451 if (r < 0)
452 return r;
453
454 r = import_make_read_only(*temp_path);
455 if (r < 0)
456 return r;
457
458 r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path);
459 if (r < 0)
460 return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path);
461
462 *temp_path = mfree(*temp_path);
463
464 return 1;
465 }
466
467 static void raw_pull_job_on_finished(PullJob *j) {
468 RawPull *i;
469 int r;
470
471 assert(j);
472 assert(j->userdata);
473
474 i = j->userdata;
475 if (j == i->roothash_job) {
476 if (j->error != 0)
477 log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
478 } else if (j == i->settings_job) {
479 if (j->error != 0)
480 log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
481 } else if (j->error != 0 && j != i->signature_job) {
482 if (j == i->checksum_job)
483 log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
484 else
485 log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
486
487 r = j->error;
488 goto finish;
489 }
490
491 /* This is invoked if either the download completed
492 * successfully, or the download was skipped because we
493 * already have the etag. In this case ->etag_exists is
494 * true.
495 *
496 * We only do something when we got all three files */
497
498 if (!raw_pull_is_done(i))
499 return;
500
501 if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) {
502 log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
503
504 r = i->signature_job->error;
505 goto finish;
506 }
507
508 if (i->roothash_job)
509 i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd);
510 if (i->settings_job)
511 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
512
513 r = raw_pull_determine_path(i, ".raw", &i->final_path);
514 if (r < 0)
515 goto finish;
516
517 if (!i->raw_job->etag_exists) {
518 /* This is a new download, verify it, and move it into place */
519 assert(i->raw_job->disk_fd >= 0);
520
521 raw_pull_report_progress(i, RAW_VERIFYING);
522
523 r = pull_verify(i->raw_job, i->roothash_job, i->settings_job, i->checksum_job, i->signature_job);
524 if (r < 0)
525 goto finish;
526
527 raw_pull_report_progress(i, RAW_UNPACKING);
528
529 r = raw_pull_maybe_convert_qcow2(i);
530 if (r < 0)
531 goto finish;
532
533 raw_pull_report_progress(i, RAW_FINALIZING);
534
535 r = import_make_read_only_fd(i->raw_job->disk_fd);
536 if (r < 0)
537 goto finish;
538
539 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
540 if (r < 0) {
541 log_error_errno(r, "Failed to move RAW file into place: %m");
542 goto finish;
543 }
544
545 i->temp_path = mfree(i->temp_path);
546
547 if (i->roothash_job &&
548 i->roothash_job->error == 0) {
549 r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
550 if (r < 0)
551 goto finish;
552 }
553
554 if (i->settings_job &&
555 i->settings_job->error == 0) {
556 r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
557 if (r < 0)
558 goto finish;
559 }
560 }
561
562 raw_pull_report_progress(i, RAW_COPYING);
563
564 r = raw_pull_make_local_copy(i);
565 if (r < 0)
566 goto finish;
567
568 r = 0;
569
570 finish:
571 if (i->on_finished)
572 i->on_finished(i, r, i->userdata);
573 else
574 sd_event_exit(i->event, r);
575 }
576
577 static int raw_pull_job_on_open_disk_generic(
578 RawPull *i,
579 PullJob *j,
580 const char *extra,
581 char **temp_path) {
582
583 int r;
584
585 assert(i);
586 assert(j);
587 assert(extra);
588 assert(temp_path);
589
590 if (!*temp_path) {
591 r = tempfn_random_child(i->image_root, extra, temp_path);
592 if (r < 0)
593 return log_oom();
594 }
595
596 (void) mkdir_parents_label(*temp_path, 0700);
597
598 j->disk_fd = open(*temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
599 if (j->disk_fd < 0)
600 return log_error_errno(errno, "Failed to create %s: %m", *temp_path);
601
602 return 0;
603 }
604
605 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
606 RawPull *i;
607 int r;
608
609 assert(j);
610 assert(j->userdata);
611
612 i = j->userdata;
613 assert(i->raw_job == j);
614
615 r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
616 if (r < 0)
617 return r;
618
619 r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
620 if (r < 0)
621 log_warning_errno(r, "Failed to set file attributes on %s, ignoring: %m", i->temp_path);
622
623 return 0;
624 }
625
626 static int raw_pull_job_on_open_disk_roothash(PullJob *j) {
627 RawPull *i;
628
629 assert(j);
630 assert(j->userdata);
631
632 i = j->userdata;
633 assert(i->roothash_job == j);
634
635 return raw_pull_job_on_open_disk_generic(i, j, "roothash", &i->roothash_temp_path);
636 }
637
638 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
639 RawPull *i;
640
641 assert(j);
642 assert(j->userdata);
643
644 i = j->userdata;
645 assert(i->settings_job == j);
646
647 return raw_pull_job_on_open_disk_generic(i, j, "settings", &i->settings_temp_path);
648 }
649
650 static void raw_pull_job_on_progress(PullJob *j) {
651 RawPull *i;
652
653 assert(j);
654 assert(j->userdata);
655
656 i = j->userdata;
657
658 raw_pull_report_progress(i, RAW_DOWNLOADING);
659 }
660
661 int raw_pull_start(
662 RawPull *i,
663 const char *url,
664 const char *local,
665 bool force_local,
666 ImportVerify verify,
667 bool settings,
668 bool roothash) {
669
670 int r;
671
672 assert(i);
673 assert(verify < _IMPORT_VERIFY_MAX);
674 assert(verify >= 0);
675
676 if (!http_url_is_valid(url))
677 return -EINVAL;
678
679 if (local && !machine_name_is_valid(local))
680 return -EINVAL;
681
682 if (i->raw_job)
683 return -EBUSY;
684
685 r = free_and_strdup(&i->local, local);
686 if (r < 0)
687 return r;
688
689 i->force_local = force_local;
690 i->verify = verify;
691 i->settings = settings;
692 i->roothash = roothash;
693
694 /* Queue job for the image itself */
695 r = pull_job_new(&i->raw_job, url, i->glue, i);
696 if (r < 0)
697 return r;
698
699 i->raw_job->on_finished = raw_pull_job_on_finished;
700 i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
701 i->raw_job->on_progress = raw_pull_job_on_progress;
702 i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
703 i->raw_job->grow_machine_directory = i->grow_machine_directory;
704
705 r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
706 if (r < 0)
707 return r;
708
709 if (roothash) {
710 r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i);
711 if (r < 0)
712 return r;
713
714 i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash;
715 i->roothash_job->on_progress = raw_pull_job_on_progress;
716 i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO;
717 }
718
719 if (settings) {
720 r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i);
721 if (r < 0)
722 return r;
723
724 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
725 i->settings_job->on_progress = raw_pull_job_on_progress;
726 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
727 }
728
729 r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
730 if (r < 0)
731 return r;
732
733 r = pull_job_begin(i->raw_job);
734 if (r < 0)
735 return r;
736
737 if (i->roothash_job) {
738 r = pull_job_begin(i->roothash_job);
739 if (r < 0)
740 return r;
741 }
742
743 if (i->settings_job) {
744 r = pull_job_begin(i->settings_job);
745 if (r < 0)
746 return r;
747 }
748
749 if (i->checksum_job) {
750 i->checksum_job->on_progress = raw_pull_job_on_progress;
751 i->checksum_job->style = VERIFICATION_PER_FILE;
752
753 r = pull_job_begin(i->checksum_job);
754 if (r < 0)
755 return r;
756 }
757
758 if (i->signature_job) {
759 i->signature_job->on_progress = raw_pull_job_on_progress;
760
761 r = pull_job_begin(i->signature_job);
762 if (r < 0)
763 return r;
764 }
765
766 return 0;
767 }