]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/import/pull-raw.c
Add SPDX license identifiers to source files under the LGPL
[thirdparty/systemd.git] / src / import / pull-raw.c
1 /* SPDX-License-Identifier: LGPL-2.1+ */
2 /***
3 This file is part of systemd.
4
5 Copyright 2014 Lennart Poettering
6
7 systemd is free software; you can redistribute it and/or modify it
8 under the terms of the GNU Lesser General Public License as published by
9 the Free Software Foundation; either version 2.1 of the License, or
10 (at your option) any later version.
11
12 systemd is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
16
17 You should have received a copy of the GNU Lesser General Public License
18 along with systemd; If not, see <http://www.gnu.org/licenses/>.
19 ***/
20
21 #include <curl/curl.h>
22 #include <linux/fs.h>
23 #include <sys/xattr.h>
24
25 #include "sd-daemon.h"
26
27 #include "alloc-util.h"
28 #include "btrfs-util.h"
29 #include "chattr-util.h"
30 #include "copy.h"
31 #include "curl-util.h"
32 #include "fd-util.h"
33 #include "fileio.h"
34 #include "fs-util.h"
35 #include "hostname-util.h"
36 #include "import-common.h"
37 #include "import-util.h"
38 #include "macro.h"
39 #include "mkdir.h"
40 #include "path-util.h"
41 #include "pull-common.h"
42 #include "pull-job.h"
43 #include "pull-raw.h"
44 #include "qcow2-util.h"
45 #include "rm-rf.h"
46 #include "string-util.h"
47 #include "strv.h"
48 #include "utf8.h"
49 #include "util.h"
50 #include "web-util.h"
51
52 typedef enum RawProgress {
53 RAW_DOWNLOADING,
54 RAW_VERIFYING,
55 RAW_UNPACKING,
56 RAW_FINALIZING,
57 RAW_COPYING,
58 } RawProgress;
59
60 struct RawPull {
61 sd_event *event;
62 CurlGlue *glue;
63
64 char *image_root;
65
66 PullJob *raw_job;
67 PullJob *roothash_job;
68 PullJob *settings_job;
69 PullJob *checksum_job;
70 PullJob *signature_job;
71
72 RawPullFinished on_finished;
73 void *userdata;
74
75 char *local;
76 bool force_local;
77 bool grow_machine_directory;
78 bool settings;
79 bool roothash;
80
81 char *final_path;
82 char *temp_path;
83
84 char *settings_path;
85 char *settings_temp_path;
86
87 char *roothash_path;
88 char *roothash_temp_path;
89
90 ImportVerify verify;
91 };
92
93 RawPull* raw_pull_unref(RawPull *i) {
94 if (!i)
95 return NULL;
96
97 pull_job_unref(i->raw_job);
98 pull_job_unref(i->settings_job);
99 pull_job_unref(i->roothash_job);
100 pull_job_unref(i->checksum_job);
101 pull_job_unref(i->signature_job);
102
103 curl_glue_unref(i->glue);
104 sd_event_unref(i->event);
105
106 if (i->temp_path) {
107 (void) unlink(i->temp_path);
108 free(i->temp_path);
109 }
110
111 if (i->roothash_temp_path) {
112 (void) unlink(i->roothash_temp_path);
113 free(i->roothash_temp_path);
114 }
115
116 if (i->settings_temp_path) {
117 (void) unlink(i->settings_temp_path);
118 free(i->settings_temp_path);
119 }
120
121 free(i->final_path);
122 free(i->roothash_path);
123 free(i->settings_path);
124 free(i->image_root);
125 free(i->local);
126 return mfree(i);
127 }
128
129 int raw_pull_new(
130 RawPull **ret,
131 sd_event *event,
132 const char *image_root,
133 RawPullFinished on_finished,
134 void *userdata) {
135
136 _cleanup_(raw_pull_unrefp) RawPull *i = NULL;
137 int r;
138
139 assert(ret);
140
141 i = new0(RawPull, 1);
142 if (!i)
143 return -ENOMEM;
144
145 i->on_finished = on_finished;
146 i->userdata = userdata;
147
148 i->image_root = strdup(image_root ?: "/var/lib/machines");
149 if (!i->image_root)
150 return -ENOMEM;
151
152 i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
153
154 if (event)
155 i->event = sd_event_ref(event);
156 else {
157 r = sd_event_default(&i->event);
158 if (r < 0)
159 return r;
160 }
161
162 r = curl_glue_new(&i->glue, i->event);
163 if (r < 0)
164 return r;
165
166 i->glue->on_finished = pull_job_curl_on_finished;
167 i->glue->userdata = i;
168
169 *ret = i;
170 i = NULL;
171
172 return 0;
173 }
174
175 static void raw_pull_report_progress(RawPull *i, RawProgress p) {
176 unsigned percent;
177
178 assert(i);
179
180 switch (p) {
181
182 case RAW_DOWNLOADING: {
183 unsigned remain = 80;
184
185 percent = 0;
186
187 if (i->settings_job) {
188 percent += i->settings_job->progress_percent * 5 / 100;
189 remain -= 5;
190 }
191
192 if (i->roothash_job) {
193 percent += i->roothash_job->progress_percent * 5 / 100;
194 remain -= 5;
195 }
196
197 if (i->checksum_job) {
198 percent += i->checksum_job->progress_percent * 5 / 100;
199 remain -= 5;
200 }
201
202 if (i->signature_job) {
203 percent += i->signature_job->progress_percent * 5 / 100;
204 remain -= 5;
205 }
206
207 if (i->raw_job)
208 percent += i->raw_job->progress_percent * remain / 100;
209 break;
210 }
211
212 case RAW_VERIFYING:
213 percent = 80;
214 break;
215
216 case RAW_UNPACKING:
217 percent = 85;
218 break;
219
220 case RAW_FINALIZING:
221 percent = 90;
222 break;
223
224 case RAW_COPYING:
225 percent = 95;
226 break;
227
228 default:
229 assert_not_reached("Unknown progress state");
230 }
231
232 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
233 log_debug("Combined progress %u%%", percent);
234 }
235
236 static int raw_pull_maybe_convert_qcow2(RawPull *i) {
237 _cleanup_close_ int converted_fd = -1;
238 _cleanup_free_ char *t = NULL;
239 int r;
240
241 assert(i);
242 assert(i->raw_job);
243
244 r = qcow2_detect(i->raw_job->disk_fd);
245 if (r < 0)
246 return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
247 if (r == 0)
248 return 0;
249
250 /* This is a QCOW2 image, let's convert it */
251 r = tempfn_random(i->final_path, NULL, &t);
252 if (r < 0)
253 return log_oom();
254
255 converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
256 if (converted_fd < 0)
257 return log_error_errno(errno, "Failed to create %s: %m", t);
258
259 r = chattr_fd(converted_fd, FS_NOCOW_FL, FS_NOCOW_FL);
260 if (r < 0)
261 log_warning_errno(r, "Failed to set file attributes on %s: %m", t);
262
263 log_info("Unpacking QCOW2 file.");
264
265 r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
266 if (r < 0) {
267 unlink(t);
268 return log_error_errno(r, "Failed to convert qcow2 image: %m");
269 }
270
271 (void) unlink(i->temp_path);
272 free(i->temp_path);
273 i->temp_path = t;
274 t = NULL;
275
276 safe_close(i->raw_job->disk_fd);
277 i->raw_job->disk_fd = converted_fd;
278 converted_fd = -1;
279
280 return 1;
281 }
282
283 static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) {
284 int r;
285
286 assert(i);
287 assert(field);
288
289 if (*field)
290 return 0;
291
292 assert(i->raw_job);
293
294 r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", suffix, field);
295 if (r < 0)
296 return log_oom();
297
298 return 1;
299 }
300
301 static int raw_pull_copy_auxiliary_file(
302 RawPull *i,
303 const char *suffix,
304 char **path) {
305
306 const char *local;
307 int r;
308
309 assert(i);
310 assert(suffix);
311 assert(path);
312
313 r = raw_pull_determine_path(i, suffix, path);
314 if (r < 0)
315 return r;
316
317 local = strjoina(i->image_root, "/", i->local, suffix);
318
319 r = copy_file_atomic(*path, local, 0644, 0, COPY_REFLINK | (i->force_local ? COPY_REPLACE : 0));
320 if (r == -EEXIST)
321 log_warning_errno(r, "File %s already exists, not replacing.", local);
322 else if (r == -ENOENT)
323 log_debug_errno(r, "Skipping creation of auxiliary file, since none was found.");
324 else if (r < 0)
325 log_warning_errno(r, "Failed to copy file %s, ignoring: %m", local);
326 else
327 log_info("Created new file %s.", local);
328
329 return 0;
330 }
331
332 static int raw_pull_make_local_copy(RawPull *i) {
333 _cleanup_free_ char *tp = NULL;
334 _cleanup_close_ int dfd = -1;
335 const char *p;
336 int r;
337
338 assert(i);
339 assert(i->raw_job);
340
341 if (!i->local)
342 return 0;
343
344 if (i->raw_job->etag_exists) {
345 /* We have downloaded this one previously, reopen it */
346
347 assert(i->raw_job->disk_fd < 0);
348
349 i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
350 if (i->raw_job->disk_fd < 0)
351 return log_error_errno(errno, "Failed to open vendor image: %m");
352 } else {
353 /* We freshly downloaded the image, use it */
354
355 assert(i->raw_job->disk_fd >= 0);
356
357 if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
358 return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
359 }
360
361 p = strjoina(i->image_root, "/", i->local, ".raw");
362
363 if (i->force_local)
364 (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
365
366 r = tempfn_random(p, NULL, &tp);
367 if (r < 0)
368 return log_oom();
369
370 dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
371 if (dfd < 0)
372 return log_error_errno(errno, "Failed to create writable copy of image: %m");
373
374 /* Turn off COW writing. This should greatly improve
375 * performance on COW file systems like btrfs, since it
376 * reduces fragmentation caused by not allowing in-place
377 * writes. */
378 r = chattr_fd(dfd, FS_NOCOW_FL, FS_NOCOW_FL);
379 if (r < 0)
380 log_warning_errno(r, "Failed to set file attributes on %s: %m", tp);
381
382 r = copy_bytes(i->raw_job->disk_fd, dfd, (uint64_t) -1, COPY_REFLINK);
383 if (r < 0) {
384 unlink(tp);
385 return log_error_errno(r, "Failed to make writable copy of image: %m");
386 }
387
388 (void) copy_times(i->raw_job->disk_fd, dfd);
389 (void) copy_xattr(i->raw_job->disk_fd, dfd);
390
391 dfd = safe_close(dfd);
392
393 r = rename(tp, p);
394 if (r < 0) {
395 r = log_error_errno(errno, "Failed to move writable image into place: %m");
396 unlink(tp);
397 return r;
398 }
399
400 log_info("Created new local image '%s'.", i->local);
401
402 if (i->roothash) {
403 r = raw_pull_copy_auxiliary_file(i, ".roothash", &i->roothash_path);
404 if (r < 0)
405 return r;
406 }
407
408 if (i->settings) {
409 r = raw_pull_copy_auxiliary_file(i, ".nspawn", &i->settings_path);
410 if (r < 0)
411 return r;
412 }
413
414 return 0;
415 }
416
417 static bool raw_pull_is_done(RawPull *i) {
418 assert(i);
419 assert(i->raw_job);
420
421 if (!PULL_JOB_IS_COMPLETE(i->raw_job))
422 return false;
423 if (i->roothash_job && !PULL_JOB_IS_COMPLETE(i->roothash_job))
424 return false;
425 if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
426 return false;
427 if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
428 return false;
429 if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
430 return false;
431
432 return true;
433 }
434
435 static int raw_pull_rename_auxiliary_file(
436 RawPull *i,
437 const char *suffix,
438 char **temp_path,
439 char **path) {
440
441 int r;
442
443 assert(i);
444 assert(temp_path);
445 assert(suffix);
446 assert(path);
447
448 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
449 * incorporate it in the file name if we can */
450 *path = mfree(*path);
451 r = raw_pull_determine_path(i, suffix, path);
452 if (r < 0)
453 return r;
454
455 r = import_make_read_only(*temp_path);
456 if (r < 0)
457 return r;
458
459 r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path);
460 if (r < 0)
461 return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path);
462
463 *temp_path = mfree(*temp_path);
464
465 return 1;
466 }
467
468 static void raw_pull_job_on_finished(PullJob *j) {
469 RawPull *i;
470 int r;
471
472 assert(j);
473 assert(j->userdata);
474
475 i = j->userdata;
476 if (j == i->roothash_job) {
477 if (j->error != 0)
478 log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
479 } else if (j == i->settings_job) {
480 if (j->error != 0)
481 log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
482 } else if (j->error != 0 && j != i->signature_job) {
483 if (j == i->checksum_job)
484 log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
485 else
486 log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
487
488 r = j->error;
489 goto finish;
490 }
491
492 /* This is invoked if either the download completed
493 * successfully, or the download was skipped because we
494 * already have the etag. In this case ->etag_exists is
495 * true.
496 *
497 * We only do something when we got all three files */
498
499 if (!raw_pull_is_done(i))
500 return;
501
502 if (i->signature_job && i->checksum_job->style == VERIFICATION_PER_DIRECTORY && i->signature_job->error != 0) {
503 log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
504
505 r = i->signature_job->error;
506 goto finish;
507 }
508
509 if (i->roothash_job)
510 i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd);
511 if (i->settings_job)
512 i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
513
514 r = raw_pull_determine_path(i, ".raw", &i->final_path);
515 if (r < 0)
516 goto finish;
517
518 if (!i->raw_job->etag_exists) {
519 /* This is a new download, verify it, and move it into place */
520 assert(i->raw_job->disk_fd >= 0);
521
522 raw_pull_report_progress(i, RAW_VERIFYING);
523
524 r = pull_verify(i->raw_job, i->roothash_job, i->settings_job, i->checksum_job, i->signature_job);
525 if (r < 0)
526 goto finish;
527
528 raw_pull_report_progress(i, RAW_UNPACKING);
529
530 r = raw_pull_maybe_convert_qcow2(i);
531 if (r < 0)
532 goto finish;
533
534 raw_pull_report_progress(i, RAW_FINALIZING);
535
536 r = import_make_read_only_fd(i->raw_job->disk_fd);
537 if (r < 0)
538 goto finish;
539
540 r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
541 if (r < 0) {
542 log_error_errno(r, "Failed to rename raw file to %s: %m", i->final_path);
543 goto finish;
544 }
545
546 i->temp_path = mfree(i->temp_path);
547
548 if (i->roothash_job &&
549 i->roothash_job->error == 0) {
550 r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
551 if (r < 0)
552 goto finish;
553 }
554
555 if (i->settings_job &&
556 i->settings_job->error == 0) {
557 r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
558 if (r < 0)
559 goto finish;
560 }
561 }
562
563 raw_pull_report_progress(i, RAW_COPYING);
564
565 r = raw_pull_make_local_copy(i);
566 if (r < 0)
567 goto finish;
568
569 r = 0;
570
571 finish:
572 if (i->on_finished)
573 i->on_finished(i, r, i->userdata);
574 else
575 sd_event_exit(i->event, r);
576 }
577
578 static int raw_pull_job_on_open_disk_generic(
579 RawPull *i,
580 PullJob *j,
581 const char *extra,
582 char **temp_path) {
583
584 int r;
585
586 assert(i);
587 assert(j);
588 assert(extra);
589 assert(temp_path);
590
591 if (!*temp_path) {
592 r = tempfn_random_child(i->image_root, extra, temp_path);
593 if (r < 0)
594 return log_oom();
595 }
596
597 (void) mkdir_parents_label(*temp_path, 0700);
598
599 j->disk_fd = open(*temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
600 if (j->disk_fd < 0)
601 return log_error_errno(errno, "Failed to create %s: %m", *temp_path);
602
603 return 0;
604 }
605
606 static int raw_pull_job_on_open_disk_raw(PullJob *j) {
607 RawPull *i;
608 int r;
609
610 assert(j);
611 assert(j->userdata);
612
613 i = j->userdata;
614 assert(i->raw_job == j);
615
616 r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
617 if (r < 0)
618 return r;
619
620 r = chattr_fd(j->disk_fd, FS_NOCOW_FL, FS_NOCOW_FL);
621 if (r < 0)
622 log_warning_errno(r, "Failed to set file attributes on %s, ignoring: %m", i->temp_path);
623
624 return 0;
625 }
626
627 static int raw_pull_job_on_open_disk_roothash(PullJob *j) {
628 RawPull *i;
629
630 assert(j);
631 assert(j->userdata);
632
633 i = j->userdata;
634 assert(i->roothash_job == j);
635
636 return raw_pull_job_on_open_disk_generic(i, j, "roothash", &i->roothash_temp_path);
637 }
638
639 static int raw_pull_job_on_open_disk_settings(PullJob *j) {
640 RawPull *i;
641
642 assert(j);
643 assert(j->userdata);
644
645 i = j->userdata;
646 assert(i->settings_job == j);
647
648 return raw_pull_job_on_open_disk_generic(i, j, "settings", &i->settings_temp_path);
649 }
650
651 static void raw_pull_job_on_progress(PullJob *j) {
652 RawPull *i;
653
654 assert(j);
655 assert(j->userdata);
656
657 i = j->userdata;
658
659 raw_pull_report_progress(i, RAW_DOWNLOADING);
660 }
661
662 int raw_pull_start(
663 RawPull *i,
664 const char *url,
665 const char *local,
666 bool force_local,
667 ImportVerify verify,
668 bool settings,
669 bool roothash) {
670
671 int r;
672
673 assert(i);
674 assert(verify < _IMPORT_VERIFY_MAX);
675 assert(verify >= 0);
676
677 if (!http_url_is_valid(url))
678 return -EINVAL;
679
680 if (local && !machine_name_is_valid(local))
681 return -EINVAL;
682
683 if (i->raw_job)
684 return -EBUSY;
685
686 r = free_and_strdup(&i->local, local);
687 if (r < 0)
688 return r;
689
690 i->force_local = force_local;
691 i->verify = verify;
692 i->settings = settings;
693 i->roothash = roothash;
694
695 /* Queue job for the image itself */
696 r = pull_job_new(&i->raw_job, url, i->glue, i);
697 if (r < 0)
698 return r;
699
700 i->raw_job->on_finished = raw_pull_job_on_finished;
701 i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
702 i->raw_job->on_progress = raw_pull_job_on_progress;
703 i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
704 i->raw_job->grow_machine_directory = i->grow_machine_directory;
705
706 r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
707 if (r < 0)
708 return r;
709
710 if (roothash) {
711 r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i);
712 if (r < 0)
713 return r;
714
715 i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash;
716 i->roothash_job->on_progress = raw_pull_job_on_progress;
717 i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO;
718 }
719
720 if (settings) {
721 r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i);
722 if (r < 0)
723 return r;
724
725 i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
726 i->settings_job->on_progress = raw_pull_job_on_progress;
727 i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
728 }
729
730 r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
731 if (r < 0)
732 return r;
733
734 r = pull_job_begin(i->raw_job);
735 if (r < 0)
736 return r;
737
738 if (i->roothash_job) {
739 r = pull_job_begin(i->roothash_job);
740 if (r < 0)
741 return r;
742 }
743
744 if (i->settings_job) {
745 r = pull_job_begin(i->settings_job);
746 if (r < 0)
747 return r;
748 }
749
750 if (i->checksum_job) {
751 i->checksum_job->on_progress = raw_pull_job_on_progress;
752 i->checksum_job->style = VERIFICATION_PER_FILE;
753
754 r = pull_job_begin(i->checksum_job);
755 if (r < 0)
756 return r;
757 }
758
759 if (i->signature_job) {
760 i->signature_job->on_progress = raw_pull_job_on_progress;
761
762 r = pull_job_begin(i->signature_job);
763 if (r < 0)
764 return r;
765 }
766
767 return 0;
768 }