]>
Commit | Line | Data |
---|---|---|
90199220 LP |
1 | /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ |
2 | ||
3 | /*** | |
4 | This file is part of systemd. | |
5 | ||
6 | Copyright 2014 Lennart Poettering | |
7 | ||
8 | systemd is free software; you can redistribute it and/or modify it | |
9 | under the terms of the GNU Lesser General Public License as published by | |
10 | the Free Software Foundation; either version 2.1 of the License, or | |
11 | (at your option) any later version. | |
12 | ||
13 | systemd is distributed in the hope that it will be useful, but | |
14 | WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | Lesser General Public License for more details. | |
17 | ||
18 | You should have received a copy of the GNU Lesser General Public License | |
19 | along with systemd; If not, see <http://www.gnu.org/licenses/>. | |
20 | ***/ | |
21 | ||
22 | #include <sys/xattr.h> | |
dfd1520d | 23 | #include <linux/fs.h> |
90199220 LP |
24 | #include <curl/curl.h> |
25 | ||
7079cfef | 26 | #include "sd-daemon.h" |
90199220 | 27 | #include "utf8.h" |
8620a9a3 LP |
28 | #include "strv.h" |
29 | #include "copy.h" | |
0d6e763b LP |
30 | #include "btrfs-util.h" |
31 | #include "util.h" | |
32 | #include "macro.h" | |
33 | #include "mkdir.h" | |
26166c88 | 34 | #include "path-util.h" |
3d7415f4 | 35 | #include "import-util.h" |
0d6e763b LP |
36 | #include "curl-util.h" |
37 | #include "qcow2-util.h" | |
dc2c282b LP |
38 | #include "pull-job.h" |
39 | #include "pull-common.h" | |
40 | #include "pull-raw.h" | |
90199220 | 41 | |
7079cfef LP |
42 | typedef enum RawProgress { |
43 | RAW_DOWNLOADING, | |
44 | RAW_VERIFYING, | |
45 | RAW_UNPACKING, | |
46 | RAW_FINALIZING, | |
47 | RAW_COPYING, | |
48 | } RawProgress; | |
90199220 | 49 | |
dc2c282b | 50 | struct RawPull { |
90199220 LP |
51 | sd_event *event; |
52 | CurlGlue *glue; | |
53 | ||
087682d1 | 54 | char *image_root; |
90199220 | 55 | |
dc2c282b LP |
56 | PullJob *raw_job; |
57 | PullJob *checksum_job; | |
58 | PullJob *signature_job; | |
90199220 | 59 | |
dc2c282b | 60 | RawPullFinished on_finished; |
0d6e763b | 61 | void *userdata; |
90199220 | 62 | |
0d6e763b LP |
63 | char *local; |
64 | bool force_local; | |
26166c88 | 65 | bool grow_machine_directory; |
8620a9a3 | 66 | |
0d6e763b LP |
67 | char *temp_path; |
68 | char *final_path; | |
8f695058 LP |
69 | |
70 | ImportVerify verify; | |
0d6e763b | 71 | }; |
49bb233b | 72 | |
dc2c282b | 73 | RawPull* raw_pull_unref(RawPull *i) { |
0d6e763b | 74 | if (!i) |
90199220 LP |
75 | return NULL; |
76 | ||
dc2c282b LP |
77 | pull_job_unref(i->raw_job); |
78 | pull_job_unref(i->checksum_job); | |
79 | pull_job_unref(i->signature_job); | |
90199220 | 80 | |
0d6e763b LP |
81 | curl_glue_unref(i->glue); |
82 | sd_event_unref(i->event); | |
90199220 | 83 | |
0d6e763b LP |
84 | if (i->temp_path) { |
85 | (void) unlink(i->temp_path); | |
86 | free(i->temp_path); | |
90199220 LP |
87 | } |
88 | ||
0d6e763b LP |
89 | free(i->final_path); |
90 | free(i->image_root); | |
91 | free(i->local); | |
92 | free(i); | |
90199220 LP |
93 | |
94 | return NULL; | |
95 | } | |
96 | ||
dc2c282b LP |
97 | int raw_pull_new( |
98 | RawPull **ret, | |
8b71fce8 LP |
99 | sd_event *event, |
100 | const char *image_root, | |
dc2c282b | 101 | RawPullFinished on_finished, |
8b71fce8 LP |
102 | void *userdata) { |
103 | ||
dc2c282b | 104 | _cleanup_(raw_pull_unrefp) RawPull *i = NULL; |
0d6e763b | 105 | int r; |
8620a9a3 | 106 | |
0d6e763b | 107 | assert(ret); |
8620a9a3 | 108 | |
dc2c282b | 109 | i = new0(RawPull, 1); |
0d6e763b | 110 | if (!i) |
8620a9a3 LP |
111 | return -ENOMEM; |
112 | ||
0d6e763b LP |
113 | i->on_finished = on_finished; |
114 | i->userdata = userdata; | |
8620a9a3 | 115 | |
0d6e763b LP |
116 | i->image_root = strdup(image_root ?: "/var/lib/machines"); |
117 | if (!i->image_root) | |
8620a9a3 LP |
118 | return -ENOMEM; |
119 | ||
26166c88 LP |
120 | i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines"); |
121 | ||
0d6e763b LP |
122 | if (event) |
123 | i->event = sd_event_ref(event); | |
124 | else { | |
125 | r = sd_event_default(&i->event); | |
dfd1520d | 126 | if (r < 0) |
0d6e763b | 127 | return r; |
2f64ba0e | 128 | } |
8620a9a3 | 129 | |
0d6e763b | 130 | r = curl_glue_new(&i->glue, i->event); |
2f64ba0e | 131 | if (r < 0) |
0d6e763b | 132 | return r; |
8620a9a3 | 133 | |
dc2c282b | 134 | i->glue->on_finished = pull_job_curl_on_finished; |
0d6e763b | 135 | i->glue->userdata = i; |
8620a9a3 | 136 | |
0d6e763b LP |
137 | *ret = i; |
138 | i = NULL; | |
8620a9a3 | 139 | |
2f64ba0e LP |
140 | return 0; |
141 | } | |
142 | ||
dc2c282b | 143 | static void raw_pull_report_progress(RawPull *i, RawProgress p) { |
7079cfef LP |
144 | unsigned percent; |
145 | ||
146 | assert(i); | |
147 | ||
148 | switch (p) { | |
149 | ||
150 | case RAW_DOWNLOADING: { | |
151 | unsigned remain = 80; | |
152 | ||
153 | percent = 0; | |
154 | ||
155 | if (i->checksum_job) { | |
156 | percent += i->checksum_job->progress_percent * 5 / 100; | |
157 | remain -= 5; | |
158 | } | |
159 | ||
160 | if (i->signature_job) { | |
161 | percent += i->signature_job->progress_percent * 5 / 100; | |
162 | remain -= 5; | |
163 | } | |
164 | ||
165 | if (i->raw_job) | |
166 | percent += i->raw_job->progress_percent * remain / 100; | |
167 | break; | |
168 | } | |
169 | ||
170 | case RAW_VERIFYING: | |
171 | percent = 80; | |
172 | break; | |
173 | ||
174 | case RAW_UNPACKING: | |
175 | percent = 85; | |
176 | break; | |
177 | ||
178 | case RAW_FINALIZING: | |
179 | percent = 90; | |
180 | break; | |
181 | ||
182 | case RAW_COPYING: | |
183 | percent = 95; | |
184 | break; | |
185 | ||
186 | default: | |
187 | assert_not_reached("Unknown progress state"); | |
188 | } | |
189 | ||
190 | sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent); | |
191 | log_debug("Combined progress %u%%", percent); | |
192 | } | |
193 | ||
dc2c282b | 194 | static int raw_pull_maybe_convert_qcow2(RawPull *i) { |
edce2aed LP |
195 | _cleanup_close_ int converted_fd = -1; |
196 | _cleanup_free_ char *t = NULL; | |
197 | int r; | |
198 | ||
0d6e763b LP |
199 | assert(i); |
200 | assert(i->raw_job); | |
edce2aed | 201 | |
0d6e763b | 202 | r = qcow2_detect(i->raw_job->disk_fd); |
edce2aed LP |
203 | if (r < 0) |
204 | return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); | |
205 | if (r == 0) | |
206 | return 0; | |
207 | ||
208 | /* This is a QCOW2 image, let's convert it */ | |
0d6e763b | 209 | r = tempfn_random(i->final_path, &t); |
edce2aed LP |
210 | if (r < 0) |
211 | return log_oom(); | |
212 | ||
213 | converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644); | |
214 | if (converted_fd < 0) | |
215 | return log_error_errno(errno, "Failed to create %s: %m", t); | |
216 | ||
0d6e763b LP |
217 | r = chattr_fd(converted_fd, true, FS_NOCOW_FL); |
218 | if (r < 0) | |
219 | log_warning_errno(errno, "Failed to set file attributes on %s: %m", t); | |
220 | ||
ec5cb56e LP |
221 | log_info("Unpacking QCOW2 file."); |
222 | ||
0d6e763b | 223 | r = qcow2_convert(i->raw_job->disk_fd, converted_fd); |
edce2aed LP |
224 | if (r < 0) { |
225 | unlink(t); | |
226 | return log_error_errno(r, "Failed to convert qcow2 image: %m"); | |
227 | } | |
228 | ||
0d6e763b LP |
229 | unlink(i->temp_path); |
230 | free(i->temp_path); | |
edce2aed | 231 | |
0d6e763b | 232 | i->temp_path = t; |
edce2aed LP |
233 | t = NULL; |
234 | ||
0d6e763b LP |
235 | safe_close(i->raw_job->disk_fd); |
236 | i->raw_job->disk_fd = converted_fd; | |
edce2aed LP |
237 | converted_fd = -1; |
238 | ||
239 | return 1; | |
240 | } | |
241 | ||
dc2c282b | 242 | static int raw_pull_make_local_copy(RawPull *i) { |
0d6e763b LP |
243 | _cleanup_free_ char *tp = NULL; |
244 | _cleanup_close_ int dfd = -1; | |
245 | const char *p; | |
90199220 LP |
246 | int r; |
247 | ||
0d6e763b LP |
248 | assert(i); |
249 | assert(i->raw_job); | |
90199220 | 250 | |
0d6e763b | 251 | if (!i->local) |
90199220 LP |
252 | return 0; |
253 | ||
85dbc41d LP |
254 | if (i->raw_job->etag_exists) { |
255 | /* We have downloaded this one previously, reopen it */ | |
256 | ||
257 | assert(i->raw_job->disk_fd < 0); | |
258 | ||
0d6e763b | 259 | if (!i->final_path) { |
dc2c282b | 260 | r = pull_make_path(i->raw_job->url, i->raw_job->etag, i->image_root, ".raw-", ".raw", &i->final_path); |
49bb233b | 261 | if (r < 0) |
0d6e763b | 262 | return log_oom(); |
49bb233b | 263 | } |
49bb233b | 264 | |
0d6e763b LP |
265 | i->raw_job->disk_fd = open(i->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC); |
266 | if (i->raw_job->disk_fd < 0) | |
267 | return log_error_errno(errno, "Failed to open vendor image: %m"); | |
85dbc41d LP |
268 | } else { |
269 | /* We freshly downloaded the image, use it */ | |
270 | ||
271 | assert(i->raw_job->disk_fd >= 0); | |
272 | ||
273 | if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1) | |
274 | return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m"); | |
90199220 LP |
275 | } |
276 | ||
63c372cb | 277 | p = strjoina(i->image_root, "/", i->local, ".raw"); |
49bb233b | 278 | |
0d6e763b LP |
279 | if (i->force_local) { |
280 | (void) btrfs_subvol_remove(p); | |
281 | (void) rm_rf_dangerous(p, false, true, false); | |
49bb233b LP |
282 | } |
283 | ||
0d6e763b | 284 | r = tempfn_random(p, &tp); |
49bb233b | 285 | if (r < 0) |
0d6e763b | 286 | return log_oom(); |
49bb233b | 287 | |
0d6e763b LP |
288 | dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); |
289 | if (dfd < 0) | |
290 | return log_error_errno(errno, "Failed to create writable copy of image: %m"); | |
49bb233b | 291 | |
0d6e763b LP |
292 | /* Turn off COW writing. This should greatly improve |
293 | * performance on COW file systems like btrfs, since it | |
294 | * reduces fragmentation caused by not allowing in-place | |
295 | * writes. */ | |
296 | r = chattr_fd(dfd, true, FS_NOCOW_FL); | |
297 | if (r < 0) | |
298 | log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp); | |
90199220 | 299 | |
0d6e763b | 300 | r = copy_bytes(i->raw_job->disk_fd, dfd, (off_t) -1, true); |
90199220 | 301 | if (r < 0) { |
0d6e763b LP |
302 | unlink(tp); |
303 | return log_error_errno(r, "Failed to make writable copy of image: %m"); | |
90199220 LP |
304 | } |
305 | ||
0d6e763b LP |
306 | (void) copy_times(i->raw_job->disk_fd, dfd); |
307 | (void) copy_xattr(i->raw_job->disk_fd, dfd); | |
1e20b411 | 308 | |
0d6e763b | 309 | dfd = safe_close(dfd); |
1e20b411 | 310 | |
0d6e763b LP |
311 | r = rename(tp, p); |
312 | if (r < 0) { | |
313 | unlink(tp); | |
314 | return log_error_errno(errno, "Failed to move writable image into place: %m"); | |
1e20b411 LP |
315 | } |
316 | ||
0d6e763b | 317 | log_info("Created new local image '%s'.", i->local); |
1e20b411 LP |
318 | return 0; |
319 | } | |
320 | ||
dc2c282b | 321 | static bool raw_pull_is_done(RawPull *i) { |
8b71fce8 LP |
322 | assert(i); |
323 | assert(i->raw_job); | |
324 | ||
dc2c282b | 325 | if (i->raw_job->state != PULL_JOB_DONE) |
8b71fce8 | 326 | return false; |
dc2c282b | 327 | if (i->checksum_job && i->checksum_job->state != PULL_JOB_DONE) |
8b71fce8 | 328 | return false; |
dc2c282b | 329 | if (i->signature_job && i->signature_job->state != PULL_JOB_DONE) |
8b71fce8 LP |
330 | return false; |
331 | ||
332 | return true; | |
333 | } | |
334 | ||
dc2c282b LP |
335 | static void raw_pull_job_on_finished(PullJob *j) { |
336 | RawPull *i; | |
8620a9a3 LP |
337 | int r; |
338 | ||
0d6e763b LP |
339 | assert(j); |
340 | assert(j->userdata); | |
8620a9a3 | 341 | |
0d6e763b LP |
342 | i = j->userdata; |
343 | if (j->error != 0) { | |
98c38001 | 344 | if (j == i->checksum_job) |
3576d631 LP |
345 | log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)"); |
346 | else if (j == i->signature_job) | |
347 | log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)"); | |
348 | else | |
349 | log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)"); | |
350 | ||
0d6e763b LP |
351 | r = j->error; |
352 | goto finish; | |
8620a9a3 LP |
353 | } |
354 | ||
0d6e763b LP |
355 | /* This is invoked if either the download completed |
356 | * successfully, or the download was skipped because we | |
85dbc41d | 357 | * already have the etag. In this case ->etag_exists is |
3576d631 LP |
358 | * true. |
359 | * | |
360 | * We only do something when we got all three files */ | |
85dbc41d | 361 | |
dc2c282b | 362 | if (!raw_pull_is_done(i)) |
3576d631 | 363 | return; |
8620a9a3 | 364 | |
3576d631 | 365 | if (!i->raw_job->etag_exists) { |
98c38001 | 366 | /* This is a new download, verify it, and move it into place */ |
3576d631 LP |
367 | assert(i->raw_job->disk_fd >= 0); |
368 | ||
dc2c282b | 369 | raw_pull_report_progress(i, RAW_VERIFYING); |
7079cfef | 370 | |
dc2c282b | 371 | r = pull_verify(i->raw_job, i->checksum_job, i->signature_job); |
0d6e763b LP |
372 | if (r < 0) |
373 | goto finish; | |
8620a9a3 | 374 | |
dc2c282b | 375 | raw_pull_report_progress(i, RAW_UNPACKING); |
7079cfef | 376 | |
dc2c282b | 377 | r = raw_pull_maybe_convert_qcow2(i); |
0d6e763b LP |
378 | if (r < 0) |
379 | goto finish; | |
85dbc41d | 380 | |
dc2c282b | 381 | raw_pull_report_progress(i, RAW_FINALIZING); |
7079cfef | 382 | |
dc2c282b | 383 | r = pull_make_read_only_fd(i->raw_job->disk_fd); |
3576d631 LP |
384 | if (r < 0) |
385 | goto finish; | |
85dbc41d | 386 | |
3576d631 LP |
387 | r = rename(i->temp_path, i->final_path); |
388 | if (r < 0) { | |
389 | r = log_error_errno(errno, "Failed to move RAW file into place: %m"); | |
390 | goto finish; | |
391 | } | |
8f695058 | 392 | |
3576d631 LP |
393 | free(i->temp_path); |
394 | i->temp_path = NULL; | |
8620a9a3 LP |
395 | } |
396 | ||
dc2c282b | 397 | raw_pull_report_progress(i, RAW_COPYING); |
7079cfef | 398 | |
dc2c282b | 399 | r = raw_pull_make_local_copy(i); |
8620a9a3 | 400 | if (r < 0) |
0d6e763b | 401 | goto finish; |
90199220 | 402 | |
0d6e763b | 403 | r = 0; |
3576d631 | 404 | |
0d6e763b | 405 | finish: |
3576d631 LP |
406 | if (i->on_finished) |
407 | i->on_finished(i, r, i->userdata); | |
408 | else | |
409 | sd_event_exit(i->event, r); | |
0d6e763b | 410 | } |
90199220 | 411 | |
dc2c282b LP |
412 | static int raw_pull_job_on_open_disk(PullJob *j) { |
413 | RawPull *i; | |
0d6e763b | 414 | int r; |
90199220 | 415 | |
0d6e763b LP |
416 | assert(j); |
417 | assert(j->userdata); | |
90199220 | 418 | |
0d6e763b | 419 | i = j->userdata; |
8b71fce8 LP |
420 | assert(i->raw_job == j); |
421 | assert(!i->final_path); | |
422 | assert(!i->temp_path); | |
90199220 | 423 | |
dc2c282b | 424 | r = pull_make_path(j->url, j->etag, i->image_root, ".raw-", ".raw", &i->final_path); |
0d6e763b LP |
425 | if (r < 0) |
426 | return log_oom(); | |
90199220 | 427 | |
0d6e763b LP |
428 | r = tempfn_random(i->final_path, &i->temp_path); |
429 | if (r <0) | |
430 | return log_oom(); | |
1e20b411 | 431 | |
0d6e763b | 432 | mkdir_parents_label(i->temp_path, 0700); |
1e20b411 | 433 | |
0d6e763b LP |
434 | j->disk_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644); |
435 | if (j->disk_fd < 0) | |
436 | return log_error_errno(errno, "Failed to create %s: %m", i->temp_path); | |
1e20b411 | 437 | |
0d6e763b | 438 | r = chattr_fd(j->disk_fd, true, FS_NOCOW_FL); |
90199220 | 439 | if (r < 0) |
0d6e763b | 440 | log_warning_errno(errno, "Failed to set file attributes on %s: %m", i->temp_path); |
90199220 LP |
441 | |
442 | return 0; | |
443 | } | |
444 | ||
dc2c282b LP |
445 | static void raw_pull_job_on_progress(PullJob *j) { |
446 | RawPull *i; | |
7079cfef LP |
447 | |
448 | assert(j); | |
449 | assert(j->userdata); | |
450 | ||
451 | i = j->userdata; | |
452 | ||
dc2c282b | 453 | raw_pull_report_progress(i, RAW_DOWNLOADING); |
7079cfef LP |
454 | } |
455 | ||
dc2c282b | 456 | int raw_pull_start(RawPull *i, const char *url, const char *local, bool force_local, ImportVerify verify) { |
90199220 LP |
457 | int r; |
458 | ||
0d6e763b | 459 | assert(i); |
8f695058 LP |
460 | assert(verify < _IMPORT_VERIFY_MAX); |
461 | assert(verify >= 0); | |
90199220 | 462 | |
0d6e763b LP |
463 | if (!http_url_is_valid(url)) |
464 | return -EINVAL; | |
90199220 | 465 | |
0d6e763b LP |
466 | if (local && !machine_name_is_valid(local)) |
467 | return -EINVAL; | |
087682d1 | 468 | |
8b71fce8 LP |
469 | if (i->raw_job) |
470 | return -EBUSY; | |
471 | ||
0d6e763b | 472 | r = free_and_strdup(&i->local, local); |
90199220 LP |
473 | if (r < 0) |
474 | return r; | |
0d6e763b | 475 | i->force_local = force_local; |
8f695058 | 476 | i->verify = verify; |
90199220 | 477 | |
85dbc41d | 478 | /* Queue job for the image itself */ |
dc2c282b | 479 | r = pull_job_new(&i->raw_job, url, i->glue, i); |
90199220 LP |
480 | if (r < 0) |
481 | return r; | |
482 | ||
dc2c282b LP |
483 | i->raw_job->on_finished = raw_pull_job_on_finished; |
484 | i->raw_job->on_open_disk = raw_pull_job_on_open_disk; | |
485 | i->raw_job->on_progress = raw_pull_job_on_progress; | |
98c38001 | 486 | i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO; |
26166c88 | 487 | i->raw_job->grow_machine_directory = i->grow_machine_directory; |
90199220 | 488 | |
dc2c282b | 489 | r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags); |
90199220 LP |
490 | if (r < 0) |
491 | return r; | |
492 | ||
dc2c282b | 493 | r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i); |
98c38001 LP |
494 | if (r < 0) |
495 | return r; | |
85dbc41d | 496 | |
dc2c282b | 497 | r = pull_job_begin(i->raw_job); |
85dbc41d LP |
498 | if (r < 0) |
499 | return r; | |
500 | ||
98c38001 | 501 | if (i->checksum_job) { |
dc2c282b | 502 | i->checksum_job->on_progress = raw_pull_job_on_progress; |
7079cfef | 503 | |
dc2c282b | 504 | r = pull_job_begin(i->checksum_job); |
3576d631 LP |
505 | if (r < 0) |
506 | return r; | |
507 | } | |
508 | ||
509 | if (i->signature_job) { | |
dc2c282b | 510 | i->signature_job->on_progress = raw_pull_job_on_progress; |
7079cfef | 511 | |
dc2c282b | 512 | r = pull_job_begin(i->signature_job); |
3576d631 LP |
513 | if (r < 0) |
514 | return r; | |
515 | } | |
516 | ||
85dbc41d | 517 | return 0; |
90199220 | 518 | } |