]> git.ipfire.org Git - thirdparty/systemd.git/blob - src/import/pull-dkr.c
projects / thirdparty / systemd.git / blob
? search:


84211d282b616bc76bfcae924bddd317952a21a9
[thirdparty/systemd.git] / src / import / pull-dkr.c
1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
2
3 /***
4 This file is part of systemd.
5
6 Copyright 2014 Lennart Poettering
7
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
12
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
17
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
20 ***/
21
22 #include <curl/curl.h>
23 #include <sys/prctl.h>
24
25 #include "sd-daemon.h"
26 #include "json.h"
27 #include "strv.h"
28 #include "btrfs-util.h"
29 #include "utf8.h"
30 #include "mkdir.h"
31 #include "rm-rf.h"
32 #include "path-util.h"
33 #include "import-util.h"
34 #include "curl-util.h"
35 #include "aufs-util.h"
36 #include "pull-job.h"
37 #include "pull-common.h"
38 #include "import-common.h"
39 #include "pull-dkr.h"
40 #include "process-util.h"
41 #include "hostname-util.h"
42
43 typedef enum DkrProgress {
44 DKR_SEARCHING,
45 DKR_RESOLVING,
46 DKR_METADATA,
47 DKR_DOWNLOADING,
48 DKR_COPYING,
49 } DkrProgress;
50
51 struct DkrPull {
52 sd_event *event;
53 CurlGlue *glue;
54
55 char *index_protocol;
56 char *index_address;
57
58 char *index_url;
59 char *image_root;
60
61 PullJob *images_job;
62 PullJob *tags_job;
63 PullJob *ancestry_job;
64 PullJob *json_job;
65 PullJob *layer_job;
66
67 char *name;
68 char *reference;
69 char *id;
70
71 char *response_digest;
72 char *response_token;
73 char **response_registries;
74
75 char **ancestry;
76 unsigned n_ancestry;
77 unsigned current_ancestry;
78
79 DkrPullFinished on_finished;
80 void *userdata;
81
82 char *local;
83 bool force_local;
84 bool grow_machine_directory;
85
86 char *temp_path;
87 char *final_path;
88
89 pid_t tar_pid;
90 };
91
92 #define PROTOCOL_PREFIX "https://"
93
94 #define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"
95 #define HEADER_REGISTRY "X-Do" /* the HTTP header for the registry */ "cker-Endpoints:"
96 #define HEADER_DIGEST "Do" /* the HTTP header for the manifest digest */ "cker-Content-Digest:"
97 #define LAYERS_MAX 127
98
99 static void dkr_pull_job_on_finished(PullJob *j);
100
101 DkrPull* dkr_pull_unref(DkrPull *i) {
102 if (!i)
103 return NULL;
104
105 if (i->tar_pid > 1) {
106 (void) kill_and_sigcont(i->tar_pid, SIGKILL);
107 (void) wait_for_terminate(i->tar_pid, NULL);
108 }
109
110 pull_job_unref(i->images_job);
111 pull_job_unref(i->tags_job);
112 pull_job_unref(i->ancestry_job);
113 pull_job_unref(i->json_job);
114 pull_job_unref(i->layer_job);
115
116 curl_glue_unref(i->glue);
117 sd_event_unref(i->event);
118
119 if (i->temp_path) {
120 (void) rm_rf(i->temp_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
121 free(i->temp_path);
122 }
123
124 free(i->name);
125 free(i->reference);
126 free(i->id);
127 free(i->response_token);
128 strv_free(i->ancestry);
129 free(i->final_path);
130 free(i->index_address);
131 free(i->index_protocol);
132 free(i->index_url);
133 free(i->image_root);
134 free(i->local);
135 free(i);
136
137 return NULL;
138 }
139
140 int dkr_pull_new(
141 DkrPull **ret,
142 sd_event *event,
143 const char *index_url,
144 const char *image_root,
145 DkrPullFinished on_finished,
146 void *userdata) {
147
148 _cleanup_(dkr_pull_unrefp) DkrPull *i = NULL;
149 char *e;
150 int r;
151
152 assert(ret);
153 assert(index_url);
154
155 if (!http_url_is_valid(index_url))
156 return -EINVAL;
157
158 i = new0(DkrPull, 1);
159 if (!i)
160 return -ENOMEM;
161
162 i->on_finished = on_finished;
163 i->userdata = userdata;
164
165 i->image_root = strdup(image_root ?: "/var/lib/machines");
166 if (!i->image_root)
167 return -ENOMEM;
168
169 i->grow_machine_directory = path_startswith(i->image_root, "/var/lib/machines");
170
171 i->index_url = strdup(index_url);
172 if (!i->index_url)
173 return -ENOMEM;
174
175 e = endswith(i->index_url, "/");
176 if (e)
177 *e = 0;
178
179 if (event)
180 i->event = sd_event_ref(event);
181 else {
182 r = sd_event_default(&i->event);
183 if (r < 0)
184 return r;
185 }
186
187 r = curl_glue_new(&i->glue, i->event);
188 if (r < 0)
189 return r;
190
191 i->glue->on_finished = pull_job_curl_on_finished;
192 i->glue->userdata = i;
193
194 *ret = i;
195 i = NULL;
196
197 return 0;
198 }
199
200 static void dkr_pull_report_progress(DkrPull *i, DkrProgress p) {
201 unsigned percent;
202
203 assert(i);
204
205 switch (p) {
206
207 case DKR_SEARCHING:
208 percent = 0;
209 if (i->images_job)
210 percent += i->images_job->progress_percent * 5 / 100;
211 break;
212
213 case DKR_RESOLVING:
214 percent = 5;
215 if (i->tags_job)
216 percent += i->tags_job->progress_percent * 5 / 100;
217 break;
218
219 case DKR_METADATA:
220 percent = 10;
221 if (i->ancestry_job)
222 percent += i->ancestry_job->progress_percent * 5 / 100;
223 if (i->json_job)
224 percent += i->json_job->progress_percent * 5 / 100;
225 break;
226
227 case DKR_DOWNLOADING:
228 percent = 20;
229 percent += 75 * i->current_ancestry / MAX(1U, i->n_ancestry);
230 if (i->layer_job)
231 percent += i->layer_job->progress_percent * 75 / MAX(1U, i->n_ancestry) / 100;
232
233 break;
234
235 case DKR_COPYING:
236 percent = 95;
237 break;
238
239 default:
240 assert_not_reached("Unknown progress state");
241 }
242
243 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
244 log_debug("Combined progress %u%%", percent);
245 }
246
247 static int parse_id(const void *payload, size_t size, char **ret) {
248 _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;
249 union json_value v = {};
250 void *json_state = NULL;
251 const char *p;
252 int t;
253
254 assert(payload);
255 assert(ret);
256
257 if (size <= 0)
258 return -EBADMSG;
259
260 if (memchr(payload, 0, size))
261 return -EBADMSG;
262
263 buf = strndup(payload, size);
264 if (!buf)
265 return -ENOMEM;
266
267 p = buf;
268 t = json_tokenize(&p, &id, &v, &json_state, NULL);
269 if (t < 0)
270 return t;
271 if (t != JSON_STRING)
272 return -EBADMSG;
273
274 t = json_tokenize(&p, &other, &v, &json_state, NULL);
275 if (t < 0)
276 return t;
277 if (t != JSON_END)
278 return -EBADMSG;
279
280 if (!dkr_id_is_valid(id))
281 return -EBADMSG;
282
283 *ret = id;
284 id = NULL;
285
286 return 0;
287 }
288
289 static int parse_ancestry(const void *payload, size_t size, char ***ret) {
290 _cleanup_free_ char *buf = NULL;
291 void *json_state = NULL;
292 const char *p;
293 enum {
294 STATE_BEGIN,
295 STATE_ITEM,
296 STATE_COMMA,
297 STATE_END,
298 } state = STATE_BEGIN;
299 _cleanup_strv_free_ char **l = NULL;
300 size_t n = 0, allocated = 0;
301
302 if (size <= 0)
303 return -EBADMSG;
304
305 if (memchr(payload, 0, size))
306 return -EBADMSG;
307
308 buf = strndup(payload, size);
309 if (!buf)
310 return -ENOMEM;
311
312 p = buf;
313 for (;;) {
314 _cleanup_free_ char *str;
315 union json_value v = {};
316 int t;
317
318 t = json_tokenize(&p, &str, &v, &json_state, NULL);
319 if (t < 0)
320 return t;
321
322 switch (state) {
323
324 case STATE_BEGIN:
325 if (t == JSON_ARRAY_OPEN)
326 state = STATE_ITEM;
327 else
328 return -EBADMSG;
329
330 break;
331
332 case STATE_ITEM:
333 if (t == JSON_STRING) {
334 if (!dkr_id_is_valid(str))
335 return -EBADMSG;
336
337 if (n+1 > LAYERS_MAX)
338 return -EFBIG;
339
340 if (!GREEDY_REALLOC(l, allocated, n + 2))
341 return -ENOMEM;
342
343 l[n++] = str;
344 str = NULL;
345 l[n] = NULL;
346
347 state = STATE_COMMA;
348
349 } else if (t == JSON_ARRAY_CLOSE)
350 state = STATE_END;
351 else
352 return -EBADMSG;
353
354 break;
355
356 case STATE_COMMA:
357 if (t == JSON_COMMA)
358 state = STATE_ITEM;
359 else if (t == JSON_ARRAY_CLOSE)
360 state = STATE_END;
361 else
362 return -EBADMSG;
363 break;
364
365 case STATE_END:
366 if (t == JSON_END) {
367
368 if (strv_isempty(l))
369 return -EBADMSG;
370
371 if (!strv_is_uniq(l))
372 return -EBADMSG;
373
374 l = strv_reverse(l);
375
376 *ret = l;
377 l = NULL;
378 return 0;
379 } else
380 return -EBADMSG;
381 }
382
383 }
384 }
385
386 static const char *dkr_pull_current_layer(DkrPull *i) {
387 assert(i);
388
389 if (strv_isempty(i->ancestry))
390 return NULL;
391
392 return i->ancestry[i->current_ancestry];
393 }
394
395 static const char *dkr_pull_current_base_layer(DkrPull *i) {
396 assert(i);
397
398 if (strv_isempty(i->ancestry))
399 return NULL;
400
401 if (i->current_ancestry <= 0)
402 return NULL;
403
404 return i->ancestry[i->current_ancestry-1];
405 }
406
407 static int dkr_pull_add_token(DkrPull *i, PullJob *j) {
408 const char *t;
409
410 assert(i);
411 assert(j);
412
413 if (i->response_token)
414 t = strjoina("Authorization: Token ", i->response_token);
415 else
416 t = HEADER_TOKEN " true";
417
418 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
419 if (!j->request_header)
420 return -ENOMEM;
421
422 return 0;
423 }
424
425 static int dkr_pull_add_bearer_token(DkrPull *i, PullJob *j) {
426 const char *t = NULL;
427
428 assert(i);
429 assert(j);
430
431 if (i->response_token)
432 t = strjoina("Authorization: Bearer ", i->response_token);
433 else
434 return -EINVAL;
435
436 j->request_header = curl_slist_new("Accept: application/json", t, NULL);
437 if (!j->request_header)
438 return -ENOMEM;
439
440 return 0;
441 }
442
443 static bool dkr_pull_is_done(DkrPull *i) {
444 assert(i);
445 assert(i->images_job);
446 if (i->images_job->state != PULL_JOB_DONE)
447 return false;
448
449 if (!i->tags_job || i->tags_job->state != PULL_JOB_DONE)
450 return false;
451
452 if (!i->ancestry_job || i->ancestry_job->state != PULL_JOB_DONE)
453 return false;
454
455 if (i->json_job && i->json_job->state != PULL_JOB_DONE)
456 return false;
457
458 if (i->layer_job && i->layer_job->state != PULL_JOB_DONE)
459 return false;
460
461 if (dkr_pull_current_layer(i))
462 return false;
463
464 return true;
465 }
466
467 static int dkr_pull_make_local_copy(DkrPull *i, DkrPullVersion version) {
468 int r;
469 _cleanup_free_ char *p = NULL;
470
471 assert(i);
472
473 if (!i->local)
474 return 0;
475
476 if (!i->final_path) {
477 i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);
478 if (!i->final_path)
479 return log_oom();
480 }
481
482 if (version == DKR_PULL_V2) {
483 r = path_get_parent(i->image_root, &p);
484 if (r < 0)
485 return r;
486 }
487
488 r = pull_make_local_copy(i->final_path, p ?: i->image_root, i->local, i->force_local);
489 if (r < 0)
490 return r;
491
492 if (version == DKR_PULL_V2) {
493 char **k;
494
495 STRV_FOREACH(k, i->ancestry) {
496 _cleanup_free_ char *d;
497
498 d = strjoin(i->image_root, "/.dkr-", *k, NULL);
499 if (!d)
500 return -ENOMEM;
501
502 r = btrfs_subvol_remove(d, BTRFS_REMOVE_QUOTA);
503 if (r < 0)
504 return r;
505 }
506
507 r = rmdir(i->image_root);
508 if (r < 0)
509 return r;
510 }
511
512 return 0;
513 }
514
515 static int dkr_pull_job_on_open_disk(PullJob *j) {
516 const char *base;
517 DkrPull *i;
518 int r;
519
520 assert(j);
521 assert(j->userdata);
522
523 i = j->userdata;
524 assert(i->layer_job == j);
525 assert(i->final_path);
526 assert(!i->temp_path);
527 assert(i->tar_pid <= 0);
528
529 r = tempfn_random(i->final_path, NULL, &i->temp_path);
530 if (r < 0)
531 return log_oom();
532
533 mkdir_parents_label(i->temp_path, 0700);
534
535 base = dkr_pull_current_base_layer(i);
536 if (base) {
537 const char *base_path;
538
539 base_path = strjoina(i->image_root, "/.dkr-", base);
540 r = btrfs_subvol_snapshot(base_path, i->temp_path, BTRFS_SNAPSHOT_FALLBACK_COPY|BTRFS_SNAPSHOT_QUOTA);
541 } else
542 r = btrfs_subvol_make(i->temp_path);
543 if (r < 0)
544 return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);
545
546 (void) import_assign_pool_quota_and_warn(i->temp_path);
547
548 j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
549 if (j->disk_fd < 0)
550 return j->disk_fd;
551
552 return 0;
553 }
554
555 static void dkr_pull_job_on_progress(PullJob *j) {
556 DkrPull *i;
557
558 assert(j);
559 assert(j->userdata);
560
561 i = j->userdata;
562
563 dkr_pull_report_progress(
564 i,
565 j == i->images_job ? DKR_SEARCHING :
566 j == i->tags_job ? DKR_RESOLVING :
567 j == i->ancestry_job || j == i->json_job ? DKR_METADATA :
568 DKR_DOWNLOADING);
569 }
570
571 static void dkr_pull_job_on_finished_v2(PullJob *j);
572
573 static int dkr_pull_pull_layer_v2(DkrPull *i) {
574 _cleanup_free_ char *path = NULL;
575 const char *url, *layer = NULL;
576 int r;
577
578 assert(i);
579 assert(!i->layer_job);
580 assert(!i->temp_path);
581 assert(!i->final_path);
582
583 for (;;) {
584 layer = dkr_pull_current_layer(i);
585 if (!layer)
586 return 0; /* no more layers */
587
588 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
589 if (!path)
590 return log_oom();
591
592 if (laccess(path, F_OK) < 0) {
593 if (errno == ENOENT)
594 break;
595
596 return log_error_errno(errno, "Failed to check for container: %m");
597 }
598
599 log_info("Layer %s already exists, skipping.", layer);
600
601 i->current_ancestry++;
602
603 path = mfree(path);
604 }
605
606 log_info("Pulling layer %s...", layer);
607
608 i->final_path = path;
609 path = NULL;
610
611 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v2/", i->name, "/blobs/", layer);
612 r = pull_job_new(&i->layer_job, url, i->glue, i);
613 if (r < 0)
614 return log_error_errno(r, "Failed to allocate layer job: %m");
615
616 r = dkr_pull_add_bearer_token(i, i->layer_job);
617 if (r < 0)
618 return log_oom();
619
620 i->layer_job->on_finished = dkr_pull_job_on_finished_v2;
621 i->layer_job->on_open_disk = dkr_pull_job_on_open_disk;
622 i->layer_job->on_progress = dkr_pull_job_on_progress;
623 i->layer_job->grow_machine_directory = i->grow_machine_directory;
624
625 r = pull_job_begin(i->layer_job);
626 if (r < 0)
627 return log_error_errno(r, "Failed to start layer job: %m");
628
629 return 0;
630 }
631
632 static int dkr_pull_pull_layer(DkrPull *i) {
633 _cleanup_free_ char *path = NULL;
634 const char *url, *layer = NULL;
635 int r;
636
637 assert(i);
638 assert(!i->layer_job);
639 assert(!i->temp_path);
640 assert(!i->final_path);
641
642 for (;;) {
643 layer = dkr_pull_current_layer(i);
644 if (!layer)
645 return 0; /* no more layers */
646
647 path = strjoin(i->image_root, "/.dkr-", layer, NULL);
648 if (!path)
649 return log_oom();
650
651 if (laccess(path, F_OK) < 0) {
652 if (errno == ENOENT)
653 break;
654
655 return log_error_errno(errno, "Failed to check for container: %m");
656 }
657
658 log_info("Layer %s already exists, skipping.", layer);
659
660 i->current_ancestry++;
661
662 path = mfree(path);
663 }
664
665 log_info("Pulling layer %s...", layer);
666
667 i->final_path = path;
668 path = NULL;
669
670 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", layer, "/layer");
671 r = pull_job_new(&i->layer_job, url, i->glue, i);
672 if (r < 0)
673 return log_error_errno(r, "Failed to allocate layer job: %m");
674
675 r = dkr_pull_add_token(i, i->layer_job);
676 if (r < 0)
677 return log_oom();
678
679 i->layer_job->on_finished = dkr_pull_job_on_finished;
680 i->layer_job->on_open_disk = dkr_pull_job_on_open_disk;
681 i->layer_job->on_progress = dkr_pull_job_on_progress;
682 i->layer_job->grow_machine_directory = i->grow_machine_directory;
683
684 r = pull_job_begin(i->layer_job);
685 if (r < 0)
686 return log_error_errno(r, "Failed to start layer job: %m");
687
688 return 0;
689 }
690
691 static int dkr_pull_job_on_header(PullJob *j, const char *header, size_t sz) {
692 _cleanup_free_ char *registry = NULL;
693 char *token, *digest;
694 DkrPull *i;
695 int r;
696
697 assert(j);
698 assert(j->userdata);
699
700 i = j->userdata;
701 r = curl_header_strdup(header, sz, HEADER_TOKEN, &token);
702 if (r < 0)
703 return log_oom();
704 if (r > 0) {
705 free(i->response_token);
706 i->response_token = token;
707 return 0;
708 }
709
710 r = curl_header_strdup(header, sz, HEADER_DIGEST, &digest);
711 if (r < 0)
712 return log_oom();
713 if (r > 0) {
714 free(i->response_digest);
715 i->response_digest = digest;
716 return 0;
717 }
718
719 r = curl_header_strdup(header, sz, HEADER_REGISTRY, &registry);
720 if (r < 0)
721 return log_oom();
722 if (r > 0) {
723 char **l, **k;
724
725 l = strv_split(registry, ",");
726 if (!l)
727 return log_oom();
728
729 STRV_FOREACH(k, l) {
730 if (!hostname_is_valid(*k, false)) {
731 log_error("Registry hostname is not valid.");
732 strv_free(l);
733 return -EBADMSG;
734 }
735 }
736
737 strv_free(i->response_registries);
738 i->response_registries = l;
739 }
740
741 return 0;
742 }
743
744 static void dkr_pull_job_on_finished_v2(PullJob *j) {
745 DkrPull *i;
746 int r;
747
748 assert(j);
749 assert(j->userdata);
750
751 i = j->userdata;
752 if (j->error != 0) {
753 if (j == i->images_job)
754 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
755 else if (j == i->ancestry_job)
756 log_error_errno(j->error, "Failed to retrieve manifest.");
757 else if (j == i->json_job)
758 log_error_errno(j->error, "Failed to retrieve json data.");
759 else
760 log_error_errno(j->error, "Failed to retrieve layer data.");
761
762 r = j->error;
763 goto finish;
764 }
765
766 if (i->images_job == j) {
767 const char *url;
768
769 assert(!i->tags_job);
770 assert(!i->ancestry_job);
771 assert(!i->json_job);
772 assert(!i->layer_job);
773
774 if (strv_isempty(i->response_registries)) {
775 r = -EBADMSG;
776 log_error("Didn't get registry information.");
777 goto finish;
778 }
779
780 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
781 dkr_pull_report_progress(i, DKR_RESOLVING);
782
783 url = strjoina(i->index_protocol, "auth.", i->index_address, "/v2/token/?scope=repository:",
784 i->name, ":pull&service=registry.", i->index_address);
785 r = pull_job_new(&i->tags_job, url, i->glue, i);
786 if (r < 0) {
787 log_error_errno(r, "Failed to allocate tags job: %m");
788 goto finish;
789 }
790
791 i->tags_job->on_finished = dkr_pull_job_on_finished_v2;
792 i->tags_job->on_progress = dkr_pull_job_on_progress;
793
794 r = pull_job_begin(i->tags_job);
795 if (r < 0) {
796 log_error_errno(r, "Failed to start tags job: %m");
797 goto finish;
798 }
799
800 } else if (i->tags_job == j) {
801 const char *url;
802 _cleanup_free_ char *buf;
803 _cleanup_json_variant_unref_ JsonVariant *doc = NULL;
804 JsonVariant *e = NULL;
805
806 assert(!i->ancestry_job);
807 assert(!i->json_job);
808 assert(!i->layer_job);
809
810 buf = strndup((const char *)j->payload, j->payload_size);
811 if (!buf) {
812 r = -ENOMEM;
813 log_oom();
814 goto finish;
815 }
816
817 r = json_parse(buf, &doc);
818 if (r < 0) {
819 log_error("Unable to parse bearer token\n%s", j->payload);
820 goto finish;
821 }
822
823 e = json_variant_value(doc, "token");
824 if (!e || e->type != JSON_VARIANT_STRING) {
825 r = -EBADMSG;
826 log_error("Invalid JSON format for Bearer token");
827 goto finish;
828 }
829
830 r = free_and_strdup(&i->response_token, json_variant_string(e));
831 if (r < 0) {
832 log_oom();
833 goto finish;
834 }
835
836 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v2/", i->name, "/manifests/", i->reference);
837 r = pull_job_new(&i->ancestry_job, url, i->glue, i);
838 if (r < 0) {
839 log_error_errno(r, "Failed to allocate ancestry job: %m");
840 goto finish;
841 }
842
843 r = dkr_pull_add_bearer_token(i, i->ancestry_job);
844 if (r < 0)
845 goto finish;
846
847 i->ancestry_job->on_finished = dkr_pull_job_on_finished_v2;
848 i->ancestry_job->on_progress = dkr_pull_job_on_progress;
849 i->ancestry_job->on_header = dkr_pull_job_on_header;
850
851
852 r = pull_job_begin(i->ancestry_job);
853 if (r < 0) {
854 log_error_errno(r, "Failed to start ancestry job: %m");
855 goto finish;
856 }
857
858 } else if (i->ancestry_job == j) {
859
860 _cleanup_json_variant_unref_ JsonVariant *doc = NULL, *compat = NULL;
861 JsonVariant *e = NULL;
862 _cleanup_strv_free_ char **ancestry = NULL;
863 size_t allocated = 0, size = 0;
864 char *path = NULL, **k = NULL;
865
866 r = json_parse((const char *)j->payload, &doc);
867 if (r < 0) {
868 log_error("Invalid JSON Manifest");
869 goto finish;
870 }
871
872 e = json_variant_value(doc, "fsLayers");
873 if (!e || e->type != JSON_VARIANT_ARRAY || e->size == 0) {
874 r = -EBADMSG;
875 goto finish;
876 }
877
878 log_info("JSON manifest with schema v%"PRIi64" for %s parsed!",
879 json_variant_integer(json_variant_value(doc, "schemaVersion")),
880 json_variant_string(json_variant_value(doc, "name")));
881
882 for (unsigned z = 0; z < e->size; z++) {
883 JsonVariant *f = json_variant_element(e, z), *g = NULL;
884 const char *layer;
885 if (f->type != JSON_VARIANT_OBJECT) {
886 r = -EBADMSG;
887 goto finish;
888 }
889
890 g = json_variant_value(f, "blobSum");
891
892 layer = json_variant_string(g);
893 if (!dkr_digest_is_valid(layer)) {
894 r = -EBADMSG;
895 goto finish;
896 }
897
898 if (!GREEDY_REALLOC(ancestry, allocated, size + 2)) {
899 r = -ENOMEM;
900 log_oom();
901 goto finish;
902 }
903
904 ancestry[size] = strdup(layer);
905 if (!ancestry[size]) {
906 r = -ENOMEM;
907 log_oom();
908 goto finish;
909 }
910
911 ancestry[size+1] = NULL;
912 size += 1;
913 }
914
915 e = json_variant_value(doc, "history");
916 if (!e || e->type != JSON_VARIANT_ARRAY) {
917 r = -EBADMSG;
918 goto finish;
919 }
920
921 e = json_variant_element(e, 0);
922 e = json_variant_value(e, "v1Compatibility");
923 r = json_parse(json_variant_string(e), &compat);
924 if (r < 0) {
925 log_error("Invalid v1Compatibility JSON");
926 goto finish;
927 }
928
929 e = json_variant_value(compat, "id");
930
931 strv_free(i->ancestry);
932 i->ancestry = strv_reverse(strv_uniq(ancestry));
933 i->n_ancestry = strv_length(i->ancestry);
934 i->current_ancestry = 0;
935 i->id = strdup(i->ancestry[i->n_ancestry - 1]);
936 if (!i->id) {
937 r = -ENOMEM;
938 log_oom();
939 goto finish;
940 }
941 path = strjoin(i->image_root, "/.dkr-", json_variant_string(e), NULL);
942 if (!path) {
943 r = -ENOMEM;
944 log_oom();
945 goto finish;
946 }
947 free(i->image_root);
948 i->image_root = path;
949 ancestry = NULL;
950
951 log_info("Required layers:\n");
952 STRV_FOREACH(k, i->ancestry)
953 log_info("\t%s", *k);
954 log_info("\nProvenance:\n\tImageID: %s\n\tDigest: %s", json_variant_string(e), i->response_digest);
955
956 dkr_pull_report_progress(i, DKR_DOWNLOADING);
957
958 r = dkr_pull_pull_layer_v2(i);
959 if (r < 0)
960 goto finish;
961
962 } else if (i->layer_job == j) {
963 assert(i->temp_path);
964 assert(i->final_path);
965
966 j->disk_fd = safe_close(j->disk_fd);
967
968 if (i->tar_pid > 0) {
969 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
970 i->tar_pid = 0;
971 if (r < 0)
972 goto finish;
973 }
974
975 r = aufs_resolve(i->temp_path);
976 if (r < 0) {
977 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
978 goto finish;
979 }
980
981 r = btrfs_subvol_set_read_only(i->temp_path, true);
982 if (r < 0) {
983 log_error_errno(r, "Failed to mark snapshot read-only: %m");
984 goto finish;
985 }
986
987 if (rename(i->temp_path, i->final_path) < 0) {
988 log_error_errno(errno, "Failed to rename snaphsot: %m");
989 goto finish;
990 }
991
992 log_info("Completed writing to layer %s.", i->final_path);
993
994 i->layer_job = pull_job_unref(i->layer_job);
995 free(i->temp_path);
996 i->temp_path = NULL;
997 free(i->final_path);
998 i->final_path = NULL;
999
1000 i->current_ancestry ++;
1001 r = dkr_pull_pull_layer_v2(i);
1002 if (r < 0)
1003 goto finish;
1004
1005 } else if (i->json_job != j)
1006 assert_not_reached("Got finished event for unknown curl object");
1007
1008 if (!dkr_pull_is_done(i))
1009 return;
1010
1011 dkr_pull_report_progress(i, DKR_COPYING);
1012
1013 r = dkr_pull_make_local_copy(i, DKR_PULL_V2);
1014 if (r < 0)
1015 goto finish;
1016
1017 r = 0;
1018
1019 finish:
1020 if (i->on_finished)
1021 i->on_finished(i, r, i->userdata);
1022 else
1023 sd_event_exit(i->event, r);
1024
1025 }
1026
1027 static void dkr_pull_job_on_finished(PullJob *j) {
1028 DkrPull *i;
1029 int r;
1030
1031 assert(j);
1032 assert(j->userdata);
1033
1034 i = j->userdata;
1035 if (j->error != 0) {
1036 if (j == i->images_job)
1037 log_error_errno(j->error, "Failed to retrieve images list. (Wrong index URL?)");
1038 else if (j == i->tags_job)
1039 log_error_errno(j->error, "Failed to retrieve tags list.");
1040 else if (j == i->ancestry_job)
1041 log_error_errno(j->error, "Failed to retrieve ancestry list.");
1042 else if (j == i->json_job)
1043 log_error_errno(j->error, "Failed to retrieve json data.");
1044 else
1045 log_error_errno(j->error, "Failed to retrieve layer data.");
1046
1047 r = j->error;
1048 goto finish;
1049 }
1050
1051 if (i->images_job == j) {
1052 const char *url;
1053
1054 assert(!i->tags_job);
1055 assert(!i->ancestry_job);
1056 assert(!i->json_job);
1057 assert(!i->layer_job);
1058
1059 if (strv_isempty(i->response_registries)) {
1060 r = -EBADMSG;
1061 log_error("Didn't get registry information.");
1062 goto finish;
1063 }
1064
1065 log_info("Index lookup succeeded, directed to registry %s.", i->response_registries[0]);
1066 dkr_pull_report_progress(i, DKR_RESOLVING);
1067
1068 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/repositories/", i->name, "/tags/", i->reference);
1069 r = pull_job_new(&i->tags_job, url, i->glue, i);
1070 if (r < 0) {
1071 log_error_errno(r, "Failed to allocate tags job: %m");
1072 goto finish;
1073 }
1074
1075 r = dkr_pull_add_token(i, i->tags_job);
1076 if (r < 0) {
1077 log_oom();
1078 goto finish;
1079 }
1080
1081 i->tags_job->on_finished = dkr_pull_job_on_finished;
1082 i->tags_job->on_progress = dkr_pull_job_on_progress;
1083
1084 r = pull_job_begin(i->tags_job);
1085 if (r < 0) {
1086 log_error_errno(r, "Failed to start tags job: %m");
1087 goto finish;
1088 }
1089
1090 } else if (i->tags_job == j) {
1091 const char *url;
1092 char *id = NULL;
1093
1094 assert(!i->ancestry_job);
1095 assert(!i->json_job);
1096 assert(!i->layer_job);
1097
1098 r = parse_id(j->payload, j->payload_size, &id);
1099 if (r < 0) {
1100 log_error_errno(r, "Failed to parse JSON id.");
1101 goto finish;
1102 }
1103
1104 free(i->id);
1105 i->id = id;
1106
1107 log_info("Tag lookup succeeded, resolved to layer %s.", i->id);
1108 dkr_pull_report_progress(i, DKR_METADATA);
1109
1110 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/ancestry");
1111 r = pull_job_new(&i->ancestry_job, url, i->glue, i);
1112 if (r < 0) {
1113 log_error_errno(r, "Failed to allocate ancestry job: %m");
1114 goto finish;
1115 }
1116
1117 r = dkr_pull_add_token(i, i->ancestry_job);
1118 if (r < 0) {
1119 log_oom();
1120 goto finish;
1121 }
1122
1123 i->ancestry_job->on_finished = dkr_pull_job_on_finished;
1124 i->ancestry_job->on_progress = dkr_pull_job_on_progress;
1125
1126 url = strjoina(PROTOCOL_PREFIX, i->response_registries[0], "/v1/images/", i->id, "/json");
1127 r = pull_job_new(&i->json_job, url, i->glue, i);
1128 if (r < 0) {
1129 log_error_errno(r, "Failed to allocate json job: %m");
1130 goto finish;
1131 }
1132
1133 r = dkr_pull_add_token(i, i->json_job);
1134 if (r < 0) {
1135 log_oom();
1136 goto finish;
1137 }
1138
1139 i->json_job->on_finished = dkr_pull_job_on_finished;
1140 i->json_job->on_progress = dkr_pull_job_on_progress;
1141
1142 r = pull_job_begin(i->ancestry_job);
1143 if (r < 0) {
1144 log_error_errno(r, "Failed to start ancestry job: %m");
1145 goto finish;
1146 }
1147
1148 r = pull_job_begin(i->json_job);
1149 if (r < 0) {
1150 log_error_errno(r, "Failed to start json job: %m");
1151 goto finish;
1152 }
1153
1154 } else if (i->ancestry_job == j) {
1155 char **ancestry = NULL, **k;
1156 unsigned n;
1157
1158 assert(!i->layer_job);
1159
1160 r = parse_ancestry(j->payload, j->payload_size, &ancestry);
1161 if (r < 0) {
1162 log_error_errno(r, "Failed to parse JSON id.");
1163 goto finish;
1164 }
1165
1166 n = strv_length(ancestry);
1167 if (n <= 0 || !streq(ancestry[n-1], i->id)) {
1168 log_error("Ancestry doesn't end in main layer.");
1169 strv_free(ancestry);
1170 r = -EBADMSG;
1171 goto finish;
1172 }
1173
1174 log_info("Ancestor lookup succeeded, requires layers:\n");
1175 STRV_FOREACH(k, ancestry)
1176 log_info("\t%s", *k);
1177
1178 strv_free(i->ancestry);
1179 i->ancestry = ancestry;
1180 i->n_ancestry = n;
1181 i->current_ancestry = 0;
1182
1183 dkr_pull_report_progress(i, DKR_DOWNLOADING);
1184
1185 r = dkr_pull_pull_layer(i);
1186 if (r < 0)
1187 goto finish;
1188
1189 } else if (i->layer_job == j) {
1190 assert(i->temp_path);
1191 assert(i->final_path);
1192
1193 j->disk_fd = safe_close(j->disk_fd);
1194
1195 if (i->tar_pid > 0) {
1196 r = wait_for_terminate_and_warn("tar", i->tar_pid, true);
1197 i->tar_pid = 0;
1198 if (r < 0)
1199 goto finish;
1200 }
1201
1202 r = aufs_resolve(i->temp_path);
1203 if (r < 0) {
1204 log_error_errno(r, "Failed to resolve aufs whiteouts: %m");
1205 goto finish;
1206 }
1207
1208 r = btrfs_subvol_set_read_only(i->temp_path, true);
1209 if (r < 0) {
1210 log_error_errno(r, "Failed to mark snapshot read-only: %m");
1211 goto finish;
1212 }
1213
1214 if (rename(i->temp_path, i->final_path) < 0) {
1215 log_error_errno(errno, "Failed to rename snaphsot: %m");
1216 goto finish;
1217 }
1218
1219 log_info("Completed writing to layer %s.", i->final_path);
1220
1221 i->layer_job = pull_job_unref(i->layer_job);
1222 i->temp_path = mfree(i->temp_path);
1223 i->final_path = mfree(i->final_path);
1224
1225 i->current_ancestry ++;
1226 r = dkr_pull_pull_layer(i);
1227 if (r < 0)
1228 goto finish;
1229
1230 } else if (i->json_job != j)
1231 assert_not_reached("Got finished event for unknown curl object");
1232
1233 if (!dkr_pull_is_done(i))
1234 return;
1235
1236 dkr_pull_report_progress(i, DKR_COPYING);
1237
1238 r = dkr_pull_make_local_copy(i, DKR_PULL_V1);
1239 if (r < 0)
1240 goto finish;
1241
1242 r = 0;
1243 finish:
1244 if (i->on_finished)
1245 i->on_finished(i, r, i->userdata);
1246 else
1247 sd_event_exit(i->event, r);
1248 }
1249
1250 static int get_protocol_address(char **protocol, char **address, const char *url) {
1251 const char *sep, *dot;
1252 _cleanup_free_ char *a = NULL, *p = NULL;
1253
1254 sep = strstr(url, "://");
1255 if (!sep)
1256 return -EINVAL;
1257
1258 dot = strrchr(url, '.');
1259 if (!dot)
1260 return -EINVAL;
1261 dot--;
1262
1263 p = strndup(url, (sep - url) + 3);
1264 if (!p)
1265 return log_oom();
1266
1267 while (dot > (sep + 3) && *dot != '.')
1268 dot--;
1269
1270 a = strdup(dot + 1);
1271 if (!a)
1272 return log_oom();
1273
1274 *address = a;
1275 *protocol = p;
1276 a = p = NULL;
1277
1278 return 0;
1279 }
1280
1281 int dkr_pull_start(DkrPull *i, const char *name, const char *reference, const char *local, bool force_local, DkrPullVersion version) {
1282 const char *url;
1283 int r;
1284
1285 assert(i);
1286
1287 if (!dkr_name_is_valid(name))
1288 return -EINVAL;
1289
1290 if (reference && !dkr_ref_is_valid(reference))
1291 return -EINVAL;
1292
1293 if (local && !machine_name_is_valid(local))
1294 return -EINVAL;
1295
1296 if (i->images_job)
1297 return -EBUSY;
1298
1299 if (!reference)
1300 reference = "latest";
1301
1302 free(i->index_protocol);
1303 free(i->index_address);
1304 r = get_protocol_address(&i->index_protocol, &i->index_address, i->index_url);
1305 if (r < 0)
1306 return r;
1307
1308 r = free_and_strdup(&i->local, local);
1309 if (r < 0)
1310 return r;
1311 i->force_local = force_local;
1312
1313 r = free_and_strdup(&i->name, name);
1314 if (r < 0)
1315 return r;
1316 r = free_and_strdup(&i->reference, reference);
1317 if (r < 0)
1318 return r;
1319
1320 url = strjoina(i->index_url, "/v1/repositories/", name, "/images");
1321
1322 r = pull_job_new(&i->images_job, url, i->glue, i);
1323 if (r < 0)
1324 return r;
1325
1326 r = dkr_pull_add_token(i, i->images_job);
1327 if (r < 0)
1328 return r;
1329
1330 if (version == DKR_PULL_V1)
1331 i->images_job->on_finished = dkr_pull_job_on_finished;
1332 else
1333 i->images_job->on_finished = dkr_pull_job_on_finished_v2;
1334
1335 i->images_job->on_header = dkr_pull_job_on_header;
1336 i->images_job->on_progress = dkr_pull_job_on_progress;
1337
1338 return pull_job_begin(i->images_job);
1339 }
Unnamed repository; edit this file 'description' to name the repository.