2 This file is part of systemd.
4 Copyright 2015 Lennart Poettering
6 systemd is free software; you can redistribute it and/or modify it
7 under the terms of the GNU Lesser General Public License as published by
8 the Free Software Foundation; either version 2.1 of the License, or
9 (at your option) any later version.
11 systemd is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public License
17 along with systemd; If not, see <http://www.gnu.org/licenses/>.
20 #include <sys/xattr.h>
22 #include "alloc-util.h"
24 #include "hexdecoct.h"
26 #include "machine-pool.h"
27 #include "parse-util.h"
29 #include "string-util.h"
31 #include "xattr-util.h"
33 PullJob
* pull_job_unref(PullJob
*j
) {
37 curl_glue_remove_and_free(j
->glue
, j
->curl
);
38 curl_slist_free_all(j
->request_header
);
40 safe_close(j
->disk_fd
);
42 import_compress_free(&j
->compress
);
44 if (j
->checksum_context
)
45 gcry_md_close(j
->checksum_context
);
49 strv_free(j
->old_etags
);
58 static void pull_job_finish(PullJob
*j
, int ret
) {
61 if (j
->state
== PULL_JOB_DONE
||
62 j
->state
== PULL_JOB_FAILED
)
66 j
->state
= PULL_JOB_DONE
;
67 j
->progress_percent
= 100;
68 log_info("Download of %s complete.", j
->url
);
70 j
->state
= PULL_JOB_FAILED
;
78 void pull_job_curl_on_finished(CurlGlue
*g
, CURL
*curl
, CURLcode result
) {
84 if (curl_easy_getinfo(curl
, CURLINFO_PRIVATE
, (char **)&j
) != CURLE_OK
)
87 if (!j
|| j
->state
== PULL_JOB_DONE
|| j
->state
== PULL_JOB_FAILED
)
90 if (result
!= CURLE_OK
) {
91 log_error("Transfer failed: %s", curl_easy_strerror(result
));
96 code
= curl_easy_getinfo(curl
, CURLINFO_RESPONSE_CODE
, &status
);
97 if (code
!= CURLE_OK
) {
98 log_error("Failed to retrieve response code: %s", curl_easy_strerror(code
));
101 } else if (status
== 304) {
102 log_info("Image already downloaded. Skipping download.");
103 j
->etag_exists
= true;
106 } else if (status
>= 300) {
107 log_error("HTTP request to %s failed with code %li.", j
->url
, status
);
110 } else if (status
< 200) {
111 log_error("HTTP request to %s finished with unexpected code %li.", j
->url
, status
);
116 if (j
->state
!= PULL_JOB_RUNNING
) {
117 log_error("Premature connection termination.");
122 if (j
->content_length
!= (uint64_t) -1 &&
123 j
->content_length
!= j
->written_compressed
) {
124 log_error("Download truncated.");
129 if (j
->checksum_context
) {
132 k
= gcry_md_read(j
->checksum_context
, GCRY_MD_SHA256
);
134 log_error("Failed to get checksum.");
139 j
->checksum
= hexmem(k
, gcry_md_get_algo_dlen(GCRY_MD_SHA256
));
145 log_debug("SHA256 of %s is %s.", j
->url
, j
->checksum
);
148 if (j
->disk_fd
>= 0 && j
->allow_sparse
) {
149 /* Make sure the file size is right, in case the file was
150 * sparse and we just seeked for the last part */
152 if (ftruncate(j
->disk_fd
, j
->written_uncompressed
) < 0) {
153 r
= log_error_errno(errno
, "Failed to truncate file: %m");
158 (void) fsetxattr(j
->disk_fd
, "user.source_etag", j
->etag
, strlen(j
->etag
), 0);
160 (void) fsetxattr(j
->disk_fd
, "user.source_url", j
->url
, strlen(j
->url
), 0);
163 struct timespec ut
[2];
165 timespec_store(&ut
[0], j
->mtime
);
167 (void) futimens(j
->disk_fd
, ut
);
169 (void) fd_setcrtime(j
->disk_fd
, j
->mtime
);
176 pull_job_finish(j
, r
);
179 static int pull_job_write_uncompressed(const void *p
, size_t sz
, void *userdata
) {
180 PullJob
*j
= userdata
;
189 if (j
->written_uncompressed
+ sz
< j
->written_uncompressed
) {
190 log_error("File too large, overflow");
194 if (j
->written_uncompressed
+ sz
> j
->uncompressed_max
) {
195 log_error("File overly large, refusing");
199 if (j
->disk_fd
>= 0) {
201 if (j
->grow_machine_directory
&& j
->written_since_last_grow
>= GROW_INTERVAL_BYTES
) {
202 j
->written_since_last_grow
= 0;
203 grow_machine_directory();
207 n
= sparse_write(j
->disk_fd
, p
, sz
, 64);
209 n
= write(j
->disk_fd
, p
, sz
);
211 return log_error_errno(errno
, "Failed to write file: %m");
212 if ((size_t) n
< sz
) {
213 log_error("Short write");
218 if (!GREEDY_REALLOC(j
->payload
, j
->payload_allocated
, j
->payload_size
+ sz
))
221 memcpy(j
->payload
+ j
->payload_size
, p
, sz
);
222 j
->payload_size
+= sz
;
225 j
->written_uncompressed
+= sz
;
226 j
->written_since_last_grow
+= sz
;
231 static int pull_job_write_compressed(PullJob
*j
, void *p
, size_t sz
) {
240 if (j
->written_compressed
+ sz
< j
->written_compressed
) {
241 log_error("File too large, overflow");
245 if (j
->written_compressed
+ sz
> j
->compressed_max
) {
246 log_error("File overly large, refusing.");
250 if (j
->content_length
!= (uint64_t) -1 &&
251 j
->written_compressed
+ sz
> j
->content_length
) {
252 log_error("Content length incorrect.");
256 if (j
->checksum_context
)
257 gcry_md_write(j
->checksum_context
, p
, sz
);
259 r
= import_uncompress(&j
->compress
, p
, sz
, pull_job_write_uncompressed
, j
);
263 j
->written_compressed
+= sz
;
268 static int pull_job_open_disk(PullJob
*j
) {
273 if (j
->on_open_disk
) {
274 r
= j
->on_open_disk(j
);
279 if (j
->disk_fd
>= 0) {
280 /* Check if we can do sparse files */
282 if (lseek(j
->disk_fd
, SEEK_SET
, 0) == 0)
283 j
->allow_sparse
= true;
286 return log_error_errno(errno
, "Failed to seek on file descriptor: %m");
288 j
->allow_sparse
= false;
292 if (j
->calc_checksum
) {
293 if (gcry_md_open(&j
->checksum_context
, GCRY_MD_SHA256
, 0) != 0) {
294 log_error("Failed to initialize hash context.");
302 static int pull_job_detect_compression(PullJob
*j
) {
303 _cleanup_free_
uint8_t *stub
= NULL
;
310 r
= import_uncompress_detect(&j
->compress
, j
->payload
, j
->payload_size
);
312 return log_error_errno(r
, "Failed to initialize compressor: %m");
316 log_debug("Stream is compressed: %s", import_compress_type_to_string(j
->compress
.type
));
318 r
= pull_job_open_disk(j
);
322 /* Now, take the payload we read so far, and decompress it */
324 stub_size
= j
->payload_size
;
328 j
->payload_allocated
= 0;
330 j
->state
= PULL_JOB_RUNNING
;
332 r
= pull_job_write_compressed(j
, stub
, stub_size
);
339 static size_t pull_job_write_callback(void *contents
, size_t size
, size_t nmemb
, void *userdata
) {
340 PullJob
*j
= userdata
;
341 size_t sz
= size
* nmemb
;
349 case PULL_JOB_ANALYZING
:
350 /* Let's first check what it actually is */
352 if (!GREEDY_REALLOC(j
->payload
, j
->payload_allocated
, j
->payload_size
+ sz
)) {
357 memcpy(j
->payload
+ j
->payload_size
, contents
, sz
);
358 j
->payload_size
+= sz
;
360 r
= pull_job_detect_compression(j
);
366 case PULL_JOB_RUNNING
:
368 r
= pull_job_write_compressed(j
, contents
, sz
);
375 case PULL_JOB_FAILED
:
380 assert_not_reached("Impossible state.");
386 pull_job_finish(j
, r
);
390 static size_t pull_job_header_callback(void *contents
, size_t size
, size_t nmemb
, void *userdata
) {
391 PullJob
*j
= userdata
;
392 size_t sz
= size
* nmemb
;
393 _cleanup_free_
char *length
= NULL
, *last_modified
= NULL
;
400 if (j
->state
== PULL_JOB_DONE
|| j
->state
== PULL_JOB_FAILED
) {
405 assert(j
->state
== PULL_JOB_ANALYZING
);
407 r
= curl_header_strdup(contents
, sz
, "ETag:", &etag
);
416 if (strv_contains(j
->old_etags
, j
->etag
)) {
417 log_info("Image already downloaded. Skipping download.");
418 j
->etag_exists
= true;
419 pull_job_finish(j
, 0);
426 r
= curl_header_strdup(contents
, sz
, "Content-Length:", &length
);
432 (void) safe_atou64(length
, &j
->content_length
);
434 if (j
->content_length
!= (uint64_t) -1) {
435 char bytes
[FORMAT_BYTES_MAX
];
437 if (j
->content_length
> j
->compressed_max
) {
438 log_error("Content too large.");
443 log_info("Downloading %s for %s.", format_bytes(bytes
, sizeof(bytes
), j
->content_length
), j
->url
);
449 r
= curl_header_strdup(contents
, sz
, "Last-Modified:", &last_modified
);
455 (void) curl_parse_http_time(last_modified
, &j
->mtime
);
460 r
= j
->on_header(j
, contents
, sz
);
468 pull_job_finish(j
, r
);
472 static int pull_job_progress_callback(void *userdata
, curl_off_t dltotal
, curl_off_t dlnow
, curl_off_t ultotal
, curl_off_t ulnow
) {
473 PullJob
*j
= userdata
;
482 percent
= ((100 * dlnow
) / dltotal
);
483 n
= now(CLOCK_MONOTONIC
);
485 if (n
> j
->last_status_usec
+ USEC_PER_SEC
&&
486 percent
!= j
->progress_percent
&&
488 char buf
[FORMAT_TIMESPAN_MAX
];
490 if (n
- j
->start_usec
> USEC_PER_SEC
&& dlnow
> 0) {
491 char y
[FORMAT_BYTES_MAX
];
494 done
= n
- j
->start_usec
;
495 left
= (usec_t
) (((double) done
* (double) dltotal
) / dlnow
) - done
;
497 log_info("Got %u%% of %s. %s left at %s/s.",
500 format_timespan(buf
, sizeof(buf
), left
, USEC_PER_SEC
),
501 format_bytes(y
, sizeof(y
), (uint64_t) ((double) dlnow
/ ((double) done
/ (double) USEC_PER_SEC
))));
503 log_info("Got %u%% of %s.", percent
, j
->url
);
505 j
->progress_percent
= percent
;
506 j
->last_status_usec
= n
;
515 int pull_job_new(PullJob
**ret
, const char *url
, CurlGlue
*glue
, void *userdata
) {
516 _cleanup_(pull_job_unrefp
) PullJob
*j
= NULL
;
522 j
= new0(PullJob
, 1);
526 j
->state
= PULL_JOB_INIT
;
528 j
->userdata
= userdata
;
530 j
->content_length
= (uint64_t) -1;
531 j
->start_usec
= now(CLOCK_MONOTONIC
);
532 j
->compressed_max
= j
->uncompressed_max
= 8LLU * 1024LLU * 1024LLU * 1024LLU; /* 8GB */
534 j
->url
= strdup(url
);
544 int pull_job_begin(PullJob
*j
) {
549 if (j
->state
!= PULL_JOB_INIT
)
552 if (j
->grow_machine_directory
)
553 grow_machine_directory();
555 r
= curl_glue_make(&j
->curl
, j
->url
, j
);
559 if (!strv_isempty(j
->old_etags
)) {
560 _cleanup_free_
char *cc
= NULL
, *hdr
= NULL
;
562 cc
= strv_join(j
->old_etags
, ", ");
566 hdr
= strappend("If-None-Match: ", cc
);
570 if (!j
->request_header
) {
571 j
->request_header
= curl_slist_new(hdr
, NULL
);
572 if (!j
->request_header
)
575 struct curl_slist
*l
;
577 l
= curl_slist_append(j
->request_header
, hdr
);
581 j
->request_header
= l
;
585 if (j
->request_header
) {
586 if (curl_easy_setopt(j
->curl
, CURLOPT_HTTPHEADER
, j
->request_header
) != CURLE_OK
)
590 if (curl_easy_setopt(j
->curl
, CURLOPT_WRITEFUNCTION
, pull_job_write_callback
) != CURLE_OK
)
593 if (curl_easy_setopt(j
->curl
, CURLOPT_WRITEDATA
, j
) != CURLE_OK
)
596 if (curl_easy_setopt(j
->curl
, CURLOPT_HEADERFUNCTION
, pull_job_header_callback
) != CURLE_OK
)
599 if (curl_easy_setopt(j
->curl
, CURLOPT_HEADERDATA
, j
) != CURLE_OK
)
602 if (curl_easy_setopt(j
->curl
, CURLOPT_XFERINFOFUNCTION
, pull_job_progress_callback
) != CURLE_OK
)
605 if (curl_easy_setopt(j
->curl
, CURLOPT_XFERINFODATA
, j
) != CURLE_OK
)
608 if (curl_easy_setopt(j
->curl
, CURLOPT_NOPROGRESS
, 0) != CURLE_OK
)
611 r
= curl_glue_add(j
->glue
, j
->curl
);
615 j
->state
= PULL_JOB_ANALYZING
;