[thirdparty/systemd.git] / src / import / pull-job.c

/***
  This file is part of systemd.

  Copyright 2015 Lennart Poettering

  systemd is free software; you can redistribute it and/or modify it
  under the terms of the GNU Lesser General Public License as published by
  the Free Software Foundation; either version 2.1 of the License, or
  (at your option) any later version.

  systemd is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public License
  along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/

#include <sys/xattr.h>

#include "alloc-util.h"
#include "fd-util.h"
#include "hexdecoct.h"
#include "import-util.h"
#include "io-util.h"
#include "machine-pool.h"
#include "parse-util.h"
#include "pull-common.h"
#include "pull-job.h"
#include "string-util.h"
#include "strv.h"
#include "xattr-util.h"

PullJob* pull_job_unref(PullJob *j) {
        if (!j)
                return NULL;

        curl_glue_remove_and_free(j->glue, j->curl);
        curl_slist_free_all(j->request_header);

        safe_close(j->disk_fd);

        import_compress_free(&j->compress);

        if (j->checksum_context)
                gcry_md_close(j->checksum_context);

        free(j->url);
        free(j->etag);
        strv_free(j->old_etags);
        free(j->payload);
        free(j->checksum);

        return mfree(j);
}

static void pull_job_finish(PullJob *j, int ret) {
        assert(j);

        if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
                return;

        if (ret == 0) {
                j->state = PULL_JOB_DONE;
                j->progress_percent = 100;
                log_info("Download of %s complete.", j->url);
        } else {
                j->state = PULL_JOB_FAILED;
                j->error = ret;
        }

        if (j->on_finished)
                j->on_finished(j);
}

static int pull_job_restart(PullJob *j) {
        int r;
        char *chksum_url = NULL;

        r = import_url_change_last_component(j->url, "SHA256SUMS", &chksum_url);
        if (r < 0)
                return r;

        free(j->url);
        j->url = chksum_url;
        j->state = PULL_JOB_INIT;
        j->payload = mfree(j->payload);
        j->payload_size = 0;
        j->payload_allocated = 0;
        j->written_compressed = 0;
        j->written_uncompressed = 0;
        j->written_since_last_grow = 0;

        r = pull_job_begin(j);
        if (r < 0)
                return r;

        return 0;
}

void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
        PullJob *j = NULL;
        CURLcode code;
        long status;
        int r;

        if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK)
                return;

        if (!j || IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
                return;

        if (result != CURLE_OK) {
                log_error("Transfer failed: %s", curl_easy_strerror(result));
                r = -EIO;
                goto finish;
        }

        code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
        if (code != CURLE_OK) {
                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
                r = -EIO;
                goto finish;
        } else if (status == 304) {
                log_info("Image already downloaded. Skipping download.");
                j->etag_exists = true;
                r = 0;
                goto finish;
        } else if (status >= 300) {
                if (status == 404 && j->style == VERIFICATION_PER_FILE) {

                        /* retry pull job with SHA256SUMS file */
                        r = pull_job_restart(j);
                        if (r < 0)
                                goto finish;

                        code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
                        if (code != CURLE_OK) {
                                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
                                r = -EIO;
                                goto finish;
                        }

                        if (status == 0) {
                                j->style = VERIFICATION_PER_DIRECTORY;
                                return;
                        }
                }

                log_error("HTTP request to %s failed with code %li.", j->url, status);
                r = -EIO;
                goto finish;
        } else if (status < 200) {
                log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
                r = -EIO;
                goto finish;
        }

        if (j->state != PULL_JOB_RUNNING) {
                log_error("Premature connection termination.");
                r = -EIO;
                goto finish;
        }

        if (j->content_length != (uint64_t) -1 &&
            j->content_length != j->written_compressed) {
                log_error("Download truncated.");
                r = -EIO;
                goto finish;
        }

        if (j->checksum_context) {
                uint8_t *k;

                k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
                if (!k) {
                        log_error("Failed to get checksum.");
                        r = -EIO;
                        goto finish;
                }

                j->checksum = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256));
                if (!j->checksum) {
                        r = log_oom();
                        goto finish;
                }

                log_debug("SHA256 of %s is %s.", j->url, j->checksum);
        }

        if (j->disk_fd >= 0 && j->allow_sparse) {
                /* Make sure the file size is right, in case the file was
                 * sparse and we just seeked for the last part */

                if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
                        r = log_error_errno(errno, "Failed to truncate file: %m");
                        goto finish;
                }

                if (j->etag)
                        (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
                if (j->url)
                        (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);

                if (j->mtime != 0) {
                        struct timespec ut[2];

                        timespec_store(&ut[0], j->mtime);
                        ut[1] = ut[0];
                        (void) futimens(j->disk_fd, ut);

                        (void) fd_setcrtime(j->disk_fd, j->mtime);
                }
        }

        r = 0;

finish:
        pull_job_finish(j, r);
}

static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) {
        PullJob *j = userdata;
        ssize_t n;

        assert(j);
        assert(p);

        if (sz <= 0)
                return 0;

        if (j->written_uncompressed + sz < j->written_uncompressed) {
                log_error("File too large, overflow");
                return -EOVERFLOW;
        }

        if (j->written_uncompressed + sz > j->uncompressed_max) {
                log_error("File overly large, refusing");
                return -EFBIG;
        }

        if (j->disk_fd >= 0) {

                if (j->grow_machine_directory && j->written_since_last_grow >= GROW_INTERVAL_BYTES) {
                        j->written_since_last_grow = 0;
                        grow_machine_directory();
                }

                if (j->allow_sparse)
                        n = sparse_write(j->disk_fd, p, sz, 64);
                else
                        n = write(j->disk_fd, p, sz);
                if (n < 0)
                        return log_error_errno(errno, "Failed to write file: %m");
                if ((size_t) n < sz) {
                        log_error("Short write");
                        return -EIO;
                }
        } else {

                if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz))
                        return log_oom();

                memcpy(j->payload + j->payload_size, p, sz);
                j->payload_size += sz;
        }

        j->written_uncompressed += sz;
        j->written_since_last_grow += sz;

        return 0;
}

static int pull_job_write_compressed(PullJob *j, void *p, size_t sz) {
        int r;

        assert(j);
        assert(p);

        if (sz <= 0)
                return 0;

        if (j->written_compressed + sz < j->written_compressed) {
                log_error("File too large, overflow");
                return -EOVERFLOW;
        }

        if (j->written_compressed + sz > j->compressed_max) {
                log_error("File overly large, refusing.");
                return -EFBIG;
        }

        if (j->content_length != (uint64_t) -1 &&
            j->written_compressed + sz > j->content_length) {
                log_error("Content length incorrect.");
                return -EFBIG;
        }

        if (j->checksum_context)
                gcry_md_write(j->checksum_context, p, sz);

        r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j);
        if (r < 0)
                return r;

        j->written_compressed += sz;

        return 0;
}

static int pull_job_open_disk(PullJob *j) {
        int r;

        assert(j);

        if (j->on_open_disk) {
                r = j->on_open_disk(j);
                if (r < 0)
                        return r;
        }

        if (j->disk_fd >= 0) {
                /* Check if we can do sparse files */

                if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
                        j->allow_sparse = true;
                else {
                        if (errno != ESPIPE)
                                return log_error_errno(errno, "Failed to seek on file descriptor: %m");

                        j->allow_sparse = false;
                }
        }

        if (j->calc_checksum) {
                if (gcry_md_open(&j->checksum_context, GCRY_MD_SHA256, 0) != 0) {
                        log_error("Failed to initialize hash context.");
                        return -EIO;
                }
        }

        return 0;
}

static int pull_job_detect_compression(PullJob *j) {
        _cleanup_free_ uint8_t *stub = NULL;
        size_t stub_size;

        int r;

        assert(j);

        r = import_uncompress_detect(&j->compress, j->payload, j->payload_size);
        if (r < 0)
                return log_error_errno(r, "Failed to initialize compressor: %m");
        if (r == 0)
                return 0;

        log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type));

        r = pull_job_open_disk(j);
        if (r < 0)
                return r;

        /* Now, take the payload we read so far, and decompress it */
        stub = j->payload;
        stub_size = j->payload_size;

        j->payload = NULL;
        j->payload_size = 0;
        j->payload_allocated = 0;

        j->state = PULL_JOB_RUNNING;

        r = pull_job_write_compressed(j, stub, stub_size);
        if (r < 0)
                return r;

        return 0;
}

static size_t pull_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
        PullJob *j = userdata;
        size_t sz = size * nmemb;
        int r;

        assert(contents);
        assert(j);

        switch (j->state) {

        case PULL_JOB_ANALYZING:
                /* Let's first check what it actually is */

                if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) {
                        r = log_oom();
                        goto fail;
                }

                memcpy(j->payload + j->payload_size, contents, sz);
                j->payload_size += sz;

                r = pull_job_detect_compression(j);
                if (r < 0)
                        goto fail;

                break;

        case PULL_JOB_RUNNING:

                r = pull_job_write_compressed(j, contents, sz);
                if (r < 0)
                        goto fail;

                break;

        case PULL_JOB_DONE:
        case PULL_JOB_FAILED:
                r = -ESTALE;
                goto fail;

        default:
                assert_not_reached("Impossible state.");
        }

        return sz;

fail:
        pull_job_finish(j, r);
        return 0;
}

static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
        PullJob *j = userdata;
        size_t sz = size * nmemb;
        _cleanup_free_ char *length = NULL, *last_modified = NULL;
        char *etag;
        int r;

        assert(contents);
        assert(j);

        if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) {
                r = -ESTALE;
                goto fail;
        }

        assert(j->state == PULL_JOB_ANALYZING);

        r = curl_header_strdup(contents, sz, "ETag:", &etag);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                free(j->etag);
                j->etag = etag;

                if (strv_contains(j->old_etags, j->etag)) {
                        log_info("Image already downloaded. Skipping download.");
                        j->etag_exists = true;
                        pull_job_finish(j, 0);
                        return sz;
                }

                return sz;
        }

        r = curl_header_strdup(contents, sz, "Content-Length:", &length);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                (void) safe_atou64(length, &j->content_length);

                if (j->content_length != (uint64_t) -1) {
                        char bytes[FORMAT_BYTES_MAX];

                        if (j->content_length > j->compressed_max) {
                                log_error("Content too large.");
                                r = -EFBIG;
                                goto fail;
                        }

                        log_info("Downloading %s for %s.", format_bytes(bytes, sizeof(bytes), j->content_length), j->url);
                }

                return sz;
        }

        r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                (void) curl_parse_http_time(last_modified, &j->mtime);
                return sz;
        }

        if (j->on_header) {
                r = j->on_header(j, contents, sz);
                if (r < 0)
                        goto fail;
        }

        return sz;

fail:
        pull_job_finish(j, r);
        return 0;
}

static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
        PullJob *j = userdata;
        unsigned percent;
        usec_t n;

        assert(j);

        if (dltotal <= 0)
                return 0;

        percent = ((100 * dlnow) / dltotal);
        n = now(CLOCK_MONOTONIC);

        if (n > j->last_status_usec + USEC_PER_SEC &&
            percent != j->progress_percent &&
            dlnow < dltotal) {
                char buf[FORMAT_TIMESPAN_MAX];

                if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) {
                        char y[FORMAT_BYTES_MAX];
                        usec_t left, done;

                        done = n - j->start_usec;
                        left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;

                        log_info("Got %u%% of %s. %s left at %s/s.",
                                 percent,
                                 j->url,
                                 format_timespan(buf, sizeof(buf), left, USEC_PER_SEC),
                                 format_bytes(y, sizeof(y), (uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC))));
                } else
                        log_info("Got %u%% of %s.", percent, j->url);

                j->progress_percent = percent;
                j->last_status_usec = n;

                if (j->on_progress)
                        j->on_progress(j);
        }

        return 0;
}

int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) {
        _cleanup_(pull_job_unrefp) PullJob *j = NULL;

        assert(url);
        assert(glue);
        assert(ret);

        j = new0(PullJob, 1);
        if (!j)
                return -ENOMEM;

        j->state = PULL_JOB_INIT;
        j->disk_fd = -1;
        j->userdata = userdata;
        j->glue = glue;
        j->content_length = (uint64_t) -1;
        j->start_usec = now(CLOCK_MONOTONIC);
        j->compressed_max = j->uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU; /* 64GB safety limit */
        j->style = VERIFICATION_STYLE_UNSET;

        j->url = strdup(url);
        if (!j->url)
                return -ENOMEM;

        *ret = j;
        j = NULL;

        return 0;
}

int pull_job_begin(PullJob *j) {
        int r;

        assert(j);

        if (j->state != PULL_JOB_INIT)
                return -EBUSY;

        if (j->grow_machine_directory)
                grow_machine_directory();

        r = curl_glue_make(&j->curl, j->url, j);
        if (r < 0)
                return r;

        if (!strv_isempty(j->old_etags)) {
                _cleanup_free_ char *cc = NULL, *hdr = NULL;

                cc = strv_join(j->old_etags, ", ");
                if (!cc)
                        return -ENOMEM;

                hdr = strappend("If-None-Match: ", cc);
                if (!hdr)
                        return -ENOMEM;

                if (!j->request_header) {
                        j->request_header = curl_slist_new(hdr, NULL);
                        if (!j->request_header)
                                return -ENOMEM;
                } else {
                        struct curl_slist *l;

                        l = curl_slist_append(j->request_header, hdr);
                        if (!l)
                                return -ENOMEM;

                        j->request_header = l;
                }
        }

        if (j->request_header) {
                if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
                        return -EIO;
        }

        if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
                return -EIO;

        r = curl_glue_add(j->glue, j->curl);
        if (r < 0)
                return r;

        j->state = PULL_JOB_ANALYZING;

        return 0;
}
Commit	Line	Data
56ebfaf1 LP	1	/***
	2	This file is part of systemd.
	3
	4	Copyright 2015 Lennart Poettering
	5
	6	systemd is free software; you can redistribute it and/or modify it
	7	under the terms of the GNU Lesser General Public License as published by
	8	the Free Software Foundation; either version 2.1 of the License, or
	9	(at your option) any later version.
	10
	11	systemd is distributed in the hope that it will be useful, but
	12	WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	Lesser General Public License for more details.
	15
	16	You should have received a copy of the GNU Lesser General Public License
	17	along with systemd; If not, see <http://www.gnu.org/licenses/>.
	18	***/
	19
	20	#include <sys/xattr.h>
	21
b5efdb8a	22	#include "alloc-util.h"
3ffd4af2	23	#include "fd-util.h"
89a5a90c	24	#include "hexdecoct.h"
1f0f4f3b	25	#include "import-util.h"
c004493c	26	#include "io-util.h"
26166c88	27	#include "machine-pool.h"
6bedfcbb	28	#include "parse-util.h"
1f0f4f3b	29	#include "pull-common.h"
3ffd4af2	30	#include "pull-job.h"
07630cea LP	31	#include "string-util.h"
07630cea LP	32	#include "strv.h"
89a5a90c	33	#include "xattr-util.h"
56ebfaf1	34
dc2c282b	35	PullJob* pull_job_unref(PullJob *j) {
56ebfaf1 LP	36	if (!j)
	37	return NULL;
	38
	39	curl_glue_remove_and_free(j->glue, j->curl);
	40	curl_slist_free_all(j->request_header);
	41
	42	safe_close(j->disk_fd);
	43
3e2cda69	44	import_compress_free(&j->compress);
56ebfaf1	45
98c38001 LP	46	if (j->checksum_context)
98c38001 LP	47	gcry_md_close(j->checksum_context);
85dbc41d	48
56ebfaf1 LP	49	free(j->url);
	50	free(j->etag);
	51	strv_free(j->old_etags);
	52	free(j->payload);
98c38001	53	free(j->checksum);
56ebfaf1	54
6b430fdb	55	return mfree(j);
56ebfaf1 LP	56	}
56ebfaf1 LP	57
dc2c282b	58	static void pull_job_finish(PullJob *j, int ret) {
56ebfaf1 LP	59	assert(j);
56ebfaf1 LP	60
3742095b	61	if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
56ebfaf1 LP	62	return;
56ebfaf1 LP	63
68c913fd	64	if (ret == 0) {
dc2c282b	65	j->state = PULL_JOB_DONE;
7079cfef	66	j->progress_percent = 100;
68c913fd LP	67	log_info("Download of %s complete.", j->url);
68c913fd LP	68	} else {
dc2c282b	69	j->state = PULL_JOB_FAILED;
56ebfaf1 LP	70	j->error = ret;
	71	}
	72
	73	if (j->on_finished)
	74	j->on_finished(j);
	75	}
	76
697be0be TB	77	static int pull_job_restart(PullJob *j) {
	78	int r;
	79	char *chksum_url = NULL;
	80
	81	r = import_url_change_last_component(j->url, "SHA256SUMS", &chksum_url);
	82	if (r < 0)
	83	return r;
	84
	85	free(j->url);
697be0be TB	86	j->url = chksum_url;
697be0be TB	87	j->state = PULL_JOB_INIT;
1f0f4f3b	88	j->payload = mfree(j->payload);
697be0be TB	89	j->payload_size = 0;
	90	j->payload_allocated = 0;
	91	j->written_compressed = 0;
	92	j->written_uncompressed = 0;
	93	j->written_since_last_grow = 0;
	94
	95	r = pull_job_begin(j);
	96	if (r < 0)
	97	return r;
	98
	99	return 0;
	100	}
	101
dc2c282b LP	102	void pull_job_curl_on_finished(CurlGlue g, CURL curl, CURLcode result) {
dc2c282b LP	103	PullJob *j = NULL;
56ebfaf1 LP	104	CURLcode code;
	105	long status;
	106	int r;
	107
a7f7d1bd	108	if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK)
56ebfaf1 LP	109	return;
56ebfaf1 LP	110
4c701096	111	if (!j \|\| IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
56ebfaf1 LP	112	return;
	113
	114	if (result != CURLE_OK) {
	115	log_error("Transfer failed: %s", curl_easy_strerror(result));
	116	r = -EIO;
	117	goto finish;
	118	}
	119
	120	code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
	121	if (code != CURLE_OK) {
	122	log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
	123	r = -EIO;
	124	goto finish;
	125	} else if (status == 304) {
	126	log_info("Image already downloaded. Skipping download.");
85dbc41d	127	j->etag_exists = true;
56ebfaf1 LP	128	r = 0;
	129	goto finish;
	130	} else if (status >= 300) {
697be0be TB	131	if (status == 404 && j->style == VERIFICATION_PER_FILE) {
	132
	133	/* retry pull job with SHA256SUMS file */
	134	r = pull_job_restart(j);
	135	if (r < 0)
	136	goto finish;
	137
	138	code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
	139	if (code != CURLE_OK) {
	140	log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
	141	r = -EIO;
	142	goto finish;
	143	}
	144
	145	if (status == 0) {
	146	j->style = VERIFICATION_PER_DIRECTORY;
	147	return;
	148	}
	149	}
	150
56ebfaf1 LP	151	log_error("HTTP request to %s failed with code %li.", j->url, status);
	152	r = -EIO;
	153	goto finish;
	154	} else if (status < 200) {
	155	log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
	156	r = -EIO;
	157	goto finish;
	158	}
	159
dc2c282b	160	if (j->state != PULL_JOB_RUNNING) {
56ebfaf1 LP	161	log_error("Premature connection termination.");
	162	r = -EIO;
	163	goto finish;
	164	}
	165
	166	if (j->content_length != (uint64_t) -1 &&
	167	j->content_length != j->written_compressed) {
	168	log_error("Download truncated.");
	169	r = -EIO;
	170	goto finish;
	171	}
	172
98c38001	173	if (j->checksum_context) {
85dbc41d LP	174	uint8_t *k;
85dbc41d LP	175
98c38001	176	k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
85dbc41d LP	177	if (!k) {
	178	log_error("Failed to get checksum.");
	179	r = -EIO;
	180	goto finish;
	181	}
	182
98c38001 LP	183	j->checksum = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256));
98c38001 LP	184	if (!j->checksum) {
85dbc41d LP	185	r = log_oom();
	186	goto finish;
	187	}
	188
98c38001	189	log_debug("SHA256 of %s is %s.", j->url, j->checksum);
85dbc41d LP	190	}
85dbc41d LP	191
56ebfaf1 LP	192	if (j->disk_fd >= 0 && j->allow_sparse) {
	193	/* Make sure the file size is right, in case the file was
	194	* sparse and we just seeked for the last part */
	195
	196	if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
b6e676ce	197	r = log_error_errno(errno, "Failed to truncate file: %m");
56ebfaf1 LP	198	goto finish;
	199	}
	200
	201	if (j->etag)
	202	(void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
	203	if (j->url)
	204	(void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
	205
	206	if (j->mtime != 0) {
	207	struct timespec ut[2];
	208
	209	timespec_store(&ut[0], j->mtime);
	210	ut[1] = ut[0];
	211	(void) futimens(j->disk_fd, ut);
	212
	213	(void) fd_setcrtime(j->disk_fd, j->mtime);
	214	}
	215	}
	216
	217	r = 0;
	218
	219	finish:
dc2c282b	220	pull_job_finish(j, r);
56ebfaf1 LP	221	}
56ebfaf1 LP	222
3e2cda69 LP	223	static int pull_job_write_uncompressed(const void p, size_t sz, void userdata) {
3e2cda69 LP	224	PullJob *j = userdata;
56ebfaf1 LP	225	ssize_t n;
	226
	227	assert(j);
	228	assert(p);
8af3cf74 LP	229
	230	if (sz <= 0)
	231	return 0;
56ebfaf1 LP	232
	233	if (j->written_uncompressed + sz < j->written_uncompressed) {
	234	log_error("File too large, overflow");
	235	return -EOVERFLOW;
	236	}
	237
	238	if (j->written_uncompressed + sz > j->uncompressed_max) {
	239	log_error("File overly large, refusing");
	240	return -EFBIG;
	241	}
	242
	243	if (j->disk_fd >= 0) {
	244
b6e676ce	245	if (j->grow_machine_directory && j->written_since_last_grow >= GROW_INTERVAL_BYTES) {
26166c88 LP	246	j->written_since_last_grow = 0;
	247	grow_machine_directory();
	248	}
	249
56ebfaf1 LP	250	if (j->allow_sparse)
	251	n = sparse_write(j->disk_fd, p, sz, 64);
	252	else
	253	n = write(j->disk_fd, p, sz);
b6e676ce LP	254	if (n < 0)
b6e676ce LP	255	return log_error_errno(errno, "Failed to write file: %m");
56ebfaf1 LP	256	if ((size_t) n < sz) {
	257	log_error("Short write");
	258	return -EIO;
	259	}
	260	} else {
	261
	262	if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz))
	263	return log_oom();
	264
8af3cf74	265	memcpy(j->payload + j->payload_size, p, sz);
56ebfaf1 LP	266	j->payload_size += sz;
	267	}
	268
	269	j->written_uncompressed += sz;
26166c88	270	j->written_since_last_grow += sz;
56ebfaf1 LP	271
	272	return 0;
	273	}
	274
dc2c282b	275	static int pull_job_write_compressed(PullJob j, void p, size_t sz) {
56ebfaf1 LP	276	int r;
	277
	278	assert(j);
	279	assert(p);
8af3cf74 LP	280
	281	if (sz <= 0)
	282	return 0;
56ebfaf1 LP	283
	284	if (j->written_compressed + sz < j->written_compressed) {
	285	log_error("File too large, overflow");
	286	return -EOVERFLOW;
	287	}
	288
	289	if (j->written_compressed + sz > j->compressed_max) {
	290	log_error("File overly large, refusing.");
	291	return -EFBIG;
	292	}
	293
	294	if (j->content_length != (uint64_t) -1 &&
	295	j->written_compressed + sz > j->content_length) {
	296	log_error("Content length incorrect.");
	297	return -EFBIG;
	298	}
	299
98c38001 LP	300	if (j->checksum_context)
98c38001 LP	301	gcry_md_write(j->checksum_context, p, sz);
85dbc41d	302
3e2cda69 LP	303	r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j);
	304	if (r < 0)
	305	return r;
56ebfaf1 LP	306
	307	j->written_compressed += sz;
	308
	309	return 0;
	310	}
	311
dc2c282b	312	static int pull_job_open_disk(PullJob *j) {
56ebfaf1 LP	313	int r;
	314
	315	assert(j);
	316
	317	if (j->on_open_disk) {
	318	r = j->on_open_disk(j);
	319	if (r < 0)
	320	return r;
	321	}
	322
	323	if (j->disk_fd >= 0) {
	324	/* Check if we can do sparse files */
	325
	326	if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
	327	j->allow_sparse = true;
	328	else {
	329	if (errno != ESPIPE)
	330	return log_error_errno(errno, "Failed to seek on file descriptor: %m");
	331
	332	j->allow_sparse = false;
	333	}
	334	}
	335
98c38001 LP	336	if (j->calc_checksum) {
98c38001 LP	337	if (gcry_md_open(&j->checksum_context, GCRY_MD_SHA256, 0) != 0) {
85dbc41d LP	338	log_error("Failed to initialize hash context.");
	339	return -EIO;
	340	}
	341	}
	342
56ebfaf1 LP	343	return 0;
	344	}
	345
dc2c282b	346	static int pull_job_detect_compression(PullJob *j) {
56ebfaf1 LP	347	_cleanup_free_ uint8_t *stub = NULL;
	348	size_t stub_size;
	349
	350	int r;
	351
	352	assert(j);
	353
3e2cda69 LP	354	r = import_uncompress_detect(&j->compress, j->payload, j->payload_size);
	355	if (r < 0)
	356	return log_error_errno(r, "Failed to initialize compressor: %m");
	357	if (r == 0)
56ebfaf1 LP	358	return 0;
56ebfaf1 LP	359
3e2cda69	360	log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type));
56ebfaf1	361
dc2c282b	362	r = pull_job_open_disk(j);
56ebfaf1 LP	363	if (r < 0)
	364	return r;
	365
	366	/* Now, take the payload we read so far, and decompress it */
	367	stub = j->payload;
	368	stub_size = j->payload_size;
	369
	370	j->payload = NULL;
	371	j->payload_size = 0;
88a1aadc	372	j->payload_allocated = 0;
56ebfaf1	373
dc2c282b	374	j->state = PULL_JOB_RUNNING;
56ebfaf1	375
dc2c282b	376	r = pull_job_write_compressed(j, stub, stub_size);
56ebfaf1 LP	377	if (r < 0)
	378	return r;
	379
	380	return 0;
	381	}
	382
dc2c282b LP	383	static size_t pull_job_write_callback(void contents, size_t size, size_t nmemb, void userdata) {
dc2c282b LP	384	PullJob *j = userdata;
56ebfaf1 LP	385	size_t sz = size * nmemb;
	386	int r;
	387
	388	assert(contents);
	389	assert(j);
	390
	391	switch (j->state) {
	392
dc2c282b	393	case PULL_JOB_ANALYZING:
56ebfaf1 LP	394	/* Let's first check what it actually is */
	395
	396	if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) {
	397	r = log_oom();
	398	goto fail;
	399	}
	400
8af3cf74	401	memcpy(j->payload + j->payload_size, contents, sz);
56ebfaf1 LP	402	j->payload_size += sz;
56ebfaf1 LP	403
dc2c282b	404	r = pull_job_detect_compression(j);
56ebfaf1 LP	405	if (r < 0)
	406	goto fail;
	407
	408	break;
	409
dc2c282b	410	case PULL_JOB_RUNNING:
56ebfaf1	411
dc2c282b	412	r = pull_job_write_compressed(j, contents, sz);
56ebfaf1 LP	413	if (r < 0)
	414	goto fail;
	415
	416	break;
	417
dc2c282b LP	418	case PULL_JOB_DONE:
dc2c282b LP	419	case PULL_JOB_FAILED:
56ebfaf1 LP	420	r = -ESTALE;
	421	goto fail;
	422
	423	default:
	424	assert_not_reached("Impossible state.");
	425	}
	426
	427	return sz;
	428
	429	fail:
dc2c282b	430	pull_job_finish(j, r);
56ebfaf1 LP	431	return 0;
	432	}
	433
dc2c282b LP	434	static size_t pull_job_header_callback(void contents, size_t size, size_t nmemb, void userdata) {
dc2c282b LP	435	PullJob *j = userdata;
56ebfaf1 LP	436	size_t sz = size * nmemb;
	437	_cleanup_free_ char length = NULL, last_modified = NULL;
	438	char *etag;
	439	int r;
	440
	441	assert(contents);
	442	assert(j);
	443
3742095b	444	if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) {
56ebfaf1 LP	445	r = -ESTALE;
	446	goto fail;
	447	}
	448
dc2c282b	449	assert(j->state == PULL_JOB_ANALYZING);
56ebfaf1 LP	450
	451	r = curl_header_strdup(contents, sz, "ETag:", &etag);
	452	if (r < 0) {
	453	log_oom();
	454	goto fail;
	455	}
	456	if (r > 0) {
	457	free(j->etag);
	458	j->etag = etag;
	459
	460	if (strv_contains(j->old_etags, j->etag)) {
	461	log_info("Image already downloaded. Skipping download.");
85dbc41d	462	j->etag_exists = true;
dc2c282b	463	pull_job_finish(j, 0);
56ebfaf1 LP	464	return sz;
	465	}
	466
	467	return sz;
	468	}
	469
	470	r = curl_header_strdup(contents, sz, "Content-Length:", &length);
	471	if (r < 0) {
	472	log_oom();
	473	goto fail;
	474	}
	475	if (r > 0) {
	476	(void) safe_atou64(length, &j->content_length);
	477
	478	if (j->content_length != (uint64_t) -1) {
	479	char bytes[FORMAT_BYTES_MAX];
	480
	481	if (j->content_length > j->compressed_max) {
	482	log_error("Content too large.");
	483	r = -EFBIG;
	484	goto fail;
	485	}
	486
68c913fd	487	log_info("Downloading %s for %s.", format_bytes(bytes, sizeof(bytes), j->content_length), j->url);
56ebfaf1 LP	488	}
	489
	490	return sz;
	491	}
	492
	493	r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
	494	if (r < 0) {
	495	log_oom();
	496	goto fail;
	497	}
	498	if (r > 0) {
	499	(void) curl_parse_http_time(last_modified, &j->mtime);
	500	return sz;
	501	}
	502
ff2670ad LP	503	if (j->on_header) {
	504	r = j->on_header(j, contents, sz);
	505	if (r < 0)
	506	goto fail;
	507	}
	508
56ebfaf1 LP	509	return sz;
	510
	511	fail:
dc2c282b	512	pull_job_finish(j, r);
56ebfaf1 LP	513	return 0;
	514	}
	515
dc2c282b LP	516	static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
dc2c282b LP	517	PullJob *j = userdata;
56ebfaf1 LP	518	unsigned percent;
	519	usec_t n;
	520
	521	assert(j);
	522
	523	if (dltotal <= 0)
	524	return 0;
	525
	526	percent = ((100 * dlnow) / dltotal);
	527	n = now(CLOCK_MONOTONIC);
	528
	529	if (n > j->last_status_usec + USEC_PER_SEC &&
68c913fd LP	530	percent != j->progress_percent &&
68c913fd LP	531	dlnow < dltotal) {
56ebfaf1 LP	532	char buf[FORMAT_TIMESPAN_MAX];
	533
	534	if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) {
90bc083b	535	char y[FORMAT_BYTES_MAX];
56ebfaf1 LP	536	usec_t left, done;
	537
	538	done = n - j->start_usec;
	539	left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
	540
90bc083b LP	541	log_info("Got %u%% of %s. %s left at %s/s.",
	542	percent,
	543	j->url,
	544	format_timespan(buf, sizeof(buf), left, USEC_PER_SEC),
	545	format_bytes(y, sizeof(y), (uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC))));
56ebfaf1 LP	546	} else
	547	log_info("Got %u%% of %s.", percent, j->url);
	548
	549	j->progress_percent = percent;
	550	j->last_status_usec = n;
7079cfef LP	551
	552	if (j->on_progress)
	553	j->on_progress(j);
56ebfaf1 LP	554	}
	555
	556	return 0;
	557	}
	558
dc2c282b LP	559	int pull_job_new(PullJob *ret, const char url, CurlGlue glue, void userdata) {
dc2c282b LP	560	_cleanup_(pull_job_unrefp) PullJob *j = NULL;
56ebfaf1 LP	561
	562	assert(url);
	563	assert(glue);
	564	assert(ret);
	565
dc2c282b	566	j = new0(PullJob, 1);
56ebfaf1 LP	567	if (!j)
	568	return -ENOMEM;
	569
dc2c282b	570	j->state = PULL_JOB_INIT;
56ebfaf1 LP	571	j->disk_fd = -1;
	572	j->userdata = userdata;
	573	j->glue = glue;
	574	j->content_length = (uint64_t) -1;
	575	j->start_usec = now(CLOCK_MONOTONIC);
055c521a	576	j->compressed_max = j->uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU; /* 64GB safety limit */
697be0be	577	j->style = VERIFICATION_STYLE_UNSET;
56ebfaf1 LP	578
	579	j->url = strdup(url);
	580	if (!j->url)
	581	return -ENOMEM;
	582
	583	*ret = j;
	584	j = NULL;
	585
	586	return 0;
	587	}
	588
dc2c282b	589	int pull_job_begin(PullJob *j) {
56ebfaf1 LP	590	int r;
	591
	592	assert(j);
	593
dc2c282b	594	if (j->state != PULL_JOB_INIT)
56ebfaf1 LP	595	return -EBUSY;
56ebfaf1 LP	596
26166c88 LP	597	if (j->grow_machine_directory)
	598	grow_machine_directory();
	599
56ebfaf1 LP	600	r = curl_glue_make(&j->curl, j->url, j);
	601	if (r < 0)
	602	return r;
	603
	604	if (!strv_isempty(j->old_etags)) {
	605	_cleanup_free_ char cc = NULL, hdr = NULL;
	606
	607	cc = strv_join(j->old_etags, ", ");
	608	if (!cc)
	609	return -ENOMEM;
	610
	611	hdr = strappend("If-None-Match: ", cc);
	612	if (!hdr)
	613	return -ENOMEM;
	614
ff2670ad LP	615	if (!j->request_header) {
	616	j->request_header = curl_slist_new(hdr, NULL);
	617	if (!j->request_header)
	618	return -ENOMEM;
	619	} else {
	620	struct curl_slist *l;
	621
	622	l = curl_slist_append(j->request_header, hdr);
	623	if (!l)
	624	return -ENOMEM;
	625
	626	j->request_header = l;
	627	}
	628	}
56ebfaf1	629
ff2670ad	630	if (j->request_header) {
56ebfaf1 LP	631	if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
	632	return -EIO;
	633	}
	634
dc2c282b	635	if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK)
56ebfaf1 LP	636	return -EIO;
	637
	638	if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
	639	return -EIO;
	640
dc2c282b	641	if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK)
56ebfaf1 LP	642	return -EIO;
	643
	644	if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
	645	return -EIO;
	646
dc2c282b	647	if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK)
56ebfaf1 LP	648	return -EIO;
	649
	650	if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK)
	651	return -EIO;
	652
	653	if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
	654	return -EIO;
	655
	656	r = curl_glue_add(j->glue, j->curl);
	657	if (r < 0)
	658	return r;
	659
dc2c282b	660	j->state = PULL_JOB_ANALYZING;
56ebfaf1 LP	661
	662	return 0;
	663	}