[thirdparty/systemd.git] / src / import / pull-job.c

/***
  This file is part of systemd.

  Copyright 2015 Lennart Poettering

  systemd is free software; you can redistribute it and/or modify it
  under the terms of the GNU Lesser General Public License as published by
  the Free Software Foundation; either version 2.1 of the License, or
  (at your option) any later version.

  systemd is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public License
  along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/

#include <sys/xattr.h>

#include "alloc-util.h"
#include "fd-util.h"
#include "hexdecoct.h"
#include "io-util.h"
#include "machine-pool.h"
#include "parse-util.h"
#include "pull-job.h"
#include "string-util.h"
#include "strv.h"
#include "xattr-util.h"
#include "pull-common.h"
#include "import-util.h"

PullJob* pull_job_unref(PullJob *j) {
        if (!j)
                return NULL;

        curl_glue_remove_and_free(j->glue, j->curl);
        curl_slist_free_all(j->request_header);

        safe_close(j->disk_fd);

        import_compress_free(&j->compress);

        if (j->checksum_context)
                gcry_md_close(j->checksum_context);

        free(j->url);
        free(j->etag);
        strv_free(j->old_etags);
        free(j->payload);
        free(j->checksum);

        return mfree(j);
}

static void pull_job_finish(PullJob *j, int ret) {
        assert(j);

        if (j->state == PULL_JOB_DONE ||
            j->state == PULL_JOB_FAILED)
                return;

        if (ret == 0) {
                j->state = PULL_JOB_DONE;
                j->progress_percent = 100;
                log_info("Download of %s complete.", j->url);
        } else {
                j->state = PULL_JOB_FAILED;
                j->error = ret;
        }

        if (j->on_finished)
                j->on_finished(j);
}

static int pull_job_restart(PullJob *j) {
        int r;
        char *chksum_url = NULL;

        r = import_url_change_last_component(j->url, "SHA256SUMS", &chksum_url);
        if (r < 0)
                return r;

        free(j->url);
        free(j->payload);
        j->url = chksum_url;
        j->state = PULL_JOB_INIT;
        j->payload = NULL;
        j->payload_size = 0;
        j->payload_allocated = 0;
        j->written_compressed = 0;
        j->written_uncompressed = 0;
        j->written_since_last_grow = 0;

        r = pull_job_begin(j);
        if (r < 0)
                return r;

        return 0;
}


void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
        PullJob *j = NULL;
        CURLcode code;
        long status;
        int r;

        if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK)
                return;

        if (!j || j->state == PULL_JOB_DONE || j->state == PULL_JOB_FAILED)
                return;

        if (result != CURLE_OK) {
                log_error("Transfer failed: %s", curl_easy_strerror(result));
                r = -EIO;
                goto finish;
        }

        code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
        if (code != CURLE_OK) {
                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
                r = -EIO;
                goto finish;
        } else if (status == 304) {
                log_info("Image already downloaded. Skipping download.");
                j->etag_exists = true;
                r = 0;
                goto finish;
        } else if (status >= 300) {
                if (status == 404 && j->style == VERIFICATION_PER_FILE) {

                        /* retry pull job with SHA256SUMS file */
                        r = pull_job_restart(j);
                        if (r < 0)
                                goto finish;

                        code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
                        if (code != CURLE_OK) {
                                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
                                r = -EIO;
                                goto finish;
                        }

                        if (status == 0) {
                                j->style = VERIFICATION_PER_DIRECTORY;
                                return;
                        }
                }

                log_error("HTTP request to %s failed with code %li.", j->url, status);
                r = -EIO;
                goto finish;
        } else if (status < 200) {
                log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
                r = -EIO;
                goto finish;
        }

        if (j->state != PULL_JOB_RUNNING) {
                log_error("Premature connection termination.");
                r = -EIO;
                goto finish;
        }

        if (j->content_length != (uint64_t) -1 &&
            j->content_length != j->written_compressed) {
                log_error("Download truncated.");
                r = -EIO;
                goto finish;
        }

        if (j->checksum_context) {
                uint8_t *k;

                k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
                if (!k) {
                        log_error("Failed to get checksum.");
                        r = -EIO;
                        goto finish;
                }

                j->checksum = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256));
                if (!j->checksum) {
                        r = log_oom();
                        goto finish;
                }

                log_debug("SHA256 of %s is %s.", j->url, j->checksum);
        }

        if (j->disk_fd >= 0 && j->allow_sparse) {
                /* Make sure the file size is right, in case the file was
                 * sparse and we just seeked for the last part */

                if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
                        r = log_error_errno(errno, "Failed to truncate file: %m");
                        goto finish;
                }

                if (j->etag)
                        (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
                if (j->url)
                        (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);

                if (j->mtime != 0) {
                        struct timespec ut[2];

                        timespec_store(&ut[0], j->mtime);
                        ut[1] = ut[0];
                        (void) futimens(j->disk_fd, ut);

                        (void) fd_setcrtime(j->disk_fd, j->mtime);
                }
        }

        r = 0;

finish:
        pull_job_finish(j, r);
}

static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) {
        PullJob *j = userdata;
        ssize_t n;

        assert(j);
        assert(p);

        if (sz <= 0)
                return 0;

        if (j->written_uncompressed + sz < j->written_uncompressed) {
                log_error("File too large, overflow");
                return -EOVERFLOW;
        }

        if (j->written_uncompressed + sz > j->uncompressed_max) {
                log_error("File overly large, refusing");
                return -EFBIG;
        }

        if (j->disk_fd >= 0) {

                if (j->grow_machine_directory && j->written_since_last_grow >= GROW_INTERVAL_BYTES) {
                        j->written_since_last_grow = 0;
                        grow_machine_directory();
                }

                if (j->allow_sparse)
                        n = sparse_write(j->disk_fd, p, sz, 64);
                else
                        n = write(j->disk_fd, p, sz);
                if (n < 0)
                        return log_error_errno(errno, "Failed to write file: %m");
                if ((size_t) n < sz) {
                        log_error("Short write");
                        return -EIO;
                }
        } else {

                if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz))
                        return log_oom();

                memcpy(j->payload + j->payload_size, p, sz);
                j->payload_size += sz;
        }

        j->written_uncompressed += sz;
        j->written_since_last_grow += sz;

        return 0;
}

static int pull_job_write_compressed(PullJob *j, void *p, size_t sz) {
        int r;

        assert(j);
        assert(p);

        if (sz <= 0)
                return 0;

        if (j->written_compressed + sz < j->written_compressed) {
                log_error("File too large, overflow");
                return -EOVERFLOW;
        }

        if (j->written_compressed + sz > j->compressed_max) {
                log_error("File overly large, refusing.");
                return -EFBIG;
        }

        if (j->content_length != (uint64_t) -1 &&
            j->written_compressed + sz > j->content_length) {
                log_error("Content length incorrect.");
                return -EFBIG;
        }

        if (j->checksum_context)
                gcry_md_write(j->checksum_context, p, sz);

        r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j);
        if (r < 0)
                return r;

        j->written_compressed += sz;

        return 0;
}

static int pull_job_open_disk(PullJob *j) {
        int r;

        assert(j);

        if (j->on_open_disk) {
                r = j->on_open_disk(j);
                if (r < 0)
                        return r;
        }

        if (j->disk_fd >= 0) {
                /* Check if we can do sparse files */

                if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
                        j->allow_sparse = true;
                else {
                        if (errno != ESPIPE)
                                return log_error_errno(errno, "Failed to seek on file descriptor: %m");

                        j->allow_sparse = false;
                }
        }

        if (j->calc_checksum) {
                if (gcry_md_open(&j->checksum_context, GCRY_MD_SHA256, 0) != 0) {
                        log_error("Failed to initialize hash context.");
                        return -EIO;
                }
        }

        return 0;
}

static int pull_job_detect_compression(PullJob *j) {
        _cleanup_free_ uint8_t *stub = NULL;
        size_t stub_size;

        int r;

        assert(j);

        r = import_uncompress_detect(&j->compress, j->payload, j->payload_size);
        if (r < 0)
                return log_error_errno(r, "Failed to initialize compressor: %m");
        if (r == 0)
                return 0;

        log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type));

        r = pull_job_open_disk(j);
        if (r < 0)
                return r;

        /* Now, take the payload we read so far, and decompress it */
        stub = j->payload;
        stub_size = j->payload_size;

        j->payload = NULL;
        j->payload_size = 0;
        j->payload_allocated = 0;

        j->state = PULL_JOB_RUNNING;

        r = pull_job_write_compressed(j, stub, stub_size);
        if (r < 0)
                return r;

        return 0;
}

static size_t pull_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
        PullJob *j = userdata;
        size_t sz = size * nmemb;
        int r;

        assert(contents);
        assert(j);

        switch (j->state) {

        case PULL_JOB_ANALYZING:
                /* Let's first check what it actually is */

                if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) {
                        r = log_oom();
                        goto fail;
                }

                memcpy(j->payload + j->payload_size, contents, sz);
                j->payload_size += sz;

                r = pull_job_detect_compression(j);
                if (r < 0)
                        goto fail;

                break;

        case PULL_JOB_RUNNING:

                r = pull_job_write_compressed(j, contents, sz);
                if (r < 0)
                        goto fail;

                break;

        case PULL_JOB_DONE:
        case PULL_JOB_FAILED:
                r = -ESTALE;
                goto fail;

        default:
                assert_not_reached("Impossible state.");
        }

        return sz;

fail:
        pull_job_finish(j, r);
        return 0;
}

static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
        PullJob *j = userdata;
        size_t sz = size * nmemb;
        _cleanup_free_ char *length = NULL, *last_modified = NULL;
        char *etag;
        int r;

        assert(contents);
        assert(j);

        if (j->state == PULL_JOB_DONE || j->state == PULL_JOB_FAILED) {
                r = -ESTALE;
                goto fail;
        }

        assert(j->state == PULL_JOB_ANALYZING);

        r = curl_header_strdup(contents, sz, "ETag:", &etag);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                free(j->etag);
                j->etag = etag;

                if (strv_contains(j->old_etags, j->etag)) {
                        log_info("Image already downloaded. Skipping download.");
                        j->etag_exists = true;
                        pull_job_finish(j, 0);
                        return sz;
                }

                return sz;
        }

        r = curl_header_strdup(contents, sz, "Content-Length:", &length);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                (void) safe_atou64(length, &j->content_length);

                if (j->content_length != (uint64_t) -1) {
                        char bytes[FORMAT_BYTES_MAX];

                        if (j->content_length > j->compressed_max) {
                                log_error("Content too large.");
                                r = -EFBIG;
                                goto fail;
                        }

                        log_info("Downloading %s for %s.", format_bytes(bytes, sizeof(bytes), j->content_length), j->url);
                }

                return sz;
        }

        r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                (void) curl_parse_http_time(last_modified, &j->mtime);
                return sz;
        }

        if (j->on_header) {
                r = j->on_header(j, contents, sz);
                if (r < 0)
                        goto fail;
        }

        return sz;

fail:
        pull_job_finish(j, r);
        return 0;
}

static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
        PullJob *j = userdata;
        unsigned percent;
        usec_t n;

        assert(j);

        if (dltotal <= 0)
                return 0;

        percent = ((100 * dlnow) / dltotal);
        n = now(CLOCK_MONOTONIC);

        if (n > j->last_status_usec + USEC_PER_SEC &&
            percent != j->progress_percent &&
            dlnow < dltotal) {
                char buf[FORMAT_TIMESPAN_MAX];

                if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) {
                        char y[FORMAT_BYTES_MAX];
                        usec_t left, done;

                        done = n - j->start_usec;
                        left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;

                        log_info("Got %u%% of %s. %s left at %s/s.",
                                 percent,
                                 j->url,
                                 format_timespan(buf, sizeof(buf), left, USEC_PER_SEC),
                                 format_bytes(y, sizeof(y), (uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC))));
                } else
                        log_info("Got %u%% of %s.", percent, j->url);

                j->progress_percent = percent;
                j->last_status_usec = n;

                if (j->on_progress)
                        j->on_progress(j);
        }

        return 0;
}

int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) {
        _cleanup_(pull_job_unrefp) PullJob *j = NULL;

        assert(url);
        assert(glue);
        assert(ret);

        j = new0(PullJob, 1);
        if (!j)
                return -ENOMEM;

        j->state = PULL_JOB_INIT;
        j->disk_fd = -1;
        j->userdata = userdata;
        j->glue = glue;
        j->content_length = (uint64_t) -1;
        j->start_usec = now(CLOCK_MONOTONIC);
        j->compressed_max = j->uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU; /* 64GB safety limit */
        j->style = VERIFICATION_STYLE_UNSET;

        j->url = strdup(url);
        if (!j->url)
                return -ENOMEM;

        *ret = j;
        j = NULL;

        return 0;
}

int pull_job_begin(PullJob *j) {
        int r;

        assert(j);

        if (j->state != PULL_JOB_INIT)
                return -EBUSY;

        if (j->grow_machine_directory)
                grow_machine_directory();

        r = curl_glue_make(&j->curl, j->url, j);
        if (r < 0)
                return r;

        if (!strv_isempty(j->old_etags)) {
                _cleanup_free_ char *cc = NULL, *hdr = NULL;

                cc = strv_join(j->old_etags, ", ");
                if (!cc)
                        return -ENOMEM;

                hdr = strappend("If-None-Match: ", cc);
                if (!hdr)
                        return -ENOMEM;

                if (!j->request_header) {
                        j->request_header = curl_slist_new(hdr, NULL);
                        if (!j->request_header)
                                return -ENOMEM;
                } else {
                        struct curl_slist *l;

                        l = curl_slist_append(j->request_header, hdr);
                        if (!l)
                                return -ENOMEM;

                        j->request_header = l;
                }
        }

        if (j->request_header) {
                if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
                        return -EIO;
        }

        if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
                return -EIO;

        r = curl_glue_add(j->glue, j->curl);
        if (r < 0)
                return r;

        j->state = PULL_JOB_ANALYZING;

        return 0;
}
Commit	Line	Data
56ebfaf1 LP	1	/***
	2	This file is part of systemd.
	3
	4	Copyright 2015 Lennart Poettering
	5
	6	systemd is free software; you can redistribute it and/or modify it
	7	under the terms of the GNU Lesser General Public License as published by
	8	the Free Software Foundation; either version 2.1 of the License, or
	9	(at your option) any later version.
	10
	11	systemd is distributed in the hope that it will be useful, but
	12	WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	Lesser General Public License for more details.
	15
	16	You should have received a copy of the GNU Lesser General Public License
	17	along with systemd; If not, see <http://www.gnu.org/licenses/>.
	18	***/
	19
	20	#include <sys/xattr.h>
	21
b5efdb8a	22	#include "alloc-util.h"
3ffd4af2	23	#include "fd-util.h"
89a5a90c	24	#include "hexdecoct.h"
c004493c	25	#include "io-util.h"
26166c88	26	#include "machine-pool.h"
6bedfcbb	27	#include "parse-util.h"
3ffd4af2	28	#include "pull-job.h"
07630cea LP	29	#include "string-util.h"
07630cea LP	30	#include "strv.h"
89a5a90c	31	#include "xattr-util.h"
697be0be TB	32	#include "pull-common.h"
697be0be TB	33	#include "import-util.h"
56ebfaf1	34
dc2c282b	35	PullJob* pull_job_unref(PullJob *j) {
56ebfaf1 LP	36	if (!j)
	37	return NULL;
	38
	39	curl_glue_remove_and_free(j->glue, j->curl);
	40	curl_slist_free_all(j->request_header);
	41
	42	safe_close(j->disk_fd);
	43
3e2cda69	44	import_compress_free(&j->compress);
56ebfaf1	45
98c38001 LP	46	if (j->checksum_context)
98c38001 LP	47	gcry_md_close(j->checksum_context);
85dbc41d	48
56ebfaf1 LP	49	free(j->url);
	50	free(j->etag);
	51	strv_free(j->old_etags);
	52	free(j->payload);
98c38001	53	free(j->checksum);
56ebfaf1	54
6b430fdb	55	return mfree(j);
56ebfaf1 LP	56	}
56ebfaf1 LP	57
dc2c282b	58	static void pull_job_finish(PullJob *j, int ret) {
56ebfaf1 LP	59	assert(j);
56ebfaf1 LP	60
dc2c282b LP	61	if (j->state == PULL_JOB_DONE \|\|
dc2c282b LP	62	j->state == PULL_JOB_FAILED)
56ebfaf1 LP	63	return;
56ebfaf1 LP	64
68c913fd	65	if (ret == 0) {
dc2c282b	66	j->state = PULL_JOB_DONE;
7079cfef	67	j->progress_percent = 100;
68c913fd LP	68	log_info("Download of %s complete.", j->url);
68c913fd LP	69	} else {
dc2c282b	70	j->state = PULL_JOB_FAILED;
56ebfaf1 LP	71	j->error = ret;
	72	}
	73
	74	if (j->on_finished)
	75	j->on_finished(j);
	76	}
	77
697be0be TB	78	static int pull_job_restart(PullJob *j) {
	79	int r;
	80	char *chksum_url = NULL;
	81
	82	r = import_url_change_last_component(j->url, "SHA256SUMS", &chksum_url);
	83	if (r < 0)
	84	return r;
	85
	86	free(j->url);
	87	free(j->payload);
	88	j->url = chksum_url;
	89	j->state = PULL_JOB_INIT;
	90	j->payload = NULL;
	91	j->payload_size = 0;
	92	j->payload_allocated = 0;
	93	j->written_compressed = 0;
	94	j->written_uncompressed = 0;
	95	j->written_since_last_grow = 0;
	96
	97	r = pull_job_begin(j);
	98	if (r < 0)
	99	return r;
	100
	101	return 0;
	102	}
	103
	104
dc2c282b LP	105	void pull_job_curl_on_finished(CurlGlue g, CURL curl, CURLcode result) {
dc2c282b LP	106	PullJob *j = NULL;
56ebfaf1 LP	107	CURLcode code;
	108	long status;
	109	int r;
	110
a7f7d1bd	111	if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK)
56ebfaf1 LP	112	return;
56ebfaf1 LP	113
dc2c282b	114	if (!j \|\| j->state == PULL_JOB_DONE \|\| j->state == PULL_JOB_FAILED)
56ebfaf1 LP	115	return;
	116
	117	if (result != CURLE_OK) {
	118	log_error("Transfer failed: %s", curl_easy_strerror(result));
	119	r = -EIO;
	120	goto finish;
	121	}
	122
	123	code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
	124	if (code != CURLE_OK) {
	125	log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
	126	r = -EIO;
	127	goto finish;
	128	} else if (status == 304) {
	129	log_info("Image already downloaded. Skipping download.");
85dbc41d	130	j->etag_exists = true;
56ebfaf1 LP	131	r = 0;
	132	goto finish;
	133	} else if (status >= 300) {
697be0be TB	134	if (status == 404 && j->style == VERIFICATION_PER_FILE) {
	135
	136	/* retry pull job with SHA256SUMS file */
	137	r = pull_job_restart(j);
	138	if (r < 0)
	139	goto finish;
	140
	141	code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
	142	if (code != CURLE_OK) {
	143	log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
	144	r = -EIO;
	145	goto finish;
	146	}
	147
	148	if (status == 0) {
	149	j->style = VERIFICATION_PER_DIRECTORY;
	150	return;
	151	}
	152	}
	153
56ebfaf1 LP	154	log_error("HTTP request to %s failed with code %li.", j->url, status);
	155	r = -EIO;
	156	goto finish;
	157	} else if (status < 200) {
	158	log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
	159	r = -EIO;
	160	goto finish;
	161	}
	162
dc2c282b	163	if (j->state != PULL_JOB_RUNNING) {
56ebfaf1 LP	164	log_error("Premature connection termination.");
	165	r = -EIO;
	166	goto finish;
	167	}
	168
	169	if (j->content_length != (uint64_t) -1 &&
	170	j->content_length != j->written_compressed) {
	171	log_error("Download truncated.");
	172	r = -EIO;
	173	goto finish;
	174	}
	175
98c38001	176	if (j->checksum_context) {
85dbc41d LP	177	uint8_t *k;
85dbc41d LP	178
98c38001	179	k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
85dbc41d LP	180	if (!k) {
	181	log_error("Failed to get checksum.");
	182	r = -EIO;
	183	goto finish;
	184	}
	185
98c38001 LP	186	j->checksum = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256));
98c38001 LP	187	if (!j->checksum) {
85dbc41d LP	188	r = log_oom();
	189	goto finish;
	190	}
	191
98c38001	192	log_debug("SHA256 of %s is %s.", j->url, j->checksum);
85dbc41d LP	193	}
85dbc41d LP	194
56ebfaf1 LP	195	if (j->disk_fd >= 0 && j->allow_sparse) {
	196	/* Make sure the file size is right, in case the file was
	197	* sparse and we just seeked for the last part */
	198
	199	if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
b6e676ce	200	r = log_error_errno(errno, "Failed to truncate file: %m");
56ebfaf1 LP	201	goto finish;
	202	}
	203
	204	if (j->etag)
	205	(void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
	206	if (j->url)
	207	(void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
	208
	209	if (j->mtime != 0) {
	210	struct timespec ut[2];
	211
	212	timespec_store(&ut[0], j->mtime);
	213	ut[1] = ut[0];
	214	(void) futimens(j->disk_fd, ut);
	215
	216	(void) fd_setcrtime(j->disk_fd, j->mtime);
	217	}
	218	}
	219
	220	r = 0;
	221
	222	finish:
dc2c282b	223	pull_job_finish(j, r);
56ebfaf1 LP	224	}
56ebfaf1 LP	225
3e2cda69 LP	226	static int pull_job_write_uncompressed(const void p, size_t sz, void userdata) {
3e2cda69 LP	227	PullJob *j = userdata;
56ebfaf1 LP	228	ssize_t n;
	229
	230	assert(j);
	231	assert(p);
8af3cf74 LP	232
	233	if (sz <= 0)
	234	return 0;
56ebfaf1 LP	235
	236	if (j->written_uncompressed + sz < j->written_uncompressed) {
	237	log_error("File too large, overflow");
	238	return -EOVERFLOW;
	239	}
	240
	241	if (j->written_uncompressed + sz > j->uncompressed_max) {
	242	log_error("File overly large, refusing");
	243	return -EFBIG;
	244	}
	245
	246	if (j->disk_fd >= 0) {
	247
b6e676ce	248	if (j->grow_machine_directory && j->written_since_last_grow >= GROW_INTERVAL_BYTES) {
26166c88 LP	249	j->written_since_last_grow = 0;
	250	grow_machine_directory();
	251	}
	252
56ebfaf1 LP	253	if (j->allow_sparse)
	254	n = sparse_write(j->disk_fd, p, sz, 64);
	255	else
	256	n = write(j->disk_fd, p, sz);
b6e676ce LP	257	if (n < 0)
b6e676ce LP	258	return log_error_errno(errno, "Failed to write file: %m");
56ebfaf1 LP	259	if ((size_t) n < sz) {
	260	log_error("Short write");
	261	return -EIO;
	262	}
	263	} else {
	264
	265	if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz))
	266	return log_oom();
	267
8af3cf74	268	memcpy(j->payload + j->payload_size, p, sz);
56ebfaf1 LP	269	j->payload_size += sz;
	270	}
	271
	272	j->written_uncompressed += sz;
26166c88	273	j->written_since_last_grow += sz;
56ebfaf1 LP	274
	275	return 0;
	276	}
	277
dc2c282b	278	static int pull_job_write_compressed(PullJob j, void p, size_t sz) {
56ebfaf1 LP	279	int r;
	280
	281	assert(j);
	282	assert(p);
8af3cf74 LP	283
	284	if (sz <= 0)
	285	return 0;
56ebfaf1 LP	286
	287	if (j->written_compressed + sz < j->written_compressed) {
	288	log_error("File too large, overflow");
	289	return -EOVERFLOW;
	290	}
	291
	292	if (j->written_compressed + sz > j->compressed_max) {
	293	log_error("File overly large, refusing.");
	294	return -EFBIG;
	295	}
	296
	297	if (j->content_length != (uint64_t) -1 &&
	298	j->written_compressed + sz > j->content_length) {
	299	log_error("Content length incorrect.");
	300	return -EFBIG;
	301	}
	302
98c38001 LP	303	if (j->checksum_context)
98c38001 LP	304	gcry_md_write(j->checksum_context, p, sz);
85dbc41d	305
3e2cda69 LP	306	r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j);
	307	if (r < 0)
	308	return r;
56ebfaf1 LP	309
	310	j->written_compressed += sz;
	311
	312	return 0;
	313	}
	314
dc2c282b	315	static int pull_job_open_disk(PullJob *j) {
56ebfaf1 LP	316	int r;
	317
	318	assert(j);
	319
	320	if (j->on_open_disk) {
	321	r = j->on_open_disk(j);
	322	if (r < 0)
	323	return r;
	324	}
	325
	326	if (j->disk_fd >= 0) {
	327	/* Check if we can do sparse files */
	328
	329	if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
	330	j->allow_sparse = true;
	331	else {
	332	if (errno != ESPIPE)
	333	return log_error_errno(errno, "Failed to seek on file descriptor: %m");
	334
	335	j->allow_sparse = false;
	336	}
	337	}
	338
98c38001 LP	339	if (j->calc_checksum) {
98c38001 LP	340	if (gcry_md_open(&j->checksum_context, GCRY_MD_SHA256, 0) != 0) {
85dbc41d LP	341	log_error("Failed to initialize hash context.");
	342	return -EIO;
	343	}
	344	}
	345
56ebfaf1 LP	346	return 0;
	347	}
	348
dc2c282b	349	static int pull_job_detect_compression(PullJob *j) {
56ebfaf1 LP	350	_cleanup_free_ uint8_t *stub = NULL;
	351	size_t stub_size;
	352
	353	int r;
	354
	355	assert(j);
	356
3e2cda69 LP	357	r = import_uncompress_detect(&j->compress, j->payload, j->payload_size);
	358	if (r < 0)
	359	return log_error_errno(r, "Failed to initialize compressor: %m");
	360	if (r == 0)
56ebfaf1 LP	361	return 0;
56ebfaf1 LP	362
3e2cda69	363	log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type));
56ebfaf1	364
dc2c282b	365	r = pull_job_open_disk(j);
56ebfaf1 LP	366	if (r < 0)
	367	return r;
	368
	369	/* Now, take the payload we read so far, and decompress it */
	370	stub = j->payload;
	371	stub_size = j->payload_size;
	372
	373	j->payload = NULL;
	374	j->payload_size = 0;
88a1aadc	375	j->payload_allocated = 0;
56ebfaf1	376
dc2c282b	377	j->state = PULL_JOB_RUNNING;
56ebfaf1	378
dc2c282b	379	r = pull_job_write_compressed(j, stub, stub_size);
56ebfaf1 LP	380	if (r < 0)
	381	return r;
	382
	383	return 0;
	384	}
	385
dc2c282b LP	386	static size_t pull_job_write_callback(void contents, size_t size, size_t nmemb, void userdata) {
dc2c282b LP	387	PullJob *j = userdata;
56ebfaf1 LP	388	size_t sz = size * nmemb;
	389	int r;
	390
	391	assert(contents);
	392	assert(j);
	393
	394	switch (j->state) {
	395
dc2c282b	396	case PULL_JOB_ANALYZING:
56ebfaf1 LP	397	/* Let's first check what it actually is */
	398
	399	if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) {
	400	r = log_oom();
	401	goto fail;
	402	}
	403
8af3cf74	404	memcpy(j->payload + j->payload_size, contents, sz);
56ebfaf1 LP	405	j->payload_size += sz;
56ebfaf1 LP	406
dc2c282b	407	r = pull_job_detect_compression(j);
56ebfaf1 LP	408	if (r < 0)
	409	goto fail;
	410
	411	break;
	412
dc2c282b	413	case PULL_JOB_RUNNING:
56ebfaf1	414
dc2c282b	415	r = pull_job_write_compressed(j, contents, sz);
56ebfaf1 LP	416	if (r < 0)
	417	goto fail;
	418
	419	break;
	420
dc2c282b LP	421	case PULL_JOB_DONE:
dc2c282b LP	422	case PULL_JOB_FAILED:
56ebfaf1 LP	423	r = -ESTALE;
	424	goto fail;
	425
	426	default:
	427	assert_not_reached("Impossible state.");
	428	}
	429
	430	return sz;
	431
	432	fail:
dc2c282b	433	pull_job_finish(j, r);
56ebfaf1 LP	434	return 0;
	435	}
	436
dc2c282b LP	437	static size_t pull_job_header_callback(void contents, size_t size, size_t nmemb, void userdata) {
dc2c282b LP	438	PullJob *j = userdata;
56ebfaf1 LP	439	size_t sz = size * nmemb;
	440	_cleanup_free_ char length = NULL, last_modified = NULL;
	441	char *etag;
	442	int r;
	443
	444	assert(contents);
	445	assert(j);
	446
dc2c282b	447	if (j->state == PULL_JOB_DONE \|\| j->state == PULL_JOB_FAILED) {
56ebfaf1 LP	448	r = -ESTALE;
	449	goto fail;
	450	}
	451
dc2c282b	452	assert(j->state == PULL_JOB_ANALYZING);
56ebfaf1 LP	453
	454	r = curl_header_strdup(contents, sz, "ETag:", &etag);
	455	if (r < 0) {
	456	log_oom();
	457	goto fail;
	458	}
	459	if (r > 0) {
	460	free(j->etag);
	461	j->etag = etag;
	462
	463	if (strv_contains(j->old_etags, j->etag)) {
	464	log_info("Image already downloaded. Skipping download.");
85dbc41d	465	j->etag_exists = true;
dc2c282b	466	pull_job_finish(j, 0);
56ebfaf1 LP	467	return sz;
	468	}
	469
	470	return sz;
	471	}
	472
	473	r = curl_header_strdup(contents, sz, "Content-Length:", &length);
	474	if (r < 0) {
	475	log_oom();
	476	goto fail;
	477	}
	478	if (r > 0) {
	479	(void) safe_atou64(length, &j->content_length);
	480
	481	if (j->content_length != (uint64_t) -1) {
	482	char bytes[FORMAT_BYTES_MAX];
	483
	484	if (j->content_length > j->compressed_max) {
	485	log_error("Content too large.");
	486	r = -EFBIG;
	487	goto fail;
	488	}
	489
68c913fd	490	log_info("Downloading %s for %s.", format_bytes(bytes, sizeof(bytes), j->content_length), j->url);
56ebfaf1 LP	491	}
	492
	493	return sz;
	494	}
	495
	496	r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
	497	if (r < 0) {
	498	log_oom();
	499	goto fail;
	500	}
	501	if (r > 0) {
	502	(void) curl_parse_http_time(last_modified, &j->mtime);
	503	return sz;
	504	}
	505
ff2670ad LP	506	if (j->on_header) {
	507	r = j->on_header(j, contents, sz);
	508	if (r < 0)
	509	goto fail;
	510	}
	511
56ebfaf1 LP	512	return sz;
	513
	514	fail:
dc2c282b	515	pull_job_finish(j, r);
56ebfaf1 LP	516	return 0;
	517	}
	518
dc2c282b LP	519	static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
dc2c282b LP	520	PullJob *j = userdata;
56ebfaf1 LP	521	unsigned percent;
	522	usec_t n;
	523
	524	assert(j);
	525
	526	if (dltotal <= 0)
	527	return 0;
	528
	529	percent = ((100 * dlnow) / dltotal);
	530	n = now(CLOCK_MONOTONIC);
	531
	532	if (n > j->last_status_usec + USEC_PER_SEC &&
68c913fd LP	533	percent != j->progress_percent &&
68c913fd LP	534	dlnow < dltotal) {
56ebfaf1 LP	535	char buf[FORMAT_TIMESPAN_MAX];
	536
	537	if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) {
90bc083b	538	char y[FORMAT_BYTES_MAX];
56ebfaf1 LP	539	usec_t left, done;
	540
	541	done = n - j->start_usec;
	542	left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
	543
90bc083b LP	544	log_info("Got %u%% of %s. %s left at %s/s.",
	545	percent,
	546	j->url,
	547	format_timespan(buf, sizeof(buf), left, USEC_PER_SEC),
	548	format_bytes(y, sizeof(y), (uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC))));
56ebfaf1 LP	549	} else
	550	log_info("Got %u%% of %s.", percent, j->url);
	551
	552	j->progress_percent = percent;
	553	j->last_status_usec = n;
7079cfef LP	554
	555	if (j->on_progress)
	556	j->on_progress(j);
56ebfaf1 LP	557	}
	558
	559	return 0;
	560	}
	561
dc2c282b LP	562	int pull_job_new(PullJob *ret, const char url, CurlGlue glue, void userdata) {
dc2c282b LP	563	_cleanup_(pull_job_unrefp) PullJob *j = NULL;
56ebfaf1 LP	564
	565	assert(url);
	566	assert(glue);
	567	assert(ret);
	568
dc2c282b	569	j = new0(PullJob, 1);
56ebfaf1 LP	570	if (!j)
	571	return -ENOMEM;
	572
dc2c282b	573	j->state = PULL_JOB_INIT;
56ebfaf1 LP	574	j->disk_fd = -1;
	575	j->userdata = userdata;
	576	j->glue = glue;
	577	j->content_length = (uint64_t) -1;
	578	j->start_usec = now(CLOCK_MONOTONIC);
055c521a	579	j->compressed_max = j->uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU; /* 64GB safety limit */
697be0be	580	j->style = VERIFICATION_STYLE_UNSET;
56ebfaf1 LP	581
	582	j->url = strdup(url);
	583	if (!j->url)
	584	return -ENOMEM;
	585
	586	*ret = j;
	587	j = NULL;
	588
	589	return 0;
	590	}
	591
dc2c282b	592	int pull_job_begin(PullJob *j) {
56ebfaf1 LP	593	int r;
	594
	595	assert(j);
	596
dc2c282b	597	if (j->state != PULL_JOB_INIT)
56ebfaf1 LP	598	return -EBUSY;
56ebfaf1 LP	599
26166c88 LP	600	if (j->grow_machine_directory)
	601	grow_machine_directory();
	602
56ebfaf1 LP	603	r = curl_glue_make(&j->curl, j->url, j);
	604	if (r < 0)
	605	return r;
	606
	607	if (!strv_isempty(j->old_etags)) {
	608	_cleanup_free_ char cc = NULL, hdr = NULL;
	609
	610	cc = strv_join(j->old_etags, ", ");
	611	if (!cc)
	612	return -ENOMEM;
	613
	614	hdr = strappend("If-None-Match: ", cc);
	615	if (!hdr)
	616	return -ENOMEM;
	617
ff2670ad LP	618	if (!j->request_header) {
	619	j->request_header = curl_slist_new(hdr, NULL);
	620	if (!j->request_header)
	621	return -ENOMEM;
	622	} else {
	623	struct curl_slist *l;
	624
	625	l = curl_slist_append(j->request_header, hdr);
	626	if (!l)
	627	return -ENOMEM;
	628
	629	j->request_header = l;
	630	}
	631	}
56ebfaf1	632
ff2670ad	633	if (j->request_header) {
56ebfaf1 LP	634	if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
	635	return -EIO;
	636	}
	637
dc2c282b	638	if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK)
56ebfaf1 LP	639	return -EIO;
	640
	641	if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
	642	return -EIO;
	643
dc2c282b	644	if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK)
56ebfaf1 LP	645	return -EIO;
	646
	647	if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
	648	return -EIO;
	649
dc2c282b	650	if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK)
56ebfaf1 LP	651	return -EIO;
	652
	653	if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK)
	654	return -EIO;
	655
	656	if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
	657	return -EIO;
	658
	659	r = curl_glue_add(j->glue, j->curl);
	660	if (r < 0)
	661	return r;
	662
dc2c282b	663	j->state = PULL_JOB_ANALYZING;
56ebfaf1 LP	664
	665	return 0;
	666	}