[thirdparty/systemd.git] / src / import / pull-job.c

/* SPDX-License-Identifier: LGPL-2.1+ */

#include <sys/xattr.h>

#include "alloc-util.h"
#include "fd-util.h"
#include "gcrypt-util.h"
#include "hexdecoct.h"
#include "import-util.h"
#include "io-util.h"
#include "machine-pool.h"
#include "parse-util.h"
#include "pull-common.h"
#include "pull-job.h"
#include "string-util.h"
#include "strv.h"
#include "xattr-util.h"

PullJob* pull_job_unref(PullJob *j) {
        if (!j)
                return NULL;

        curl_glue_remove_and_free(j->glue, j->curl);
        curl_slist_free_all(j->request_header);

        safe_close(j->disk_fd);

        import_compress_free(&j->compress);

        if (j->checksum_context)
                gcry_md_close(j->checksum_context);

        free(j->url);
        free(j->etag);
        strv_free(j->old_etags);
        free(j->payload);
        free(j->checksum);

        return mfree(j);
}

static void pull_job_finish(PullJob *j, int ret) {
        assert(j);

        if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
                return;

        if (ret == 0) {
                j->state = PULL_JOB_DONE;
                j->progress_percent = 100;
                log_info("Download of %s complete.", j->url);
        } else {
                j->state = PULL_JOB_FAILED;
                j->error = ret;
        }

        if (j->on_finished)
                j->on_finished(j);
}

static int pull_job_restart(PullJob *j) {
        int r;
        char *chksum_url = NULL;

        r = import_url_change_last_component(j->url, "SHA256SUMS", &chksum_url);
        if (r < 0)
                return r;

        free(j->url);
        j->url = chksum_url;
        j->state = PULL_JOB_INIT;
        j->payload = mfree(j->payload);
        j->payload_size = 0;
        j->payload_allocated = 0;
        j->written_compressed = 0;
        j->written_uncompressed = 0;

        r = pull_job_begin(j);
        if (r < 0)
                return r;

        return 0;
}

void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
        PullJob *j = NULL;
        CURLcode code;
        long status;
        int r;

        if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK)
                return;

        if (!j || IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
                return;

        if (result != CURLE_OK) {
                log_error("Transfer failed: %s", curl_easy_strerror(result));
                r = -EIO;
                goto finish;
        }

        code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
        if (code != CURLE_OK) {
                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
                r = -EIO;
                goto finish;
        } else if (status == 304) {
                log_info("Image already downloaded. Skipping download.");
                j->etag_exists = true;
                r = 0;
                goto finish;
        } else if (status >= 300) {
                if (status == 404 && j->style == VERIFICATION_PER_FILE) {

                        /* retry pull job with SHA256SUMS file */
                        r = pull_job_restart(j);
                        if (r < 0)
                                goto finish;

                        code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
                        if (code != CURLE_OK) {
                                log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
                                r = -EIO;
                                goto finish;
                        }

                        if (status == 0) {
                                j->style = VERIFICATION_PER_DIRECTORY;
                                return;
                        }
                }

                log_error("HTTP request to %s failed with code %li.", j->url, status);
                r = -EIO;
                goto finish;
        } else if (status < 200) {
                log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
                r = -EIO;
                goto finish;
        }

        if (j->state != PULL_JOB_RUNNING) {
                log_error("Premature connection termination.");
                r = -EIO;
                goto finish;
        }

        if (j->content_length != (uint64_t) -1 &&
            j->content_length != j->written_compressed) {
                log_error("Download truncated.");
                r = -EIO;
                goto finish;
        }

        if (j->checksum_context) {
                uint8_t *k;

                k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
                if (!k) {
                        log_error("Failed to get checksum.");
                        r = -EIO;
                        goto finish;
                }

                j->checksum = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256));
                if (!j->checksum) {
                        r = log_oom();
                        goto finish;
                }

                log_debug("SHA256 of %s is %s.", j->url, j->checksum);
        }

        if (j->disk_fd >= 0 && j->allow_sparse) {
                /* Make sure the file size is right, in case the file was
                 * sparse and we just seeked for the last part */

                if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
                        r = log_error_errno(errno, "Failed to truncate file: %m");
                        goto finish;
                }

                if (j->etag)
                        (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
                if (j->url)
                        (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);

                if (j->mtime != 0) {
                        struct timespec ut[2];

                        timespec_store(&ut[0], j->mtime);
                        ut[1] = ut[0];
                        (void) futimens(j->disk_fd, ut);

                        (void) fd_setcrtime(j->disk_fd, j->mtime);
                }
        }

        r = 0;

finish:
        pull_job_finish(j, r);
}

static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) {
        PullJob *j = userdata;
        ssize_t n;

        assert(j);
        assert(p);

        if (sz <= 0)
                return 0;

        if (j->written_uncompressed + sz < j->written_uncompressed)
                return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW),
                                       "File too large, overflow");

        if (j->written_uncompressed + sz > j->uncompressed_max)
                return log_error_errno(SYNTHETIC_ERRNO(EFBIG),
                                       "File overly large, refusing");

        if (j->disk_fd >= 0) {

                if (j->allow_sparse)
                        n = sparse_write(j->disk_fd, p, sz, 64);
                else {
                        n = write(j->disk_fd, p, sz);
                        if (n < 0)
                                n = -errno;
                }
                if (n < 0)
                        return log_error_errno((int) n, "Failed to write file: %m");
                if ((size_t) n < sz)
                        return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
        } else {

                if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz))
                        return log_oom();

                memcpy(j->payload + j->payload_size, p, sz);
                j->payload_size += sz;
        }

        j->written_uncompressed += sz;

        return 0;
}

static int pull_job_write_compressed(PullJob *j, void *p, size_t sz) {
        int r;

        assert(j);
        assert(p);

        if (sz <= 0)
                return 0;

        if (j->written_compressed + sz < j->written_compressed)
                return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");

        if (j->written_compressed + sz > j->compressed_max)
                return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing.");

        if (j->content_length != (uint64_t) -1 &&
            j->written_compressed + sz > j->content_length)
                return log_error_errno(SYNTHETIC_ERRNO(EFBIG),
                                       "Content length incorrect.");

        if (j->checksum_context)
                gcry_md_write(j->checksum_context, p, sz);

        r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j);
        if (r < 0)
                return r;

        j->written_compressed += sz;

        return 0;
}

static int pull_job_open_disk(PullJob *j) {
        int r;

        assert(j);

        if (j->on_open_disk) {
                r = j->on_open_disk(j);
                if (r < 0)
                        return r;
        }

        if (j->disk_fd >= 0) {
                /* Check if we can do sparse files */

                if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
                        j->allow_sparse = true;
                else {
                        if (errno != ESPIPE)
                                return log_error_errno(errno, "Failed to seek on file descriptor: %m");

                        j->allow_sparse = false;
                }
        }

        if (j->calc_checksum) {
                initialize_libgcrypt(false);

                if (gcry_md_open(&j->checksum_context, GCRY_MD_SHA256, 0) != 0)
                        return log_error_errno(SYNTHETIC_ERRNO(EIO),
                                               "Failed to initialize hash context.");
        }

        return 0;
}

static int pull_job_detect_compression(PullJob *j) {
        _cleanup_free_ uint8_t *stub = NULL;
        size_t stub_size;

        int r;

        assert(j);

        r = import_uncompress_detect(&j->compress, j->payload, j->payload_size);
        if (r < 0)
                return log_error_errno(r, "Failed to initialize compressor: %m");
        if (r == 0)
                return 0;

        log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type));

        r = pull_job_open_disk(j);
        if (r < 0)
                return r;

        /* Now, take the payload we read so far, and decompress it */
        stub = j->payload;
        stub_size = j->payload_size;

        j->payload = NULL;
        j->payload_size = 0;
        j->payload_allocated = 0;

        j->state = PULL_JOB_RUNNING;

        r = pull_job_write_compressed(j, stub, stub_size);
        if (r < 0)
                return r;

        return 0;
}

static size_t pull_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
        PullJob *j = userdata;
        size_t sz = size * nmemb;
        int r;

        assert(contents);
        assert(j);

        switch (j->state) {

        case PULL_JOB_ANALYZING:
                /* Let's first check what it actually is */

                if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) {
                        r = log_oom();
                        goto fail;
                }

                memcpy(j->payload + j->payload_size, contents, sz);
                j->payload_size += sz;

                r = pull_job_detect_compression(j);
                if (r < 0)
                        goto fail;

                break;

        case PULL_JOB_RUNNING:

                r = pull_job_write_compressed(j, contents, sz);
                if (r < 0)
                        goto fail;

                break;

        case PULL_JOB_DONE:
        case PULL_JOB_FAILED:
                r = -ESTALE;
                goto fail;

        default:
                assert_not_reached("Impossible state.");
        }

        return sz;

fail:
        pull_job_finish(j, r);
        return 0;
}

static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
        PullJob *j = userdata;
        size_t sz = size * nmemb;
        _cleanup_free_ char *length = NULL, *last_modified = NULL;
        char *etag;
        int r;

        assert(contents);
        assert(j);

        if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) {
                r = -ESTALE;
                goto fail;
        }

        assert(j->state == PULL_JOB_ANALYZING);

        r = curl_header_strdup(contents, sz, "ETag:", &etag);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                free(j->etag);
                j->etag = etag;

                if (strv_contains(j->old_etags, j->etag)) {
                        log_info("Image already downloaded. Skipping download.");
                        j->etag_exists = true;
                        pull_job_finish(j, 0);
                        return sz;
                }

                return sz;
        }

        r = curl_header_strdup(contents, sz, "Content-Length:", &length);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                (void) safe_atou64(length, &j->content_length);

                if (j->content_length != (uint64_t) -1) {
                        char bytes[FORMAT_BYTES_MAX];

                        if (j->content_length > j->compressed_max) {
                                log_error("Content too large.");
                                r = -EFBIG;
                                goto fail;
                        }

                        log_info("Downloading %s for %s.", format_bytes(bytes, sizeof(bytes), j->content_length), j->url);
                }

                return sz;
        }

        r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
        if (r < 0) {
                log_oom();
                goto fail;
        }
        if (r > 0) {
                (void) curl_parse_http_time(last_modified, &j->mtime);
                return sz;
        }

        if (j->on_header) {
                r = j->on_header(j, contents, sz);
                if (r < 0)
                        goto fail;
        }

        return sz;

fail:
        pull_job_finish(j, r);
        return 0;
}

static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
        PullJob *j = userdata;
        unsigned percent;
        usec_t n;

        assert(j);

        if (dltotal <= 0)
                return 0;

        percent = ((100 * dlnow) / dltotal);
        n = now(CLOCK_MONOTONIC);

        if (n > j->last_status_usec + USEC_PER_SEC &&
            percent != j->progress_percent &&
            dlnow < dltotal) {
                char buf[FORMAT_TIMESPAN_MAX];

                if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) {
                        char y[FORMAT_BYTES_MAX];
                        usec_t left, done;

                        done = n - j->start_usec;
                        left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;

                        log_info("Got %u%% of %s. %s left at %s/s.",
                                 percent,
                                 j->url,
                                 format_timespan(buf, sizeof(buf), left, USEC_PER_SEC),
                                 format_bytes(y, sizeof(y), (uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC))));
                } else
                        log_info("Got %u%% of %s.", percent, j->url);

                j->progress_percent = percent;
                j->last_status_usec = n;

                if (j->on_progress)
                        j->on_progress(j);
        }

        return 0;
}

int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) {
        _cleanup_(pull_job_unrefp) PullJob *j = NULL;
        _cleanup_free_ char *u = NULL;

        assert(url);
        assert(glue);
        assert(ret);

        u = strdup(url);
        if (!u)
                return -ENOMEM;

        j = new(PullJob, 1);
        if (!j)
                return -ENOMEM;

        *j = (PullJob) {
                .state = PULL_JOB_INIT,
                .disk_fd = -1,
                .userdata = userdata,
                .glue = glue,
                .content_length = (uint64_t) -1,
                .start_usec = now(CLOCK_MONOTONIC),
                .compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
                .uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
                .style = VERIFICATION_STYLE_UNSET,
                .url = TAKE_PTR(u),
        };

        *ret = TAKE_PTR(j);

        return 0;
}

int pull_job_begin(PullJob *j) {
        int r;

        assert(j);

        if (j->state != PULL_JOB_INIT)
                return -EBUSY;

        r = curl_glue_make(&j->curl, j->url, j);
        if (r < 0)
                return r;

        if (!strv_isempty(j->old_etags)) {
                _cleanup_free_ char *cc = NULL, *hdr = NULL;

                cc = strv_join(j->old_etags, ", ");
                if (!cc)
                        return -ENOMEM;

                hdr = strappend("If-None-Match: ", cc);
                if (!hdr)
                        return -ENOMEM;

                if (!j->request_header) {
                        j->request_header = curl_slist_new(hdr, NULL);
                        if (!j->request_header)
                                return -ENOMEM;
                } else {
                        struct curl_slist *l;

                        l = curl_slist_append(j->request_header, hdr);
                        if (!l)
                                return -ENOMEM;

                        j->request_header = l;
                }
        }

        if (j->request_header) {
                if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
                        return -EIO;
        }

        if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK)
                return -EIO;

        if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
                return -EIO;

        r = curl_glue_add(j->glue, j->curl);
        if (r < 0)
                return r;

        j->state = PULL_JOB_ANALYZING;

        return 0;
}
Commit	Line	Data
53e1b683	1	/* SPDX-License-Identifier: LGPL-2.1+ */
56ebfaf1 LP	2
	3	#include <sys/xattr.h>
	4
b5efdb8a	5	#include "alloc-util.h"
3ffd4af2	6	#include "fd-util.h"
eef836ed	7	#include "gcrypt-util.h"
89a5a90c	8	#include "hexdecoct.h"
1f0f4f3b	9	#include "import-util.h"
c004493c	10	#include "io-util.h"
26166c88	11	#include "machine-pool.h"
6bedfcbb	12	#include "parse-util.h"
1f0f4f3b	13	#include "pull-common.h"
3ffd4af2	14	#include "pull-job.h"
07630cea LP	15	#include "string-util.h"
07630cea LP	16	#include "strv.h"
89a5a90c	17	#include "xattr-util.h"
56ebfaf1	18
dc2c282b	19	PullJob* pull_job_unref(PullJob *j) {
56ebfaf1 LP	20	if (!j)
	21	return NULL;
	22
	23	curl_glue_remove_and_free(j->glue, j->curl);
	24	curl_slist_free_all(j->request_header);
	25
	26	safe_close(j->disk_fd);
	27
3e2cda69	28	import_compress_free(&j->compress);
56ebfaf1	29
98c38001 LP	30	if (j->checksum_context)
98c38001 LP	31	gcry_md_close(j->checksum_context);
85dbc41d	32
56ebfaf1 LP	33	free(j->url);
	34	free(j->etag);
	35	strv_free(j->old_etags);
	36	free(j->payload);
98c38001	37	free(j->checksum);
56ebfaf1	38
6b430fdb	39	return mfree(j);
56ebfaf1 LP	40	}
56ebfaf1 LP	41
dc2c282b	42	static void pull_job_finish(PullJob *j, int ret) {
56ebfaf1 LP	43	assert(j);
56ebfaf1 LP	44
3742095b	45	if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
56ebfaf1 LP	46	return;
56ebfaf1 LP	47
68c913fd	48	if (ret == 0) {
dc2c282b	49	j->state = PULL_JOB_DONE;
7079cfef	50	j->progress_percent = 100;
68c913fd LP	51	log_info("Download of %s complete.", j->url);
68c913fd LP	52	} else {
dc2c282b	53	j->state = PULL_JOB_FAILED;
56ebfaf1 LP	54	j->error = ret;
	55	}
	56
	57	if (j->on_finished)
	58	j->on_finished(j);
	59	}
	60
697be0be TB	61	static int pull_job_restart(PullJob *j) {
	62	int r;
	63	char *chksum_url = NULL;
	64
	65	r = import_url_change_last_component(j->url, "SHA256SUMS", &chksum_url);
	66	if (r < 0)
	67	return r;
	68
	69	free(j->url);
697be0be TB	70	j->url = chksum_url;
697be0be TB	71	j->state = PULL_JOB_INIT;
1f0f4f3b	72	j->payload = mfree(j->payload);
697be0be TB	73	j->payload_size = 0;
	74	j->payload_allocated = 0;
	75	j->written_compressed = 0;
	76	j->written_uncompressed = 0;
697be0be TB	77
	78	r = pull_job_begin(j);
	79	if (r < 0)
	80	return r;
	81
	82	return 0;
	83	}
	84
dc2c282b LP	85	void pull_job_curl_on_finished(CurlGlue g, CURL curl, CURLcode result) {
dc2c282b LP	86	PullJob *j = NULL;
56ebfaf1 LP	87	CURLcode code;
	88	long status;
	89	int r;
	90
a7f7d1bd	91	if (curl_easy_getinfo(curl, CURLINFO_PRIVATE, (char **)&j) != CURLE_OK)
56ebfaf1 LP	92	return;
56ebfaf1 LP	93
4c701096	94	if (!j \|\| IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED))
56ebfaf1 LP	95	return;
	96
	97	if (result != CURLE_OK) {
	98	log_error("Transfer failed: %s", curl_easy_strerror(result));
	99	r = -EIO;
	100	goto finish;
	101	}
	102
	103	code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
	104	if (code != CURLE_OK) {
	105	log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
	106	r = -EIO;
	107	goto finish;
	108	} else if (status == 304) {
	109	log_info("Image already downloaded. Skipping download.");
85dbc41d	110	j->etag_exists = true;
56ebfaf1 LP	111	r = 0;
	112	goto finish;
	113	} else if (status >= 300) {
697be0be TB	114	if (status == 404 && j->style == VERIFICATION_PER_FILE) {
	115
	116	/* retry pull job with SHA256SUMS file */
	117	r = pull_job_restart(j);
	118	if (r < 0)
	119	goto finish;
	120
	121	code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
	122	if (code != CURLE_OK) {
	123	log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
	124	r = -EIO;
	125	goto finish;
	126	}
	127
	128	if (status == 0) {
	129	j->style = VERIFICATION_PER_DIRECTORY;
	130	return;
	131	}
	132	}
	133
56ebfaf1 LP	134	log_error("HTTP request to %s failed with code %li.", j->url, status);
	135	r = -EIO;
	136	goto finish;
	137	} else if (status < 200) {
	138	log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
	139	r = -EIO;
	140	goto finish;
	141	}
	142
dc2c282b	143	if (j->state != PULL_JOB_RUNNING) {
56ebfaf1 LP	144	log_error("Premature connection termination.");
	145	r = -EIO;
	146	goto finish;
	147	}
	148
	149	if (j->content_length != (uint64_t) -1 &&
	150	j->content_length != j->written_compressed) {
	151	log_error("Download truncated.");
	152	r = -EIO;
	153	goto finish;
	154	}
	155
98c38001	156	if (j->checksum_context) {
85dbc41d LP	157	uint8_t *k;
85dbc41d LP	158
98c38001	159	k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
85dbc41d LP	160	if (!k) {
	161	log_error("Failed to get checksum.");
	162	r = -EIO;
	163	goto finish;
	164	}
	165
98c38001 LP	166	j->checksum = hexmem(k, gcry_md_get_algo_dlen(GCRY_MD_SHA256));
98c38001 LP	167	if (!j->checksum) {
85dbc41d LP	168	r = log_oom();
	169	goto finish;
	170	}
	171
98c38001	172	log_debug("SHA256 of %s is %s.", j->url, j->checksum);
85dbc41d LP	173	}
85dbc41d LP	174
56ebfaf1 LP	175	if (j->disk_fd >= 0 && j->allow_sparse) {
	176	/* Make sure the file size is right, in case the file was
	177	* sparse and we just seeked for the last part */
	178
	179	if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
b6e676ce	180	r = log_error_errno(errno, "Failed to truncate file: %m");
56ebfaf1 LP	181	goto finish;
	182	}
	183
	184	if (j->etag)
	185	(void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
	186	if (j->url)
	187	(void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
	188
	189	if (j->mtime != 0) {
	190	struct timespec ut[2];
	191
	192	timespec_store(&ut[0], j->mtime);
	193	ut[1] = ut[0];
	194	(void) futimens(j->disk_fd, ut);
	195
	196	(void) fd_setcrtime(j->disk_fd, j->mtime);
	197	}
	198	}
	199
	200	r = 0;
	201
	202	finish:
dc2c282b	203	pull_job_finish(j, r);
56ebfaf1 LP	204	}
56ebfaf1 LP	205
3e2cda69 LP	206	static int pull_job_write_uncompressed(const void p, size_t sz, void userdata) {
3e2cda69 LP	207	PullJob *j = userdata;
56ebfaf1 LP	208	ssize_t n;
	209
	210	assert(j);
	211	assert(p);
8af3cf74 LP	212
	213	if (sz <= 0)
	214	return 0;
56ebfaf1	215
baaa35ad ZJS	216	if (j->written_uncompressed + sz < j->written_uncompressed)
	217	return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW),
	218	"File too large, overflow");
56ebfaf1	219
baaa35ad ZJS	220	if (j->written_uncompressed + sz > j->uncompressed_max)
	221	return log_error_errno(SYNTHETIC_ERRNO(EFBIG),
	222	"File overly large, refusing");
56ebfaf1 LP	223
	224	if (j->disk_fd >= 0) {
	225
	226	if (j->allow_sparse)
	227	n = sparse_write(j->disk_fd, p, sz, 64);
e986910e	228	else {
56ebfaf1	229	n = write(j->disk_fd, p, sz);
e986910e YW	230	if (n < 0)
	231	n = -errno;
	232	}
b6e676ce	233	if (n < 0)
e986910e	234	return log_error_errno((int) n, "Failed to write file: %m");
baaa35ad ZJS	235	if ((size_t) n < sz)
baaa35ad ZJS	236	return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
56ebfaf1 LP	237	} else {
	238
	239	if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz))
	240	return log_oom();
	241
8af3cf74	242	memcpy(j->payload + j->payload_size, p, sz);
56ebfaf1 LP	243	j->payload_size += sz;
	244	}
	245
	246	j->written_uncompressed += sz;
	247
	248	return 0;
	249	}
	250
dc2c282b	251	static int pull_job_write_compressed(PullJob j, void p, size_t sz) {
56ebfaf1 LP	252	int r;
	253
	254	assert(j);
	255	assert(p);
8af3cf74 LP	256
	257	if (sz <= 0)
	258	return 0;
56ebfaf1	259
baaa35ad ZJS	260	if (j->written_compressed + sz < j->written_compressed)
baaa35ad ZJS	261	return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
56ebfaf1	262
baaa35ad ZJS	263	if (j->written_compressed + sz > j->compressed_max)
baaa35ad ZJS	264	return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing.");
56ebfaf1 LP	265
56ebfaf1 LP	266	if (j->content_length != (uint64_t) -1 &&
baaa35ad ZJS	267	j->written_compressed + sz > j->content_length)
	268	return log_error_errno(SYNTHETIC_ERRNO(EFBIG),
	269	"Content length incorrect.");
56ebfaf1	270
98c38001 LP	271	if (j->checksum_context)
98c38001 LP	272	gcry_md_write(j->checksum_context, p, sz);
85dbc41d	273
3e2cda69 LP	274	r = import_uncompress(&j->compress, p, sz, pull_job_write_uncompressed, j);
	275	if (r < 0)
	276	return r;
56ebfaf1 LP	277
	278	j->written_compressed += sz;
	279
	280	return 0;
	281	}
	282
dc2c282b	283	static int pull_job_open_disk(PullJob *j) {
56ebfaf1 LP	284	int r;
	285
	286	assert(j);
	287
	288	if (j->on_open_disk) {
	289	r = j->on_open_disk(j);
	290	if (r < 0)
	291	return r;
	292	}
	293
	294	if (j->disk_fd >= 0) {
	295	/* Check if we can do sparse files */
	296
	297	if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
	298	j->allow_sparse = true;
	299	else {
	300	if (errno != ESPIPE)
	301	return log_error_errno(errno, "Failed to seek on file descriptor: %m");
	302
	303	j->allow_sparse = false;
	304	}
	305	}
	306
98c38001	307	if (j->calc_checksum) {
eef836ed YW	308	initialize_libgcrypt(false);
eef836ed YW	309
baaa35ad ZJS	310	if (gcry_md_open(&j->checksum_context, GCRY_MD_SHA256, 0) != 0)
	311	return log_error_errno(SYNTHETIC_ERRNO(EIO),
	312	"Failed to initialize hash context.");
85dbc41d LP	313	}
85dbc41d LP	314
56ebfaf1 LP	315	return 0;
	316	}
	317
dc2c282b	318	static int pull_job_detect_compression(PullJob *j) {
56ebfaf1 LP	319	_cleanup_free_ uint8_t *stub = NULL;
	320	size_t stub_size;
	321
	322	int r;
	323
	324	assert(j);
	325
3e2cda69 LP	326	r = import_uncompress_detect(&j->compress, j->payload, j->payload_size);
	327	if (r < 0)
	328	return log_error_errno(r, "Failed to initialize compressor: %m");
	329	if (r == 0)
56ebfaf1 LP	330	return 0;
56ebfaf1 LP	331
3e2cda69	332	log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type));
56ebfaf1	333
dc2c282b	334	r = pull_job_open_disk(j);
56ebfaf1 LP	335	if (r < 0)
	336	return r;
	337
	338	/* Now, take the payload we read so far, and decompress it */
	339	stub = j->payload;
	340	stub_size = j->payload_size;
	341
	342	j->payload = NULL;
	343	j->payload_size = 0;
88a1aadc	344	j->payload_allocated = 0;
56ebfaf1	345
dc2c282b	346	j->state = PULL_JOB_RUNNING;
56ebfaf1	347
dc2c282b	348	r = pull_job_write_compressed(j, stub, stub_size);
56ebfaf1 LP	349	if (r < 0)
	350	return r;
	351
	352	return 0;
	353	}
	354
dc2c282b LP	355	static size_t pull_job_write_callback(void contents, size_t size, size_t nmemb, void userdata) {
dc2c282b LP	356	PullJob *j = userdata;
56ebfaf1 LP	357	size_t sz = size * nmemb;
	358	int r;
	359
	360	assert(contents);
	361	assert(j);
	362
	363	switch (j->state) {
	364
dc2c282b	365	case PULL_JOB_ANALYZING:
56ebfaf1 LP	366	/* Let's first check what it actually is */
	367
	368	if (!GREEDY_REALLOC(j->payload, j->payload_allocated, j->payload_size + sz)) {
	369	r = log_oom();
	370	goto fail;
	371	}
	372
8af3cf74	373	memcpy(j->payload + j->payload_size, contents, sz);
56ebfaf1 LP	374	j->payload_size += sz;
56ebfaf1 LP	375
dc2c282b	376	r = pull_job_detect_compression(j);
56ebfaf1 LP	377	if (r < 0)
	378	goto fail;
	379
	380	break;
	381
dc2c282b	382	case PULL_JOB_RUNNING:
56ebfaf1	383
dc2c282b	384	r = pull_job_write_compressed(j, contents, sz);
56ebfaf1 LP	385	if (r < 0)
	386	goto fail;
	387
	388	break;
	389
dc2c282b LP	390	case PULL_JOB_DONE:
dc2c282b LP	391	case PULL_JOB_FAILED:
56ebfaf1 LP	392	r = -ESTALE;
	393	goto fail;
	394
	395	default:
	396	assert_not_reached("Impossible state.");
	397	}
	398
	399	return sz;
	400
	401	fail:
dc2c282b	402	pull_job_finish(j, r);
56ebfaf1 LP	403	return 0;
	404	}
	405
dc2c282b LP	406	static size_t pull_job_header_callback(void contents, size_t size, size_t nmemb, void userdata) {
dc2c282b LP	407	PullJob *j = userdata;
56ebfaf1 LP	408	size_t sz = size * nmemb;
	409	_cleanup_free_ char length = NULL, last_modified = NULL;
	410	char *etag;
	411	int r;
	412
	413	assert(contents);
	414	assert(j);
	415
3742095b	416	if (IN_SET(j->state, PULL_JOB_DONE, PULL_JOB_FAILED)) {
56ebfaf1 LP	417	r = -ESTALE;
	418	goto fail;
	419	}
	420
dc2c282b	421	assert(j->state == PULL_JOB_ANALYZING);
56ebfaf1 LP	422
	423	r = curl_header_strdup(contents, sz, "ETag:", &etag);
	424	if (r < 0) {
	425	log_oom();
	426	goto fail;
	427	}
	428	if (r > 0) {
	429	free(j->etag);
	430	j->etag = etag;
	431
	432	if (strv_contains(j->old_etags, j->etag)) {
	433	log_info("Image already downloaded. Skipping download.");
85dbc41d	434	j->etag_exists = true;
dc2c282b	435	pull_job_finish(j, 0);
56ebfaf1 LP	436	return sz;
	437	}
	438
	439	return sz;
	440	}
	441
	442	r = curl_header_strdup(contents, sz, "Content-Length:", &length);
	443	if (r < 0) {
	444	log_oom();
	445	goto fail;
	446	}
	447	if (r > 0) {
	448	(void) safe_atou64(length, &j->content_length);
	449
	450	if (j->content_length != (uint64_t) -1) {
	451	char bytes[FORMAT_BYTES_MAX];
	452
	453	if (j->content_length > j->compressed_max) {
	454	log_error("Content too large.");
	455	r = -EFBIG;
	456	goto fail;
	457	}
	458
68c913fd	459	log_info("Downloading %s for %s.", format_bytes(bytes, sizeof(bytes), j->content_length), j->url);
56ebfaf1 LP	460	}
	461
	462	return sz;
	463	}
	464
	465	r = curl_header_strdup(contents, sz, "Last-Modified:", &last_modified);
	466	if (r < 0) {
	467	log_oom();
	468	goto fail;
	469	}
	470	if (r > 0) {
	471	(void) curl_parse_http_time(last_modified, &j->mtime);
	472	return sz;
	473	}
	474
ff2670ad LP	475	if (j->on_header) {
	476	r = j->on_header(j, contents, sz);
	477	if (r < 0)
	478	goto fail;
	479	}
	480
56ebfaf1 LP	481	return sz;
	482
	483	fail:
dc2c282b	484	pull_job_finish(j, r);
56ebfaf1 LP	485	return 0;
	486	}
	487
dc2c282b LP	488	static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
dc2c282b LP	489	PullJob *j = userdata;
56ebfaf1 LP	490	unsigned percent;
	491	usec_t n;
	492
	493	assert(j);
	494
	495	if (dltotal <= 0)
	496	return 0;
	497
	498	percent = ((100 * dlnow) / dltotal);
	499	n = now(CLOCK_MONOTONIC);
	500
	501	if (n > j->last_status_usec + USEC_PER_SEC &&
68c913fd LP	502	percent != j->progress_percent &&
68c913fd LP	503	dlnow < dltotal) {
56ebfaf1 LP	504	char buf[FORMAT_TIMESPAN_MAX];
	505
	506	if (n - j->start_usec > USEC_PER_SEC && dlnow > 0) {
90bc083b	507	char y[FORMAT_BYTES_MAX];
56ebfaf1 LP	508	usec_t left, done;
	509
	510	done = n - j->start_usec;
	511	left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
	512
90bc083b LP	513	log_info("Got %u%% of %s. %s left at %s/s.",
	514	percent,
	515	j->url,
	516	format_timespan(buf, sizeof(buf), left, USEC_PER_SEC),
	517	format_bytes(y, sizeof(y), (uint64_t) ((double) dlnow / ((double) done / (double) USEC_PER_SEC))));
56ebfaf1 LP	518	} else
	519	log_info("Got %u%% of %s.", percent, j->url);
	520
	521	j->progress_percent = percent;
	522	j->last_status_usec = n;
7079cfef LP	523
	524	if (j->on_progress)
	525	j->on_progress(j);
56ebfaf1 LP	526	}
	527
	528	return 0;
	529	}
	530
dc2c282b LP	531	int pull_job_new(PullJob *ret, const char url, CurlGlue glue, void userdata) {
dc2c282b LP	532	_cleanup_(pull_job_unrefp) PullJob *j = NULL;
0d94088e	533	_cleanup_free_ char *u = NULL;
56ebfaf1 LP	534
	535	assert(url);
	536	assert(glue);
	537	assert(ret);
	538
0d94088e	539	u = strdup(url);
9b5b4bed	540	if (!u)
56ebfaf1 LP	541	return -ENOMEM;
56ebfaf1 LP	542
0d94088e YW	543	j = new(PullJob, 1);
0d94088e YW	544	if (!j)
56ebfaf1 LP	545	return -ENOMEM;
56ebfaf1 LP	546
0d94088e YW	547	*j = (PullJob) {
	548	.state = PULL_JOB_INIT,
	549	.disk_fd = -1,
	550	.userdata = userdata,
	551	.glue = glue,
	552	.content_length = (uint64_t) -1,
	553	.start_usec = now(CLOCK_MONOTONIC),
	554	.compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
	555	.uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
	556	.style = VERIFICATION_STYLE_UNSET,
	557	.url = TAKE_PTR(u),
	558	};
	559
1cc6c93a	560	*ret = TAKE_PTR(j);
56ebfaf1 LP	561
	562	return 0;
	563	}
	564
dc2c282b	565	int pull_job_begin(PullJob *j) {
56ebfaf1 LP	566	int r;
	567
	568	assert(j);
	569
dc2c282b	570	if (j->state != PULL_JOB_INIT)
56ebfaf1 LP	571	return -EBUSY;
	572
	573	r = curl_glue_make(&j->curl, j->url, j);
	574	if (r < 0)
	575	return r;
	576
	577	if (!strv_isempty(j->old_etags)) {
	578	_cleanup_free_ char cc = NULL, hdr = NULL;
	579
	580	cc = strv_join(j->old_etags, ", ");
	581	if (!cc)
	582	return -ENOMEM;
	583
	584	hdr = strappend("If-None-Match: ", cc);
	585	if (!hdr)
	586	return -ENOMEM;
	587
ff2670ad LP	588	if (!j->request_header) {
	589	j->request_header = curl_slist_new(hdr, NULL);
	590	if (!j->request_header)
	591	return -ENOMEM;
	592	} else {
	593	struct curl_slist *l;
	594
	595	l = curl_slist_append(j->request_header, hdr);
	596	if (!l)
	597	return -ENOMEM;
	598
	599	j->request_header = l;
	600	}
	601	}
56ebfaf1	602
ff2670ad	603	if (j->request_header) {
56ebfaf1 LP	604	if (curl_easy_setopt(j->curl, CURLOPT_HTTPHEADER, j->request_header) != CURLE_OK)
	605	return -EIO;
	606	}
	607
dc2c282b	608	if (curl_easy_setopt(j->curl, CURLOPT_WRITEFUNCTION, pull_job_write_callback) != CURLE_OK)
56ebfaf1 LP	609	return -EIO;
	610
	611	if (curl_easy_setopt(j->curl, CURLOPT_WRITEDATA, j) != CURLE_OK)
	612	return -EIO;
	613
dc2c282b	614	if (curl_easy_setopt(j->curl, CURLOPT_HEADERFUNCTION, pull_job_header_callback) != CURLE_OK)
56ebfaf1 LP	615	return -EIO;
	616
	617	if (curl_easy_setopt(j->curl, CURLOPT_HEADERDATA, j) != CURLE_OK)
	618	return -EIO;
	619
dc2c282b	620	if (curl_easy_setopt(j->curl, CURLOPT_XFERINFOFUNCTION, pull_job_progress_callback) != CURLE_OK)
56ebfaf1 LP	621	return -EIO;
	622
	623	if (curl_easy_setopt(j->curl, CURLOPT_XFERINFODATA, j) != CURLE_OK)
	624	return -EIO;
	625
	626	if (curl_easy_setopt(j->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
	627	return -EIO;
	628
	629	r = curl_glue_add(j->glue, j->curl);
	630	if (r < 0)
	631	return r;
	632
dc2c282b	633	j->state = PULL_JOB_ANALYZING;
56ebfaf1 LP	634
	635	return 0;
	636	}