From: Lennart Poettering Date: Wed, 5 Nov 2025 16:31:24 +0000 (+0100) Subject: import: rework pull logic to store download digests in binary form rather than string X-Git-Tag: v259-rc1~127^2~6 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=e825635af62552f51a9ce1fe2dd132c7be4bbf87;p=thirdparty%2Fsystemd.git import: rework pull logic to store download digests in binary form rather than string We generally want to store data in parsed form, not formatted form, hence let's follow our own rules on this, and store the message digest as "struct iovec" rather than as string. This is generally more efficient and safer, simply because of case issues. --- diff --git a/src/import/pull-common.c b/src/import/pull-common.c index 08a5a6192d1..61ac57e5555 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -6,6 +6,7 @@ #include "dirent-util.h" #include "escape.h" #include "fd-util.h" +#include "hexdecoct.h" #include "io-util.h" #include "log.h" #include "memory-util.h" @@ -247,7 +248,7 @@ int pull_make_verification_jobs( PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, - const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */ + const struct iovec *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */ const char *url, CurlGlue *glue, PullJobFinished on_finished, @@ -267,7 +268,7 @@ int pull_make_verification_jobs( /* If verification is turned off, or if the checksum to validate is already specified we don't need * to download a checksum file or signature, hence shortcut things */ - if (verify == IMPORT_VERIFY_NO || checksum) { + if (verify == IMPORT_VERIFY_NO || iovec_is_set(checksum)) { *ret_checksum_job = *ret_signature_job = NULL; return 0; } @@ -351,7 +352,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) { return 0; assert(job->calc_checksum); - assert(job->checksum); + assert(iovec_is_set(&job->checksum)); r = import_url_last_component(job->url, &fn); if (r < 0) @@ -365,6 +366,10 @@ static int verify_one(PullJob *checksum_job, PullJob *job) { return log_error_errno(SYNTHETIC_ERRNO(ELOOP), "Cannot verify checksum/signature files via themselves."); + _cleanup_free_ char *cs = hexmem(job->checksum.iov_base, job->checksum.iov_len); + if (!cs) + return log_oom(); + const char *p = NULL; FOREACH_STRING(separator, " *", /* separator for binary mode */ @@ -372,7 +377,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) { " " /* non-standard separator used by linuxcontainers.org */) { _cleanup_free_ char *line = NULL; - line = strjoin(job->checksum, separator, fn, "\n"); + line = strjoin(cs, separator, fn, "\n"); if (!line) return log_oom(); @@ -510,7 +515,7 @@ finish: } int pull_verify(ImportVerify verify, - const char *checksum, /* Verify with literal checksum */ + const struct iovec *checksum, /* Verify with literal checksum */ PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, @@ -543,9 +548,9 @@ int pull_verify(ImportVerify verify, assert(!verity_job); assert(main_job->calc_checksum); - assert(main_job->checksum); + assert(iovec_is_set(&main_job->checksum)); - if (!strcaseeq(checksum, main_job->checksum)) + if (iovec_memcmp(checksum, &main_job->checksum) != 0) return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", main_job->url); @@ -566,7 +571,7 @@ int pull_verify(ImportVerify verify, verify_job = main_job; } else { assert(main_job->calc_checksum); - assert(main_job->checksum); + assert(iovec_is_set(&main_job->checksum)); assert(checksum_job); assert(checksum_job->state == PULL_JOB_DONE); diff --git a/src/import/pull-common.h b/src/import/pull-common.h index 160c035a212..48cb6c5ec4e 100644 --- a/src/import/pull-common.h +++ b/src/import/pull-common.h @@ -14,9 +14,9 @@ int pull_find_old_etags(const char *url, const char *root, int dt, const char *p int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret); int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata); -int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); +int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const struct iovec *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); -int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job); +int pull_verify(ImportVerify verify, const struct iovec *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job); typedef enum VerificationStyle { VERIFICATION_PER_FILE, /* SUSE-style ".sha256" files with detached gpg signature */ diff --git a/src/import/pull-job.c b/src/import/pull-job.c index 9459a89ceaa..57e9e95939a 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -48,7 +48,7 @@ PullJob* pull_job_unref(PullJob *j) { free(j->etag); strv_free(j->old_etags); free(j->payload); - free(j->checksum); + iovec_done(&j->checksum); return mfree(j); } @@ -92,7 +92,7 @@ static int pull_job_restart(PullJob *j, const char *new_url) { j->etag = mfree(j->etag); j->etag_exists = false; j->mtime = 0; - j->checksum = mfree(j->checksum); + iovec_done(&j->checksum); j->expected_content_length = UINT64_MAX; curl_glue_remove_and_free(j->glue, j->curl); @@ -233,22 +233,31 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { if (j->checksum_ctx) { unsigned checksum_len; - uint8_t k[EVP_MAX_MD_SIZE]; - r = EVP_DigestFinal_ex(j->checksum_ctx, k, &checksum_len); - if (r == 0) { - r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum."); + iovec_done(&j->checksum); + j->checksum.iov_base = malloc(EVP_MAX_MD_SIZE); + if (!j->checksum.iov_base) { + r = log_oom(); goto finish; } - assert(checksum_len <= sizeof k); - j->checksum = hexmem(k, checksum_len); - if (!j->checksum) { - r = log_oom(); + r = EVP_DigestFinal_ex(j->checksum_ctx, j->checksum.iov_base, &checksum_len); + if (r == 0) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum."); goto finish; } + assert(checksum_len <= EVP_MAX_MD_SIZE); + j->checksum.iov_len = checksum_len; - log_debug("SHA256 of %s is %s.", j->url, j->checksum); + if (DEBUG_LOGGING) { + _cleanup_free_ char *h = hexmem(j->checksum.iov_base, j->checksum.iov_len); + if (!h) { + r = log_oom(); + goto finish; + } + + log_debug("%s of %s is %s.", EVP_MD_CTX_get0_name(j->checksum_ctx), j->url, h); + } } /* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no diff --git a/src/import/pull-job.h b/src/import/pull-job.h index 34156e9dd6f..7437aab01a8 100644 --- a/src/import/pull-job.h +++ b/src/import/pull-job.h @@ -78,7 +78,8 @@ typedef struct PullJob { bool calc_checksum; EVP_MD_CTX *checksum_ctx; - char *checksum; + struct iovec checksum; + bool sync; bool force_memory; } PullJob; diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index 6ba8f6cdc90..274bd3ec0d6 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -68,7 +68,7 @@ typedef struct RawPull { char *verity_path; char *verity_temp_path; - char *checksum; + struct iovec checksum; } RawPull; RawPull* raw_pull_unref(RawPull *i) { @@ -99,7 +99,7 @@ RawPull* raw_pull_unref(RawPull *i) { free(i->verity_path); free(i->image_root); free(i->local); - free(i->checksum); + iovec_done(&i->checksum); return mfree(i); } @@ -585,7 +585,7 @@ static void raw_pull_job_on_finished(PullJob *j) { raw_pull_report_progress(i, RAW_VERIFYING); r = pull_verify(i->verify, - i->checksum, + &i->checksum, i->raw_job, i->checksum_job, i->signature_job, @@ -827,7 +827,7 @@ int raw_pull_start( uint64_t size_max, ImportFlags flags, ImportVerify verify, - const char *checksum) { + const struct iovec *checksum) { int r; @@ -835,11 +835,11 @@ int raw_pull_start( assert(url); assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); - assert((verify < 0) || !checksum); + assert((verify < 0) || !iovec_is_set(checksum)); assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_RAW)); assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT)); assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !(flags & IMPORT_DIRECT)); - assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !checksum); + assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !iovec_is_set(checksum)); if (!http_url_is_valid(url) && !file_url_is_valid(url)) return -EINVAL; @@ -854,9 +854,8 @@ int raw_pull_start( if (r < 0) return r; - r = free_and_strdup(&i->checksum, checksum); - if (r < 0) - return r; + if (!iovec_memdup(checksum, &i->checksum)) + return -ENOMEM; i->flags = flags; i->verify = verify; @@ -869,7 +868,7 @@ int raw_pull_start( i->raw_job->on_finished = raw_pull_job_on_finished; i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw; - if (checksum) + if (iovec_is_set(checksum)) i->raw_job->calc_checksum = true; else if (verify != IMPORT_VERIFY_NO) { /* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its @@ -899,7 +898,7 @@ int raw_pull_start( &i->checksum_job, &i->signature_job, verify, - i->checksum, + &i->checksum, url, i->glue, raw_pull_job_on_finished, diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h index c8126d4ff1b..0dbd4dcd20c 100644 --- a/src/import/pull-raw.h +++ b/src/import/pull-raw.h @@ -14,4 +14,4 @@ RawPull* raw_pull_unref(RawPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref); -int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const char *checksum); +int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const struct iovec *checksum); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index 23bdfb186a7..a899e9a225d 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -65,7 +65,7 @@ typedef struct TarPull { char *settings_path; char *settings_temp_path; - char *checksum; + struct iovec checksum; int tree_fd; int userns_fd; @@ -98,7 +98,7 @@ TarPull* tar_pull_unref(TarPull *i) { free(i->settings_path); free(i->image_root); free(i->local); - free(i->checksum); + iovec_done(&i->checksum); safe_close(i->tree_fd); safe_close(i->userns_fd); @@ -478,7 +478,7 @@ static void tar_pull_job_on_finished(PullJob *j) { clear_progress_bar(/* prefix= */ NULL); r = pull_verify(i->verify, - i->checksum, + &i->checksum, i->tar_job, i->checksum_job, i->signature_job, @@ -698,17 +698,17 @@ int tar_pull_start( const char *local, ImportFlags flags, ImportVerify verify, - const char *checksum) { + const struct iovec *checksum) { int r; assert(i); assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); - assert((verify < 0) || !checksum); + assert((verify < 0) || !iovec_is_set(checksum)); assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_TAR)); assert(!(flags & IMPORT_PULL_SETTINGS) || !(flags & IMPORT_DIRECT)); - assert(!(flags & IMPORT_PULL_SETTINGS) || !checksum); + assert(!(flags & IMPORT_PULL_SETTINGS) || !iovec_is_set(checksum)); if (!http_url_is_valid(url) && !file_url_is_valid(url)) return -EINVAL; @@ -723,9 +723,8 @@ int tar_pull_start( if (r < 0) return r; - r = free_and_strdup(&i->checksum, checksum); - if (r < 0) - return r; + if (!iovec_memdup(checksum, &i->checksum)) + return -ENOMEM; i->flags = flags; i->verify = verify; diff --git a/src/import/pull-tar.h b/src/import/pull-tar.h index 8f622dff3ac..d2118272ff1 100644 --- a/src/import/pull-tar.h +++ b/src/import/pull-tar.h @@ -14,4 +14,4 @@ TarPull* tar_pull_unref(TarPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref); -int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const char *checksum); +int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const struct iovec *checksum); diff --git a/src/import/pull.c b/src/import/pull.c index a1b9b940fdf..a57c1c039d9 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -15,6 +15,7 @@ #include "import-common.h" #include "import-util.h" #include "io-util.h" +#include "iovec-util.h" #include "log.h" #include "main-func.h" #include "parse-argument.h" @@ -32,11 +33,11 @@ static char *arg_image_root = NULL; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; static ImportFlags arg_import_flags = IMPORT_PULL_SETTINGS | IMPORT_PULL_ROOTHASH | IMPORT_PULL_ROOTHASH_SIGNATURE | IMPORT_PULL_VERITY | IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC; static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; -static char *arg_checksum = NULL; +static struct iovec arg_checksum = {}; static ImageClass arg_class = IMAGE_MACHINE; static RuntimeScope arg_runtime_scope = _RUNTIME_SCOPE_INVALID; -STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep); +STATIC_DESTRUCTOR_REGISTER(arg_checksum, iovec_done); STATIC_DESTRUCTOR_REGISTER(arg_image_root, freep); static int normalize_local(const char *local, const char *url, char **ret) { @@ -162,7 +163,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { normalized, arg_import_flags & IMPORT_PULL_FLAGS_MASK_TAR, arg_verify, - arg_checksum); + &arg_checksum); if (r < 0) return log_error_errno(r, "Failed to pull image: %m"); @@ -231,7 +232,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { arg_size_max, arg_import_flags & IMPORT_PULL_FLAGS_MASK_RAW, arg_verify, - arg_checksum); + &arg_checksum); if (r < 0) return log_error_errno(r, "Failed to pull image: %m"); @@ -371,7 +372,6 @@ static int parse_argv(int argc, char *argv[]) { v = import_verify_from_string(optarg); if (v < 0) { _cleanup_free_ void *h = NULL; - char *hh; size_t n; /* If this is not a valid verification mode, maybe it's a literally specified @@ -385,11 +385,10 @@ static int parse_argv(int argc, char *argv[]) { return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2); - hh = hexmem(h, n); /* bring into canonical (lowercase) form */ - if (!hh) - return log_oom(); + iovec_done(&arg_checksum); + arg_checksum.iov_base = TAKE_PTR(h); + arg_checksum.iov_len = n; - free_and_replace(arg_checksum, hh); arg_import_flags &= ~(IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY); arg_verify = _IMPORT_VERIFY_INVALID; } else @@ -542,7 +541,7 @@ static int parse_argv(int argc, char *argv[]) { if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); - if (arg_checksum && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0) + if (iovec_is_set(&arg_checksum) && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded."); if (!arg_image_root) {