import: rework pull logic to store download digests in binary form rather than string

We generally want to store data in parsed form, not formatted form,
hence let's follow our own rules on this, and store the message digest
as "struct iovec" rather than as string. This is generally more
efficient and safer, simply because of case issues.

diff --git a/src/import/pull-common.c b/src/import/pull-common.c
index 08a5a6192d1fa455dcb672d9df8e28ec8633afc5..61ac57e5555fee59cb4d96a8914c8cf0d2f2aa1d 100644 (file)
--- a/src/import/pull-common.c
+++ b/src/import/pull-common.c
@@ -6,6 +6,7 @@
 #include "dirent-util.h"
 #include "escape.h"
 #include "fd-util.h"
+#include "hexdecoct.h"
 #include "io-util.h"
 #include "log.h"
 #include "memory-util.h"
@@ -247,7 +248,7 @@ int pull_make_verification_jobs(
                 PullJob **ret_checksum_job,
                 PullJob **ret_signature_job,
                 ImportVerify verify,
-                const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */
+                const struct iovec *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */
                 const char *url,
                 CurlGlue *glue,
                 PullJobFinished on_finished,
@@ -267,7 +268,7 @@ int pull_make_verification_jobs(
 
         /* If verification is turned off, or if the checksum to validate is already specified we don't need
          * to download a checksum file or signature, hence shortcut things */
-        if (verify == IMPORT_VERIFY_NO || checksum) {
+        if (verify == IMPORT_VERIFY_NO || iovec_is_set(checksum)) {
                 *ret_checksum_job = *ret_signature_job = NULL;
                 return 0;
         }
@@ -351,7 +352,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
                 return 0;
 
         assert(job->calc_checksum);
-        assert(job->checksum);
+        assert(iovec_is_set(&job->checksum));
 
         r = import_url_last_component(job->url, &fn);
         if (r < 0)
@@ -365,6 +366,10 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
                 return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
                                        "Cannot verify checksum/signature files via themselves.");
 
+        _cleanup_free_ char *cs = hexmem(job->checksum.iov_base, job->checksum.iov_len);
+        if (!cs)
+                return log_oom();
+
         const char *p = NULL;
         FOREACH_STRING(separator,
                        " *", /* separator for binary mode */
@@ -372,7 +377,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
                        " "   /* non-standard separator used by linuxcontainers.org */) {
                 _cleanup_free_ char *line = NULL;
 
-                line = strjoin(job->checksum, separator, fn, "\n");
+                line = strjoin(cs, separator, fn, "\n");
                 if (!line)
                         return log_oom();
 
@@ -510,7 +515,7 @@ finish:
 }
 
 int pull_verify(ImportVerify verify,
-                const char *checksum, /* Verify with literal checksum */
+                const struct iovec *checksum, /* Verify with literal checksum */
                 PullJob *main_job,
                 PullJob *checksum_job,
                 PullJob *signature_job,
@@ -543,9 +548,9 @@ int pull_verify(ImportVerify verify,
                 assert(!verity_job);
 
                 assert(main_job->calc_checksum);
-                assert(main_job->checksum);
+                assert(iovec_is_set(&main_job->checksum));
 
-                if (!strcaseeq(checksum, main_job->checksum))
+                if (iovec_memcmp(checksum, &main_job->checksum) != 0)
                         return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
                                                "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.",
                                                main_job->url);
@@ -566,7 +571,7 @@ int pull_verify(ImportVerify verify,
                 verify_job = main_job;
         } else {
                 assert(main_job->calc_checksum);
-                assert(main_job->checksum);
+                assert(iovec_is_set(&main_job->checksum));
                 assert(checksum_job);
                 assert(checksum_job->state == PULL_JOB_DONE);
 

diff --git a/src/import/pull-common.h b/src/import/pull-common.h
index 160c035a2126b93404b55df1bf7fffc6dc2a31d4..48cb6c5ec4ea06cb89b379b1b7ec9c9c432a6b23 100644 (file)
--- a/src/import/pull-common.h
+++ b/src/import/pull-common.h
@@ -14,9 +14,9 @@ int pull_find_old_etags(const char *url, const char *root, int dt, const char *p
 int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret);
 
 int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata);
-int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
+int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const struct iovec *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
 
-int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
+int pull_verify(ImportVerify verify, const struct iovec *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
 
 typedef enum VerificationStyle {
         VERIFICATION_PER_FILE,      /* SUSE-style ".sha256" files with detached gpg signature */

diff --git a/src/import/pull-job.c b/src/import/pull-job.c
index 9459a89ceaaab8423ca57ccdaa1fdf7f95ec9c5e..57e9e95939aa5a3c494a916feef349f201c6ebf1 100644 (file)
--- a/src/import/pull-job.c
+++ b/src/import/pull-job.c
@@ -48,7 +48,7 @@ PullJob* pull_job_unref(PullJob *j) {
         free(j->etag);
         strv_free(j->old_etags);
         free(j->payload);
-        free(j->checksum);
+        iovec_done(&j->checksum);
 
         return mfree(j);
 }
@@ -92,7 +92,7 @@ static int pull_job_restart(PullJob *j, const char *new_url) {
         j->etag = mfree(j->etag);
         j->etag_exists = false;
         j->mtime = 0;
-        j->checksum = mfree(j->checksum);
+        iovec_done(&j->checksum);
         j->expected_content_length = UINT64_MAX;
 
         curl_glue_remove_and_free(j->glue, j->curl);
@@ -233,22 +233,31 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
 
         if (j->checksum_ctx) {
                 unsigned checksum_len;
-                uint8_t k[EVP_MAX_MD_SIZE];
 
-                r = EVP_DigestFinal_ex(j->checksum_ctx, k, &checksum_len);
-                if (r == 0) {
-                        r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum.");
+                iovec_done(&j->checksum);
+                j->checksum.iov_base = malloc(EVP_MAX_MD_SIZE);
+                if (!j->checksum.iov_base) {
+                        r = log_oom();
                         goto finish;
                 }
-                assert(checksum_len <= sizeof k);
 
-                j->checksum = hexmem(k, checksum_len);
-                if (!j->checksum) {
-                        r = log_oom();
+                r = EVP_DigestFinal_ex(j->checksum_ctx, j->checksum.iov_base, &checksum_len);
+                if (r == 0) {
+                        r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum.");
                         goto finish;
                 }
+                assert(checksum_len <= EVP_MAX_MD_SIZE);
+                j->checksum.iov_len = checksum_len;
 
-                log_debug("SHA256 of %s is %s.", j->url, j->checksum);
+                if (DEBUG_LOGGING) {
+                        _cleanup_free_ char *h = hexmem(j->checksum.iov_base, j->checksum.iov_len);
+                        if (!h) {
+                                r = log_oom();
+                                goto finish;
+                        }
+
+                        log_debug("%s of %s is %s.", EVP_MD_CTX_get0_name(j->checksum_ctx), j->url, h);
+                }
         }
 
         /* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no

diff --git a/src/import/pull-job.h b/src/import/pull-job.h
index 34156e9dd6fc25985fe053d381f0c60ed0572594..7437aab01a866c83912c0757b077ea42ba8dadfd 100644 (file)
--- a/src/import/pull-job.h
+++ b/src/import/pull-job.h
@@ -78,7 +78,8 @@ typedef struct PullJob {
         bool calc_checksum;
         EVP_MD_CTX *checksum_ctx;
 
-        char *checksum;
+        struct iovec checksum;
+
         bool sync;
         bool force_memory;
 } PullJob;

diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c
index 6ba8f6cdc903c1efacc15e740930f1215fd39f3b..274bd3ec0d6cbc44f250ed8456ba10788c098457 100644 (file)
--- a/src/import/pull-raw.c
+++ b/src/import/pull-raw.c
@@ -68,7 +68,7 @@ typedef struct RawPull {
         char *verity_path;
         char *verity_temp_path;
 
-        char *checksum;
+        struct iovec checksum;
 } RawPull;
 
 RawPull* raw_pull_unref(RawPull *i) {
@@ -99,7 +99,7 @@ RawPull* raw_pull_unref(RawPull *i) {
         free(i->verity_path);
         free(i->image_root);
         free(i->local);
-        free(i->checksum);
+        iovec_done(&i->checksum);
 
         return mfree(i);
 }
@@ -585,7 +585,7 @@ static void raw_pull_job_on_finished(PullJob *j) {
                 raw_pull_report_progress(i, RAW_VERIFYING);
 
                 r = pull_verify(i->verify,
-                                i->checksum,
+                                &i->checksum,
                                 i->raw_job,
                                 i->checksum_job,
                                 i->signature_job,
@@ -827,7 +827,7 @@ int raw_pull_start(
                 uint64_t size_max,
                 ImportFlags flags,
                 ImportVerify verify,
-                const char *checksum) {
+                const struct iovec *checksum) {
 
         int r;
 
@@ -835,11 +835,11 @@ int raw_pull_start(
         assert(url);
         assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
         assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
-        assert((verify < 0) || !checksum);
+        assert((verify < 0) || !iovec_is_set(checksum));
         assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_RAW));
         assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT));
         assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !(flags & IMPORT_DIRECT));
-        assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !checksum);
+        assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !iovec_is_set(checksum));
 
         if (!http_url_is_valid(url) && !file_url_is_valid(url))
                 return -EINVAL;
@@ -854,9 +854,8 @@ int raw_pull_start(
         if (r < 0)
                 return r;
 
-        r = free_and_strdup(&i->checksum, checksum);
-        if (r < 0)
-                return r;
+        if (!iovec_memdup(checksum, &i->checksum))
+                return -ENOMEM;
 
         i->flags = flags;
         i->verify = verify;
@@ -869,7 +868,7 @@ int raw_pull_start(
         i->raw_job->on_finished = raw_pull_job_on_finished;
         i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
 
-        if (checksum)
+        if (iovec_is_set(checksum))
                 i->raw_job->calc_checksum = true;
         else if (verify != IMPORT_VERIFY_NO) {
                 /* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its
@@ -899,7 +898,7 @@ int raw_pull_start(
                         &i->checksum_job,
                         &i->signature_job,
                         verify,
-                        i->checksum,
+                        &i->checksum,
                         url,
                         i->glue,
                         raw_pull_job_on_finished,

diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h
index c8126d4ff1b320ccb4230e895bead417e2209976..0dbd4dcd20c73f9cee6b9737fb3764639bf93bd5 100644 (file)
--- a/src/import/pull-raw.h
+++ b/src/import/pull-raw.h
@@ -14,4 +14,4 @@ RawPull* raw_pull_unref(RawPull *pull);
 
 DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref);
 
-int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const char *checksum);
+int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const struct iovec *checksum);

diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c
index 23bdfb186a7e8701593d5e5ba254556aad4d9a97..a899e9a225de67b0eac617075a3e0929cd181785 100644 (file)
--- a/src/import/pull-tar.c
+++ b/src/import/pull-tar.c
@@ -65,7 +65,7 @@ typedef struct TarPull {
         char *settings_path;
         char *settings_temp_path;
 
-        char *checksum;
+        struct iovec checksum;
 
         int tree_fd;
         int userns_fd;
@@ -98,7 +98,7 @@ TarPull* tar_pull_unref(TarPull *i) {
         free(i->settings_path);
         free(i->image_root);
         free(i->local);
-        free(i->checksum);
+        iovec_done(&i->checksum);
 
         safe_close(i->tree_fd);
         safe_close(i->userns_fd);
@@ -478,7 +478,7 @@ static void tar_pull_job_on_finished(PullJob *j) {
 
                 clear_progress_bar(/* prefix= */ NULL);
                 r = pull_verify(i->verify,
-                                i->checksum,
+                                &i->checksum,
                                 i->tar_job,
                                 i->checksum_job,
                                 i->signature_job,
@@ -698,17 +698,17 @@ int tar_pull_start(
                 const char *local,
                 ImportFlags flags,
                 ImportVerify verify,
-                const char *checksum) {
+                const struct iovec *checksum) {
 
         int r;
 
         assert(i);
         assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
         assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
-        assert((verify < 0) || !checksum);
+        assert((verify < 0) || !iovec_is_set(checksum));
         assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_TAR));
         assert(!(flags & IMPORT_PULL_SETTINGS) || !(flags & IMPORT_DIRECT));
-        assert(!(flags & IMPORT_PULL_SETTINGS) || !checksum);
+        assert(!(flags & IMPORT_PULL_SETTINGS) || !iovec_is_set(checksum));
 
         if (!http_url_is_valid(url) && !file_url_is_valid(url))
                 return -EINVAL;
@@ -723,9 +723,8 @@ int tar_pull_start(
         if (r < 0)
                 return r;
 
-        r = free_and_strdup(&i->checksum, checksum);
-        if (r < 0)
-                return r;
+        if (!iovec_memdup(checksum, &i->checksum))
+                return -ENOMEM;
 
         i->flags = flags;
         i->verify = verify;

diff --git a/src/import/pull-tar.h b/src/import/pull-tar.h
index 8f622dff3ac9524aaf298c86c8dd5285626cd8b6..d2118272ff16e28b131031556367ac5986bcd04e 100644 (file)
--- a/src/import/pull-tar.h
+++ b/src/import/pull-tar.h
@@ -14,4 +14,4 @@ TarPull* tar_pull_unref(TarPull *pull);
 
 DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref);
 
-int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const char *checksum);
+int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const struct iovec *checksum);

diff --git a/src/import/pull.c b/src/import/pull.c
index a1b9b940fdfbdc1b412cb2c3efbe98ee84aff43d..a57c1c039d916910078908a258985e3900cdbe48 100644 (file)
--- a/src/import/pull.c
+++ b/src/import/pull.c
@@ -15,6 +15,7 @@
 #include "import-common.h"
 #include "import-util.h"
 #include "io-util.h"
+#include "iovec-util.h"
 #include "log.h"
 #include "main-func.h"
 #include "parse-argument.h"
@@ -32,11 +33,11 @@ static char *arg_image_root = NULL;
 static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE;
 static ImportFlags arg_import_flags = IMPORT_PULL_SETTINGS | IMPORT_PULL_ROOTHASH | IMPORT_PULL_ROOTHASH_SIGNATURE | IMPORT_PULL_VERITY | IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC;
 static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
-static char *arg_checksum = NULL;
+static struct iovec arg_checksum = {};
 static ImageClass arg_class = IMAGE_MACHINE;
 static RuntimeScope arg_runtime_scope = _RUNTIME_SCOPE_INVALID;
 
-STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep);
+STATIC_DESTRUCTOR_REGISTER(arg_checksum, iovec_done);
 STATIC_DESTRUCTOR_REGISTER(arg_image_root, freep);
 
 static int normalize_local(const char *local, const char *url, char **ret) {
@@ -162,7 +163,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) {
                         normalized,
                         arg_import_flags & IMPORT_PULL_FLAGS_MASK_TAR,
                         arg_verify,
-                        arg_checksum);
+                        &arg_checksum);
         if (r < 0)
                 return log_error_errno(r, "Failed to pull image: %m");
 
@@ -231,7 +232,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
                         arg_size_max,
                         arg_import_flags & IMPORT_PULL_FLAGS_MASK_RAW,
                         arg_verify,
-                        arg_checksum);
+                        &arg_checksum);
         if (r < 0)
                 return log_error_errno(r, "Failed to pull image: %m");
 
@@ -371,7 +372,6 @@ static int parse_argv(int argc, char *argv[]) {
                         v = import_verify_from_string(optarg);
                         if (v < 0) {
                                 _cleanup_free_ void *h = NULL;
-                                char *hh;
                                 size_t n;
 
                                 /* If this is not a valid verification mode, maybe it's a literally specified
@@ -385,11 +385,10 @@ static int parse_argv(int argc, char *argv[]) {
                                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                                                "64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2);
 
-                                hh = hexmem(h, n); /* bring into canonical (lowercase) form */
-                                if (!hh)
-                                        return log_oom();
+                                iovec_done(&arg_checksum);
+                                arg_checksum.iov_base = TAKE_PTR(h);
+                                arg_checksum.iov_len = n;
 
-                                free_and_replace(arg_checksum, hh);
                                 arg_import_flags &= ~(IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY);
                                 arg_verify = _IMPORT_VERIFY_INVALID;
                         } else
@@ -542,7 +541,7 @@ static int parse_argv(int argc, char *argv[]) {
         if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT))
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
 
-        if (arg_checksum && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0)
+        if (iovec_is_set(&arg_checksum) && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0)
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded.");
 
         if (!arg_image_root) {