#include "dirent-util.h"
#include "escape.h"
#include "fd-util.h"
+#include "hostname-util.h"
#include "io-util.h"
#include "memory-util.h"
#include "path-util.h"
return 0;
}
-int pull_make_local_copy(const char *final, const char *image_root, const char *local, PullFlags flags) {
- const char *p;
- int r;
-
- assert(final);
- assert(local);
-
- if (!image_root)
- image_root = "/var/lib/machines";
-
- p = prefix_roota(image_root, local);
-
- if (FLAGS_SET(flags, PULL_FORCE))
- (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
-
- r = btrfs_subvol_snapshot(final, p,
- BTRFS_SNAPSHOT_QUOTA|
- BTRFS_SNAPSHOT_FALLBACK_COPY|
- BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
- BTRFS_SNAPSHOT_RECURSIVE);
- if (r < 0)
- return log_error_errno(r, "Failed to create local image: %m");
-
- log_info("Created new local image '%s'.", local);
-
- return 0;
-}
-
static int hash_url(const char *url, char **ret) {
uint64_t h;
static const sd_id128_t k = SD_ID128_ARRAY(df,89,16,87,01,cc,42,30,98,ab,4a,19,a6,a5,63,4f);
const char *url,
int (*strip_suffixes)(const char *name, char **ret),
const char *suffix,
+ ImportVerify verify,
CurlGlue *glue,
+ PullJobOpenDisk on_open_disk,
PullJobFinished on_finished,
void *userdata) {
if (r < 0)
return r;
+ job->on_open_disk = on_open_disk;
job->on_finished = on_finished;
job->compressed_max = job->uncompressed_max = 1ULL * 1024ULL * 1024ULL;
+ job->calc_checksum = IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
*ret = TAKE_PTR(job);
-
return 0;
}
+static bool is_checksum_file(const char *fn) {
+ /* Returns true if the specified filename refers to a checksum file we grok */
+
+ if (!fn)
+ return false;
+
+ return streq(fn, "SHA256SUMS") || endswith(fn, ".sha256");
+}
+
+static bool is_signature_file(const char *fn) {
+ /* Returns true if the specified filename refers to a signature file we grok (reminder:
+ * suse-style .sha256 files are inline signed) */
+
+ if (!fn)
+ return false;
+
+ return streq(fn, "SHA256SUMS.gpg") || endswith(fn, ".sha256");
+}
+
int pull_make_verification_jobs(
PullJob **ret_checksum_job,
PullJob **ret_signature_job,
ImportVerify verify,
+ const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */
const char *url,
CurlGlue *glue,
PullJobFinished on_finished,
void *userdata) {
_cleanup_(pull_job_unrefp) PullJob *checksum_job = NULL, *signature_job = NULL;
+ _cleanup_free_ char *fn = NULL;
int r;
assert(ret_checksum_job);
assert(ret_signature_job);
- assert(verify >= 0);
- assert(verify < _IMPORT_VERIFY_MAX);
+ assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
+ assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+ assert((verify < 0) || !checksum);
assert(url);
assert(glue);
- if (verify != IMPORT_VERIFY_NO) {
- _cleanup_free_ char *checksum_url = NULL, *fn = NULL;
- const char *chksums = NULL;
+ /* If verification is turned off, or if the checksum to validate is already specified we don't need
+ * to download a checksum file or signature, hence shortcut things */
+ if (verify == IMPORT_VERIFY_NO || checksum) {
+ *ret_checksum_job = *ret_signature_job = NULL;
+ return 0;
+ }
+
+ r = import_url_last_component(url, &fn);
+ if (r < 0 && r != -EADDRNOTAVAIL) /* EADDRNOTAVAIL means there was no last component, which is OK for
+ * us, we'll just assume it's not a checksum/signature file */
+ return r;
+
+ /* Acquire the checksum file if verification or signature verification is requested and the main file
+ * to acquire isn't a checksum or signature file anyway */
+ if (verify != IMPORT_VERIFY_NO && !is_checksum_file(fn) && !is_signature_file(fn)) {
+ _cleanup_free_ char *checksum_url = NULL;
+ const char *suffixed = NULL;
/* Queue jobs for the checksum file for the image. */
- r = import_url_last_component(url, &fn);
- if (r < 0)
- return r;
- chksums = strjoina(fn, ".sha256");
+ if (fn)
+ suffixed = strjoina(fn, ".sha256"); /* Start with the suse-style checksum (if there's a base filename) */
+ else
+ suffixed = "SHA256SUMS";
- r = import_url_change_last_component(url, chksums, &checksum_url);
+ r = import_url_change_last_component(url, suffixed, &checksum_url);
if (r < 0)
return r;
checksum_job->on_finished = on_finished;
checksum_job->uncompressed_max = checksum_job->compressed_max = 1ULL * 1024ULL * 1024ULL;
+ checksum_job->on_not_found = pull_job_restart_with_sha256sum; /* if this fails, look for ubuntu-style checksum */
}
- if (verify == IMPORT_VERIFY_SIGNATURE) {
+ if (verify == IMPORT_VERIFY_SIGNATURE && !is_signature_file(fn)) {
_cleanup_free_ char *signature_url = NULL;
/* Queue job for the SHA256SUMS.gpg file for the image. */
*ret_checksum_job = TAKE_PTR(checksum_job);
*ret_signature_job = TAKE_PTR(signature_job);
-
return 0;
}
r = import_url_last_component(job->url, &fn);
if (r < 0)
- return log_oom();
+ return log_error_errno(r, "Failed to extract filename from URL '%s': %m", job->url);
if (!filename_is_valid(fn))
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Cannot verify checksum, could not determine server-side file name.");
- line = strjoina(job->checksum, " *", fn, "\n");
+ if (is_checksum_file(fn) || is_signature_file(fn)) /* We cannot verify checksum files or signature files with a checksum file */
+ return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
+ "Cannot verify checksum/signature files via themselves.");
+ line = strjoina(job->checksum, " *", fn, "\n"); /* string for binary mode */
p = memmem_safe(checksum_job->payload,
checksum_job->payload_size,
line,
strlen(line));
-
if (!p) {
- line = strjoina(job->checksum, " ", fn, "\n");
-
+ line = strjoina(job->checksum, " ", fn, "\n"); /* string for text mode */
p = memmem_safe(checksum_job->payload,
checksum_job->payload_size,
line,
strlen(line));
}
+ /* Only counts if found at beginning of a line */
if (!p || (p != (char*) checksum_job->payload && p[-1] != '\n'))
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
- "DOWNLOAD INVALID: Checksum of %s file did not checkout, file has been tampered with.", fn);
+ "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", fn);
log_info("SHA256 checksum of %s is valid.", job->url);
return 1;
pid = 0;
if (r < 0)
goto finish;
- if (r != EXIT_SUCCESS) {
- log_error("DOWNLOAD INVALID: Signature verification failed.");
- r = -EBADMSG;
- } else {
+ if (r != EXIT_SUCCESS)
+ r = log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+ "DOWNLOAD INVALID: Signature verification failed.");
+ else {
log_info("Signature verification succeeded.");
r = 0;
}
}
int pull_verify(ImportVerify verify,
+ const char *checksum, /* Verify with literal checksum */
PullJob *main_job,
PullJob *checksum_job,
PullJob *signature_job,
PullJob *roothash_signature_job,
PullJob *verity_job) {
+ _cleanup_free_ char *fn = NULL;
VerificationStyle style;
- PullJob *j;
+ PullJob *verify_job;
int r;
+ assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
+ assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+ assert((verify < 0) || !checksum);
assert(main_job);
assert(main_job->state == PULL_JOB_DONE);
- if (verify == IMPORT_VERIFY_NO)
+ if (verify == IMPORT_VERIFY_NO) /* verification turned off */
return 0;
- assert(main_job->calc_checksum);
- assert(main_job->checksum);
- assert(checksum_job);
- assert(checksum_job->state == PULL_JOB_DONE);
+ if (checksum) {
+ /* Verification by literal checksum */
+ assert(!checksum_job);
+ assert(!signature_job);
+ assert(!settings_job);
+ assert(!roothash_job);
+ assert(!roothash_signature_job);
+ assert(!verity_job);
- if (!checksum_job->payload || checksum_job->payload_size <= 0)
- return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
- "Checksum is empty, cannot verify.");
+ assert(main_job->calc_checksum);
+ assert(main_job->checksum);
- FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) {
- r = verify_one(checksum_job, j);
- if (r < 0)
- return r;
+ if (!strcaseeq(checksum, main_job->checksum))
+ return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+ "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.",
+ main_job->url);
+
+ return 0;
}
- if (verify == IMPORT_VERIFY_CHECKSUM)
+ r = import_url_last_component(main_job->url, &fn);
+ if (r < 0)
+ return log_error_errno(r, "Failed to extract filename from URL '%s': %m", main_job->url);
+
+ if (is_signature_file(fn))
+ return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
+ "Main download is a signature file, can't verify it.");
+
+ if (is_checksum_file(fn)) {
+ log_debug("Main download is a checksum file, can't validate its checksum with itself, skipping.");
+ verify_job = main_job;
+ } else {
+ PullJob *j;
+ assert(main_job->calc_checksum);
+ assert(main_job->checksum);
+ assert(checksum_job);
+ assert(checksum_job->state == PULL_JOB_DONE);
+
+ if (!checksum_job->payload || checksum_job->payload_size <= 0)
+ return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
+ "Checksum is empty, cannot verify.");
+
+ FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) {
+ r = verify_one(checksum_job, j);
+ if (r < 0)
+ return r;
+ }
+
+ verify_job = checksum_job;
+ }
+
+ if (verify != IMPORT_VERIFY_SIGNATURE)
return 0;
- r = verification_style_from_url(checksum_job->url, &style);
+ assert(verify_job);
+
+ r = verification_style_from_url(verify_job->url, &style);
if (r < 0)
- return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", checksum_job->url);
+ return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", verify_job->url);
if (style == VERIFICATION_PER_DIRECTORY) {
assert(signature_job);
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Signature is empty, cannot verify.");
- return verify_gpg(checksum_job->payload, checksum_job->payload_size, signature_job->payload, signature_job->payload_size);
+ return verify_gpg(verify_job->payload, verify_job->payload_size, signature_job->payload, signature_job->payload_size);
} else
- return verify_gpg(checksum_job->payload, checksum_job->payload_size, NULL, 0);
+ return verify_gpg(verify_job->payload, verify_job->payload_size, NULL, 0);
}
int verification_style_from_url(const char *url, VerificationStyle *ret) {
return 1;
}
+
+bool pull_validate_local(const char *name, PullFlags flags) {
+
+ if (FLAGS_SET(flags, PULL_DIRECT))
+ return path_is_valid(name);
+
+ return hostname_is_valid(name, 0);
+}
+
+int pull_url_needs_checksum(const char *url) {
+ _cleanup_free_ char *fn = NULL;
+ int r;
+
+ /* Returns true if we need to validate this resource via a hash value. This returns true for all
+ * files — except for gpg signature files and SHA256SUMS files and the like, which are validated with
+ * a validation tool like gpg. */
+
+ r = import_url_last_component(url, &fn);
+ if (r == -EADDRNOTAVAIL) /* no last component? then let's assume it's not a signature/checksum file */
+ return false;
+ if (r < 0)
+ return r;
+
+ return !is_checksum_file(fn) && !is_signature_file(fn);
+}
#include "pull-job.h"
typedef enum PullFlags {
- PULL_FORCE = 1 << 0, /* replace existing image */
- PULL_SETTINGS = 1 << 1, /* .nspawn settings file */
- PULL_ROOTHASH = 1 << 2, /* only for raw: .roothash file for verity */
- PULL_ROOTHASH_SIGNATURE = 1 << 3, /* only for raw: .roothash.p7s file for verity */
- PULL_VERITY = 1 << 4, /* only for raw: .verity file for verity */
+ PULL_FORCE = 1 << 0, /* replace existing image */
+ PULL_READ_ONLY = 1 << 1, /* make generated image read-only */
+ PULL_SETTINGS = 1 << 1, /* download .nspawn settings file */
+ PULL_ROOTHASH = 1 << 2, /* only for raw: download .roothash file for verity */
+ PULL_ROOTHASH_SIGNATURE = 1 << 3, /* only for raw: download .roothash.p7s file for verity */
+ PULL_VERITY = 1 << 4, /* only for raw: download .verity file for verity */
+ PULL_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */
+ PULL_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */
+ PULL_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */
+ PULL_DIRECT = 1 << 5, /* download without rename games */
+ PULL_SYNC = 1 << 6, /* fsync() right before we are done */
/* The supported flags for the tar and the raw pulling */
- PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_SETTINGS,
- PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY,
+ PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_BTRFS_SUBVOL|PULL_BTRFS_QUOTA|PULL_DIRECT|PULL_SYNC,
+ PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY|PULL_CONVERT_QCOW2|PULL_DIRECT|PULL_SYNC,
} PullFlags;
-int pull_make_local_copy(const char *final, const char *root, const char *local, PullFlags flags);
-
int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags);
int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret);
-int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
-int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
+int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata);
+int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
-int pull_verify(ImportVerify verify, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
+int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
typedef enum VerificationStyle {
VERIFICATION_PER_FILE, /* SuSE-style ".sha256" files with inline gpg signature */
int verification_style_from_url(const char *url, VerificationStyle *style);
int pull_job_restart_with_sha256sum(PullJob *job, char **ret);
+
+bool pull_validate_local(const char *name, PullFlags flags);
+
+int pull_url_needs_checksum(const char *url);
#include "alloc-util.h"
#include "fd-util.h"
#include "format-util.h"
+#include "fs-util.h"
#include "gcrypt-util.h"
#include "hexdecoct.h"
#include "import-util.h"
#include "strv.h"
#include "xattr-util.h"
+void pull_job_close_disk_fd(PullJob *j) {
+ if (!j)
+ return;
+
+ if (j->close_disk_fd)
+ safe_close(j->disk_fd);
+
+ j->disk_fd = -1;
+}
+
PullJob* pull_job_unref(PullJob *j) {
if (!j)
return NULL;
+ pull_job_close_disk_fd(j);
+
curl_glue_remove_and_free(j->glue, j->curl);
curl_slist_free_all(j->request_header);
- safe_close(j->disk_fd);
-
import_compress_free(&j->compress);
if (j->checksum_context)
return;
if (result != CURLE_OK) {
- log_error("Transfer failed: %s", curl_easy_strerror(result));
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Transfer failed: %s", curl_easy_strerror(result));
goto finish;
}
code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
if (code != CURLE_OK) {
- log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
goto finish;
} else if (status == 304) {
log_info("Image already downloaded. Skipping download.");
code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
if (code != CURLE_OK) {
- log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
goto finish;
}
}
}
- log_error("HTTP request to %s failed with code %li.", j->url, status);
- r = -EIO;
+ r = log_error_errno(
+ status == 404 ? SYNTHETIC_ERRNO(ENOMEDIUM) : SYNTHETIC_ERRNO(EIO), /* Make the most common error recognizable */
+ "HTTP request to %s failed with code %li.", j->url, status);
goto finish;
} else if (status < 200) {
- log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "HTTP request to %s finished with unexpected code %li.", j->url, status);
goto finish;
}
if (j->state != PULL_JOB_RUNNING) {
- log_error("Premature connection termination.");
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination.");
goto finish;
}
if (j->content_length != UINT64_MAX &&
j->content_length != j->written_compressed) {
- log_error("Download truncated.");
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Download truncated.");
goto finish;
}
k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
if (!k) {
- log_error("Failed to get checksum.");
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum.");
goto finish;
}
log_debug("SHA256 of %s is %s.", j->url, j->checksum);
}
- if (j->disk_fd >= 0 && j->allow_sparse) {
- /* Make sure the file size is right, in case the file was
- * sparse and we just seeked for the last part */
+ /* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no
+ * offset is specified, which indicates we only own the file partially) */
- if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
- r = log_error_errno(errno, "Failed to truncate file: %m");
- goto finish;
- }
+ if (j->disk_fd >= 0) {
- if (j->etag)
- (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
- if (j->url)
- (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
+ if (S_ISREG(j->disk_stat.st_mode)) {
- if (j->mtime != 0) {
- struct timespec ut[2];
+ if (j->offset == UINT64_MAX) {
+
+ if (j->written_compressed > 0) {
+ /* Make sure the file size is right, in case the file was sparse and we just seeked
+ * for the last part */
+ if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
+ r = log_error_errno(errno, "Failed to truncate file: %m");
+ goto finish;
+ }
+ }
+
+ if (j->etag)
+ (void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
+ if (j->url)
+ (void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
+
+ if (j->mtime != 0) {
+ struct timespec ut;
+
+ timespec_store(&ut, j->mtime);
+
+ if (futimens(j->disk_fd, (struct timespec[]) { ut, ut }) < 0)
+ log_debug_errno(errno, "Failed to adjust atime/mtime of created image, ignoring: %m");
+
+ r = fd_setcrtime(j->disk_fd, j->mtime);
+ if (r < 0)
+ log_debug_errno(r, "Failed to adjust crtime of created image, ignoring: %m");
+ }
+ }
+
+ if (j->sync) {
+ r = fsync_full(j->disk_fd);
+ if (r < 0) {
+ log_error_errno(r, "Failed to synchronize file to disk: %m");
+ goto finish;
+ }
+ }
- timespec_store(&ut[0], j->mtime);
- ut[1] = ut[0];
- (void) futimens(j->disk_fd, ut);
+ } else if (S_ISBLK(j->disk_stat.st_mode) && j->sync) {
- (void) fd_setcrtime(j->disk_fd, j->mtime);
+ if (fsync(j->disk_fd) < 0) {
+ r = log_error_errno(errno, "Failed to synchronize block device: %m");
+ goto finish;
+ }
}
}
static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) {
PullJob *j = userdata;
- ssize_t n;
+ bool too_much = false;
+ int r;
assert(j);
assert(p);
+ assert(sz > 0);
- if (sz <= 0)
- return 0;
+ if (j->written_uncompressed > UINT64_MAX - sz)
+ return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
- if (j->written_uncompressed + sz < j->written_uncompressed)
- return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW),
- "File too large, overflow");
+ if (j->written_uncompressed >= j->uncompressed_max) {
+ too_much = true;
+ goto finish;
+ }
- if (j->written_uncompressed + sz > j->uncompressed_max)
- return log_error_errno(SYNTHETIC_ERRNO(EFBIG),
- "File overly large, refusing");
+ if (j->written_uncompressed + sz > j->uncompressed_max) {
+ too_much = true;
+ sz = j->uncompressed_max - j->written_uncompressed; /* since we have the data in memory
+ * already, we might as well write it to
+ * disk to the max */
+ }
if (j->disk_fd >= 0) {
- if (j->allow_sparse)
+ if (S_ISREG(j->disk_stat.st_mode)) {
+ ssize_t n;
+
n = sparse_write(j->disk_fd, p, sz, 64);
- else {
- n = write(j->disk_fd, p, sz);
if (n < 0)
- n = -errno;
+ return log_error_errno((int) n, "Failed to write file: %m");
+ if ((size_t) n < sz)
+ return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
+ } else {
+ r = loop_write(j->disk_fd, p, sz, false);
+ if (r < 0)
+ return log_error_errno(r, "Failed to write file: %m");
}
- if (n < 0)
- return log_error_errno((int) n, "Failed to write file: %m");
- if ((size_t) n < sz)
- return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
- } else {
+ }
+ if (j->disk_fd < 0 || j->force_memory) {
if (!GREEDY_REALLOC(j->payload, j->payload_size + sz))
return log_oom();
j->written_uncompressed += sz;
+finish:
+ if (too_much)
+ return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing.");
+
return 0;
}
}
if (j->disk_fd >= 0) {
- /* Check if we can do sparse files */
+ if (fstat(j->disk_fd, &j->disk_stat) < 0)
+ return log_error_errno(errno, "Failed to stat disk file: %m");
- if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
- j->allow_sparse = true;
- else {
- if (errno != ESPIPE)
+ if (j->offset != UINT64_MAX) {
+ if (lseek(j->disk_fd, j->offset, SEEK_SET) == (off_t) -1)
return log_error_errno(errno, "Failed to seek on file descriptor: %m");
-
- j->allow_sparse = false;
}
}
code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
if (code != CURLE_OK) {
- log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
- r = -EIO;
+ r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
goto fail;
}
return 0;
}
-int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) {
+int pull_job_new(
+ PullJob **ret,
+ const char *url,
+ CurlGlue *glue,
+ void *userdata) {
+
_cleanup_(pull_job_unrefp) PullJob *j = NULL;
_cleanup_free_ char *u = NULL;
*j = (PullJob) {
.state = PULL_JOB_INIT,
.disk_fd = -1,
+ .close_disk_fd = true,
.userdata = userdata,
.glue = glue,
.content_length = UINT64_MAX,
.compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
.uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
.url = TAKE_PTR(u),
+ .offset = UINT64_MAX,
+ .sync = true,
};
*ret = TAKE_PTR(j);
#pragma once
#include <gcrypt.h>
+#include <sys/stat.h>
#include "curl-util.h"
#include "import-compress.h"
#include "macro.h"
+#include "pull-common.h"
typedef struct PullJob PullJob;
uint64_t content_length;
uint64_t written_compressed;
uint64_t written_uncompressed;
+ uint64_t offset;
uint64_t uncompressed_max;
uint64_t compressed_max;
size_t payload_size;
int disk_fd;
+ bool close_disk_fd;
+ struct stat disk_stat;
usec_t mtime;
usec_t start_usec;
usec_t last_status_usec;
- bool allow_sparse;
-
bool calc_checksum;
gcry_md_hd_t checksum_context;
char *checksum;
+ bool sync;
+ bool force_memory;
};
int pull_job_new(PullJob **job, const char *url, CurlGlue *glue, void *userdata);
void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result);
+void pull_job_close_disk_fd(PullJob *j);
+
DEFINE_TRIVIAL_CLEANUP_FUNC(PullJob*, pull_job_unref);
#include "hostname-util.h"
#include "import-common.h"
#include "import-util.h"
+#include "install-file.h"
#include "macro.h"
#include "mkdir.h"
#include "path-util.h"
ImportVerify verify;
char *image_root;
+ uint64_t offset;
+
PullJob *raw_job;
PullJob *checksum_job;
PullJob *signature_job;
RawPullFinished on_finished;
void *userdata;
- char *local;
+ char *local; /* In PULL_DIRECT mode the path we are supposed to place things in, otherwise the
+ * machine name of the final copy we make */
char *final_path;
char *temp_path;
char *verity_path;
char *verity_temp_path;
+
+ char *checksum;
};
RawPull* raw_pull_unref(RawPull *i) {
free(i->verity_path);
free(i->image_root);
free(i->local);
+ free(i->checksum);
return mfree(i);
}
.image_root = TAKE_PTR(root),
.event = TAKE_PTR(e),
.glue = TAKE_PTR(g),
+ .offset = UINT64_MAX,
};
i->glue->on_finished = pull_job_curl_on_finished;
}
static int raw_pull_maybe_convert_qcow2(RawPull *i) {
+ _cleanup_(unlink_and_freep) char *t = NULL;
_cleanup_close_ int converted_fd = -1;
- _cleanup_free_ char *t = NULL;
+ _cleanup_free_ char *f = NULL;
int r;
assert(i);
assert(i->raw_job);
+ assert(!FLAGS_SET(i->flags, PULL_DIRECT));
+
+ if (!FLAGS_SET(i->flags, PULL_CONVERT_QCOW2))
+ return 0;
+
+ assert(i->final_path);
+ assert(i->raw_job->close_disk_fd);
r = qcow2_detect(i->raw_job->disk_fd);
if (r < 0)
return 0;
/* This is a QCOW2 image, let's convert it */
- r = tempfn_random(i->final_path, NULL, &t);
+ r = tempfn_random(i->final_path, NULL, &f);
if (r < 0)
return log_oom();
- converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+ converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (converted_fd < 0)
- return log_error_errno(errno, "Failed to create %s: %m", t);
+ return log_error_errno(errno, "Failed to create %s: %m", f);
+
+ t = TAKE_PTR(f);
(void) import_set_nocow_and_log(converted_fd, t);
log_info("Unpacking QCOW2 file.");
r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
- if (r < 0) {
- (void) unlink(t);
+ if (r < 0)
return log_error_errno(r, "Failed to convert qcow2 image: %m");
- }
- (void) unlink(i->temp_path);
- free_and_replace(i->temp_path, t);
+ unlink_and_free(i->temp_path);
+ i->temp_path = TAKE_PTR(t);
CLOSE_AND_REPLACE(i->raw_job->disk_fd, converted_fd);
return 1;
}
-static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) {
+static int raw_pull_determine_path(
+ RawPull *i,
+ const char *suffix,
+ char **field /* input + output (!) */) {
int r;
assert(i);
static int raw_pull_copy_auxiliary_file(
RawPull *i,
const char *suffix,
- char **path) {
+ char **path /* input + output (!) */) {
const char *local;
int r;
local = strjoina(i->image_root, "/", i->local, suffix);
- r = copy_file_atomic(*path, local, 0644, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0));
+ r = copy_file_atomic(
+ *path,
+ local,
+ 0644,
+ 0, 0,
+ COPY_REFLINK |
+ (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
+ (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
if (r == -EEXIST)
log_warning_errno(r, "File %s already exists, not replacing.", local);
else if (r == -ENOENT)
}
static int raw_pull_make_local_copy(RawPull *i) {
- _cleanup_free_ char *tp = NULL;
+ _cleanup_(unlink_and_freep) char *tp = NULL;
+ _cleanup_free_ char *f = NULL;
_cleanup_close_ int dfd = -1;
const char *p;
int r;
assert(i);
assert(i->raw_job);
+ assert(!FLAGS_SET(i->flags, PULL_DIRECT));
if (!i->local)
return 0;
/* We freshly downloaded the image, use it */
assert(i->raw_job->disk_fd >= 0);
+ assert(i->offset == UINT64_MAX);
if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
p = strjoina(i->image_root, "/", i->local, ".raw");
- if (FLAGS_SET(i->flags, PULL_FORCE))
- (void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
-
- r = tempfn_random(p, NULL, &tp);
+ r = tempfn_random(p, NULL, &f);
if (r < 0)
return log_oom();
- dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+ dfd = open(f, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (dfd < 0)
return log_error_errno(errno, "Failed to create writable copy of image: %m");
+ tp = TAKE_PTR(f);
+
/* Turn off COW writing. This should greatly improve performance on COW file systems like btrfs,
* since it reduces fragmentation caused by not allowing in-place writes. */
(void) import_set_nocow_and_log(dfd, tp);
r = copy_bytes(i->raw_job->disk_fd, dfd, UINT64_MAX, COPY_REFLINK);
- if (r < 0) {
- (void) unlink(tp);
+ if (r < 0)
return log_error_errno(r, "Failed to make writable copy of image: %m");
- }
(void) copy_times(i->raw_job->disk_fd, dfd, COPY_CRTIME);
(void) copy_xattr(i->raw_job->disk_fd, dfd, 0);
dfd = safe_close(dfd);
- r = rename(tp, p);
- if (r < 0) {
- r = log_error_errno(errno, "Failed to move writable image into place: %m");
- (void) unlink(tp);
- return r;
- }
+ r = install_file(AT_FDCWD, tp,
+ AT_FDCWD, p,
+ (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
+ (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
+ (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
+ if (r < 0)
+ return log_error_errno(errno, "Failed to move local image into place '%s': %m", p);
+
+ tp = mfree(tp);
log_info("Created new local image '%s'.", i->local);
int r;
assert(i);
+ assert(path);
assert(temp_path);
+ assert(*temp_path);
assert(suffix);
- assert(path);
/* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
* incorporate it in the file name if we can */
if (r < 0)
return r;
- r = import_make_read_only(*temp_path);
+ r = install_file(
+ AT_FDCWD, *temp_path,
+ AT_FDCWD, *path,
+ INSTALL_READ_ONLY|
+ (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0)
- return r;
-
- r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path);
- if (r < 0)
- return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path);
+ return log_error_errno(r, "Failed to move '%s' into place: %m", *path);
*temp_path = mfree(*temp_path);
-
return 1;
}
static void raw_pull_job_on_finished(PullJob *j) {
RawPull *i;
+ PullJob *jj;
int r;
assert(j);
assert(j->userdata);
i = j->userdata;
- if (j == i->settings_job) {
- if (j->error != 0)
+
+ if (j->error != 0) {
+ /* Only the main job and the checksum job are fatal if they fail. The other fails are just
+ * "decoration", that we'll download if we can. The signature job isn't fatal here because we
+ * might not actually need it in case Suse style signatures are used, that are inline in the
+ * checksum file. */
+
+ if (j == i->raw_job) {
+ if (j->error == ENOMEDIUM) /* HTTP 404 */
+ r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
+ else
+ r = log_error_errno(j->error, "Failed to retrieve image file.");
+ goto finish;
+ } else if (j == i->checksum_job) {
+ r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
+ goto finish;
+ } else if (j == i->signature_job)
+ log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
+ else if (j == i->settings_job)
log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
- } else if (j == i->roothash_job) {
- if (j->error != 0)
+ else if (j == i->roothash_job)
log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
- } else if (j == i->roothash_signature_job) {
- if (j->error != 0)
+ else if (j == i->roothash_signature_job)
log_info_errno(j->error, "Root hash signature file could not be retrieved, proceeding without.");
- } else if (j == i->verity_job) {
- if (j->error != 0)
- log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without. %s", j->url);
- } else if (j->error != 0 && j != i->signature_job) {
- if (j == i->checksum_job)
- log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
+ else if (j == i->verity_job)
+ log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without.");
else
- log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
-
- r = j->error;
- goto finish;
+ assert_not_reached();
}
/* This is invoked if either the download completed successfully, or the download was skipped because
* we already have the etag. In this case ->etag_exists is true.
*
- * We only do something when we got all three files */
+ * We only do something when we got all files */
if (!raw_pull_is_done(i))
return;
if (i->signature_job && i->signature_job->error != 0) {
VerificationStyle style;
+ PullJob *verify_job;
+
+ /* The signature job failed. Let's see if we actually need it */
+
+ verify_job = i->checksum_job ?: i->raw_job; /* if the checksum job doesn't exist this must be
+ * because the main job is the checksum file
+ * itself */
+
+ assert(verify_job);
- r = verification_style_from_url(i->checksum_job->url, &style);
+ r = verification_style_from_url(verify_job->url, &style);
if (r < 0) {
log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
goto finish;
* in per-directory verification mode, since only
* then the signature is detached, and thus a file
* of its own. */
- log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
- r = i->signature_job->error;
+ r = log_error_errno(i->signature_job->error,
+ "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
goto finish;
}
}
- if (i->settings_job)
- i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
- if (i->roothash_job)
- i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd);
- if (i->roothash_signature_job)
- i->roothash_signature_job->disk_fd = safe_close(i->roothash_signature_job->disk_fd);
- if (i->verity_job)
- i->verity_job->disk_fd = safe_close(i->verity_job->disk_fd);
-
- r = raw_pull_determine_path(i, ".raw", &i->final_path);
- if (r < 0)
- goto finish;
+ /* Let's close these auxiliary files now, we don't need access to them anymore. */
+ FOREACH_POINTER(jj, i->settings_job, i->roothash_job, i->roothash_signature_job, i->verity_job)
+ pull_job_close_disk_fd(jj);
if (!i->raw_job->etag_exists) {
- /* This is a new download, verify it, and move it into place */
- assert(i->raw_job->disk_fd >= 0);
-
raw_pull_report_progress(i, RAW_VERIFYING);
r = pull_verify(i->verify,
+ i->checksum,
i->raw_job,
i->checksum_job,
i->signature_job,
i->verity_job);
if (r < 0)
goto finish;
+ }
- raw_pull_report_progress(i, RAW_UNPACKING);
+ if (i->flags & PULL_DIRECT) {
+ assert(!i->settings_job);
+ assert(!i->roothash_job);
+ assert(!i->roothash_signature_job);
+ assert(!i->verity_job);
+
+ raw_pull_report_progress(i, RAW_FINALIZING);
- r = raw_pull_maybe_convert_qcow2(i);
+ if (i->local) {
+ r = install_file(AT_FDCWD, i->local,
+ AT_FDCWD, NULL,
+ ((i->flags & PULL_READ_ONLY) && i->offset == UINT64_MAX ? INSTALL_READ_ONLY : 0) |
+ (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
+ if (r < 0) {
+ log_error_errno(r, "Failed to finalize raw file to '%s': %m", i->local);
+ goto finish;
+ }
+ }
+ } else {
+ r = raw_pull_determine_path(i, ".raw", &i->final_path);
if (r < 0)
goto finish;
- raw_pull_report_progress(i, RAW_FINALIZING);
+ if (!i->raw_job->etag_exists) {
+ /* This is a new download, verify it, and move it into place */
+
+ assert(i->temp_path);
+ assert(i->final_path);
- if (i->raw_job->etag) {
- /* Only make a read-only copy if ETag header is set. */
- r = import_make_read_only_fd(i->raw_job->disk_fd);
+ raw_pull_report_progress(i, RAW_UNPACKING);
+
+ r = raw_pull_maybe_convert_qcow2(i);
if (r < 0)
goto finish;
- r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
+ raw_pull_report_progress(i, RAW_FINALIZING);
+
+ r = install_file(AT_FDCWD, i->temp_path,
+ AT_FDCWD, i->final_path,
+ INSTALL_READ_ONLY|
+ (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0) {
- log_error_errno(r, "Failed to rename raw file to %s: %m", i->final_path);
+ log_error_errno(r, "Failed to move raw file to '%s': %m", i->final_path);
goto finish;
}
- }
- i->temp_path = mfree(i->temp_path);
+ i->temp_path = mfree(i->temp_path);
- if (i->roothash_job &&
- i->roothash_job->error == 0) {
- r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
- if (r < 0)
- goto finish;
- }
+ if (i->settings_job &&
+ i->settings_job->error == 0) {
+ r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
+ if (r < 0)
+ goto finish;
+ }
- if (i->settings_job &&
- i->settings_job->error == 0) {
- r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
- if (r < 0)
- goto finish;
+ if (i->roothash_job &&
+ i->roothash_job->error == 0) {
+ r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
+ if (r < 0)
+ goto finish;
+ }
+
+ if (i->roothash_signature_job &&
+ i->roothash_signature_job->error == 0) {
+ r = raw_pull_rename_auxiliary_file(i, ".roothash.p7s", &i->roothash_signature_temp_path, &i->roothash_signature_path);
+ if (r < 0)
+ goto finish;
+ }
+
+ if (i->verity_job &&
+ i->verity_job->error == 0) {
+ r = raw_pull_rename_auxiliary_file(i, ".verity", &i->verity_temp_path, &i->verity_path);
+ if (r < 0)
+ goto finish;
+ }
}
- }
- raw_pull_report_progress(i, RAW_COPYING);
+ raw_pull_report_progress(i, RAW_COPYING);
- r = raw_pull_make_local_copy(i);
- if (r < 0)
- goto finish;
+ r = raw_pull_make_local_copy(i);
+ if (r < 0)
+ goto finish;
+ }
r = 0;
RawPull *i,
PullJob *j,
const char *extra,
- char **temp_path) {
+ char **temp_path /* input + output */) {
int r;
assert(extra);
assert(temp_path);
+ assert(!FLAGS_SET(i->flags, PULL_DIRECT));
+
if (!*temp_path) {
r = tempfn_random_child(i->image_root, extra, temp_path);
if (r < 0)
i = j->userdata;
assert(i->raw_job == j);
+ assert(j->disk_fd < 0);
- r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
- if (r < 0)
- return r;
+ if (i->flags & PULL_DIRECT) {
+
+ if (!i->local) { /* If no local name specified, the pull job will write its data to stdout */
+ j->disk_fd = STDOUT_FILENO;
+ j->close_disk_fd = false;
+ return 0;
+ }
+
+ (void) mkdir_parents_label(i->local, 0700);
+
+ j->disk_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664);
+ if (j->disk_fd < 0)
+ return log_error_errno(errno, "Failed to open destination '%s': %m", i->local);
+
+ if (i->offset == UINT64_MAX)
+ (void) import_set_nocow_and_log(j->disk_fd, i->local);
+
+ } else {
+ r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
+ if (r < 0)
+ return r;
+
+ assert(i->offset == UINT64_MAX);
+ (void) import_set_nocow_and_log(j->disk_fd, i->temp_path);
+ }
- (void) import_set_nocow_and_log(j->disk_fd, i->temp_path);
return 0;
}
RawPull *i,
const char *url,
const char *local,
+ uint64_t offset,
+ uint64_t size_max,
PullFlags flags,
- ImportVerify verify) {
+ ImportVerify verify,
+ const char *checksum) {
+ PullJob *j;
int r;
assert(i);
- assert(verify < _IMPORT_VERIFY_MAX);
- assert(verify >= 0);
+ assert(url);
+ assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
+ assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+ assert((verify < 0) || !checksum);
assert(!(flags & ~PULL_FLAGS_MASK_RAW));
+ assert(offset == UINT64_MAX || FLAGS_SET(flags, PULL_DIRECT));
+ assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !(flags & PULL_DIRECT));
+ assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !checksum);
if (!http_url_is_valid(url))
return -EINVAL;
- if (local && !hostname_is_valid(local, 0))
+ if (local && !pull_validate_local(local, flags))
return -EINVAL;
if (i->raw_job)
if (r < 0)
return r;
+ r = free_and_strdup(&i->checksum, checksum);
+ if (r < 0)
+ return r;
+
i->flags = flags;
i->verify = verify;
i->raw_job->on_finished = raw_pull_job_on_finished;
i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
- i->raw_job->on_progress = raw_pull_job_on_progress;
- i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
-
- r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
- if (r < 0)
- return r;
-
- r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
- if (r < 0)
- return r;
-
- if (FLAGS_SET(flags, PULL_SETTINGS)) {
- r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i);
- if (r < 0)
- return r;
- i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
- i->settings_job->on_progress = raw_pull_job_on_progress;
- i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
- }
+ if (checksum)
+ i->raw_job->calc_checksum = true;
+ else if (verify != IMPORT_VERIFY_NO) {
+ /* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its
+ * signature, which we let gpg verify instead. */
- if (FLAGS_SET(flags, PULL_ROOTHASH)) {
- r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i);
+ r = pull_url_needs_checksum(url);
if (r < 0)
return r;
- i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash;
- i->roothash_job->on_progress = raw_pull_job_on_progress;
- i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO;
+ i->raw_job->calc_checksum = r;
+ i->raw_job->force_memory = true; /* make sure this is both written to disk if that's
+ * requested and into memory, since we need to verify it */
}
- if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) {
- r = pull_make_auxiliary_job(&i->roothash_signature_job, url, raw_strip_suffixes, ".roothash.p7s", i->glue, raw_pull_job_on_finished, i);
- if (r < 0)
- return r;
-
- i->roothash_signature_job->on_open_disk = raw_pull_job_on_open_disk_roothash_signature;
- i->roothash_signature_job->on_progress = raw_pull_job_on_progress;
- i->roothash_signature_job->calc_checksum = verify != IMPORT_VERIFY_NO;
- }
+ if (size_max != UINT64_MAX)
+ i->raw_job->uncompressed_max = size_max;
+ if (offset != UINT64_MAX)
+ i->raw_job->offset = i->offset = offset;
- if (FLAGS_SET(flags, PULL_VERITY)) {
- r = pull_make_auxiliary_job(&i->verity_job, url, raw_strip_suffixes, ".verity", i->glue, raw_pull_job_on_finished, i);
+ if (!FLAGS_SET(flags, PULL_DIRECT)) {
+ r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
if (r < 0)
return r;
-
- i->verity_job->on_open_disk = raw_pull_job_on_open_disk_verity;
- i->verity_job->on_progress = raw_pull_job_on_progress;
- i->verity_job->calc_checksum = verify != IMPORT_VERIFY_NO;
}
- r = pull_job_begin(i->raw_job);
+ r = pull_make_verification_jobs(
+ &i->checksum_job,
+ &i->signature_job,
+ verify,
+ i->checksum,
+ url,
+ i->glue,
+ raw_pull_job_on_finished,
+ i);
if (r < 0)
return r;
- if (i->checksum_job) {
- i->checksum_job->on_progress = raw_pull_job_on_progress;
- i->checksum_job->on_not_found = pull_job_restart_with_sha256sum;
-
- r = pull_job_begin(i->checksum_job);
+ if (FLAGS_SET(flags, PULL_SETTINGS)) {
+ r = pull_make_auxiliary_job(
+ &i->settings_job,
+ url,
+ raw_strip_suffixes,
+ ".nspawn",
+ verify,
+ i->glue,
+ raw_pull_job_on_open_disk_settings,
+ raw_pull_job_on_finished,
+ i);
if (r < 0)
return r;
}
- if (i->signature_job) {
- i->signature_job->on_progress = raw_pull_job_on_progress;
-
- r = pull_job_begin(i->signature_job);
+ if (FLAGS_SET(flags, PULL_ROOTHASH)) {
+ r = pull_make_auxiliary_job(
+ &i->roothash_job,
+ url,
+ raw_strip_suffixes,
+ ".roothash",
+ verify,
+ i->glue,
+ raw_pull_job_on_open_disk_roothash,
+ raw_pull_job_on_finished,
+ i);
if (r < 0)
return r;
}
- if (i->settings_job) {
- r = pull_job_begin(i->settings_job);
+ if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) {
+ r = pull_make_auxiliary_job(
+ &i->roothash_signature_job,
+ url,
+ raw_strip_suffixes,
+ ".roothash.p7s",
+ verify,
+ i->glue,
+ raw_pull_job_on_open_disk_roothash_signature,
+ raw_pull_job_on_finished,
+ i);
if (r < 0)
return r;
}
- if (i->roothash_job) {
- r = pull_job_begin(i->roothash_job);
+ if (FLAGS_SET(flags, PULL_VERITY)) {
+ r = pull_make_auxiliary_job(
+ &i->verity_job,
+ url,
+ raw_strip_suffixes,
+ ".verity",
+ verify,
+ i->glue,
+ raw_pull_job_on_open_disk_verity,
+ raw_pull_job_on_finished,
+ i);
if (r < 0)
return r;
}
- if (i->roothash_signature_job) {
- r = pull_job_begin(i->roothash_signature_job);
- if (r < 0)
- return r;
- }
+ FOREACH_POINTER(j,
+ i->raw_job,
+ i->checksum_job,
+ i->signature_job,
+ i->settings_job,
+ i->roothash_job,
+ i->roothash_signature_job,
+ i->verity_job) {
+
+ if (!j)
+ continue;
+
+ j->on_progress = raw_pull_job_on_progress;
+ j->sync = FLAGS_SET(flags, PULL_SYNC);
- if (i->verity_job) {
- r = pull_job_begin(i->verity_job);
+ r = pull_job_begin(j);
if (r < 0)
return r;
}
DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref);
-int raw_pull_start(RawPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify);
+int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, PullFlags flags, ImportVerify verify, const char *checksum);
#include "hostname-util.h"
#include "import-common.h"
#include "import-util.h"
+#include "install-file.h"
#include "macro.h"
#include "mkdir.h"
#include "path-util.h"
#include "string-util.h"
#include "strv.h"
#include "tmpfile-util.h"
+#include "user-util.h"
#include "utf8.h"
#include "util.h"
#include "web-util.h"
char *settings_path;
char *settings_temp_path;
+
+ char *checksum;
};
TarPull* tar_pull_unref(TarPull *i) {
free(i->settings_path);
free(i->image_root);
free(i->local);
+ free(i->checksum);
return mfree(i);
}
log_debug("Combined progress %u%%", percent);
}
-static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) {
+static int tar_pull_determine_path(
+ TarPull *i,
+ const char *suffix,
+ char **field /* input + output (!) */) {
int r;
assert(i);
}
static int tar_pull_make_local_copy(TarPull *i) {
+ _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL;
+ const char *p;
int r;
assert(i);
if (!i->local)
return 0;
- r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->flags);
+ assert(i->final_path);
+
+ p = prefix_roota(i->image_root, i->local);
+
+ r = tempfn_random(p, NULL, &t);
if (r < 0)
- return r;
+ return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p);
+
+ if (i->flags & PULL_BTRFS_SUBVOL)
+ r = btrfs_subvol_snapshot(
+ i->final_path,
+ t,
+ (i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)|
+ BTRFS_SNAPSHOT_FALLBACK_COPY|
+ BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
+ BTRFS_SNAPSHOT_RECURSIVE);
+ else
+ r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS);
+ if (r < 0)
+ return log_error_errno(r, "Failed to create local image: %m");
+
+ r = install_file(AT_FDCWD, t,
+ AT_FDCWD, p,
+ (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
+ (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
+ (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
+ if (r < 0)
+ return log_error_errno(r, "Failed to install local image '%s': %m", p);
+
+ t = mfree(t);
+
+ log_info("Created new local image '%s'.", i->local);
if (FLAGS_SET(i->flags, PULL_SETTINGS)) {
const char *local_settings;
local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
- r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0));
+ r = copy_file_atomic(
+ i->settings_path,
+ local_settings,
+ 0664,
+ 0, 0,
+ COPY_REFLINK |
+ (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
+ (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
if (r == -EEXIST)
log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
else if (r == -ENOENT)
i = j->userdata;
- if (j == i->settings_job) {
- if (j->error != 0)
+ if (j->error != 0) {
+ if (j == i->tar_job) {
+ if (j->error == ENOMEDIUM) /* HTTP 404 */
+ r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
+ else
+ r = log_error_errno(j->error, "Failed to retrieve image file.");
+ goto finish;
+ } else if (j == i->checksum_job) {
+ r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
+ goto finish;
+ } else if (j == i->signature_job)
+ log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
+ else if (j == i->settings_job)
log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
- } else if (j->error != 0 && j != i->signature_job) {
- if (j == i->checksum_job)
- log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
else
- log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
-
- r = j->error;
- goto finish;
+ assert("unexpected job");
}
/* This is invoked if either the download completed successfully, or the download was skipped because
if (i->signature_job && i->signature_job->error != 0) {
VerificationStyle style;
+ assert(i->checksum_job);
+
r = verification_style_from_url(i->checksum_job->url, &style);
if (r < 0) {
log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
* in per-directory verification mode, since only
* then the signature is detached, and thus a file
* of its own. */
- log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
- r = i->signature_job->error;
+ r = log_error_errno(i->signature_job->error,
+ "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
goto finish;
}
}
- i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
- if (i->settings_job)
- i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
-
- r = tar_pull_determine_path(i, NULL, &i->final_path);
- if (r < 0)
- goto finish;
+ pull_job_close_disk_fd(i->tar_job);
+ pull_job_close_disk_fd(i->settings_job);
if (i->tar_pid > 0) {
r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG);
tar_pull_report_progress(i, TAR_VERIFYING);
r = pull_verify(i->verify,
+ i->checksum,
i->tar_job,
i->checksum_job,
i->signature_job,
/* verity_job = */ NULL);
if (r < 0)
goto finish;
+ }
- tar_pull_report_progress(i, TAR_FINALIZING);
+ if (i->flags & PULL_DIRECT) {
+ assert(!i->settings_job);
+ assert(i->local);
+ assert(!i->temp_path);
- r = import_mangle_os_tree(i->temp_path);
- if (r < 0)
- goto finish;
+ tar_pull_report_progress(i, TAR_FINALIZING);
- r = import_make_read_only(i->temp_path);
+ r = import_mangle_os_tree(i->local);
if (r < 0)
goto finish;
- r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
+ r = install_file(
+ AT_FDCWD, i->local,
+ AT_FDCWD, NULL,
+ (i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 |
+ (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
if (r < 0) {
- log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
+ log_error_errno(r, "Failed to finalize '%s': %m", i->local);
goto finish;
}
+ } else {
+ r = tar_pull_determine_path(i, NULL, &i->final_path);
+ if (r < 0)
+ goto finish;
- i->temp_path = mfree(i->temp_path);
-
- if (i->settings_job &&
- i->settings_job->error == 0) {
+ if (!i->tar_job->etag_exists) {
+ /* This is a new download, verify it, and move it into place */
- /* Also move the settings file into place, if it exists. Note that we do so only if we also
- * moved the tar file in place, to keep things strictly in sync. */
- assert(i->settings_temp_path);
+ assert(i->temp_path);
+ assert(i->final_path);
- /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
- * we should incorporate it in the file name if we can */
- i->settings_path = mfree(i->settings_path);
+ tar_pull_report_progress(i, TAR_FINALIZING);
- r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
+ r = import_mangle_os_tree(i->temp_path);
if (r < 0)
goto finish;
- r = import_make_read_only(i->settings_temp_path);
- if (r < 0)
- goto finish;
-
- r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
+ r = install_file(
+ AT_FDCWD, i->temp_path,
+ AT_FDCWD, i->final_path,
+ INSTALL_READ_ONLY|
+ (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
if (r < 0) {
- log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
+ log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
goto finish;
}
- i->settings_temp_path = mfree(i->settings_temp_path);
+ i->temp_path = mfree(i->temp_path);
+
+ if (i->settings_job &&
+ i->settings_job->error == 0) {
+
+ /* Also move the settings file into place, if it exists. Note that we do so only if we also
+ * moved the tar file in place, to keep things strictly in sync. */
+ assert(i->settings_temp_path);
+
+ /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
+ * we should incorporate it in the file name if we can */
+ i->settings_path = mfree(i->settings_path);
+
+ r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
+ if (r < 0)
+ goto finish;
+
+ r = install_file(
+ AT_FDCWD, i->settings_temp_path,
+ AT_FDCWD, i->settings_path,
+ INSTALL_READ_ONLY|
+ (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
+ if (r < 0) {
+ log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
+ goto finish;
+ }
+
+ i->settings_temp_path = mfree(i->settings_temp_path);
+ }
}
- }
- tar_pull_report_progress(i, TAR_COPYING);
+ tar_pull_report_progress(i, TAR_COPYING);
- r = tar_pull_make_local_copy(i);
- if (r < 0)
- goto finish;
+ r = tar_pull_make_local_copy(i);
+ if (r < 0)
+ goto finish;
+ }
r = 0;
}
static int tar_pull_job_on_open_disk_tar(PullJob *j) {
+ const char *where;
TarPull *i;
int r;
assert(i->tar_job == j);
assert(i->tar_pid <= 0);
- if (!i->temp_path) {
- r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
- if (r < 0)
- return log_oom();
+ if (i->flags & PULL_DIRECT)
+ where = i->local;
+ else {
+ if (!i->temp_path) {
+ r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
+ if (r < 0)
+ return log_oom();
+ }
+
+ where = i->temp_path;
}
- mkdir_parents_label(i->temp_path, 0700);
+ (void) mkdir_parents_label(where, 0700);
+
+ if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE))
+ (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
- r = btrfs_subvol_make_fallback(i->temp_path, 0755);
+ if (i->flags & PULL_BTRFS_SUBVOL)
+ r = btrfs_subvol_make_fallback(where, 0755);
+ else
+ r = mkdir(where, 0755) < 0 ? -errno : 0;
+ if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
+ * because in that case our temporary path collided */
+ r = 0;
if (r < 0)
- return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path);
- if (r > 0) { /* actually btrfs subvol */
- (void) import_assign_pool_quota_and_warn(i->image_root);
- (void) import_assign_pool_quota_and_warn(i->temp_path);
+ return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where);
+ if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */
+ if (!(i->flags & PULL_DIRECT))
+ (void) import_assign_pool_quota_and_warn(i->image_root);
+ (void) import_assign_pool_quota_and_warn(where);
}
- j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
+ j->disk_fd = import_fork_tar_x(where, &i->tar_pid);
if (j->disk_fd < 0)
return j->disk_fd;
return log_oom();
}
- mkdir_parents_label(i->settings_temp_path, 0700);
+ (void) mkdir_parents_label(i->settings_temp_path, 0700);
j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (j->disk_fd < 0)
const char *url,
const char *local,
PullFlags flags,
- ImportVerify verify) {
+ ImportVerify verify,
+ const char *checksum) {
+ PullJob *j;
int r;
assert(i);
- assert(verify < _IMPORT_VERIFY_MAX);
- assert(verify >= 0);
+ assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
+ assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
+ assert((verify < 0) || !checksum);
assert(!(flags & ~PULL_FLAGS_MASK_TAR));
+ assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT));
+ assert(!(flags & PULL_SETTINGS) || !checksum);
if (!http_url_is_valid(url))
return -EINVAL;
- if (local && !hostname_is_valid(local, 0))
+ if (local && !pull_validate_local(local, flags))
return -EINVAL;
if (i->tar_job)
if (r < 0)
return r;
+ r = free_and_strdup(&i->checksum, checksum);
+ if (r < 0)
+ return r;
+
i->flags = flags;
i->verify = verify;
i->tar_job->on_finished = tar_pull_job_on_finished;
i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
- i->tar_job->on_progress = tar_pull_job_on_progress;
- i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
+ i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
- r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
- if (r < 0)
- return r;
+ if (!FLAGS_SET(flags, PULL_DIRECT)) {
+ r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
+ if (r < 0)
+ return r;
+ }
/* Set up download of checksum/signature files */
- r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
+ r = pull_make_verification_jobs(
+ &i->checksum_job,
+ &i->signature_job,
+ verify,
+ checksum,
+ url,
+ i->glue,
+ tar_pull_job_on_finished,
+ i);
if (r < 0)
return r;
/* Set up download job for the settings file (.nspawn) */
if (FLAGS_SET(flags, PULL_SETTINGS)) {
- r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i);
+ r = pull_make_auxiliary_job(
+ &i->settings_job,
+ url,
+ tar_strip_suffixes,
+ ".nspawn",
+ verify,
+ i->glue,
+ tar_pull_job_on_open_disk_settings,
+ tar_pull_job_on_finished,
+ i);
if (r < 0)
return r;
-
- i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
- i->settings_job->on_progress = tar_pull_job_on_progress;
- i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
}
- r = pull_job_begin(i->tar_job);
- if (r < 0)
- return r;
-
- if (i->checksum_job) {
- i->checksum_job->on_progress = tar_pull_job_on_progress;
- i->checksum_job->on_not_found = pull_job_restart_with_sha256sum;
-
- r = pull_job_begin(i->checksum_job);
- if (r < 0)
- return r;
- }
+ FOREACH_POINTER(j,
+ i->tar_job,
+ i->checksum_job,
+ i->signature_job,
+ i->settings_job) {
- if (i->signature_job) {
- i->signature_job->on_progress = tar_pull_job_on_progress;
+ if (!j)
+ continue;
- r = pull_job_begin(i->signature_job);
- if (r < 0)
- return r;
- }
+ j->on_progress = tar_pull_job_on_progress;
+ j->sync = FLAGS_SET(flags, PULL_SYNC);
- if (i->settings_job) {
- r = pull_job_begin(i->settings_job);
+ r = pull_job_begin(j);
if (r < 0)
return r;
}
DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref);
-int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify);
+int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify, const char *checksum);
#include "alloc-util.h"
#include "discover-image.h"
+#include "env-util.h"
+#include "hexdecoct.h"
#include "hostname-util.h"
+#include "import-common.h"
#include "import-util.h"
+#include "io-util.h"
#include "main-func.h"
+#include "parse-argument.h"
#include "parse-util.h"
#include "pull-raw.h"
#include "pull-tar.h"
#include "signal-util.h"
#include "string-util.h"
+#include "terminal-util.h"
#include "verbs.h"
#include "web-util.h"
static const char *arg_image_root = "/var/lib/machines";
static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE;
-static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY;
+static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY | PULL_BTRFS_SUBVOL | PULL_BTRFS_QUOTA | PULL_CONVERT_QCOW2 | PULL_SYNC;
+static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
+static char *arg_checksum = NULL;
-static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
- log_notice("Transfer aborted.");
- sd_event_exit(sd_event_source_get_event(s), EINTR);
+STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep);
+
+static int normalize_local(const char *local, const char *url, char **ret) {
+ _cleanup_free_ char *ll = NULL;
+ int r;
+
+ if (arg_pull_flags & PULL_DIRECT) {
+
+ if (!local)
+ log_debug("Writing downloaded data to STDOUT.");
+ else {
+ if (!path_is_absolute(local)) {
+ ll = path_join(arg_image_root, local);
+ if (!ll)
+ return log_oom();
+
+ local = ll;
+ }
+
+ if (!path_is_valid(local))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+ "Local path name '%s' is not valid.", local);
+ }
+
+ } else if (local) {
+
+ if (!hostname_is_valid(local, 0))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+ "Local image name '%s' is not valid.",
+ local);
+
+ if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
+ r = image_find(IMAGE_MACHINE, local, NULL, NULL);
+ if (r < 0) {
+ if (r != -ENOENT)
+ return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
+ } else
+ return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
+ "Image '%s' already exists.",
+ local);
+ }
+ }
+
+ if (!ll && local) {
+ ll = strdup(local);
+ if (!ll)
+ return log_oom();
+ }
+
+ if (ll) {
+ if (arg_offset != UINT64_MAX)
+ log_info("Pulling '%s', saving at offset %" PRIu64 " in '%s'.", url, arg_offset, ll);
+ else
+ log_info("Pulling '%s', saving as '%s'.", url, ll);
+ } else
+ log_info("Pulling '%s'.", url);
+
+ *ret = TAKE_PTR(ll);
return 0;
}
}
static int pull_tar(int argc, char *argv[], void *userdata) {
- _cleanup_(tar_pull_unrefp) TarPull *pull = NULL;
+ _cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
+ _cleanup_(tar_pull_unrefp) TarPull *pull = NULL;
const char *url, *local;
- _cleanup_free_ char *l = NULL, *ll = NULL;
int r;
url = argv[1];
if (!http_url_is_valid(url))
- return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
- "URL '%s' is not valid.", url);
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url);
if (argc >= 3)
- local = argv[2];
+ local = empty_or_dash_to_null(argv[2]);
else {
+ _cleanup_free_ char *l = NULL;
+
r = import_url_last_component(url, &l);
if (r < 0)
- return log_error_errno(r, "Failed get final component of URL: %m");
-
- local = l;
- }
+ return log_error_errno(r, "Failed to get final component of URL: %m");
- local = empty_or_dash_to_null(local);
-
- if (local) {
- r = tar_strip_suffixes(local, &ll);
+ r = tar_strip_suffixes(l, &ll);
if (r < 0)
return log_oom();
local = ll;
+ }
- if (!hostname_is_valid(local, 0))
- return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
- "Local image name '%s' is not valid.",
- local);
-
- if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
- r = image_find(IMAGE_MACHINE, local, NULL, NULL);
- if (r < 0) {
- if (r != -ENOENT)
- return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
- } else {
- return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
- "Image '%s' already exists.",
- local);
- }
- }
+ if (!local && FLAGS_SET(arg_pull_flags, PULL_DIRECT))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Pulling tar images to STDOUT is not supported.");
- log_info("Pulling '%s', saving as '%s'.", url, local);
- } else
- log_info("Pulling '%s'.", url);
+ r = normalize_local(local, url, &normalized);
+ if (r < 0)
+ return r;
- r = sd_event_default(&event);
+ r = import_allocate_event_with_signals(&event);
if (r < 0)
- return log_error_errno(r, "Failed to allocate event loop: %m");
+ return r;
- assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
- (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
- (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
+ if (!FLAGS_SET(arg_pull_flags, PULL_SYNC))
+ log_info("File system synchronization on completion is off.");
r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event);
if (r < 0)
return log_error_errno(r, "Failed to allocate puller: %m");
- r = tar_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_TAR, arg_verify);
+ r = tar_pull_start(
+ pull,
+ url,
+ normalized,
+ arg_pull_flags & PULL_FLAGS_MASK_TAR,
+ arg_verify,
+ arg_checksum);
if (r < 0)
return log_error_errno(r, "Failed to pull image: %m");
}
static int pull_raw(int argc, char *argv[], void *userdata) {
- _cleanup_(raw_pull_unrefp) RawPull *pull = NULL;
+ _cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
+ _cleanup_(raw_pull_unrefp) RawPull *pull = NULL;
const char *url, *local;
- _cleanup_free_ char *l = NULL, *ll = NULL;
int r;
url = argv[1];
if (!http_url_is_valid(url))
- return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
- "URL '%s' is not valid.", url);
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url);
if (argc >= 3)
- local = argv[2];
+ local = empty_or_dash_to_null(argv[2]);
else {
+ _cleanup_free_ char *l = NULL;
+
r = import_url_last_component(url, &l);
if (r < 0)
- return log_error_errno(r, "Failed get final component of URL: %m");
-
- local = l;
- }
+ return log_error_errno(r, "Failed to get final component of URL: %m");
- local = empty_or_dash_to_null(local);
-
- if (local) {
- r = raw_strip_suffixes(local, &ll);
+ r = raw_strip_suffixes(l, &ll);
if (r < 0)
return log_oom();
local = ll;
+ }
- if (!hostname_is_valid(local, 0))
- return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
- "Local image name '%s' is not valid.",
- local);
-
- if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
- r = image_find(IMAGE_MACHINE, local, NULL, NULL);
- if (r < 0) {
- if (r != -ENOENT)
- return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
- } else {
- return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
- "Image '%s' already exists.",
- local);
- }
- }
-
- log_info("Pulling '%s', saving as '%s'.", url, local);
- } else
- log_info("Pulling '%s'.", url);
-
- r = sd_event_default(&event);
+ r = normalize_local(local, url, &normalized);
if (r < 0)
- return log_error_errno(r, "Failed to allocate event loop: %m");
+ return r;
- assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
- (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
- (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
+ r = import_allocate_event_with_signals(&event);
+ if (r < 0)
+ return r;
- r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event);
+ if (!FLAGS_SET(arg_pull_flags, PULL_SYNC))
+ log_info("File system synchronization on completion is off.");
+ r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event);
if (r < 0)
return log_error_errno(r, "Failed to allocate puller: %m");
- r = raw_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_RAW, arg_verify);
+ r = raw_pull_start(
+ pull,
+ url,
+ normalized,
+ arg_offset,
+ arg_size_max,
+ arg_pull_flags & PULL_FLAGS_MASK_RAW,
+ arg_verify,
+ arg_checksum);
if (r < 0)
return log_error_errno(r, "Failed to pull image: %m");
static int help(int argc, char *argv[], void *userdata) {
- printf("%s [OPTIONS...] {COMMAND} ...\n\n"
- "Download container or virtual machine images.\n\n"
+ printf("%1$s [OPTIONS...] {COMMAND} ...\n"
+ "\n%4$sDownload container or virtual machine images.%5$s\n"
+ "\n%2$sCommands:%3$s\n"
+ " tar URL [NAME] Download a TAR image\n"
+ " raw URL [NAME] Download a RAW image\n"
+ "\n%2$sOptions:%3$s\n"
" -h --help Show this help\n"
" --version Show package version\n"
" --force Force creation of image\n"
" --verify=MODE Verify downloaded image, one of: 'no',\n"
- " 'checksum', 'signature'\n"
+ " 'checksum', 'signature' or literal SHA256 hash\n"
" --settings=BOOL Download settings file with image\n"
" --roothash=BOOL Download root hash file with image\n"
" --roothash-signature=BOOL\n"
" Download root hash signature file with image\n"
" --verity=BOOL Download verity file with image\n"
" --image-root=PATH Image root directory\n\n"
- "Commands:\n"
- " tar URL [NAME] Download a TAR image\n"
- " raw URL [NAME] Download a RAW image\n",
- program_invocation_short_name);
+ " --read-only Create a read-only image\n"
+ " --direct Download directly to specified file\n"
+ " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n"
+ " instead of a directory\n"
+ " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n"
+ " subvolume\n"
+ " --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n"
+ " regular disk images\n"
+ " --sync=BOOL Controls whether to sync() before completing\n"
+ " --offset=BYTES Offset to seek to in destination\n"
+ " --size-max=BYTES Maximum number of bytes to write to destination\n",
+ program_invocation_short_name,
+ ansi_underline(),
+ ansi_normal(),
+ ansi_highlight(),
+ ansi_normal());
return 0;
}
ARG_ROOTHASH,
ARG_ROOTHASH_SIGNATURE,
ARG_VERITY,
+ ARG_READ_ONLY,
+ ARG_DIRECT,
+ ARG_BTRFS_SUBVOL,
+ ARG_BTRFS_QUOTA,
+ ARG_CONVERT_QCOW2,
+ ARG_SYNC,
+ ARG_OFFSET,
+ ARG_SIZE_MAX,
};
static const struct option options[] = {
{ "roothash", required_argument, NULL, ARG_ROOTHASH },
{ "roothash-signature", required_argument, NULL, ARG_ROOTHASH_SIGNATURE },
{ "verity", required_argument, NULL, ARG_VERITY },
+ { "read-only", no_argument, NULL, ARG_READ_ONLY },
+ { "direct", no_argument, NULL, ARG_DIRECT },
+ { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL },
+ { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA },
+ { "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 },
+ { "sync", required_argument, NULL, ARG_SYNC },
+ { "offset", required_argument, NULL, ARG_OFFSET },
+ { "size-max", required_argument, NULL, ARG_SIZE_MAX },
{}
};
arg_image_root = optarg;
break;
- case ARG_VERIFY:
- arg_verify = import_verify_from_string(optarg);
- if (arg_verify < 0)
- return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
- "Invalid verification setting '%s'", optarg);
+ case ARG_VERIFY: {
+ ImportVerify v;
+
+ v = import_verify_from_string(optarg);
+ if (v < 0) {
+ _cleanup_free_ void *h = NULL;
+ char *hh;
+ size_t n;
+
+ /* If this is not a vaid verification mode, maybe it's a literally specified
+ * SHA256 hash? We can handle that too... */
+
+ r = unhexmem(optarg, (size_t) -1, &h, &n);
+ if (r < 0 || n == 0)
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+ "Invalid verification setting: %s", optarg);
+ if (n != 32)
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+ "64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2);
+
+ hh = hexmem(h, n); /* bring into canonical (lowercase) form */
+ if (!hh)
+ return log_oom();
+
+ free_and_replace(arg_checksum, hh);
+ arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY);
+ arg_verify = _IMPORT_VERIFY_INVALID;
+ } else
+ arg_verify = v;
break;
+ }
case ARG_SETTINGS:
- r = parse_boolean(optarg);
+ r = parse_boolean_argument("--settings=", optarg, NULL);
if (r < 0)
- return log_error_errno(r, "Failed to parse --settings= parameter '%s': %m", optarg);
+ return r;
SET_FLAG(arg_pull_flags, PULL_SETTINGS, r);
break;
case ARG_ROOTHASH:
- r = parse_boolean(optarg);
+ r = parse_boolean_argument("--roothash=", optarg, NULL);
if (r < 0)
- return log_error_errno(r, "Failed to parse --roothash= parameter '%s': %m", optarg);
+ return r;
SET_FLAG(arg_pull_flags, PULL_ROOTHASH, r);
break;
case ARG_ROOTHASH_SIGNATURE:
- r = parse_boolean(optarg);
+ r = parse_boolean_argument("--roothash-signature=", optarg, NULL);
if (r < 0)
- return log_error_errno(r, "Failed to parse --roothash-signature= parameter '%s': %m", optarg);
+ return r;
SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, r);
break;
case ARG_VERITY:
- r = parse_boolean(optarg);
+ r = parse_boolean_argument("--verity=", optarg, NULL);
if (r < 0)
- return log_error_errno(r, "Failed to parse --verity= parameter '%s': %m", optarg);
+ return r;
SET_FLAG(arg_pull_flags, PULL_VERITY, r);
break;
+ case ARG_READ_ONLY:
+ arg_pull_flags |= PULL_READ_ONLY;
+ break;
+
+ case ARG_DIRECT:
+ arg_pull_flags |= PULL_DIRECT;
+ arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY);
+ break;
+
+ case ARG_BTRFS_SUBVOL:
+ r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL);
+ if (r < 0)
+ return r;
+
+ SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r);
+ break;
+
+ case ARG_BTRFS_QUOTA:
+ r = parse_boolean_argument("--btrfs-quota=", optarg, NULL);
+ if (r < 0)
+ return r;
+
+ SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r);
+ break;
+
+ case ARG_CONVERT_QCOW2:
+ r = parse_boolean_argument("--convert-qcow2=", optarg, NULL);
+ if (r < 0)
+ return r;
+
+ SET_FLAG(arg_pull_flags, PULL_CONVERT_QCOW2, r);
+ break;
+
+ case ARG_SYNC:
+ r = parse_boolean_argument("--sync=", optarg, NULL);
+ if (r < 0)
+ return r;
+
+ SET_FLAG(arg_pull_flags, PULL_SYNC, r);
+ break;
+
+ case ARG_OFFSET: {
+ uint64_t u;
+
+ r = safe_atou64(optarg, &u);
+ if (r < 0)
+ return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg);
+ if (!FILE_SIZE_VALID(u))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg);
+
+ arg_offset = u;
+ break;
+ }
+
+ case ARG_SIZE_MAX: {
+ uint64_t u;
+
+ r = parse_size(optarg, 1024, &u);
+ if (r < 0)
+ return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg);
+ if (!FILE_SIZE_VALID(u))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg);
+
+ arg_size_max = u;
+ break;
+ }
+
case '?':
return -EINVAL;
assert_not_reached();
}
+ /* Make sure offset+size is still in the valid range if both set */
+ if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX &&
+ ((arg_size_max > (UINT64_MAX - arg_offset)) ||
+ !FILE_SIZE_VALID(arg_offset + arg_size_max)))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range.");
+
+ if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_pull_flags, PULL_DIRECT))
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
+
+ if (arg_checksum && (arg_pull_flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) != 0)
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded.");
+
return 1;
}
+static void parse_env(void) {
+ int r;
+
+ /* Let's make these relatively low-level settings also controllable via env vars. User can then set
+ * them for systemd-importd.service if they like to tweak behaviour */
+
+ r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL");
+ if (r >= 0)
+ SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r);
+ else if (r != -ENXIO)
+ log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m");
+
+ r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA");
+ if (r >= 0)
+ SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r);
+ else if (r != -ENXIO)
+ log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m");
+
+ r = getenv_bool("SYSTEMD_IMPORT_SYNC");
+ if (r >= 0)
+ SET_FLAG(arg_pull_flags, PULL_SYNC, r);
+ else if (r != -ENXIO)
+ log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m");
+}
+
static int pull_main(int argc, char *argv[]) {
static const Verb verbs[] = {
{ "help", VERB_ANY, VERB_ANY, 0, help },
log_parse_environment();
log_open();
+ parse_env();
+
r = parse_argv(argc, argv);
if (r <= 0)
return r;
projects / thirdparty / systemd.git / commitdiff
