import/export: add support for zstd

diff --git a/man/importctl.xml b/man/importctl.xml
index 56a4c9055c588f2f50260fe355c14ae8662c4105..f67565bc05454544e17627832a8fa54d1aa01c2a 100644 (file)
--- a/man/importctl.xml
+++ b/man/importctl.xml
@@ -182,11 +182,12 @@
         archive, possibly compressed with
         <citerefentry project='die-net'><refentrytitle>xz</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
         <citerefentry project='die-net'><refentrytitle>gzip</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
+        <citerefentry project='die-net'><refentrytitle>zstd</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
         or
         <citerefentry project='die-net'><refentrytitle>bzip2</refentrytitle><manvolnum>1</manvolnum></citerefentry>.
         It will then be unpacked into its own
         subvolume/directory. When <command>import-raw</command> is used, the file should be a qcow2 or raw
-        disk image, possibly compressed with xz, gzip or bzip2. If the second argument (the resulting image
+        disk image, possibly compressed with xz, gzip, zstd or bzip2. If the second argument (the resulting image
         name) is not specified, it is automatically derived from the file name. If the filename is passed as
         <literal>-</literal>, the image is read from standard input, in which case the second argument is
         mandatory.</para>
@@ -222,6 +223,8 @@
         <citerefentry project='die-net'><refentrytitle>gzip</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
         if it ends in <literal>.xz</literal>, with
         <citerefentry project='die-net'><refentrytitle>xz</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
+        if it ends in <literal>.zst</literal>, with
+        <citerefentry project='die-net'><refentrytitle>zstd</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
         and if it ends in <literal>.bz2</literal>, with
         <citerefentry project='die-net'><refentrytitle>bzip2</refentrytitle><manvolnum>1</manvolnum></citerefentry>.
         If the path ends in neither, the file is left uncompressed. If the second argument is missing, the image
@@ -315,8 +318,8 @@
         <listitem><para>When used with the <option>export-tar</option> or <option>export-raw</option>
         commands, specifies the compression format to use for the resulting file. Takes one of
         <literal>uncompressed</literal>, <literal>xz</literal>, <literal>gzip</literal>,
-        <literal>bzip2</literal>. By default, the format is determined automatically from the output image
-        file name passed.</para>
+        <literal>zst</literal>, <literal>bzip2</literal>. By default, the format is determined
+        automatically from the output image file name passed.</para>
 
         <xi:include href="version-info.xml" xpointer="v256"/></listitem>
       </varlistentry>
@@ -450,6 +453,7 @@
       <member><citerefentry project='die-net'><refentrytitle>tar</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
       <member><citerefentry project='die-net'><refentrytitle>xz</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
       <member><citerefentry project='die-net'><refentrytitle>gzip</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
+      <member><citerefentry project='die-net'><refentrytitle>zstd</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
       <member><citerefentry project='die-net'><refentrytitle>bzip2</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
     </simplelist></para>
   </refsect1>

diff --git a/man/machinectl.xml b/man/machinectl.xml
index 7e6161b31cbc213d8a429e09d4a679e9811c2657..eab1a7d99e3f20cff0580567cd25523a03d33f15 100644 (file)
--- a/man/machinectl.xml
+++ b/man/machinectl.xml
@@ -872,6 +872,7 @@
       <member><citerefentry project='die-net'><refentrytitle>xz</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
       <member><citerefentry project='die-net'><refentrytitle>gzip</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
       <member><citerefentry project='die-net'><refentrytitle>bzip2</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
+      <member><citerefentry project='die-net'><refentrytitle>zstd</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
     </simplelist></para>
   </refsect1>
 

diff --git a/man/org.freedesktop.import1.xml b/man/org.freedesktop.import1.xml
index 146adf6bcf0e0424045ca2e1d49f029569d34f14..cece4cad0b44f02d1141f40dcac6fe950c595b7c 100644 (file)
--- a/man/org.freedesktop.import1.xml
+++ b/man/org.freedesktop.import1.xml
@@ -214,12 +214,13 @@ node /org/freedesktop/import1 {
       to the tar or raw file to import. It should reference a file on disk, a pipe or a socket. When
       <function>ImportTar()</function>/<function>ImportTarEx()</function> is used the file descriptor should
       refer to a tar file, optionally compressed with <citerefentry project="die-net"><refentrytitle>gzip</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
+      <citerefentry project="die-net"><refentrytitle>zstd</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
       <citerefentry project="die-net"><refentrytitle>bzip2</refentrytitle><manvolnum>1</manvolnum></citerefentry>, or
       <citerefentry project="die-net"><refentrytitle>xz</refentrytitle><manvolnum>1</manvolnum></citerefentry>.
       <command>systemd-importd</command> will detect the used compression scheme (if any) automatically. When
       <function>ImportRaw()</function>/<function>ImportRawEx()</function> is used the file descriptor should
       refer to a raw or qcow2 disk image containing an MBR or GPT disk label, also optionally compressed with
-      gzip, bzip2 or xz. In either case, if the file is specified as a file descriptor on disk, progress
+      gzip, zstd, bzip2 or xz. In either case, if the file is specified as a file descriptor on disk, progress
       information is generated for the import operation (as in that case we know the total size on disk). If
       a socket or pipe is specified, progress information is not available. The file descriptor argument is
       followed by a local name for the image. This should be a name suitable as a hostname and will be used
@@ -250,9 +251,9 @@ node /org/freedesktop/import1 {
       name to export as their first parameter, followed by a file descriptor (opened for writing) where the
       tar or raw file will be written. It may either reference a file on disk or a pipe/socket. The third
       argument specifies in which compression format to write the image. It takes one of
-      <literal>uncompressed</literal>, <literal>xz</literal>, <literal>bzip2</literal> or
-      <literal>gzip</literal>, depending on which compression scheme is required. The image written to the
-      specified file descriptor will be a tar file in case of
+      <literal>uncompressed</literal>, <literal>xz</literal>, <literal>bzip2</literal>,
+      <literal>gzip</literal> or <literal>zstd</literal>, depending on which compression scheme is required.
+      The image written to the specified file descriptor will be a tar file in case of
       <function>ExportTar()</function>/<function>ExportTarEx()</function> or a raw disk image in case of
       <function>ExportRaw()</function>/<function>ExportRawEx()</function>. Note that currently raw disk
       images may not be exported as tar files, and vice versa. This restriction might be lifted
@@ -267,8 +268,8 @@ node /org/freedesktop/import1 {
       <function>PullRaw()</function>/<function>PullRawEx()</function> may be used to download, verify and
       import a system image from a URL. They take a URL argument which should point to a tar or raw file on
       the <literal>http://</literal> or <literal>https://</literal> protocols, possibly compressed with xz,
-      bzip2 or gzip. The second argument is a local name for the image. It should be suitable as a hostname,
-      similarly to the matching argument of the
+      bzip2, gzip or zstd. The second argument is a local name for the image. It should be suitable as a
+      hostname, similarly to the matching argument of the
       <function>ImportTar()</function>/<function>ImportTarEx()</function> and
       <function>ImportRaw()</function>/<function>ImportRawEx()</function> methods above. The third argument
       indicates the verification mode for the image. It may be one of <literal>no</literal>,

diff --git a/shell-completion/bash/importctl b/shell-completion/bash/importctl
index d8187d0fff8c515a2258bbb7a34283a7858133bf..0a13842445390df9893b285d614c5bfe9e01f0c1 100644 (file)
--- a/shell-completion/bash/importctl
+++ b/shell-completion/bash/importctl
@@ -73,7 +73,7 @@ _importctl() {
                 comps='no checksum signature'
                 ;;
             --format)
-                comps='uncompressed xz gzip bzip2'
+                comps='uncompressed xz gzip bzip2 zstd'
                 ;;
             --class)
                 comps='machine portable sysext confext'

diff --git a/shell-completion/bash/machinectl b/shell-completion/bash/machinectl
index b3c2e501641f6839b0f6e352698a2aad522db283..10a59fc923787ccb78e45eb89875187195016746 100644 (file)
--- a/shell-completion/bash/machinectl
+++ b/shell-completion/bash/machinectl
@@ -85,7 +85,7 @@ _machinectl() {
                 comps=$( machinectl --verify=help 2>/dev/null )
                 ;;
             --format)
-                comps='uncompressed xz gzip bzip2'
+                comps='uncompressed xz gzip bzip2 zstd'
                 ;;
         esac
         COMPREPLY=( $(compgen -W '$comps' -- "$cur") )

diff --git a/src/import/export.c b/src/import/export.c
index 346aca9446036dd0f685073f08ab4dca93323db2..0ee2a1db71dde9ae1a4229b812c2ed1e60cfa202 100644 (file)
--- a/src/import/export.c
+++ b/src/import/export.c
@@ -43,6 +43,8 @@ static void determine_compression_from_filename(const char *p) {
                 arg_compress = IMPORT_COMPRESS_GZIP;
         else if (endswith(p, ".bz2"))
                 arg_compress = IMPORT_COMPRESS_BZIP2;
+        else if (endswith(p, ".zst"))
+                arg_compress = IMPORT_COMPRESS_ZSTD;
         else
                 arg_compress = IMPORT_COMPRESS_UNCOMPRESSED;
 }
@@ -254,6 +256,8 @@ static int parse_argv(int argc, char *argv[]) {
                                 arg_compress = IMPORT_COMPRESS_GZIP;
                         else if (streq(optarg, "bzip2"))
                                 arg_compress = IMPORT_COMPRESS_BZIP2;
+                        else if (streq(optarg, "zstd"))
+                                arg_compress = IMPORT_COMPRESS_ZSTD;
                         else
                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                                        "Unknown format: %s", optarg);

diff --git a/src/import/import-compress.c b/src/import/import-compress.c
index 28cf6f841a2b39e71c95d941eafcce557e158794..67dd446ddafb10a25ea833fadc2f0527ddbb38f9 100644 (file)
--- a/src/import/import-compress.c
+++ b/src/import/import-compress.c
@@ -19,6 +19,16 @@ void import_compress_free(ImportCompress *c) {
                         BZ2_bzCompressEnd(&c->bzip2);
                 else
                         BZ2_bzDecompressEnd(&c->bzip2);
+#endif
+#if HAVE_ZSTD
+        } else if (c->type == IMPORT_COMPRESS_ZSTD) {
+                if (c->encoding) {
+                        ZSTD_freeCCtx(c->c_zstd);
+                        c->c_zstd = NULL;
+                } else {
+                        ZSTD_freeDCtx(c->d_zstd);
+                        c->d_zstd = NULL;
+                }
 #endif
         }
 
@@ -35,6 +45,9 @@ int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) {
         static const uint8_t bzip2_signature[] = {
                 'B', 'Z', 'h'
         };
+        static const uint8_t zstd_signature[] = {
+                0x28, 0xb5, 0x2f, 0xfd
+        };
 
         int r;
 
@@ -43,8 +56,9 @@ int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) {
         if (c->type != IMPORT_COMPRESS_UNKNOWN)
                 return 1;
 
-        if (size < MAX3(sizeof(xz_signature),
+        if (size < MAX4(sizeof(xz_signature),
                         sizeof(gzip_signature),
+                        sizeof(zstd_signature),
                         sizeof(bzip2_signature)))
                 return 0;
 
@@ -73,6 +87,14 @@ int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) {
                         return -EIO;
 
                 c->type = IMPORT_COMPRESS_BZIP2;
+#endif
+#if HAVE_ZSTD
+        } else if (memcmp(data, zstd_signature, sizeof(zstd_signature)) == 0) {
+                c->d_zstd = ZSTD_createDCtx();
+                if (!c->d_zstd)
+                        return -ENOMEM;
+
+                c->type = IMPORT_COMPRESS_ZSTD;
 #endif
         } else
                 c->type = IMPORT_COMPRESS_UNCOMPRESSED;
@@ -187,6 +209,35 @@ int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCo
 
                 break;
 #endif
+#if HAVE_ZSTD
+        case IMPORT_COMPRESS_ZSTD: {
+                ZSTD_inBuffer input = {
+                        .src =  (void*) data,
+                        .size = size,
+                };
+
+                while (input.pos < input.size) {
+                        uint8_t buffer[16 * 1024];
+                        ZSTD_outBuffer output = {
+                                .dst = buffer,
+                                .size = sizeof(buffer),
+                        };
+                        size_t res;
+
+                        res = ZSTD_decompressStream(c->d_zstd, &output, &input);
+                        if (ZSTD_isError(res))
+                                return -EIO;
+
+                        if (output.pos > 0) {
+                                r = callback(output.dst, output.pos, userdata);
+                                if (r < 0)
+                                        return r;
+                        }
+                }
+
+                break;
+        }
+#endif
 
         default:
                 assert_not_reached();
@@ -231,6 +282,20 @@ int import_compress_init(ImportCompress *c, ImportCompressType t) {
                 break;
 #endif
 
+#if HAVE_ZSTD
+        case IMPORT_COMPRESS_ZSTD:
+                c->c_zstd = ZSTD_createCCtx();
+                if (!c->c_zstd)
+                        return -ENOMEM;
+
+                r = ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT);
+                if (ZSTD_isError(r))
+                        return -EIO;
+
+                c->type = IMPORT_COMPRESS_ZSTD;
+                break;
+#endif
+
         case IMPORT_COMPRESS_UNCOMPRESSED:
                 c->type = IMPORT_COMPRESS_UNCOMPRESSED;
                 break;
@@ -351,6 +416,35 @@ int import_compress(ImportCompress *c, const void *data, size_t size, void **buf
                 break;
 #endif
 
+#if HAVE_ZSTD
+        case IMPORT_COMPRESS_ZSTD: {
+                ZSTD_inBuffer input = {
+                        .src = data,
+                        .size = size,
+                };
+
+                while (input.pos < input.size) {
+                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
+                        if (r < 0)
+                                return r;
+
+                        ZSTD_outBuffer output = {
+                                .dst = ((uint8_t *) *buffer + *buffer_size),
+                                .size = *buffer_allocated - *buffer_size,
+                        };
+                        size_t res;
+
+                        res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue);
+                        if (ZSTD_isError(res))
+                                return -EIO;
+
+                        *buffer_size += output.pos;
+                }
+
+                break;
+        }
+#endif
+
         case IMPORT_COMPRESS_UNCOMPRESSED:
 
                 if (*buffer_allocated < size) {
@@ -455,6 +549,32 @@ int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size
                 break;
 #endif
 
+#if HAVE_ZSTD
+        case IMPORT_COMPRESS_ZSTD: {
+                ZSTD_inBuffer input = {};
+                size_t res;
+
+                do {
+                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated);
+                        if (r < 0)
+                                return r;
+
+                        ZSTD_outBuffer output = {
+                                .dst = ((uint8_t *) *buffer + *buffer_size),
+                                .size = *buffer_allocated - *buffer_size,
+                        };
+
+                        res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end);
+                        if (ZSTD_isError(res))
+                                return -EIO;
+
+                        *buffer_size += output.pos;
+                } while (res != 0);
+
+                break;
+        }
+#endif
+
         case IMPORT_COMPRESS_UNCOMPRESSED:
                 break;
 
@@ -473,6 +593,9 @@ static const char* const import_compress_type_table[_IMPORT_COMPRESS_TYPE_MAX] =
 #if HAVE_BZIP2
         [IMPORT_COMPRESS_BZIP2] = "bzip2",
 #endif
+#if HAVE_ZSTD
+        [IMPORT_COMPRESS_ZSTD] = "zstd",
+#endif
 };
 
 DEFINE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType);

diff --git a/src/import/import-compress.h b/src/import/import-compress.h
index 0a4210378ac817e9b1b6165bca25992b43adba52..6ad6cf7a676114613418d2b1ea1373275d08ab13 100644 (file)
--- a/src/import/import-compress.h
+++ b/src/import/import-compress.h
@@ -7,6 +7,10 @@
 #include <lzma.h>
 #include <sys/types.h>
 #include <zlib.h>
+#if HAVE_ZSTD
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
 
 #include "macro.h"
 
@@ -16,6 +20,7 @@ typedef enum ImportCompressType {
         IMPORT_COMPRESS_XZ,
         IMPORT_COMPRESS_GZIP,
         IMPORT_COMPRESS_BZIP2,
+        IMPORT_COMPRESS_ZSTD,
         _IMPORT_COMPRESS_TYPE_MAX,
         _IMPORT_COMPRESS_TYPE_INVALID = -EINVAL,
 } ImportCompressType;
@@ -28,6 +33,10 @@ typedef struct ImportCompress {
                 z_stream gzip;
 #if HAVE_BZIP2
                 bz_stream bzip2;
+#endif
+#if HAVE_ZSTD
+                ZSTD_CCtx *c_zstd;
+                ZSTD_DCtx *d_zstd;
 #endif
         };
 } ImportCompress;

diff --git a/src/import/importctl.c b/src/import/importctl.c
index 1ddba76b09cc0e631eee05043b47c2f71eb66cf9..97a95d3612aec34f70b3e48bdba862971cd10957 100644 (file)
--- a/src/import/importctl.c
+++ b/src/import/importctl.c
@@ -491,6 +491,8 @@ static void determine_compression_from_filename(const char *p) {
                 arg_format = "gzip";
         else if (endswith(p, ".bz2"))
                 arg_format = "bzip2";
+        else if (endswith(p, ".zst"))
+                arg_format = "zstd";
 }
 
 static int export_tar(int argc, char *argv[], void *userdata) {
@@ -1018,7 +1020,8 @@ static int help(int argc, char *argv[], void *userdata) {
                "                              otherwise\n"
                "     --verify=MODE            Verification mode for downloaded images (no,\n"
                "                               checksum, signature)\n"
-               "     --format=xz|gzip|bzip2   Desired output format for export\n"
+               "     --format=xz|gzip|bzip2|zstd\n"
+               "                              Desired output format for export\n"
                "     --force                  Install image even if already exists\n"
                "  -m --class=machine          Install as machine image\n"
                "  -P --class=portable         Install as portable service image\n"
@@ -1139,7 +1142,7 @@ static int parse_argv(int argc, char *argv[]) {
                         break;
 
                 case ARG_FORMAT:
-                        if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2"))
+                        if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2", "zstd"))
                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                                        "Unknown format: %s", optarg);
 

diff --git a/src/import/meson.build b/src/import/meson.build
index 45500edb433b9dcc58bf973b48a63c0d58778159..a9dc051394190bf6350a3b507d056922e9d88536 100644 (file)
--- a/src/import/meson.build
+++ b/src/import/meson.build
@@ -47,6 +47,7 @@ lib_import_common = static_library(
                 libbzip2,
                 libxz,
                 libz,
+                libzstd,
                 userspace,
         ],
         build_by_default : false)
@@ -61,6 +62,7 @@ common_deps = [
         libcurl,
         libxz,
         libz,
+        libzstd,
 ]
 
 executables += [

diff --git a/src/machine/machinectl.c b/src/machine/machinectl.c
index 366446c3062937e22a69eeeae9b5d6f539e078a5..5a6052549d31745f4101ddb10a9c9baa1ce5cc25 100644 (file)
--- a/src/machine/machinectl.c
+++ b/src/machine/machinectl.c
@@ -2334,7 +2334,7 @@ static int parse_argv(int argc, char *argv[]) {
                         break;
 
                 case ARG_FORMAT:
-                        if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2"))
+                        if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2", "zstd"))
                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                                        "Unknown format: %s", optarg);
 

diff --git a/src/shared/import-util.c b/src/shared/import-util.c
index 1a396a6570c5c030e32128022eb941a9bbc34f45..0c61ea449f5a1f45c36404a027435ac2ed9a0e87 100644 (file)
--- a/src/shared/import-util.c
+++ b/src/shared/import-util.c
@@ -152,6 +152,8 @@ int tar_strip_suffixes(const char *name, char **ret) {
                 e = endswith(name, ".tar.gz");
         if (!e)
                 e = endswith(name, ".tar.bz2");
+        if (!e)
+                e = endswith(name, ".tar.zst");
         if (!e)
                 e = endswith(name, ".tgz");
         if (!e)
@@ -174,6 +176,7 @@ int raw_strip_suffixes(const char *p, char **ret) {
                 ".xz\0"
                 ".gz\0"
                 ".bz2\0"
+                ".zst\0"
                 ".sysext.raw\0"
                 ".confext.raw\0"
                 ".raw\0"

diff --git a/test/units/TEST-13-NSPAWN.importctl.sh b/test/units/TEST-13-NSPAWN.importctl.sh
index e4931a9bfda0aef130f3631a9cfe8678b1694b98..080b42625cbfba54e360a3cac956a6dc269eb3f9 100755 (executable)
--- a/test/units/TEST-13-NSPAWN.importctl.sh
+++ b/test/units/TEST-13-NSPAWN.importctl.sh
@@ -84,6 +84,10 @@ systemctl daemon-reload
 systemctl start systemd-import@var-lib-confexts-importtest9.service
 cmp /var/tmp/importtest /var/lib/confexts/importtest9/importtest
 
+importctl export-raw --class=confext --format zstd importtest /var/tmp/importtest10.raw.zst
+importctl import-raw --class=confext /var/tmp/importtest10.raw.zst
+cmp /var/tmp/importtest /var/lib/confexts/importtest10.raw
+
 # Verify generic service calls, too
 varlinkctl call --more /run/systemd/io.systemd.Import io.systemd.service.Ping '{}'
 varlinkctl call --more /run/systemd/io.systemd.Import io.systemd.service.SetLogLevel '{"level":"7"}'