]> git.ipfire.org Git - thirdparty/kmod.git/commitdiff
add Zstandard compression support
authorTorge Matthies <openglfreak@googlemail.com>
Tue, 8 Sep 2020 19:59:20 +0000 (21:59 +0200)
committerLucas De Marchi <lucas.demarchi@intel.com>
Fri, 11 Sep 2020 04:55:01 +0000 (21:55 -0700)
I changed the style of the hackargs variable in autogen.sh to multiline
because said line was becoming a bit long with the new --with-zstd arg
added.

A previous version of this patch has been running on my two Arch Linux
installations (with an accompanying mkinitcpio patch) for several months
over many kernel updates without any issues.
Any additional testing and/or patch review would of course be appreciated.

Signed-off-by: Torge Matthies <openglfreak@googlemail.com>
.semaphore/semaphore.yml
.travis.yml
Makefile.am
autogen.sh
configure.ac
libkmod/libkmod-file.c
libkmod/libkmod.pc.in
shared/util.c
testsuite/mkosi/mkosi.fedora
testsuite/test-util.c

index db47ca1594f8093d6c1c3f580815f791c2846186..fe1d78a9b8780e7b6f4b8daa4b28951e94e040c4 100644 (file)
@@ -22,7 +22,7 @@ blocks:
       prologue:
         commands:
           - sudo apt update
-          - sudo apt --yes install docbook-xsl liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
+          - sudo apt --yes install docbook-xsl libzstd-dev liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
           - checkout
 
       epilogue:
@@ -42,5 +42,5 @@ blocks:
       prologue:
         commands:
           - sudo apt update
-          - sudo apt --yes install docbook-xsl liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
+          - sudo apt --yes install docbook-xsl libzstd-dev liblzma-dev zlib1g-dev cython linux-headers-generic libssl-dev
           - checkout
index 02c753e88b8376ae4b3f9ddd9d9a4f1d4fac4774..0fcce14e13fe6becf79edf1b84d697e4250364ec 100644 (file)
@@ -14,6 +14,7 @@ matrix:
 before_install:
   - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
   - sudo apt-get update -qq
+  - sudo apt-get install -qq libzstd-dev
   - sudo apt-get install -qq liblzma-dev
   - sudo apt-get install -qq zlib1g-dev
   - sudo apt-get install -qq xsltproc docbook-xsl
index 8eadb99787c0bc44e2d1d139cf1c16c6cfe5804d..37d840b98f87b41ce6ec22bcb0cd71750338f3ba 100644 (file)
@@ -31,6 +31,8 @@ SED_PROCESS = \
        -e 's,@exec_prefix\@,$(exec_prefix),g' \
        -e 's,@libdir\@,$(libdir),g' \
        -e 's,@includedir\@,$(includedir),g' \
+       -e 's,@libzstd_CFLAGS\@,${libzstd_CFLAGS},g' \
+       -e 's,@libzstd_LIBS\@,${libzstd_LIBS},g' \
        -e 's,@liblzma_CFLAGS\@,${liblzma_CFLAGS},g' \
        -e 's,@liblzma_LIBS\@,${liblzma_LIBS},g' \
        -e 's,@zlib_CFLAGS\@,${zlib_CFLAGS},g' \
@@ -90,7 +92,7 @@ libkmod_libkmod_la_DEPENDENCIES = \
        ${top_srcdir}/libkmod/libkmod.sym
 libkmod_libkmod_la_LIBADD = \
        shared/libshared.la \
-       ${liblzma_LIBS} ${zlib_LIBS} ${libcrypto_LIBS}
+       ${libzstd_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libcrypto_LIBS}
 
 noinst_LTLIBRARIES += libkmod/libkmod-internal.la
 libkmod_libkmod_internal_la_SOURCES = $(libkmod_libkmod_la_SOURCES)
index 67b119f6fc6e092c5d21f87088109ffaca9cdbf9..e4997c4c53e37309bbed97cd8e3faaa32f6563f6 100755 (executable)
@@ -32,7 +32,14 @@ fi
 
 cd $oldpwd
 
-hackargs="--enable-debug --enable-python --with-xz --with-zlib --with-openssl"
+hackargs="\
+--enable-debug \
+--enable-python \
+--with-zstd \
+--with-xz \
+--with-zlib \
+--with-openssl \
+"
 
 if [ "x$1" = "xc" ]; then
         shift
index 4a65d6ba7a3790eae54b9b82656af77492b4820f..a49f6b98fc2ad70320d2dd3d4d5d3fddd06062ed 100644 (file)
@@ -83,6 +83,17 @@ AC_ARG_WITH([rootlibdir],
         [], [with_rootlibdir=$libdir])
 AC_SUBST([rootlibdir], [$with_rootlibdir])
 
+AC_ARG_WITH([zstd],
+       AS_HELP_STRING([--with-zstd], [handle Zstandard-compressed modules @<:@default=disabled@:>@]),
+       [], [with_zstd=no])
+AS_IF([test "x$with_zstd" != "xno"], [
+       PKG_CHECK_MODULES([libzstd], [libzstd >= 1.4.4])
+       AC_DEFINE([ENABLE_ZSTD], [1], [Enable Zstandard for modules.])
+], [
+       AC_MSG_NOTICE([Zstandard support not requested])
+])
+CC_FEATURE_APPEND([with_features], [with_zstd], [ZSTD])
+
 AC_ARG_WITH([xz],
        AS_HELP_STRING([--with-xz], [handle Xz-compressed modules @<:@default=disabled@:>@]),
        [], [with_xz=no])
@@ -307,7 +318,7 @@ AC_MSG_RESULT([
        tools:                  ${enable_tools}
        python bindings:        ${enable_python}
        logging:                ${enable_logging}
-       compression:            xz=${with_xz}  zlib=${with_zlib}
+       compression:            zstd=${with_zstd}  xz=${with_xz}  zlib=${with_zlib}
        debug:                  ${enable_debug}
        coverage:               ${enable_coverage}
        doc:                    ${enable_gtk_doc}
index 5eeba6a912a2ff987ec3e471da83b92d158182d3..b6a8cc9579f73d38681ee1d4461f9e758c54dbaa 100644 (file)
@@ -26,6 +26,9 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#ifdef ENABLE_ZSTD
+#include <zstd.h>
+#endif
 #ifdef ENABLE_XZ
 #include <lzma.h>
 #endif
@@ -45,6 +48,9 @@ struct file_ops {
 };
 
 struct kmod_file {
+#ifdef ENABLE_ZSTD
+       bool zstd_used;
+#endif
 #ifdef ENABLE_XZ
        bool xz_used;
 #endif
@@ -60,6 +66,141 @@ struct kmod_file {
        struct kmod_elf *elf;
 };
 
+#ifdef ENABLE_ZSTD
+static int zstd_read_block(struct kmod_file *file, size_t block_size,
+                          ZSTD_inBuffer *input, size_t *input_capacity)
+{
+       ssize_t rdret;
+       int ret;
+
+       if (*input_capacity < block_size) {
+               free((void *)input->src);
+               input->src = malloc(block_size);
+               if (input->src == NULL) {
+                       ret = -errno;
+                       ERR(file->ctx, "zstd: %m\n");
+                       return ret;
+               }
+               *input_capacity = block_size;
+       }
+
+       rdret = read(file->fd, (void *)input->src, block_size);
+       if (rdret < 0) {
+               ret = -errno;
+               ERR(file->ctx, "zstd: %m\n");
+               return ret;
+       }
+
+       input->pos = 0;
+       input->size = rdret;
+       return 0;
+}
+
+static int zstd_ensure_outbuffer_space(ZSTD_outBuffer *buffer, size_t min_free)
+{
+       uint8_t *old_buffer = buffer->dst;
+       int ret = 0;
+
+       if (buffer->size - buffer->pos >= min_free)
+               return 0;
+
+       buffer->size += min_free;
+       buffer->dst = realloc(buffer->dst, buffer->size);
+       if (buffer->dst == NULL) {
+               ret = -errno;
+               free(old_buffer);
+       }
+
+       return ret;
+}
+
+static int zstd_decompress_block(struct kmod_file *file, ZSTD_DStream *dstr,
+                                ZSTD_inBuffer *input, ZSTD_outBuffer *output,
+                                size_t *next_block_size)
+{
+       size_t out_buf_min_size = ZSTD_DStreamOutSize();
+       int ret = 0;
+
+       do {
+               ssize_t dsret;
+
+               ret = zstd_ensure_outbuffer_space(output, out_buf_min_size);
+               if (ret) {
+                       ERR(file->ctx, "zstd: %s\n", strerror(-ret));
+                       break;
+               }
+
+               dsret = ZSTD_decompressStream(dstr, output, input);
+               if (ZSTD_isError(dsret)) {
+                       ret = -EINVAL;
+                       ERR(file->ctx, "zstd: %s\n", ZSTD_getErrorName(dsret));
+                       break;
+               }
+               if (dsret > 0)
+                       *next_block_size = (size_t)dsret;
+       } while (input->pos < input->size
+                || output->pos > output->size
+                || output->size - output->pos < out_buf_min_size);
+
+       return ret;
+}
+
+static int load_zstd(struct kmod_file *file)
+{
+       ZSTD_DStream *dstr;
+       size_t next_block_size;
+       size_t zst_inb_capacity = 0;
+       ZSTD_inBuffer zst_inb = { 0 };
+       ZSTD_outBuffer zst_outb = { 0 };
+       int ret;
+
+       dstr = ZSTD_createDStream();
+       if (dstr == NULL) {
+               ret = -EINVAL;
+               ERR(file->ctx, "zstd: Failed to create decompression stream\n");
+               goto out;
+       }
+
+       next_block_size = ZSTD_initDStream(dstr);
+
+       while (true) {
+               ret = zstd_read_block(file, next_block_size, &zst_inb,
+                                     &zst_inb_capacity);
+               if (ret != 0)
+                       goto out;
+               if (zst_inb.size == 0) /* EOF */
+                       break;
+
+               ret = zstd_decompress_block(file, dstr, &zst_inb, &zst_outb,
+                                           &next_block_size);
+               if (ret != 0)
+                       goto out;
+       }
+
+       ZSTD_freeDStream(dstr);
+       free((void *)zst_inb.src);
+       file->zstd_used = true;
+       file->memory = zst_outb.dst;
+       file->size = zst_outb.pos;
+       return 0;
+out:
+       if (dstr != NULL)
+               ZSTD_freeDStream(dstr);
+       free((void *)zst_inb.src);
+       free((void *)zst_outb.dst);
+       return ret;
+}
+
+static void unload_zstd(struct kmod_file *file)
+{
+       if (!file->zstd_used)
+               return;
+       free(file->memory);
+}
+
+static const char magic_zstd[] = {0x28, 0xB5, 0x2F, 0xFD};
+#endif
+
 #ifdef ENABLE_XZ
 static void xz_uncompress_belch(struct kmod_file *file, lzma_ret ret)
 {
@@ -238,6 +379,9 @@ static const struct comp_type {
        const char *magic_bytes;
        const struct file_ops ops;
 } comp_types[] = {
+#ifdef ENABLE_ZSTD
+       {sizeof(magic_zstd), magic_zstd, {load_zstd, unload_zstd}},
+#endif
 #ifdef ENABLE_XZ
        {sizeof(magic_xz), magic_xz, {load_xz, unload_xz}},
 #endif
index e4fdf21dcd8f1ddbfd2ccc41b74c2656aa0175f8..3acca6a6f135ef1da99b9c2ec2613e5eaacaf7bd 100644 (file)
@@ -7,5 +7,5 @@ Name: libkmod
 Description: Library to deal with kernel modules
 Version: @VERSION@
 Libs: -L${libdir} -lkmod
-Libs.private: @liblzma_LIBS@ @zlib_LIBS@
+Libs.private: @libzstd_LIBS@ @liblzma_LIBS@ @zlib_LIBS@
 Cflags: -I${includedir}
index fd2028df5a603007ee4233f2bdf850c6e4224e6d..b487b5f5d44b196bfadc3354daea5949861537e5 100644 (file)
@@ -45,6 +45,9 @@ static const struct kmod_ext {
 #endif
 #ifdef ENABLE_XZ
        {".ko.xz", sizeof(".ko.xz") - 1},
+#endif
+#ifdef ENABLE_ZSTD
+       {".ko.zst", sizeof(".ko.zst") - 1},
 #endif
        { }
 };
index 5252f8775f69334eda2f199bbd1c00a6597a05c5..7a2ee5e7390751cf22deb36d3478a663d4e77447 100644 (file)
@@ -19,6 +19,7 @@ BuildPackages =
        make
        pkgconf-pkg-config
        xml-common
+       libzstd-devel
        xz-devel
        zlib-devel
        openssl-devel
index 5e25e587d69bdc4e0dd8f00f37bfb3405f770e3f..621446b10503908371c5b820fe0f6dc8a4a5e1f0 100644 (file)
@@ -156,6 +156,9 @@ static int test_path_ends_with_kmod_ext(const struct test *t)
 #endif
 #ifdef ENABLE_XZ
                { "/bla.ko.xz", true },
+#endif
+#ifdef ENABLE_ZSTD
+               { "/bla.ko.zst", true },
 #endif
                { "/bla.ko.x", false },
                { "/bla.ko.", false },