repart: Add compression support

Now that mkfs.btrfs is adding support for compressing the generated
filesystem (https://github.com/kdave/btrfs-progs/pull/882), let's
add general support for specifying the compression algorithm and
compression level to use.

We opt to not parse the specified compression algorithm and instead
pass it on as is to the mkfs tool. This has a few benefits:

- We support every compression algorithm supported by every tool
  automatically.
- Users don't need to modify systemd-repart if a mkfs tool learns a
  new compression algorithm in the future
- We don't need to maintain a bunch of tables for filesystem to map
  from our generic compression algorithm enum to the filesystem specific
  names.

We don't add support for btrfs just yet until the corresponding PR
in btrfs-progs is merged.

diff --git a/man/repart.d.xml b/man/repart.d.xml
index 2e3aa68f446f4597fc438e798ba309c9d83abc3f..a3c42e15e0143b9c63af3e67fe47e7849db846b5 100644 (file)
--- a/man/repart.d.xml
+++ b/man/repart.d.xml
@@ -811,6 +811,68 @@
 
         <xi:include href="version-info.xml" xpointer="v256"/></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><varname>Compression=</varname></term>
+
+        <listitem><para>Specify the compression algorithm to use for the filesystem configured with
+        <varname>Format=</varname>. Takes a single argument specifying the compression algorithm.</para>
+
+        <para>Note that this setting is only taken into account when the filesystem configured with
+        <varname>Format=</varname> supports compression (btrfs, squashfs, erofs). Here's an incomplete list
+        of compression algorithms supported by the filesystems known to
+        <command>systemd-repart</command>:</para>
+
+        <table>
+          <title>File System Compression Algorithms</title>
+
+          <tgroup cols='3' align='left' colsep='1' rowsep='1'>
+            <colspec colname="filesystem" />
+            <colspec colname="algorithms" />
+            <colspec colname="manpage" />
+
+            <thead>
+              <row>
+                <entry>File System</entry>
+                <entry>Compression Algorithms</entry>
+                <entry>Documentation</entry>
+              </row>
+            </thead>
+
+            <tbody>
+              <row>
+                <entry><constant>squashfs</constant></entry>
+                <entry>gzip, lzo, lz4, xz, zstd, lzma</entry>
+                <entry><member><citerefentry project='man-pages'><refentrytitle>mksquashfs</refentrytitle><manvolnum>1</manvolnum></citerefentry></member></entry>
+              </row>
+
+              <row>
+                <entry><constant>erofs</constant></entry>
+                <entry>lz4, lz4hc, lzma, deflate, libdeflate, zstd</entry>
+                <entry><member><citerefentry project='man-pages'><refentrytitle>mkfs.erofs</refentrytitle><manvolnum>1</manvolnum></citerefentry></member></entry>
+              </row>
+            </tbody>
+          </tgroup>
+        </table>
+
+        <xi:include href="version-info.xml" xpointer="v257"/></listitem>
+      </varlistentry>
+
+      <varlistentry>
+        <term><varname>CompressionLevel=</varname></term>
+
+        <listitem><para>Specify the compression level to use for the filesystem configured with
+        <varname>Format=</varname>. Takes a single argument specifying the compression level to use for the
+        configured compression algorithm. The possible compression levels and their meaning are filesystem
+        specific (refer to the filesystem's documentation for the exact meaning of a particular compression
+        level).</para>
+
+        <para>Note that this setting is only taken into account when the filesystem configured with
+        <varname>Format=</varname> supports compression and the <varname>Compression=</varname> setting is
+        configured explicitly.</para>
+
+        <xi:include href="version-info.xml" xpointer="v257"/></listitem>
+      </varlistentry>
     </variablelist>
   </refsect1>
 

diff --git a/src/home/homework-luks.c b/src/home/homework-luks.c
index ed5ee930c121c25e6b1046bdc92260e5c68ae322..b859658f00d7abe35e97de40955a811fc4f3c51a 100644 (file)
--- a/src/home/homework-luks.c
+++ b/src/home/homework-luks.c
@@ -2378,6 +2378,8 @@ int home_create_luks(
                             user_record_luks_discard(h),
                             /* quiet = */ true,
                             /* sector_size = */ 0,
+                            /* compression = */ NULL,
+                            /* compression_level= */ NULL,
                             extra_mkfs_options);
         if (r < 0)
                 return r;

diff --git a/src/partition/makefs.c b/src/partition/makefs.c
index 53439a4bbc5f042602a0b66efed3709aed232c06..c76e81d97625b22c748d390a3129e12b90757e55 100644 (file)
--- a/src/partition/makefs.c
+++ b/src/partition/makefs.c
@@ -78,6 +78,8 @@ static int run(int argc, char *argv[]) {
                                /* discard = */ true,
                                /* quiet = */ true,
                                /* sector_size = */ 0,
+                               /* compression = */ NULL,
+                               /* compression_level = */ NULL,
                                /* extra_mkfs_options = */ NULL);
 }
 

diff --git a/src/partition/repart.c b/src/partition/repart.c
index d7703827897bce8b17dfb8588212ef18c4502e19..6e76cb011a05de7917c526e5dd5e2338ea8a8bcb 100644 (file)
--- a/src/partition/repart.c
+++ b/src/partition/repart.c
@@ -385,6 +385,8 @@ typedef struct Partition {
         MinimizeMode minimize;
         uint64_t verity_data_block_size;
         uint64_t verity_hash_block_size;
+        char *compression;
+        char *compression_level;
 
         uint64_t gpt_flags;
         int no_auto;
@@ -545,6 +547,8 @@ static Partition* partition_free(Partition *p) {
         ordered_hashmap_free(p->subvolumes);
         free(p->default_subvolume);
         free(p->verity_match_key);
+        free(p->compression);
+        free(p->compression_level);
 
         iovec_done(&p->roothash);
 
@@ -581,6 +585,8 @@ static void partition_foreignize(Partition *p) {
         p->subvolumes = ordered_hashmap_free(p->subvolumes);
         p->default_subvolume = mfree(p->default_subvolume);
         p->verity_match_key = mfree(p->verity_match_key);
+        p->compression = mfree(p->compression);
+        p->compression_level = mfree(p->compression_level);
 
         p->priority = 0;
         p->weight = 1000;
@@ -2088,38 +2094,40 @@ static int partition_finalize_fstype(Partition *p, const char *path) {
 static int partition_read_definition(Partition *p, const char *path, const char *const *conf_file_dirs) {
 
         ConfigTableItem table[] = {
-                { "Partition", "Type",                     config_parse_type,              0, &p->type                    },
-                { "Partition", "Label",                    config_parse_label,             0, &p->new_label               },
-                { "Partition", "UUID",                     config_parse_uuid,              0, p                           },
-                { "Partition", "Priority",                 config_parse_int32,             0, &p->priority                },
-                { "Partition", "Weight",                   config_parse_weight,            0, &p->weight                  },
-                { "Partition", "PaddingWeight",            config_parse_weight,            0, &p->padding_weight          },
-                { "Partition", "SizeMinBytes",             config_parse_size4096,         -1, &p->size_min                },
-                { "Partition", "SizeMaxBytes",             config_parse_size4096,          1, &p->size_max                },
-                { "Partition", "PaddingMinBytes",          config_parse_size4096,         -1, &p->padding_min             },
-                { "Partition", "PaddingMaxBytes",          config_parse_size4096,          1, &p->padding_max             },
-                { "Partition", "FactoryReset",             config_parse_bool,              0, &p->factory_reset           },
-                { "Partition", "CopyBlocks",               config_parse_copy_blocks,       0, p                           },
-                { "Partition", "Format",                   config_parse_fstype,            0, &p->format                  },
-                { "Partition", "CopyFiles",                config_parse_copy_files,        0, &p->copy_files              },
-                { "Partition", "ExcludeFiles",             config_parse_exclude_files,     0, &p->exclude_files_source    },
-                { "Partition", "ExcludeFilesTarget",       config_parse_exclude_files,     0, &p->exclude_files_target    },
-                { "Partition", "MakeDirectories",          config_parse_make_dirs,         0, &p->make_directories        },
-                { "Partition", "Encrypt",                  config_parse_encrypt,           0, &p->encrypt                 },
-                { "Partition", "Verity",                   config_parse_verity,            0, &p->verity                  },
-                { "Partition", "VerityMatchKey",           config_parse_string,            0, &p->verity_match_key        },
-                { "Partition", "Flags",                    config_parse_gpt_flags,         0, &p->gpt_flags               },
-                { "Partition", "ReadOnly",                 config_parse_tristate,          0, &p->read_only               },
-                { "Partition", "NoAuto",                   config_parse_tristate,          0, &p->no_auto                 },
-                { "Partition", "GrowFileSystem",           config_parse_tristate,          0, &p->growfs                  },
-                { "Partition", "SplitName",                config_parse_string,            0, &p->split_name_format       },
-                { "Partition", "Minimize",                 config_parse_minimize,          0, &p->minimize                },
-                { "Partition", "Subvolumes",               config_parse_subvolumes,        0, &p->subvolumes              },
-                { "Partition", "DefaultSubvolume",         config_parse_default_subvolume, 0, &p->default_subvolume       },
-                { "Partition", "VerityDataBlockSizeBytes", config_parse_block_size,        0, &p->verity_data_block_size  },
-                { "Partition", "VerityHashBlockSizeBytes", config_parse_block_size,        0, &p->verity_hash_block_size  },
-                { "Partition", "MountPoint",               config_parse_mountpoint,        0, p                           },
-                { "Partition", "EncryptedVolume",          config_parse_encrypted_volume,  0, p                           },
+                { "Partition", "Type",                     config_parse_type,              0,                                  &p->type                    },
+                { "Partition", "Label",                    config_parse_label,             0,                                  &p->new_label               },
+                { "Partition", "UUID",                     config_parse_uuid,              0,                                  p                           },
+                { "Partition", "Priority",                 config_parse_int32,             0,                                  &p->priority                },
+                { "Partition", "Weight",                   config_parse_weight,            0,                                  &p->weight                  },
+                { "Partition", "PaddingWeight",            config_parse_weight,            0,                                  &p->padding_weight          },
+                { "Partition", "SizeMinBytes",             config_parse_size4096,         -1,                                  &p->size_min                },
+                { "Partition", "SizeMaxBytes",             config_parse_size4096,          1,                                  &p->size_max                },
+                { "Partition", "PaddingMinBytes",          config_parse_size4096,         -1,                                  &p->padding_min             },
+                { "Partition", "PaddingMaxBytes",          config_parse_size4096,          1,                                  &p->padding_max             },
+                { "Partition", "FactoryReset",             config_parse_bool,              0,                                  &p->factory_reset           },
+                { "Partition", "CopyBlocks",               config_parse_copy_blocks,       0,                                  p                           },
+                { "Partition", "Format",                   config_parse_fstype,            0,                                  &p->format                  },
+                { "Partition", "CopyFiles",                config_parse_copy_files,        0,                                  &p->copy_files              },
+                { "Partition", "ExcludeFiles",             config_parse_exclude_files,     0,                                  &p->exclude_files_source    },
+                { "Partition", "ExcludeFilesTarget",       config_parse_exclude_files,     0,                                  &p->exclude_files_target    },
+                { "Partition", "MakeDirectories",          config_parse_make_dirs,         0,                                  &p->make_directories        },
+                { "Partition", "Encrypt",                  config_parse_encrypt,           0,                                  &p->encrypt                 },
+                { "Partition", "Verity",                   config_parse_verity,            0,                                  &p->verity                  },
+                { "Partition", "VerityMatchKey",           config_parse_string,            0,                                  &p->verity_match_key        },
+                { "Partition", "Flags",                    config_parse_gpt_flags,         0,                                  &p->gpt_flags               },
+                { "Partition", "ReadOnly",                 config_parse_tristate,          0,                                  &p->read_only               },
+                { "Partition", "NoAuto",                   config_parse_tristate,          0,                                  &p->no_auto                 },
+                { "Partition", "GrowFileSystem",           config_parse_tristate,          0,                                  &p->growfs                  },
+                { "Partition", "SplitName",                config_parse_string,            0,                                  &p->split_name_format       },
+                { "Partition", "Minimize",                 config_parse_minimize,          0,                                  &p->minimize                },
+                { "Partition", "Subvolumes",               config_parse_subvolumes,        0,                                  &p->subvolumes              },
+                { "Partition", "DefaultSubvolume",         config_parse_default_subvolume, 0,                                  &p->default_subvolume       },
+                { "Partition", "VerityDataBlockSizeBytes", config_parse_block_size,        0,                                  &p->verity_data_block_size  },
+                { "Partition", "VerityHashBlockSizeBytes", config_parse_block_size,        0,                                  &p->verity_hash_block_size  },
+                { "Partition", "MountPoint",               config_parse_mountpoint,        0,                                  p                           },
+                { "Partition", "EncryptedVolume",          config_parse_encrypted_volume,  0,                                  p                           },
+                { "Partition", "Compression",              config_parse_string,            CONFIG_PARSE_STRING_SAFE_AND_ASCII, &p->compression             },
+                { "Partition", "CompressionLevel",         config_parse_string,            CONFIG_PARSE_STRING_SAFE_AND_ASCII, &p->compression_level       },
                 {}
         };
         _cleanup_free_ char *filename = NULL;
@@ -5469,7 +5477,8 @@ static int context_mkfs(Context *context) {
 
                 r = make_filesystem(partition_target_path(t), p->format, strempty(p->new_label), root,
                                     p->fs_uuid, arg_discard, /* quiet = */ false,
-                                    context->fs_sector_size, extra_mkfs_options);
+                                    context->fs_sector_size, p->compression, p->compression_level,
+                                    extra_mkfs_options);
                 if (r < 0)
                         return r;
 
@@ -7033,6 +7042,8 @@ static int context_minimize(Context *context) {
                                     fs_uuid,
                                     arg_discard, /* quiet = */ false,
                                     context->fs_sector_size,
+                                    p->compression,
+                                    p->compression_level,
                                     extra_mkfs_options);
                 if (r < 0)
                         return r;
@@ -7114,6 +7125,8 @@ static int context_minimize(Context *context) {
                                     arg_discard,
                                     /* quiet = */ false,
                                     context->fs_sector_size,
+                                    p->compression,
+                                    p->compression_level,
                                     extra_mkfs_options);
                 if (r < 0)
                         return r;

diff --git a/src/shared/mkfs-util.c b/src/shared/mkfs-util.c
index 76249899292c008b7e736e96fe912d32bf5b1bd4..15652cf57a0d173069ab14664d286a4d2b5c18db 100644 (file)
--- a/src/shared/mkfs-util.c
+++ b/src/shared/mkfs-util.c
@@ -323,6 +323,8 @@ int make_filesystem(
                 bool discard,
                 bool quiet,
                 uint64_t sector_size,
+                char *compression,
+                char *compression_level,
                 char * const *extra_mkfs_args) {
 
         _cleanup_free_ char *mkfs = NULL, *mangled_label = NULL;
@@ -570,12 +572,19 @@ int make_filesystem(
                                 root, node,
                                 "-noappend");
 
+                if (compression) {
+                        if (strv_extend_many(&argv, "-comp", compression) < 0)
+                                return log_oom();
+
+                        if (compression_level && strv_extend_many(&argv, "-Xcompression-level", compression_level) < 0)
+                                return log_oom();
+                }
+
                 /* mksquashfs -quiet option is pretty new so let's redirect stdout to /dev/null instead. */
                 if (quiet)
                         stdio_fds[1] = -EBADF;
 
         } else if (streq(fstype, "erofs")) {
-
                 argv = strv_new(mkfs,
                                 "-U", vol_id,
                                 node, root);
@@ -583,6 +592,20 @@ int make_filesystem(
                 if (quiet && strv_extend(&argv, "--quiet") < 0)
                         return log_oom();
 
+                if (compression) {
+                        _cleanup_free_ char *c = NULL;
+
+                        c = strjoin("-z", compression);
+                        if (!c)
+                                return log_oom();
+
+                        if (compression_level && !strextend(&c, ",level=", compression_level))
+                                return log_oom();
+
+                        if (strv_extend(&argv, c) < 0)
+                                return log_oom();
+                }
+
         } else
                 /* Generic fallback for all other file systems */
                 argv = strv_new(mkfs, node);

diff --git a/src/shared/mkfs-util.h b/src/shared/mkfs-util.h
index 9a1cb585d6cec3abcfc9fdbc977ca0b60e25e9f2..165862e7e1ddc32caa3a03bd1ddb798f24e30aab 100644 (file)
--- a/src/shared/mkfs-util.h
+++ b/src/shared/mkfs-util.h
@@ -20,6 +20,8 @@ int make_filesystem(
                 bool discard,
                 bool quiet,
                 uint64_t sector_size,
+                char *compression,
+                char *compression_level,
                 char * const *extra_mkfs_args);
 
 int mkfs_options_from_env(const char *component, const char *fstype, char ***ret);

diff --git a/src/test/test-loop-block.c b/src/test/test-loop-block.c
index 15c635781b31fb770af2e1e5a22dbbb1b9ee0dcc..e69c0d5caf4a2d2c311fe94be93d7e5a6ca2d9e8 100644 (file)
--- a/src/test/test-loop-block.c
+++ b/src/test/test-loop-block.c
@@ -251,16 +251,16 @@ static int run(int argc, char *argv[]) {
         assert_se(r >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_ESP].node, "vfat", "EFI", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_ESP].node, "vfat", "EFI", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_XBOOTLDR].node, "vfat", "xbootldr", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_XBOOTLDR].node, "vfat", "xbootldr", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_ROOT].node, "ext4", "root", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_ROOT].node, "ext4", "root", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);
 
         assert_se(sd_id128_randomize(&id) >= 0);
-        assert_se(make_filesystem(dissected->partitions[PARTITION_HOME].node, "ext4", "home", NULL, id, true, false, 0, NULL) >= 0);
+        assert_se(make_filesystem(dissected->partitions[PARTITION_HOME].node, "ext4", "home", NULL, id, true, false, 0, NULL, NULL, NULL) >= 0);
 
         dissected = dissected_image_unref(dissected);
 

diff --git a/test/units/TEST-58-REPART.sh b/test/units/TEST-58-REPART.sh
index 968d332f068e54fbd7808e0e4fb35c5dec0f7630..b2a3be686cab31034272d6b0a6f433a421d8a044 100755 (executable)
--- a/test/units/TEST-58-REPART.sh
+++ b/test/units/TEST-58-REPART.sh
@@ -1312,6 +1312,41 @@ testcase_list_devices() {
     systemd-repart --list-devices
 }
 
+testcase_compression() {
+    local workdir image defs
+
+    workdir="$(mktemp --directory "/tmp/test-repart.compression.XXXXXXXXXX")"
+    # shellcheck disable=SC2064
+    trap "rm -rf '${workdir:?}'" RETURN
+
+    image="$workdir/image.img"
+    defs="$workdir/defs"
+    mkdir "$defs"
+
+    # TODO: add btrfs once btrfs-progs v6.11 is available in distributions.
+    for format in squashfs erofs; do
+        if ! command -v "mkfs.$format" && ! command -v mksquashfs >/dev/null; then
+            continue
+        fi
+
+        [[ "$format" == "squashfs" ]] && compression=zstd
+        [[ "$format" == "erofs" ]] && compression=lz4hc
+
+        tee "$defs/10-root.conf" <<EOF
+[Partition]
+Type=root
+Format=$format
+Compression=$compression
+CompressionLevel=3
+CopyFiles=$defs:/def
+SizeMinBytes=48M
+EOF
+
+        rm -f "$image"
+        systemd-repart --empty=create --size=auto --pretty=yes --dry-run=no --definitions="$defs" "$image"
+    done
+}
+
 OFFLINE="yes"
 run_testcases