+From 599f8f2aaace3df939cb145368574a52268d82d0 Mon Sep 17 00:00:00 2001
+From: Nick Terrell <terrelln@fb.com>
+Date: Wed, 21 Jun 2017 17:31:39 -0700
+Subject: [PATCH 3/4] btrfs: Add zstd support
+
+Add zstd compression and decompression support to BtrFS. zstd at its
+fastest level compresses almost as well as zlib, while offering much
+faster compression and decompression, approaching lzo speeds.
+
+I benchmarked btrfs with zstd compression against no compression, lzo
+compression, and zlib compression. I benchmarked two scenarios. Copying
+a set of files to btrfs, and then reading the files. Copying a tarball
+to btrfs, extracting it to btrfs, and then reading the extracted files.
+After every operation, I call `sync` and include the sync time.
+Between every pair of operations I unmount and remount the filesystem
+to avoid caching. The benchmark files can be found in the upstream
+zstd source repository under
+`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}`
+[1] [2].
+
+I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
+The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
+16 GB of RAM, and a SSD.
+
+The first compression benchmark is copying 10 copies of the unzipped
+Silesia corpus [3] into a BtrFS filesystem mounted with
+`-o compress-force=Method`. The decompression benchmark times how long
+it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is
+measured by comparing the output of `df` and `du`. See the benchmark file
+[1] for details. I benchmarked multiple zstd compression levels, although
+the patch uses zstd level 1.
+
+| Method | Ratio | Compression MB/s | Decompression speed |
+|---------|-------|------------------|---------------------|
+| None | 0.99 | 504 | 686 |
+| lzo | 1.66 | 398 | 442 |
+| zlib | 2.58 | 65 | 241 |
+| zstd 1 | 2.57 | 260 | 383 |
+| zstd 3 | 2.71 | 174 | 408 |
+| zstd 6 | 2.87 | 70 | 398 |
+| zstd 9 | 2.92 | 43 | 406 |
+| zstd 12 | 2.93 | 21 | 408 |
+| zstd 15 | 3.01 | 11 | 354 |
+
+The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it
+measures the compression ratio, extracts the tar, and deletes the tar.
+Then it measures the compression ratio again, and `tar`s the extracted
+files into `/dev/null`. See the benchmark file [2] for details.
+
+| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) |
+|--------|-----------|---------------|----------|------------|----------|
+| None | 0.97 | 0.78 | 0.981 | 5.501 | 8.807 |
+| lzo | 2.06 | 1.38 | 1.631 | 8.458 | 8.585 |
+| zlib | 3.40 | 1.86 | 7.750 | 21.544 | 11.744 |
+| zstd 1 | 3.57 | 1.85 | 2.579 | 11.479 | 9.389 |
+
+[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh
+[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh
+[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
+[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz
+
+zstd source repository: https://github.com/facebook/zstd
+
+Signed-off-by: Nick Terrell <terrelln@fb.com>
+---
+ fs/btrfs/Kconfig | 2 +
+ fs/btrfs/Makefile | 2 +-
+ fs/btrfs/compression.c | 1 +
+ fs/btrfs/compression.h | 6 +-
+ fs/btrfs/ctree.h | 1 +
+ fs/btrfs/disk-io.c | 2 +
+ fs/btrfs/ioctl.c | 6 +-
+ fs/btrfs/props.c | 6 +
+ fs/btrfs/super.c | 12 +-
+ fs/btrfs/sysfs.c | 2 +
+ fs/btrfs/zstd.c | 433 +++++++++++++++++++++++++++++++++++++++++++++
+ include/uapi/linux/btrfs.h | 8 +-
+ 12 files changed, 469 insertions(+), 12 deletions(-)
+ create mode 100644 fs/btrfs/zstd.c
+
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
index 80e9c18..a26c63b 100644
--- a/fs/btrfs/Kconfig
reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
uuid-tree.o props.o hash.o free-space-tree.o
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
-index c7721a6..66d4ced 100644
+index 10e6b28..3beb0d0 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -761,6 +761,7 @@ static struct {
&btrfs_lzo_compress,
+ &btrfs_zstd_compress,
};
-
+
void __init btrfs_init_compress(void)
diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
index 39ec43a..d99fc21 100644
+ BTRFS_COMPRESS_TYPES = 3,
+ BTRFS_COMPRESS_LAST = 4,
};
-
+
struct btrfs_compress_op {
@@ -92,5 +93,6 @@ struct btrfs_compress_op {
-
+
extern const struct btrfs_compress_op btrfs_zlib_compress;
extern const struct btrfs_compress_op btrfs_lzo_compress;
+extern const struct btrfs_compress_op btrfs_zstd_compress;
-
+
#endif
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
-index 29b7fc2..878b23b9 100644
+index 4f8f75d..61dd3dd 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
-@@ -270,6 +270,7 @@ struct btrfs_super_block {
+@@ -271,6 +271,7 @@ struct btrfs_super_block {
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
-index 08b74da..0c43e4e 100644
+index 5f678dc..49c0e91 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
-@@ -2853,6 +2853,8 @@ int open_ctree(struct super_block *sb,
+@@ -2831,6 +2831,8 @@ int open_ctree(struct super_block *sb,
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
-+ else if (tree_root->fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
++ else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
+ features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
-
+
if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
btrfs_info(fs_info, "has skinny extents");
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
-index dabfc7a..d8ea727 100644
+index e176375..f732cfd 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
-
+
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
comp = "lzo";
- else
comp, strlen(comp), 0);
if (ret)
@@ -1463,6 +1465,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
-
+
if (range->compress_type == BTRFS_COMPRESS_LZO) {
btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
+ } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) {
+ btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD);
}
-
+
ret = defrag_count;
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
index d6cb155..162105f 100644
return 0;
+ else if (!strncmp("zstd", value, len))
+ return 0;
-
+
return -EINVAL;
}
@@ -405,6 +407,8 @@ static int prop_compression_apply(struct inode *inode,
+ type = BTRFS_COMPRESS_ZSTD;
else
return -EINVAL;
-
+
@@ -422,6 +426,8 @@ static const char *prop_compression_extract(struct inode *inode)
return "zlib";
case BTRFS_COMPRESS_LZO:
+ case BTRFS_COMPRESS_ZSTD:
+ return "zstd";
}
-
+
return NULL;
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
-index da687dc..b064456 100644
+index 4f1cdd5..4f792d5 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
} else if (strncmp(args[0].from, "no", 2) == 0) {
compress_type = "no";
btrfs_clear_opt(info->mount_opt, COMPRESS);
-@@ -1230,8 +1238,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
+@@ -1240,8 +1248,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
if (btrfs_test_opt(info, COMPRESS)) {
if (info->compress_type == BTRFS_COMPRESS_ZLIB)
compress_type = "zlib";
+ .decompress = zstd_decompress,
+};
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
-index db4c253..f26c34f 100644
+index a456e53..992c150 100644
--- a/include/uapi/linux/btrfs.h
+++ b/include/uapi/linux/btrfs.h
@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
- */
-#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
+#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4)
-
+
/*
* older kernels tried to do bigger metadata blocks, but the
+--
+2.9.3
+From 5ff6a64abaea7b7f11d37cb0fdf08642316a3a90 Mon Sep 17 00:00:00 2001
+From: Nick Terrell <terrelln@fb.com>
+Date: Mon, 12 Jun 2017 12:18:23 -0700
+Subject: [PATCH 4/4] squashfs: Add zstd support
+
+Add zstd compression and decompression support to SquashFS. zstd is a
+great fit for SquashFS because it can compress at ratios approaching xz,
+while decompressing twice as fast as zlib. For SquashFS in particular,
+it can decompress as fast as lzo and lz4. It also has the flexibility
+to turn down the compression ratio for faster compression times.
+
+The compression benchmark is run on the file tree from the SquashFS archive
+found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the
+default block size (128 KB) and and various compression algorithms/levels.
+xz and zstd are also benchmarked with 256 KB blocks. The decompression
+benchmark times how long it takes to `tar` the file tree into `/dev/null`.
+See the benchmark file in the upstream zstd source repository located under
+`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details.
+
+I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
+The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
+16 GB of RAM, and a SSD.
+
+| Method | Ratio | Compression MB/s | Decompression MB/s |
+|----------------|-------|------------------|--------------------|
+| gzip | 2.92 | 15 | 128 |
+| lzo | 2.64 | 9.5 | 217 |
+| lz4 | 2.12 | 94 | 218 |
+| xz | 3.43 | 5.5 | 35 |
+| xz 256 KB | 3.53 | 5.4 | 40 |
+| zstd 1 | 2.71 | 96 | 210 |
+| zstd 5 | 2.93 | 69 | 198 |
+| zstd 10 | 3.01 | 41 | 225 |
+| zstd 15 | 3.13 | 11.4 | 224 |
+| zstd 16 256 KB | 3.24 | 8.1 | 210 |
+
+This patch was written by Sean Purcell <me@seanp.xyz>, but I will be
+taking over the submission process.
+
+[1] http://releases.ubuntu.com/16.10/
+[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh
+
+zstd source repository: https://github.com/facebook/zstd
+
+Cc: Sean Purcell <me@seanp.xyz>
+Signed-off-by: Nick Terrell <terrelln@fb.com>
+---
+ fs/squashfs/Kconfig | 14 +++++
+ fs/squashfs/Makefile | 1 +
+ fs/squashfs/decompressor.c | 7 +++
+ fs/squashfs/decompressor.h | 4 ++
+ fs/squashfs/squashfs_fs.h | 1 +
+ fs/squashfs/zstd_wrapper.c | 150 +++++++++++++++++++++++++++++++++++++++++++++
+ 6 files changed, 177 insertions(+)
+ create mode 100644 fs/squashfs/zstd_wrapper.c
+
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
index ffb093e..1adb334 100644
--- a/fs/squashfs/Kconfig
+++ b/fs/squashfs/Kconfig
@@ -165,6 +165,20 @@ config SQUASHFS_XZ
-
+
If unsure, say N.
-
+
+config SQUASHFS_ZSTD
+ bool "Include support for ZSTD compressed file systems"
+ depends on SQUASHFS
@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
};
#endif
-
+
+#ifndef CONFIG_SQUASHFS_ZSTD
+static const struct squashfs_decompressor squashfs_zstd_comp_ops = {
+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0
+ &squashfs_zstd_comp_ops,
&squashfs_unknown_comp_ops
};
-
+
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
index a25713c..0f5a8e4 100644
--- a/fs/squashfs/decompressor.h
@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
extern const struct squashfs_decompressor squashfs_zlib_comp_ops;
#endif
-
+
+#ifdef CONFIG_SQUASHFS_ZSTD
+extern const struct squashfs_decompressor squashfs_zstd_comp_ops;
+#endif
#define XZ_COMPRESSION 4
#define LZ4_COMPRESSION 5
+#define ZSTD_COMPRESSION 6
-
+
struct squashfs_super_block {
__le32 s_magic;
diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
+ .name = "zstd",
+ .supported = 1
+};
+--
+2.9.3
--- /dev/null
+# !/bin/sh
+set -e
+
+# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
+# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
+# 16 GB of RAM and an SSD.
+
+# silesia is a directory that can be downloaded from
+# http://mattmahoney.net/dc/silesia.html
+# ls -l silesia/
+# total 203M
+# -rwxr-xr-x 1 terrelln 9.8M Apr 12 2002 dickens
+# -rwxr-xr-x 1 terrelln 49M May 31 2002 mozilla
+# -rwxr-xr-x 1 terrelln 9.6M Mar 20 2003 mr
+# -rwxr-xr-x 1 terrelln 32M Apr 2 2002 nci
+# -rwxr-xr-x 1 terrelln 5.9M Jul 4 2002 ooffice
+# -rwxr-xr-x 1 terrelln 9.7M Apr 11 2002 osdb
+# -rwxr-xr-x 1 terrelln 6.4M Apr 2 2002 reymont
+# -rwxr-xr-x 1 terrelln 21M Mar 25 2002 samba
+# -rwxr-xr-x 1 terrelln 7.0M Mar 24 2002 sao
+# -rwxr-xr-x 1 terrelln 40M Mar 25 2002 webster
+# -rwxr-xr-x 1 terrelln 8.1M Apr 4 2002 x-ray
+# -rwxr-xr-x 1 terrelln 5.1M Nov 30 2000 xml
+
+# $HOME is on a ext4 filesystem
+BENCHMARK_FILE="linux-4.11.6.tar"
+BENCHMARK_DIR="$HOME/$BENCHMARK_FILE"
+
+# Normalize the environment
+sudo umount /mnt/btrfs 2> /dev/null > /dev/null || true
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+sudo rm -rf /mnt/btrfs/*
+sync
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+# Run the benchmark
+echo "Copy"
+time sh -c "sudo cp -r $BENCHMARK_DIR /mnt/btrfs/$BENCHMARK_FILE && sync"
+
+echo "Approximate tarred compression ratio"
+printf "%d / %d\n" \
+ $(df /mnt/btrfs --output=used -B 1 | tail -n 1) \
+ $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);
+
+# Unmount and remount to avoid any caching
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+echo "Extract"
+time sh -c "sudo tar -C /mnt/btrfs -xf /mnt/btrfs/$BENCHMARK_FILE && sync"
+
+# Remove the tarball, leaving only the extracted data
+sudo rm /mnt/btrfs/$BENCHMARK_FILE
+# Unmount and remount to avoid any caching
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+echo "Approximate extracted compression ratio"
+printf "%d / %d\n" \
+ $(df /mnt/btrfs --output=used -B 1 | tail -n 1) \
+ $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);
+
+echo "Read"
+time sudo tar -c /mnt/btrfs 2> /dev/null | wc -c > /dev/null
+
+sudo rm -rf /mnt/btrfs/*
+sudo umount /mnt/btrfs
+
+# Run for each of -o compress-force={none, lzo, zlib, zstd} 5 times and take the
+# min time and ratio.
+
+# none
+# copy: 0.981 s
+# extract: 5.501 s
+# read: 8.807 s
+# tarball ratio: 0.97
+# extracted ratio: 0.78
+
+# lzo
+# copy: 1.631 s
+# extract: 8.458 s
+# read: 8.585 s
+# tarball ratio: 2.06
+# extracted ratio: 1.38
+
+# zlib
+# copy: 7.750 s
+# extract: 21.544 s
+# read: 11.744 s
+# tarball ratio : 3.40
+# extracted ratio: 1.86
+
+# zstd 1
+# copy: 2.579 s
+# extract: 11.479 s
+# read: 9.389 s
+# tarball ratio : 3.57
+# extracted ratio: 1.85
projects / thirdparty / zstd.git / commitdiff
