From: Yann Collet Date: Fri, 19 Jan 2018 19:26:35 +0000 (-0800) Subject: add `zcat` symlink support, suggested by @wtarreau X-Git-Tag: v1.3.4~1^2~70^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cb5eba8e206fb4a28c23809b7da949efd4603abc;p=thirdparty%2Fzstd.git add `zcat` symlink support, suggested by @wtarreau added some test also updated relevant doc + fixed a mistake in `lz4` symlink support : lz4 utility doesn't remove source files by default (like zstd, but unlike gzip). The symlink must behave the same. --- diff --git a/Makefile b/Makefile index d2c71ad39..47746ab1d 100644 --- a/Makefile +++ b/Makefile @@ -72,14 +72,18 @@ zstdmt: zlibwrapper: $(MAKE) -C $(ZWRAPDIR) test -.PHONY: check -check: shortest - -.PHONY: test shortest -test shortest: +.PHONY: test +test: $(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DZSTD_DEBUG=1" $(MAKE) -C $(TESTDIR) $@ +.PHONY: shortest +shortest: + $(MAKE) -C $(TESTDIR) $@ + +.PHONY: check +check: shortest + .PHONY: examples examples: CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all diff --git a/programs/README.md b/programs/README.md index 2da9a6db1..1331a684a 100644 --- a/programs/README.md +++ b/programs/README.md @@ -11,61 +11,74 @@ There are however other Makefile targets that create different variations of CLI #### Compilation variables -`zstd` scope can be altered by modifying the following compilation variables : +`zstd` scope can be altered by modifying the following `make` variables : - __HAVE_THREAD__ : multithreading is automatically enabled when `pthread` is detected. - It's possible to disable multithread support, by setting HAVE_THREAD=0 . - Example : make zstd HAVE_THREAD=0 - It's also possible to force compilation with multithread support, using HAVE_THREAD=1. - In which case, linking stage will fail if `pthread` library cannot be found. - This might be useful to prevent silent feature disabling. + It's possible to disable multithread support, by setting `HAVE_THREAD=0`. + Example : `make zstd HAVE_THREAD=0` + It's also possible to force multithread support, using `HAVE_THREAD=1`. + In which case, linking stage will fail if neither `pthread` nor `windows.h` library can be found. + This is useful to ensure this feature is not silently disabled. + +- __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`. + Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible. + But older versions (< v0.8.0) produced different, incompatible, frames. + By default, `zstd` supports decoding legacy formats >= v0.4.0 (`ZSTD_LEGACY_SUPPORT=4`). + This can be altered by modifying this compilation variable. + `ZSTD_LEGACY_SUPPORT=1` means "support all formats >= v0.1.0". + `ZSTD_LEGACY_SUPPORT=2` means "support all formats >= v0.2.0", and so on. + `ZSTD_LEGACY_SUPPORT=0` means _DO NOT_ support any legacy format. + if `ZSTD_LEGACY_SUPPORT >= 8`, it's the same as `0`, since there is no legacy format after `7`. + Note : `zstd` only supports decoding older formats, and cannot generate any legacy format. - __HAVE_ZLIB__ : `zstd` can compress and decompress files in `.gz` format. This is ordered through command `--format=gzip`. Alternatively, symlinks named `gzip` or `gunzip` will mimic intended behavior. `.gz` support is automatically enabled when `zlib` library is detected at build time. - It's possible to disable `.gz` support, by setting HAVE_ZLIB=0. - Example : make zstd HAVE_ZLIB=0 - It's also possible to force compilation with zlib support, using HAVE_ZLIB=1. + It's possible to disable `.gz` support, by setting `HAVE_ZLIB=0`. + Example : `make zstd HAVE_ZLIB=0` + It's also possible to force compilation with zlib support, `using HAVE_ZLIB=1`. In which case, linking stage will fail if `zlib` library cannot be found. - This might be useful to prevent silent feature disabling. + This is useful to prevent silent feature disabling. - __HAVE_LZMA__ : `zstd` can compress and decompress files in `.xz` and `.lzma` formats. This is ordered through commands `--format=xz` and `--format=lzma` respectively. Alternatively, symlinks named `xz`, `unxz`, `lzma`, or `unlzma` will mimic intended behavior. `.xz` and `.lzma` support is automatically enabled when `lzma` library is detected at build time. - It's possible to disable `.xz` and `.lzma` support, by setting HAVE_LZMA=0 . - Example : make zstd HAVE_LZMA=0 - It's also possible to force compilation with lzma support, using HAVE_LZMA=1. + It's possible to disable `.xz` and `.lzma` support, by setting `HAVE_LZMA=0` . + Example : `make zstd HAVE_LZMA=0` + It's also possible to force compilation with lzma support, using `HAVE_LZMA=1`. In which case, linking stage will fail if `lzma` library cannot be found. - This might be useful to prevent silent feature disabling. + This is useful to prevent silent feature disabling. - __HAVE_LZ4__ : `zstd` can compress and decompress files in `.lz4` formats. This is ordered through commands `--format=lz4`. Alternatively, symlinks named `lz4`, or `unlz4` will mimic intended behavior. `.lz4` support is automatically enabled when `lz4` library is detected at build time. - It's possible to disable `.lz4` support, by setting HAVE_LZ4=0 . - Example : make zstd HAVE_LZ4=0 - It's also possible to force compilation with lz4 support, using HAVE_LZ4=1. + It's possible to disable `.lz4` support, by setting `HAVE_LZ4=0` . + Example : `make zstd HAVE_LZ4=0` + It's also possible to force compilation with lz4 support, using `HAVE_LZ4=1`. In which case, linking stage will fail if `lz4` library cannot be found. - This might be useful to prevent silent feature disabling. - -- __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`. - Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible. - But older versions (< v0.8.0) produced different, incompatible, frames. - By default, `zstd` supports decoding legacy formats >= v0.4.0 (`ZSTD_LEGACY_SUPPORT=4`). - This can be altered by modifying this compilation variable. - `ZSTD_LEGACY_SUPPORT=1` means "support all formats >= v0.1.0". - `ZSTD_LEGACY_SUPPORT=2` means "support all formats >= v0.2.0", and so on. - `ZSTD_LEGACY_SUPPORT=0` means _DO NOT_ support any legacy format. - if `ZSTD_LEGACY_SUPPORT >= 8`, it's the same as `0`, since there is no legacy format after `7`. - Note : `zstd` only supports decoding older formats, and cannot generate any legacy format. + This is useful to prevent silent feature disabling. #### Aggregation of parameters CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. +#### Symlink shortcuts +It's possible to invoke `zstd` through a symlink. +When the name of the symlink has a specific value, it triggers an associated behavior. +- `zstdmt` : compress using all cores available on local system. +- `zcat` : will decompress and output target file using any of the supported formats. `gzcat` and `zstdcat` are also equivalent. +- `gzip` : if zlib support is enabled, will mimic `gzip` by compressing file using `.gz` format, removing source file by default (use `--keep` to preserve). If zlib is not supported, triggers an error. +- `xz` : if lzma support is enabled, will mimic `xz` by compressing file using `.xz` format, removing source file by default (use `--keep` to preserve). If xz is not supported, triggers an error. +- `lzma` : if lzma support is enabled, will mimic `lzma` by compressing file using `.lzma` format, removing source file by default (use `--keep` to preserve). If lzma is not supported, triggers an error. +- `lz4` : if lz4 support is enabled, will mimic `lz4` by compressing file using `.lz4` format. If lz4 is not supported, triggers an error. +- `unzstd` and `unlz4` will decompress any of the supported format. +- `ungz`, `unxz` and `unlzma` will do the same, and will also remove source file by default (use `--keep` to preserve). + + #### Dictionary builder in Command Line Interface Zstd offers a training mode, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored @@ -155,7 +168,7 @@ Benchmark arguments : #### Long distance matching mode The long distance matching mode, enabled with `--long`, is designed to improve the compression ratio for files with long matches at a large distance (up to the -maximum window size, `128 MiB`) while still maintaining compression speed. +maximum window size, `128 MiB`) while still maintaining compression speed. Enabling this mode sets the window size to `128 MiB` and thus increases the memory usage for both the compressor and decompressor. Performance in terms of speed is @@ -168,7 +181,7 @@ decompression speed with and without long distance matching on an ideal use case: a tar of four versions of clang (versions `3.4.1`, `3.4.2`, `3.5.0`, `3.5.1`) with a total size of `244889600 B`. This is an ideal use case as there are many long distance matches within the maximum window size of `128 MiB` (each -version is less than `128 MiB`). +version is less than `128 MiB`). Compression Speed vs Ratio | Decompression Speed ---------------------------|--------------------- @@ -202,8 +215,3 @@ The below table illustrates this on the [Silesia compression corpus]. | `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` | | `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` | | `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` | - - - - - diff --git a/programs/zstdcli.c b/programs/zstdcli.c index eae208c52..7beb2cb4c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -54,6 +54,7 @@ #define ZSTD_ZSTDMT "zstdmt" #define ZSTD_UNZSTD "unzstd" #define ZSTD_CAT "zstdcat" +#define ZSTD_ZCAT "zcat" #define ZSTD_GZ "gzip" #define ZSTD_GUNZIP "gunzip" #define ZSTD_GZCAT "gzcat" @@ -423,16 +424,17 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbThreads=0; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; - if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } - if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ - if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ - if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat */ - if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ - if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma */ - if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ - if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz */ - if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like xz */ - if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like unxz */ + if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ + if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ + if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ + if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ + if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); } /* behave like lz4 */ + if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */ memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ diff --git a/tests/.gitignore b/tests/.gitignore index a06fa20be..4911b2d62 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -24,6 +24,8 @@ pool poolTests invalidDictionaries checkTag +zcat +zstdcat # Tmp test directory zstdtest @@ -53,6 +55,7 @@ tmp* *.gz result out +*.zstd # fuzzer afl diff --git a/tests/playTests.sh b/tests/playTests.sh index 1bd39545b..0e12ec8e7 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -239,8 +239,18 @@ $ZSTD -c hello.tmp > hello.zstd --no-check $ZSTD -c world.tmp > world.zstd --no-check cat hello.zstd world.zstd > helloworld.zstd $ZSTD -dc helloworld.zstd > result.tmp -cat result.tmp $DIFF helloworld.tmp result.tmp +$ECHO "testing zstdcat symlink" +ln -sf $ZSTD zstdcat +./zstdcat helloworld.zstd > result.tmp +$DIFF helloworld.tmp result.tmp +rm zstdcat +rm result.tmp +$ECHO "testing zcat symlink" +ln -sf $ZSTD zcat +./zcat helloworld.zstd > result.tmp +$DIFF helloworld.tmp result.tmp +rm zcat rm ./*.tmp ./*.zstd $ECHO "frame concatenation tests completed"