From: Yann Collet
Introduction
- zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
- at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
- decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+ zstd, short for Zstandard, is a fast lossless compression algorithm,
+ targeting real-time compression scenarios at zlib-level and better compression ratios.
+ The zstd compression library provides in-memory compression and decompression functions.
+ The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
Compression can be done in:
- a single step (described as Simple API)
@@ -73,21 +74,17 @@
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); -NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize. - ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single - frame, but distinguishes empty frames from frames with an unknown size, or errors. - - Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple - concatenated frames in one buffer, and so is more general. - As a result however, it requires more computation and entire frames to be passed to it, - as opposed to ZSTD_getFrameContentSize which requires only a single frame's header. +
NOTE: This function is planned to be obsolete, in favor of ZSTD_getFrameContentSize(). + ZSTD_getFrameContentSize() works the same way, + returning the decompressed size of a single frame, + but distinguishes empty frames from frames with an unknown size, or errors. 'src' is the start of a zstd compressed frame. @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. - note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + note 1 : decompressed size is an optional field, it may not be present, typically in streaming mode. When `return==0`, data to decompress could be any size. In which case, it's necessary to use streaming mode to decompress data. - Optionally, application can still use ZSTD_decompress() while relying on implied limits. + Optionally, application can use ZSTD_decompress() while relying on implied limits. (For example, data may be necessarily cut into blocks <= 16 KB). note 2 : decompressed size is always present when compression is done with ZSTD_compress() note 3 : decompressed size can be very large (64-bits value), @@ -114,20 +111,26 @@ const char* ZSTD_getErrorName(size_t code); /*!< provides readable strin ZSTD_CCtx* ZSTD_createCCtx(void); size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); +size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel);Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
Decompression context
When decompressing many times, - it is recommended to allocate a context just once, and re-use it for each successive compression operation. + it is recommended to allocate a context only once, + and re-use it for each successive compression operation. This will make workload friendlier for system's memory. - Use one context per thread for parallel execution in multi-threaded environments. + Use one context per thread for parallel execution.typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTD_DCtx* ZSTD_createDCtx(void); size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). +
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); +Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx())
Simple dictionary API
@@ -154,7 +157,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);Fast dictionary API
-ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); +ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel);When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only. @@ -298,7 +302,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
Advanced types
-typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */ +typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, + ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
typedef struct { unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ @@ -509,7 +514,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize); ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition. * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility. * The higher the value of selected strategy, the more complex it is, - * resulting in stronger and slower compression. */ + * resulting in stronger and slower compression. + * Special: value 0 means "do not change strategy". */ #if 0 ZSTD_p_windowSize, /* Maximum allowed back-reference distance. * Can be set to a more precise value than windowLog. diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 754bb8b08..565cf04a1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -273,7 +273,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v return 0; case ZSTD_p_compressionStrategy : - if (value > (unsigned)ZSTD_btultra) return ERROR(compressionParameter_unsupported); + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); ZSTD_cLevelToCParams(cctx); cctx->params.cParams.strategy = (ZSTD_strategy)value; return 0; @@ -2575,14 +2576,17 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) { - static const ZSTD_blockCompressor blockCompressor[2][8] = { - { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, + static const ZSTD_blockCompressor blockCompressor[2][(unsigned)ZSTD_btultra+1] = { + { NULL, + ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btultra }, - { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, + { NULL, + ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict } }; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); return blockCompressor[extDict][(U32)strat]; } diff --git a/lib/zstd.h b/lib/zstd.h index c1ae9da54..b6e4b263c 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -36,9 +36,10 @@ extern "C" { /******************************************************************************************************* Introduction - zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios - at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and - decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22. + zstd, short for Zstandard, is a fast lossless compression algorithm, + targeting real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression functions. + The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22. Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory. Compression can be done in: - a single step (described as Simple API) @@ -89,21 +90,17 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); /*! ZSTD_getDecompressedSize() : - * NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize. - * ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single - * frame, but distinguishes empty frames from frames with an unknown size, or errors. - * - * Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple - * concatenated frames in one buffer, and so is more general. - * As a result however, it requires more computation and entire frames to be passed to it, - * as opposed to ZSTD_getFrameContentSize which requires only a single frame's header. + * NOTE: This function is planned to be obsolete, in favor of ZSTD_getFrameContentSize(). + * ZSTD_getFrameContentSize() works the same way, + * returning the decompressed size of a single frame, + * but distinguishes empty frames from frames with an unknown size, or errors. * * 'src' is the start of a zstd compressed frame. * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. - * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * note 1 : decompressed size is an optional field, it may not be present, typically in streaming mode. * When `return==0`, data to decompress could be any size. * In which case, it's necessary to use streaming mode to decompress data. - * Optionally, application can still use ZSTD_decompress() while relying on implied limits. + * Optionally, application can use ZSTD_decompress() while relying on implied limits. * (For example, data may be necessarily cut into blocks <= 16 KB). * note 2 : decompressed size is always present when compression is done with ZSTD_compress() * note 3 : decompressed size can be very large (64-bits value), @@ -137,20 +134,26 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /*! ZSTD_compressCCtx() : * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */ -ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); /*= Decompression context * When decompressing many times, - * it is recommended to allocate a context just once, and re-use it for each successive compression operation. + * it is recommended to allocate a context only once, + * and re-use it for each successive compression operation. * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution in multi-threaded environments. */ + * Use one context per thread for parallel execution. */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /*! ZSTD_decompressDCtx() : - * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); /************************** @@ -187,7 +190,8 @@ typedef struct ZSTD_CDict_s ZSTD_CDict; * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. * ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only. * `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); /*! ZSTD_freeCDict() : * Function frees memory allocated by ZSTD_createCDict(). */ @@ -375,7 +379,8 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f /*--- Advanced types ---*/ -typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */ +typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, + ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */ typedef struct { unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ @@ -623,7 +628,8 @@ typedef enum { ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition. * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility. * The higher the value of selected strategy, the more complex it is, - * resulting in stronger and slower compression. */ + * resulting in stronger and slower compression. + * Special: value 0 means "do not change strategy". */ #if 0 ZSTD_p_windowSize, /* Maximum allowed back-reference distance. * Can be set to a more precise value than windowLog. diff --git a/programs/fileio.c b/programs/fileio.c index ba15555dd..2c12b924c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -369,7 +369,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); /* 0 means : do not change */ + if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy; #ifdef ZSTD_MULTITHREAD { size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 4b9cd97a5..6814d5476 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -12,9 +12,9 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif @@ -25,7 +25,7 @@ #include/* fgets, sscanf */ #include /* strcmp */ #include /* clock_t */ -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "zstd.h" /* ZSTD_VERSION_STRING */ #include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zstdmt_compress.h" diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 5c166bfb8..6c4900ebb 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -12,9 +12,9 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -26,7 +26,7 @@ #include /* clock_t, clock() */ #include /* strcmp */ #include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */ #include "zstd.h" /* ZSTD_compressBound */ #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */ #include "zstdmt_compress.h"