From: Yann Collet Date: Wed, 21 Nov 2018 00:06:00 +0000 (-0800) Subject: completed implementation of ZSTD_cParam_getBounds() X-Git-Tag: v1.3.8~33^2~22 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5c6d4b18ac1d81407354f19e2d2c84303793c559;p=thirdparty%2Fzstd.git completed implementation of ZSTD_cParam_getBounds() for all parameters --- diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index c77b090e2..04951071b 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -459,7 +459,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * Special: value 0 means "use default value" (default: 64). */ ZSTD_p_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. * Larger values improve collision resolution but decrease compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX . + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. * Special: value 0 means "use default value" (default: 3). */ ZSTD_p_ldmHashEveryLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). @@ -490,8 +490,13 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * 0 means default, which is dynamically determined based on compression parameters. * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest. * The minimum size is automatically and transparently enforced */ - ZSTD_p_overlapSizeLog=402, /* Size of previous input reloaded at the beginning of each job, as a fraction of window size. - * 0 : no overlap; 6(default) : use 1/8th of windowSize; >=9 : use full windowSize */ + ZSTD_p_overlapSizeLog=402, /* Size of previous job reloaded at the beginning of each job, as a fraction of window size. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Values range from 0 (no overlap) to 9 (overlap a full windowSize). + * Each rank (except 0) increase/decrease load size by a factor 2 + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:w/256; + * default value is 6 : use 1/8th of windowSize */ /* =================================================================== */ /* experimental parameters - no stability guaranteed */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index edc799e21..ce65cef25 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -269,7 +269,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_p_compressionStrategy: bounds.lowerBound = (int)ZSTD_fast; - bounds.upperBound = (int)ZSTD_btultra; + bounds.upperBound = (int)ZSTD_btultra; /* note : how to ensure at compile time that this is the highest value strategy ? */ return bounds; case ZSTD_p_contentSizeFlag: @@ -297,21 +297,65 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_p_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + case ZSTD_p_overlapSizeLog: - case ZSTD_p_rsyncable: + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; + return bounds; + case ZSTD_p_enableLongDistanceMatching: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_p_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + case ZSTD_p_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + case ZSTD_p_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + case ZSTD_p_ldmHashEveryLog: + bounds.lowerBound = ZSTD_LDM_HASHEVERYLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHEVERYLOG_MAX; + return bounds; /* experimental parameters */ + case ZSTD_p_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_p_forceMaxWindow : bounds.lowerBound = 0; bounds.upperBound = 1; return bounds; + case ZSTD_p_format: + ZSTD_STATIC_ASSERT((int)ZSTD_f_zstd1 < (int)ZSTD_f_zstd1_magicless); + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + return bounds; + case ZSTD_p_forceAttachDict: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -402,7 +446,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_nbWorkers: - if ((value>0) && cctx->staticSize) { + if ((value!=0) && cctx->staticSize) { return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ } return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); @@ -447,31 +491,31 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, } case ZSTD_p_windowLog : - if (value>0) /* 0 => use default */ + if (value!=0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CCtxParams->cParams.windowLog = value; return CCtxParams->cParams.windowLog; case ZSTD_p_hashLog : - if (value>0) /* 0 => use default */ + if (value!=0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CCtxParams->cParams.hashLog = value; return CCtxParams->cParams.hashLog; case ZSTD_p_chainLog : - if (value>0) /* 0 => use default */ + if (value!=0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CCtxParams->cParams.chainLog = value; return CCtxParams->cParams.chainLog; case ZSTD_p_searchLog : - if (value>0) /* 0 => use default */ + if (value!=0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CCtxParams->cParams.searchLog = value; return value; case ZSTD_p_minMatch : - if (value>0) /* 0 => use default */ + if (value!=0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX); CCtxParams->cParams.minMatch = value; return CCtxParams->cParams.minMatch; @@ -482,29 +526,29 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, return CCtxParams->cParams.targetLength; case ZSTD_p_compressionStrategy : - if (value>0) /* 0 => use default */ + if (value!=0) /* 0 => use default */ CLAMPCHECK(value, (int)ZSTD_fast, (int)ZSTD_btultra); CCtxParams->cParams.strategy = (ZSTD_strategy)value; return (size_t)CCtxParams->cParams.strategy; case ZSTD_p_contentSizeFlag : /* Content size written in frame header _when known_ (default:1) */ - DEBUGLOG(4, "set content size flag = %u", (value>0)); - CCtxParams->fParams.contentSizeFlag = value > 0; + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; return CCtxParams->fParams.contentSizeFlag; case ZSTD_p_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ - CCtxParams->fParams.checksumFlag = value > 0; + CCtxParams->fParams.checksumFlag = value != 0; return CCtxParams->fParams.checksumFlag; case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ - DEBUGLOG(4, "set dictIDFlag = %u", (value>0)); + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); CCtxParams->fParams.noDictIDFlag = !value; return !CCtxParams->fParams.noDictIDFlag; case ZSTD_p_forceMaxWindow : - CCtxParams->forceWindow = (value > 0); + CCtxParams->forceWindow = (value != 0); return CCtxParams->forceWindow; case ZSTD_p_forceAttachDict : { @@ -516,7 +560,7 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_p_nbWorkers : #ifndef ZSTD_MULTITHREAD - if (value>0) return ERROR(parameter_unsupported); + if (value!=0) return ERROR(parameter_unsupported); return 0; #else return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value); @@ -544,24 +588,24 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, #endif case ZSTD_p_enableLongDistanceMatching : - CCtxParams->ldmParams.enableLdm = (value>0); + CCtxParams->ldmParams.enableLdm = (value!=0); return CCtxParams->ldmParams.enableLdm; case ZSTD_p_ldmHashLog : - if (value>0) /* 0 ==> auto */ + if (value!=0) /* 0 ==> auto */ CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CCtxParams->ldmParams.hashLog = value; return CCtxParams->ldmParams.hashLog; case ZSTD_p_ldmMinMatch : - if (value>0) /* 0 ==> default */ + if (value!=0) /* 0 ==> default */ CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX); CCtxParams->ldmParams.minMatchLength = value; return CCtxParams->ldmParams.minMatchLength; case ZSTD_p_ldmBucketSizeLog : - if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) - return ERROR(parameter_outOfBound); + if (value!=0) /* 0 ==> default */ + CLAMPCHECK(value, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX); CCtxParams->ldmParams.bucketSizeLog = value; return CCtxParams->ldmParams.bucketSizeLog; diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index eff0eb3ac..477c20883 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -10,7 +10,6 @@ /* ====== Tuning parameters ====== */ -#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */ #define ZSTDMT_OVERLAPLOG_DEFAULT 6 diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 6ecd75a58..666b506e8 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -30,6 +30,10 @@ /* === Constants === */ #define ZSTDMT_NBWORKERS_MAX 200 +#ifndef ZSTDMT_JOBSIZE_MIN +# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */ +#endif +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1 GB)) /* note : limited by `jobSize` type, which is `int` */ /* === Memory management === */ @@ -64,10 +68,6 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /* === Advanced functions and parameters === */ -#ifndef ZSTDMT_JOBSIZE_MIN -# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */ -#endif - ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, diff --git a/lib/zstd.h b/lib/zstd.h index 5738fdad1..492b7fddd 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -553,7 +553,7 @@ typedef enum { * Special: value 0 means "use default value" (default: 64). */ ZSTD_p_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. * Larger values improve collision resolution but decrease compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX . + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. * Special: value 0 means "use default value" (default: 3). */ ZSTD_p_ldmHashEveryLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). @@ -584,8 +584,13 @@ typedef enum { * 0 means default, which is dynamically determined based on compression parameters. * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest. * The minimum size is automatically and transparently enforced */ - ZSTD_p_overlapSizeLog=402, /* Size of previous input reloaded at the beginning of each job, as a fraction of window size. - * 0 : no overlap; 6(default) : use 1/8th of windowSize; >=9 : use full windowSize */ + ZSTD_p_overlapSizeLog=402, /* Size of previous job reloaded at the beginning of each job, as a fraction of window size. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Values range from 0 (no overlap) to 9 (overlap a full windowSize). + * Each rank (except 0) increase/decrease load size by a factor 2 + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:w/256; + * default value is 6 : use 1/8th of windowSize */ /* =================================================================== */ /* experimental parameters - no stability guaranteed */ @@ -886,8 +891,6 @@ ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ #define ZSTD_SKIPPABLEHEADERSIZE 8 -#define ZSTD_HASHLOG3_MAX 17 - /* note : matches --ultra and --long default (27) ? */ #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame * requiring larger than (1<