From: Yann Collet Date: Sat, 3 Feb 2018 00:31:20 +0000 (-0800) Subject: ensure compression parameters are updated when only compression level is changed X-Git-Tag: v1.3.4~1^2~60^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5188749e1cf38c7fc1faddc5415940c0bf74c449;p=thirdparty%2Fzstd.git ensure compression parameters are updated when only compression level is changed --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index fe916b50c..9e768edd3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -140,8 +140,6 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) /* private API call, for dictBuilder only */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } -#define ZSTD_CLEVEL_CUSTOM 999 - static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize) { @@ -3010,7 +3008,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, /** ZSTD_compressStream_generic(): * internal function for all *compressStream*() variants and *compress_generic() - * non-static, because can be called from zstdmt.c + * non-static, because can be called from zstdmt_compress.c * @return : hint size for next input */ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, ZSTD_outBuffer* output, @@ -3192,6 +3190,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ } if (params.nbWorkers > 0) { + /* mt context creation */ if (cctx->mtctx == NULL || (params.nbWorkers != ZSTDMT_getNbWorkers(cctx->mtctx))) { DEBUGLOG(4, "ZSTD_compress_generic: creating new mtctx for nbWorkers=%u", params.nbWorkers); @@ -3202,6 +3201,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem); if (cctx->mtctx == NULL) return ERROR(memory_allocation); } + /* mt compression */ DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); CHECK_F( ZSTDMT_initCStream_internal( cctx->mtctx, @@ -3258,7 +3258,7 @@ size_t ZSTD_compress_generic_simpleArgs ( /*====== Finalize ======*/ /*! ZSTD_flushStream() : -* @return : amount of data remaining to flush */ + * @return : amount of data remaining to flush */ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { ZSTD_inBuffer input = { NULL, 0, 0 }; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index ad2a4be0d..22213c225 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -30,8 +30,9 @@ extern "C" { /*-************************************* * Constants ***************************************/ -static const U32 g_searchStrength = 8; -#define HASH_READ_SIZE 8 +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_CLEVEL_CUSTOM 999 /*-************************************* diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index 834a64b00..6415480e2 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -113,7 +113,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } } else { - ip += ((ip-anchor) >> g_searchStrength) + 1; + ip += ((ip-anchor) >> kSearchStrength) + 1; continue; } @@ -264,7 +264,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { - ip += ((ip-anchor) >> g_searchStrength) + 1; + ip += ((ip-anchor) >> kSearchStrength) + 1; continue; } } diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index 8941326e1..b60f21580 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -79,7 +79,7 @@ size_t ZSTD_compressBlock_fast_generic( } else { U32 offset; if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { - ip += ((ip-anchor) >> g_searchStrength) + 1; + ip += ((ip-anchor) >> kSearchStrength) + 1; continue; } mLength = ZSTD_count(ip+4, match+4, iend) + 4; @@ -185,7 +185,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( } else { if ( (matchIndex < lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { - ip += ((ip-anchor) >> g_searchStrength) + 1; + ip += ((ip-anchor) >> kSearchStrength) + 1; continue; } { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 7f59eb34a..7cb53b2ef 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -537,7 +537,7 @@ size_t ZSTD_compressBlock_lazy_generic( } if (matchLength < 4) { - ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ continue; } @@ -710,7 +710,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( } if (matchLength < 4) { - ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ continue; } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index a2deac145..4f327c07c 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -672,10 +672,14 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) * New parameters will be applied to next compression job. */ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, int compressionLevel, ZSTD_compressionParameters cParams) { - U32 const wlog = cParams.windowLog; - mtctx->params.cParams = cParams; - mtctx->params.cParams.windowLog = wlog; /* Do not modify windowLog ! Frame must keep same wlog during the whole process ! */ + U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", + compressionLevel); mtctx->params.compressionLevel = compressionLevel; + if (compressionLevel != ZSTD_CLEVEL_CUSTOM) + cParams = ZSTD_getCParams(compressionLevel, mtctx->frameContentSize, 0 /* should be dictSize */ ); + cParams. windowLog = saved_wlog; + mtctx->params.cParams = cParams; } /* ZSTDMT_getNbWorkers(): @@ -912,7 +916,8 @@ size_t ZSTDMT_initCStream_internal( assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(mtctx->cctxPool->totalCCtx == params.nbWorkers); - mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ + + /* init */ if (params.jobSize == 0) { if (params.cParams.windowLog >= 29) params.jobSize = ZSTDMT_JOBSIZE_MAX; @@ -921,6 +926,7 @@ size_t ZSTDMT_initCStream_internal( } if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; + mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(params); DEBUGLOG(4, "ZSTDMT_initCStream_internal: switch to single blocking thread mode"); @@ -929,6 +935,7 @@ size_t ZSTDMT_initCStream_internal( dict, dictSize, cdict, singleThreadParams, pledgedSrcSize); } + DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */