From: Nick Terrell Date: Thu, 14 Jun 2018 23:24:18 +0000 (-0700) Subject: Adjust advanced parameters to source size X-Git-Tag: v1.3.5~3^2~13^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3841dbac84ae8d2e2d288a200fd18c28fafed0ae;p=thirdparty%2Fzstd.git Adjust advanced parameters to source size In the new advanced API, adjust the parameters even if they are explicitly set. This mainly applies to the `windowLog`, and accordingly the `hashLog` and `chainLog`, when the source size is known. --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 807b1df5b..0bf6c4c3f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -150,21 +150,6 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) /* private API call, for dictBuilder only */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } -ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) -{ - ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); - if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; - if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog; - if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; - if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog; - if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog; - if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength; - if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength; - if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy; - return cParams; -} - static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( ZSTD_compressionParameters cParams) { @@ -761,6 +746,22 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); } +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); + if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog; + if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; + if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog; + if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog; + if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength; + if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength; + if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy; + assert(!ZSTD_checkCParams(cParams)); + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize); +} + static size_t ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, const U32 forCCtx) @@ -1078,7 +1079,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (params.ldmParams.enableLdm) { /* Adjust long distance matching parameters */ - params.ldmParams.windowLog = params.cParams.windowLog; ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); assert(params.ldmParams.hashEveryLog < 32); @@ -3638,6 +3638,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, params.cParams = ZSTD_getCParamsFromCCtxParams( &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); + #ifdef ZSTD_MULTITHREAD if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 819c62aa3..215f55cf4 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -9,6 +9,7 @@ #include "zstd_ldm.h" +#include "debug.h" #include "zstd_fast.h" /* ZSTD_fillHashTable() */ #include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ @@ -20,7 +21,7 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, ZSTD_compressionParameters const* cParams) { - U32 const windowLog = cParams->windowLog; + params->windowLog = cParams->windowLog; ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; @@ -33,12 +34,13 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->minMatchLength = minMatch; } if (params->hashLog == 0) { - params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG); + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); assert(params->hashLog <= ZSTD_HASHLOG_MAX); } if (params->hashEveryLog == 0) { - params->hashEveryLog = - windowLog < params->hashLog ? 0 : windowLog - params->hashLog; + params->hashEveryLog = params->windowLog < params->hashLog + ? 0 + : params->windowLog - params->hashLog; } params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 0c59c6a72..d93c1262f 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -429,7 +429,6 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* /* Adjust parameters */ if (params.ldmParams.enableLdm) { DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); - params.ldmParams.windowLog = params.cParams.windowLog; ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); assert(params.ldmParams.hashEveryLog < 32); diff --git a/programs/fileio.c b/programs/fileio.c index e8d25ab9d..d9e972040 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -759,8 +759,9 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, DISPLAYLEVEL(6, "compression using zstd format \n"); /* init */ - if (fileSize != UTIL_FILESIZE_UNKNOWN) - ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); + if (fileSize != UTIL_FILESIZE_UNKNOWN) { + CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); + } (void)compressionLevel; (void)srcFileName; /* Main compression loop */ @@ -1755,7 +1756,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles && strcmp(suffixPtr, LZMA_EXTENSION) && strcmp(suffixPtr, LZ4_EXTENSION)) ) { const char* suffixlist = ZSTD_EXTENSION - #ifdef ZSTD_GZCOMPRESS + #ifdef ZSTD_GZCOMPRESS "/" GZ_EXTENSION #endif #ifdef ZSTD_LZMACOMPRESS