From 787b76904a82998a7bdd95a40f4e442c98ae6461 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 13 Mar 2019 15:23:24 -0700 Subject: [PATCH] [libzstd] Allow compression parameters to be set with a cdict The order you set parameters in the advanced API is not supposed to matter. However, once you call `ZSTD_CCtx_refCDict()` the compression parameters cannot be changed. Remove that restriction, and document what parameters are used when using a CDict. If the CCtx is in dictionary mode, then the CDict's parameters are used. If the CCtx is not in dictionary mode, then its requested parameters are used. --- lib/compress/zstd_compress.c | 20 ++++---------------- lib/zstd.h | 9 +++++++-- tests/fuzzer.c | 11 +++++++++++ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8de58b709..987c7b90a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -450,11 +450,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) switch(param) { - case ZSTD_c_compressionLevel: - RETURN_ERROR_IF(cctx->cdict, stage_wrong, - "compression level is configured in cdict"); + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); break; + case ZSTD_c_compressionLevel: case ZSTD_c_windowLog: case ZSTD_c_hashLog: case ZSTD_c_chainLog: @@ -462,20 +463,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_minMatch: case ZSTD_c_targetLength: case ZSTD_c_strategy: - RETURN_ERROR_IF(cctx->cdict, stage_wrong, - "cparams are configured in cdict"); - break; - - case ZSTD_c_nbWorkers: - RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, - "MT not compatible with static alloc"); - break; - case ZSTD_c_ldmHashRateLog: - RETURN_ERROR_IF(cctx->cdict, stage_wrong, - "LDM hash rate log is configured in cdict"); - break; - case ZSTD_c_format: case ZSTD_c_contentSizeFlag: case ZSTD_c_checksumFlag: diff --git a/lib/zstd.h b/lib/zstd.h index a29c9669a..4373d6dfd 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -503,7 +503,10 @@ typedef enum { ZSTD_fast=1, typedef enum { - /* compression parameters */ + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict() + * for more info (superseded-by-cdict). */ ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. @@ -705,7 +708,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s /*! ZSTD_CCtx_refCDict() : * Reference a prepared dictionary, to be used for all next compressed frames. * Note that compression parameters are enforced from within CDict, - * and supercede any compression parameter previously set within CCtx. + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. * The dictionary will remain valid for future compressed frames using same CCtx. * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special : Referencing a NULL CDict means "return to no-dictionary mode". diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 375313795..feb16c806 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1294,6 +1294,17 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++); { U32 u; for (u=0; u