From: Yann Collet Date: Tue, 4 Dec 2018 23:35:37 +0000 (-0800) Subject: implemented ZSTD_dParam_getBounds() X-Git-Tag: v1.3.8~33^2~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=aec945f0dcb261753a4f1ef646586146386dec48;p=thirdparty%2Fzstd.git implemented ZSTD_dParam_getBounds() and ZSTD_DCtx_setParameter() --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 06a3f101e..00307c074 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -128,7 +128,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) #ifdef ZSTD_MULTITHREAD return ZSTDMT_sizeof_CCtx(cctx->mtctx); #else - (void) cctx; + (void)cctx; return 0; #endif } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 68acebc6b..251de122f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1256,6 +1256,80 @@ size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) return 0; } +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + default: + bounds.error = ERROR(parameter_unsupported); + return bounds; + } + assert(0); + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + if (!ZSTD_dParam_withinBounds(p, v)) \ + return ERROR(parameter_outOfBound); \ +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); + switch(dParam) { + case ZSTD_d_windowLogMax: + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = 1 << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + default: + return ERROR(parameter_unsupported); + } + assert(0); /* should be unreachable */ + return ERROR(parameter_unsupported); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + (void)ZSTD_initDStream(dctx); + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + if (dctx->streamStage != zdss_init) + return ERROR(stage_wrong); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + } + return 0; +} + size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) { @@ -1578,20 +1652,3 @@ size_t ZSTD_decompressStream_simpleArgs ( *srcPos = input.pos; return cErr; } - - -size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) -{ - if ( (reset == ZSTD_reset_session_only) - || (reset == ZSTD_reset_session_and_parameters) ) { - (void)ZSTD_initDStream(dctx); - } - if ( (reset == ZSTD_reset_parameters) - || (reset == ZSTD_reset_session_and_parameters) ) { - if (dctx->streamStage != zdss_init) - return ERROR(stage_wrong); - dctx->format = ZSTD_f_zstd1; - dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; - } - return 0; -} diff --git a/lib/zstd.h b/lib/zstd.h index 0cc431c14..4c4a7cd09 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -640,7 +640,7 @@ typedef struct { * otherwise they will either trigger an error or be automatically clamped. * @return : a structure, ZSTD_bounds, which contains * - an error status field, which must be tested using ZSTD_isError() - * - both lower and upper bounds, inclusive + * - lower and upper bounds, both inclusive */ ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); @@ -834,16 +834,16 @@ typedef enum { * - an error status field, which must be tested using ZSTD_isError() * - both lower and upper bounds, inclusive */ -ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); /* not implemented yet */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); /*! ZSTD_DCtx_setParameter() : * Set one compression parameter, selected by enum ZSTD_dParameter. * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). * Setting a parameter is only possible during frame initialization (before starting decompression). - * @return : an error code (which can be tested using ZSTD_isError()). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); /* not implemented yet */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); /*! ZSTD_DCtx_loadDictionary() : @@ -895,7 +895,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, /*! ZSTD_DCtx_reset() : * Return a DCtx to clean state. - * Session and parameters can be reset jointly or separately + * Session and parameters can be reset jointly or separately. * Parameters can only be reset when no active frame is being decompressed. * @return : 0, or an error code, which can be tested with ZSTD_isError() */ @@ -1003,7 +1003,7 @@ typedef enum { ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number. * Useful to save 4 bytes per generated frame. - * Decoder cannot recognise automatically this format, requiring instructions. */ + * Decoder cannot recognise automatically this format, requiring this instruction. */ } ZSTD_format_e; typedef enum { @@ -1440,6 +1440,12 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre */ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 + /*! ZSTD_DCtx_setFormat() : * Instruct the decoder context about what kind of data to decode next. * This instruction is mandatory to decode data without a fully-formed header, diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fc323cff4..7fba9e0ae 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -309,7 +309,6 @@ static int basicUnitTests(U32 seed, double compressibility) size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); void* const compressedBuffer = malloc(compressedBufferSize); void* const decodedBuffer = malloc(CNBuffSize); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); int testResult = 0; U32 testNb=0; size_t cSize; @@ -381,13 +380,19 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); - { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0); - if (r != CNBuffSize) goto _output_error; } + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0); + if (r != CNBuffSize) goto _output_error; + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); - { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL); - if (r != CNBuffSize) goto _output_error; } + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL); + if (r != CNBuffSize) goto _output_error; + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); @@ -544,8 +549,8 @@ static int basicUnitTests(U32 seed, double compressibility) /* this test is really too long, and should be made faster */ DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0); - size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 1; /* ensure U32 overflow protection is triggered */ + ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); + size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ size_t cnb; assert(cctx != NULL); params.fParams.contentSizeFlag = 0; @@ -775,7 +780,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* Dictionary and CCtx Duplication tests */ { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 551; + assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL); DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++); { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); @@ -859,6 +866,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); + ZSTD_freeDCtx(dctx); } /* Dictionary and dictBuilder tests */ @@ -903,7 +911,7 @@ static int basicUnitTests(U32 seed, double compressibility) coverParams.nbThreads = 4; dictSize = ZDICT_optimizeTrainFromBuffer_cover( dictBuffer, dictBufferCapacity, - CNBuffer, samplesSizes, nbSamples, + CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ &coverParams); if (ZDICT_isError(dictSize)) goto _output_error; } @@ -948,11 +956,14 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); @@ -981,11 +992,14 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); @@ -1034,11 +1048,14 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK (unknown)\n"); DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); @@ -1052,19 +1069,24 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); - { - const size_t ret = ZSTD_decompress_usingDict( - dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, - "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); - if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error; + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + const size_t ret = ZSTD_decompress_usingDict( + dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, + "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); + if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) + goto _output_error; + ZSTD_freeDCtx(dctx); } DISPLAYLEVEL(3, "OK \n"); @@ -1133,10 +1155,12 @@ static int basicUnitTests(U32 seed, double compressibility) */ { size_t dSize; BYTE data[1024]; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); + assert(dctx != NULL); assert(cdict != NULL); memset(data, 'x', sizeof(data)); cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, data, sizeof(data), cdict); @@ -1145,6 +1169,7 @@ static int basicUnitTests(U32 seed, double compressibility) dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } + ZSTD_freeDCtx(dctx); } DISPLAYLEVEL(3, "OK \n"); @@ -1300,9 +1325,43 @@ static int basicUnitTests(U32 seed, double compressibility) } } + /* advanced parameters for decompression */ + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); + + DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); + { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + CHECK(bounds.error); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); + { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); + if (!ZSTD_isError(sr)) goto _output_error; + } + { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); + if (!ZSTD_isError(bounds.error)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); + { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); + if (!ZSTD_isError(sr)) goto _output_error; + } + { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); + if (!ZSTD_isError(sr)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_freeDCtx(dctx); + } + + /* custom formats tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ + assert(dctx != NULL); assert(cctx != NULL); /* basic block compression */ DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); @@ -1339,13 +1398,16 @@ static int basicUnitTests(U32 seed, double compressibility) } ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); } /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 65 KB; static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ size_t cSize2; + assert(cctx != NULL); assert(dctx != NULL); /* basic block compression */ DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); @@ -1363,13 +1425,14 @@ static int basicUnitTests(U32 seed, double compressibility) /* very long stream of block compression */ DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); - CHECK( ZSTD_compressBegin(cctx, -99) ); /* we just want to quickly overflow internal U32 index */ + CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ CHECK( ZSTD_getBlockSize(cctx) >= blockSize); { U64 const toCompress = 5000000000ULL; /* > 4 GB */ U64 compressed = 0; while (compressed < toCompress) { size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); - if (ZSTD_isError(cSize)) goto _output_error; + assert(blockCSize != 0); + if (ZSTD_isError(blockCSize)) goto _output_error; compressed += blockCSize; } } @@ -1407,8 +1470,8 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); } - ZSTD_freeDCtx(dctx); /* long rle test */ { size_t sampleSize = 0;