From: Danielle Rozenblit Date: Mon, 23 Jan 2023 15:55:11 +0000 (-0800) Subject: modify sequence compression api fuzzer X-Git-Tag: v1.5.4^2~28^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=638d502002b6fbecb3e8ccc79b159766e1244a83;p=thirdparty%2Fzstd.git modify sequence compression api fuzzer --- diff --git a/.gitignore b/.gitignore index a136ea394..cefdfe923 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ tmp* *.zstd dictionary. dictionary +sequence_fuzz_dictionary NUL # Build artefacts diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b12cacc77..3a48e7dcd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -288,6 +288,15 @@ static int ZSTD_resolveExternalSequenceValidation(int mode) { #endif } +/* Resolves maxBlockSize to the default if no value is present. */ +static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { + if (maxBlockSize == 0) { + return ZSTD_BLOCKSIZE_MAX; + } else { + return maxBlockSize; + } +} + /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { @@ -312,6 +321,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); + cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); assert(!ZSTD_checkCParams(cParams)); return cctxParams; } @@ -377,6 +387,7 @@ ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); + cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm); } @@ -604,6 +615,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = 1; return bounds; + case ZSTD_c_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + default: bounds.error = ERROR(parameter_unsupported); return bounds; @@ -670,6 +686,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_deterministicRefPrefix: case ZSTD_c_prefetchCDictTables: case ZSTD_c_enableMatchFinderFallback: + case ZSTD_c_maxBlockSize: default: return 0; } @@ -727,6 +744,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_deterministicRefPrefix: case ZSTD_c_prefetchCDictTables: case ZSTD_c_enableMatchFinderFallback: + case ZSTD_c_maxBlockSize: break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); @@ -964,6 +982,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->enableMatchFinderFallback = value; return CCtxParams->enableMatchFinderFallback; + case ZSTD_c_maxBlockSize: + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_maxBlockSize, value); + CCtxParams->maxBlockSize = value; + return CCtxParams->maxBlockSize; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -1102,6 +1126,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_enableMatchFinderFallback: *value = CCtxParams->enableMatchFinderFallback; break; + case ZSTD_c_maxBlockSize: + *value = (int)CCtxParams->maxBlockSize; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -1546,10 +1573,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( const size_t buffInSize, const size_t buffOutSize, const U64 pledgedSrcSize, - int useExternalMatchFinder) + int useExternalMatchFinder, + size_t maxBlockSize) { size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder); size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) @@ -1571,7 +1599,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); size_t const externalSeqSpace = useExternalMatchFinder - ? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) : 0; size_t const neededSpace = @@ -1601,7 +1629,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) * be needed. However, we still allocate two 0-sized buffers, which can * take space under ASAN. */ return ZSTD_estimateCCtxSize_usingCCtxParams_internal( - &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder); + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize); } size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) @@ -1651,7 +1679,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) ? ((size_t)1 << cParams.windowLog) + blockSize : 0; @@ -1662,7 +1690,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) return ZSTD_estimateCCtxSize_usingCCtxParams_internal( &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, - ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder); + ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize); } } @@ -1936,6 +1964,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, assert(params->useRowMatchFinder != ZSTD_ps_auto); assert(params->useBlockSplitter != ZSTD_ps_auto); assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + assert(params->maxBlockSize != 0); if (params->ldmParams.enableLdm == ZSTD_ps_enable) { /* Adjust long distance matching parameters */ ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); @@ -1944,7 +1973,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, } { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + size_t const blockSize = MIN(params->maxBlockSize, windowSize); size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder); size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) ? ZSTD_compressBound(blockSize) + 1 @@ -1962,7 +1991,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const neededSpace = ZSTD_estimateCCtxSize_usingCCtxParams_internal( ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, - buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder); + buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder, params->maxBlockSize); int resizeWorkspace; FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); @@ -2340,6 +2369,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; params.ldmParams = srcCCtx->appliedParams.ldmParams; params.fParams = fParams; + params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff); @@ -3128,7 +3158,6 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, zc->appliedParams.useRowMatchFinder, dictMode); - assert(zc->externalMatchCtx.mFinder == NULL); ms->ldmSeqStore = NULL; lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); } @@ -4474,7 +4503,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) { ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; assert(!ZSTD_checkCParams(cParams)); - return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); + return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); } size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) @@ -5912,8 +5941,16 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); + params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); #ifdef ZSTD_MULTITHREAD + /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ + RETURN_ERROR_IF( + params.useExternalMatchFinder == 1 && params.nbWorkers >= 1, + parameter_combination_unsupported, + "External matchfinder isn't supported with nbWorkers >= 1" + ); + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ } @@ -6103,6 +6140,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, /* Reset to the original values. */ cctx->requestedParams.inBufferMode = originalInBufferMode; cctx->requestedParams.outBufferMode = originalOutBufferMode; + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); @@ -6742,14 +6780,19 @@ void ZSTD_registerExternalMatchFinder( ZSTD_CCtx* zc, void* mState, ZSTD_externalMatchFinder_F* mFinder ) { - ZSTD_externalMatchCtx emctx = { - mState, - mFinder, - - /* seqBuffer is allocated later (from the cwskp) */ - NULL, /* seqBuffer */ - 0 /* seqBufferCapacity */ - }; - zc->externalMatchCtx = emctx; - zc->requestedParams.useExternalMatchFinder = 1; + if (mFinder != NULL) { + ZSTD_externalMatchCtx emctx = { + mState, + mFinder, + + /* seqBuffer is allocated later (from the cwskp) */ + NULL, /* seqBuffer */ + 0 /* seqBufferCapacity */ + }; + zc->externalMatchCtx = emctx; + zc->requestedParams.useExternalMatchFinder = 1; + } else { + ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx)); + zc->requestedParams.useExternalMatchFinder = 0; + } } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index bb5e25083..3b888acfa 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -355,6 +355,9 @@ struct ZSTD_CCtx_params_s { * Users can't set this externally. * It is set internally in ZSTD_registerExternalMatchFinder(). */ int useExternalMatchFinder; + + /* Adjust the max block size*/ + size_t maxBlockSize; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2)) diff --git a/lib/zstd.h b/lib/zstd.h index f47c24676..a2e023452 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -479,6 +479,7 @@ typedef enum { * ZSTD_c_useRowMatchFinder * ZSTD_c_prefetchCDictTables * ZSTD_c_enableMatchFinderFallback + * ZSTD_c_maxBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -499,7 +500,8 @@ typedef enum { ZSTD_c_experimentalParam14=1011, ZSTD_c_experimentalParam15=1012, ZSTD_c_experimentalParam16=1013, - ZSTD_c_experimentalParam17=1014 + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015 } ZSTD_cParameter; typedef struct { @@ -1200,6 +1202,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ #define ZSTD_STRATEGY_MIN ZSTD_fast #define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ #define ZSTD_OVERLAPLOG_MIN 0 @@ -2112,6 +2115,18 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo * documentation (below) before setting this parameter. */ #define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17 +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() innacurate). Only currently meant to be used for testing. + * + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -2825,8 +2840,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* bloc * externalMatchState. * * *** LIMITATIONS *** - * External matchfinders are compatible with all zstd compression APIs. There are - * only two limitations. + * External matchfinders are compatible with all zstd compression APIs which respect + * advanced parameters. However, there are three limitations: * * First, the ZSTD_c_enableLongDistanceMatching cParam is not supported. * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an @@ -2848,7 +2863,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* bloc * APIs, work with the external matchfinder, but the external matchfinder won't * receive any history from the previous block. Each block is an independent chunk. * - * Long-term, we plan to overcome both limitations. There is no technical blocker to + * Third, multi-threading within a single compression is not supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external matchfinder is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to * overcoming them. It is purely a question of engineering effort. */ @@ -2871,6 +2890,17 @@ typedef size_t ZSTD_externalMatchFinder_F ( * compressions. It will remain set until the user explicitly resets compression * parameters. * + * External matchfinder registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on + * compression APIs which respect advanced parameters, such as compress2() and + * compressStream(). Older compression APIs such as compressCCtx(), which predate + * the introduction of "advanced parameters", will ignore any external matchfinder + * setting. + * + * The external matchfinder can be "cleared" by registering a NULL external + * matchfinder function pointer. This removes all limitations described above in + * the "LIMITATIONS" section of the API docs. + * * The user is strongly encouraged to read the full API documentation (above) * before calling this function. */ ZSTDLIB_STATIC_API void diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 285e401ef..7770dd2a5 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -387,12 +387,9 @@ BMK_benchMemAdvancedNoAlloc( RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); } -#if defined(UTIL_TIME_USES_C90_CLOCK) - if (adv->nbWorkers > 1) { - OUTPUTLEVEL(2, "Warning : time measurements restricted to C90 clock_t. \n") - OUTPUTLEVEL(2, "Warning : using C90 clock_t leads to incorrect measurements in multithreading mode. \n") + if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) { + OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n") } -#endif /* Bench */ { U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0); diff --git a/programs/dibio.c b/programs/dibio.c index b21338cd8..26ebe5ca1 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -274,21 +274,20 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t int n; memset(&fs, 0, sizeof(fs)); - // We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX + /* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */ assert( chunkSize <= SAMPLESIZE_MAX ); for (n=0; n 0) - { - // TODO: is there a minimum sample size? Can we have a 1-byte sample? + if (chunkSize > 0) { + /* TODO: is there a minimum sample size? Can we have a 1-byte sample? */ fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize); fs.totalSizeToLoad += fileSize; } diff --git a/programs/fileio_common.h b/programs/fileio_common.h index 827a5a06b..55491b8e3 100644 --- a/programs/fileio_common.h +++ b/programs/fileio_common.h @@ -122,4 +122,4 @@ extern UTIL_time_t g_displayClock; #if defined (__cplusplus) } #endif -#endif //ZSTD_FILEIO_COMMON_H +#endif /* ZSTD_FILEIO_COMMON_H */ diff --git a/programs/timefn.c b/programs/timefn.c index 08aa1cfcb..f941e57e6 100644 --- a/programs/timefn.c +++ b/programs/timefn.c @@ -12,7 +12,8 @@ /* === Dependencies === */ #include "timefn.h" - +#include "platform.h" /* set _POSIX_C_SOURCE */ +#include /* CLOCK_MONOTONIC, TIME_UTC */ /*-**************************************** * Time functions @@ -20,12 +21,11 @@ #if defined(_WIN32) /* Windows */ +#include /* LARGE_INTEGER */ #include /* abort */ #include /* perror */ -UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; } - -PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) +UTIL_time_t UTIL_getTime(void) { static LARGE_INTEGER ticksPerSecond; static int init = 0; @@ -36,16 +36,20 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) } init = 1; } - return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; + { UTIL_time_t r; + LARGE_INTEGER x; + QueryPerformanceCounter(&x); + r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart); + return r; + } } - #elif defined(__APPLE__) && defined(__MACH__) -UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); } +#include /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */ -PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) +UTIL_time_t UTIL_getTime(void) { static mach_timebase_info_data_t rate; static int init = 0; @@ -53,12 +57,39 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) mach_timebase_info(&rate); init = 1; } - return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom); + { UTIL_time_t r; + r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom; + return r; + } } +/* POSIX.1-2001 (optional) */ +#elif defined(CLOCK_MONOTONIC) + +#include /* abort */ +#include /* perror */ -/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance. - Android also lacks it but does define TIME_UTC. */ +UTIL_time_t UTIL_getTime(void) +{ + /* time must be initialized, othersize it may fail msan test. + * No good reason, likely a limitation of timespec_get() for some target */ + struct timespec time = { 0, 0 }; + if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) { + perror("timefn::clock_gettime(CLOCK_MONOTONIC)"); + abort(); + } + { UTIL_time_t r; + r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec; + return r; + } +} + + +/* C11 requires support of timespec_get(). + * However, FreeBSD 11 claims C11 compliance while lacking timespec_get(). + * Double confirm timespec_get() support by checking the definition of TIME_UTC. + * However, some versions of Android manage to simultanously define TIME_UTC + * and lack timespec_get() support... */ #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \ && defined(TIME_UTC) && !defined(__ANDROID__) @@ -69,46 +100,38 @@ UTIL_time_t UTIL_getTime(void) { /* time must be initialized, othersize it may fail msan test. * No good reason, likely a limitation of timespec_get() for some target */ - UTIL_time_t time = UTIL_TIME_INITIALIZER; + struct timespec time = { 0, 0 }; if (timespec_get(&time, TIME_UTC) != TIME_UTC) { - perror("timefn::timespec_get"); + perror("timefn::timespec_get(TIME_UTC)"); abort(); } - return time; -} - -static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end) -{ - UTIL_time_t diff; - if (end.tv_nsec < begin.tv_nsec) { - diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec; - diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec; - } else { - diff.tv_sec = end.tv_sec - begin.tv_sec; - diff.tv_nsec = end.tv_nsec - begin.tv_nsec; + { UTIL_time_t r; + r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec; + return r; } - return diff; -} - -PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end) -{ - UTIL_time_t const diff = UTIL_getSpanTime(begin, end); - PTime nano = 0; - nano += 1000000000ULL * diff.tv_sec; - nano += diff.tv_nsec; - return nano; } #else /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */ -UTIL_time_t UTIL_getTime(void) { return clock(); } -PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } +UTIL_time_t UTIL_getTime(void) +{ + UTIL_time_t r; + r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC; + return r; +} + +#define TIME_MT_MEASUREMENTS_NOT_SUPPORTED #endif /* ==== Common functions, valid for all time API ==== */ +PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) +{ + return clockEnd.t - clockStart.t; +} + PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end) { return UTIL_getSpanTimeNano(begin, end) / 1000ULL; @@ -134,3 +157,12 @@ void UTIL_waitForNextTick(void) clockEnd = UTIL_getTime(); } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0); } + +int UTIL_support_MT_measurements(void) +{ +# if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED) + return 0; +# else + return 1; +# endif +} diff --git a/programs/timefn.h b/programs/timefn.h index 3e4476a4e..b814ff8d8 100644 --- a/programs/timefn.h +++ b/programs/timefn.h @@ -18,7 +18,7 @@ extern "C" { /*-**************************************** -* Local Types +* Types ******************************************/ #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) @@ -32,40 +32,11 @@ extern "C" { typedef unsigned long long PTime; /* does not support compilers without long long support */ #endif - - -/*-**************************************** -* Time types (note: OS dependent) -******************************************/ -#include /* TIME_UTC, then struct timespec and clock_t */ - -#if defined(_WIN32) /* Windows */ - - #include /* LARGE_INTEGER */ - typedef LARGE_INTEGER UTIL_time_t; - #define UTIL_TIME_INITIALIZER { { 0, 0 } } - -#elif defined(__APPLE__) && defined(__MACH__) - - #include - typedef PTime UTIL_time_t; - #define UTIL_TIME_INITIALIZER 0 - -/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance. - Android also lacks it but does define TIME_UTC. */ -#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \ - && defined(TIME_UTC) && !defined(__ANDROID__) - - typedef struct timespec UTIL_time_t; - #define UTIL_TIME_INITIALIZER { 0, 0 } - -#else /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */ - - #define UTIL_TIME_USES_C90_CLOCK - typedef clock_t UTIL_time_t; - #define UTIL_TIME_INITIALIZER 0 - -#endif +/* UTIL_time_t contains a nanosecond time counter. + * The absolute value is not meaningful. + * It's only valid to compute the difference between 2 measurements. */ +typedef struct { PTime t; } UTIL_time_t; +#define UTIL_TIME_INITIALIZER { 0 } /*-**************************************** @@ -73,16 +44,23 @@ extern "C" { ******************************************/ UTIL_time_t UTIL_getTime(void); + +/* Timer resolution can be low on some platforms. + * To improve accuracy, it's recommended to wait for a new tick + * before starting benchmark measurements */ void UTIL_waitForNextTick(void); +/* tells if timefn will return correct time measurements + * in presence of multi-threaded workload. + * note : this is not the case if only C90 clock_t measurements are available */ +int UTIL_support_MT_measurements(void); PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd); PTime UTIL_clockSpanNano(UTIL_time_t clockStart); -#define SEC_TO_MICRO ((PTime)1000000) PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd); PTime UTIL_clockSpanMicro(UTIL_time_t clockStart); - +#define SEC_TO_MICRO ((PTime)1000000) /* nb of microseconds in a second */ #if defined (__cplusplus) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 15e89d51f..86bbaf8bf 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -25,21 +25,13 @@ #include "zdict.h" /* Direct access to internal compression functions is required */ -#include "zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */ +#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64 */ -#ifndef MIN - #define MIN(a, b) ((a) < (b) ? (a) : (b)) -#endif - -#ifndef MAX_PATH - #ifdef PATH_MAX - #define MAX_PATH PATH_MAX - #else - #define MAX_PATH 256 - #endif +#if !(defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) +# define inline /* disable */ #endif /*-************************************ @@ -71,6 +63,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; } \ } while (0) + /*-******************************************************* * Random function *********************************************************/ @@ -176,6 +169,14 @@ const char* BLOCK_TYPES[] = {"raw", "rle", "compressed"}; #define MIN_SEQ_LEN (3) #define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN) +#ifndef MAX_PATH + #ifdef PATH_MAX + #define MAX_PATH PATH_MAX + #else + #define MAX_PATH 256 + #endif +#endif + BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE]; BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2]; BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; @@ -241,6 +242,10 @@ typedef enum { gt_block, /* generate compressed blocks without block/frame headers */ } genType_e; +#ifndef MIN + #define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + /*-******************************************************* * Global variables (set from command line) *********************************************************/ diff --git a/tests/external_matchfinder.h b/tests/external_matchfinder.h index 041f73e4d..e4f7c95f0 100644 --- a/tests/external_matchfinder.h +++ b/tests/external_matchfinder.h @@ -32,4 +32,4 @@ size_t zstreamExternalMatchFinder( size_t windowSize ); -#endif // EXTERNAL_MATCHFINDER +#endif /* EXTERNAL_MATCHFINDER */ diff --git a/tests/fuzz/sequence_compression_api.c b/tests/fuzz/sequence_compression_api.c index ec8fef488..0985f4cc8 100644 --- a/tests/fuzz/sequence_compression_api.c +++ b/tests/fuzz/sequence_compression_api.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "fuzz_helpers.h" #include "zstd_helpers.h" #include "fuzz_data_producer.h" @@ -32,11 +33,17 @@ static void* literalsBuffer = NULL; static void* generatedSrc = NULL; static ZSTD_Sequence* generatedSequences = NULL; +static void* dictBuffer = NULL; +static ZSTD_CDict* cdict = NULL; +static ZSTD_DDict* ddict = NULL; + #define ZSTD_FUZZ_GENERATED_SRC_MAXSIZE (1 << 20) /* Allow up to 1MB generated data */ +#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 20) /* Fixed size 1MB literals buffer */ #define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */ -#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << 18) /* Allow up to a 256KB dict */ -#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 18) /* Fixed size 256KB literals buffer */ +#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << ZSTD_WINDOWLOG_MAX_32) /* Allow up to 1 << ZSTD_WINDOWLOG_MAX_32 dictionary */ #define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */ +#define ZSTD_FUZZ_DICT_FILE "sequence_fuzz_dictionary" + /* Deterministic random number generator */ #define FUZZ_RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) @@ -55,9 +62,9 @@ static uint32_t FUZZ_RDG_rand(uint32_t* src) /* Make a pseudorandom string - this simple function exists to avoid * taking a dependency on datagen.h to have RDG_genBuffer(). */ -static char* generatePseudoRandomString(char* str, size_t size) { +static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProducer_t* producer) { const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_"; - uint32_t seed = 0; + uint32_t seed = FUZZ_dataProducer_uint32(producer); if (size) { for (size_t n = 0; n < size; n++) { int key = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1); @@ -67,6 +74,26 @@ static char* generatePseudoRandomString(char* str, size_t size) { return str; } +/* + * Create large dictionary file + */ +static void generateDictFile(size_t size, FUZZ_dataProducer_t* producer) { + char c; + FILE *dictFile; + const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_"; + uint32_t seed = FUZZ_dataProducer_uint32(producer); + + dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "w"); + FUZZ_ASSERT(dictFile); + + while (size) { + c = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1); + fputc(c, dictFile); + size--; + } + fclose(dictFile); +} + /* Returns size of source buffer */ static size_t decodeSequences(void* dst, size_t nbSequences, size_t literalsSize, @@ -100,14 +127,14 @@ static size_t decodeSequences(void* dst, size_t nbSequences, size_t j = 0; size_t k = 0; if (dictSize != 0) { - if (generatedSequences[i].offset > bytesWritten) { - /* Offset goes into the dictionary */ - size_t offsetFromEndOfDict = generatedSequences[i].offset - bytesWritten; - for (; k < offsetFromEndOfDict && k < matchLength; ++k) { - op[k] = dictPtr[dictSize - offsetFromEndOfDict + k]; + if (generatedSequences[i].offset > bytesWritten) { /* Offset goes into the dictionary */ + size_t dictOffset = generatedSequences[i].offset - bytesWritten; + size_t matchInDict = MIN(matchLength, dictOffset); + for (; k < matchInDict; ++k) { + op[k] = dictPtr[dictSize - dictOffset + k]; } - matchLength -= k; - op += k; + matchLength -= matchInDict; + op += matchInDict; } } for (; j < matchLength; ++j) { @@ -138,9 +165,9 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer, size_t literalsSizeLimit, size_t dictSize, size_t windowLog, ZSTD_sequenceFormat_e mode) { - const uint32_t repCode = 0; /* not used by sequence ingestion api */ - const uint32_t windowSize = 1 << windowLog; - const uint32_t blockSizeMax = MIN(128 << 10, 1 << windowLog); + const uint32_t repCode = 0; /* Not used by sequence ingestion api */ + size_t windowSize = 1ULL << windowLog; + size_t blockSizeMax = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE; uint32_t bytesGenerated = 0; uint32_t nbSeqGenerated = 0; @@ -148,12 +175,12 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer, uint32_t blockSize = 0; if (mode == ZSTD_sf_explicitBlockDelimiters) { - /* ensure that no sequence can be larger than one block */ + /* Ensure that no sequence can be larger than one block */ literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2); matchLengthMax = MIN(matchLengthMax, blockSizeMax/2); } - while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* extra room for explicit delimiters */ + while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* Extra room for explicit delimiters */ && bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE && !FUZZ_dataProducer_empty(producer)) { uint32_t matchLength; @@ -210,38 +237,31 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer, } generatedSequences[nbSeqGenerated++] = seq; isFirstSequence = 0; - } } + } + } + if (mode == ZSTD_sf_explicitBlockDelimiters) { /* always end sequences with a block delimiter */ const ZSTD_Sequence endBlock = {0, 0, 0, 0}; assert(nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ); generatedSequences[nbSeqGenerated++] = endBlock; } - return nbSeqGenerated; } static size_t roundTripTest(void* result, size_t resultCapacity, void* compressed, size_t compressedCapacity, const void* src, size_t srcSize, - const void* dict, size_t dictSize, const ZSTD_Sequence* seqs, size_t seqSize, - int wLog, int cLevel, unsigned hasDict, + unsigned hasDict, ZSTD_sequenceFormat_e mode) { size_t cSize; size_t dSize; - ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode); if (hasDict) { - FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize)); - FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize)); + FUZZ_ZASSERT(ZSTD_CCtx_refCDict(cctx, cdict)); + FUZZ_ZASSERT(ZSTD_DCtx_refDDict(dctx, ddict)); } cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity, @@ -272,7 +292,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size) size_t cBufSize; size_t generatedSrcSize; size_t nbSequences; - void* dictBuffer = NULL; size_t dictSize = 0; unsigned hasDict; unsigned wLog; @@ -281,23 +300,66 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size) FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size); FUZZ_ASSERT(producer); - if (literalsBuffer == NULL) { + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + /* Generate window log first so we don't generate offsets too large */ + wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22); + mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1); + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach); + + if (!literalsBuffer) { literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE); FUZZ_ASSERT(literalsBuffer); - literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE); + literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, producer); + } + + if (!dictBuffer) { /* Generate global dictionary buffer */ + FILE* dictFile; + ZSTD_compressionParameters cParams; + + /* Generate a large dictionary file and mmap to buffer */ + generateDictFile(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, producer); + dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "r"); + dictBuffer = mmap(NULL, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, PROT_READ, MAP_PRIVATE, fileno(dictFile), 0); + FUZZ_ASSERT(dictBuffer); + fclose(dictFile); + + /* Create global cdict and ddict*/ + cParams = ZSTD_getCParams(1, ZSTD_FUZZ_GENERATED_SRC_MAXSIZE, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE); + cParams.minMatch = ZSTD_MINMATCH_MIN; + cParams.hashLog = ZSTD_HASHLOG_MIN; + cParams.chainLog = ZSTD_CHAINLOG_MIN; + + cdict = ZSTD_createCDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem); + ddict = ZSTD_createDDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem); + FUZZ_ASSERT(cdict); + FUZZ_ASSERT(ddict); } + FUZZ_ASSERT(cdict); + FUZZ_ASSERT(ddict); + hasDict = FUZZ_dataProducer_uint32Range(producer, 0, 1); if (hasDict) { - dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE); - dictBuffer = FUZZ_malloc(dictSize); - FUZZ_ASSERT(dictBuffer); - dictBuffer = generatePseudoRandomString(dictBuffer, dictSize); + dictSize = ZSTD_FUZZ_GENERATED_DICT_MAXSIZE; } - /* Generate window log first so we don't generate offsets too large */ - wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32); - cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22); - mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1); if (!generatedSequences) { generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ); @@ -305,8 +367,10 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size) if (!generatedSrc) { generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE); } + nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog, mode); generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize, mode); + /* Note : in explicit block delimiters mode, * the fuzzer might generate a lot of small blocks. * In which case, the final compressed size might be > ZSTD_compressBound(). @@ -318,30 +382,17 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size) rBufSize = generatedSrcSize; rBuf = FUZZ_malloc(rBufSize); - if (!cctx) { - cctx = ZSTD_createCCtx(); - FUZZ_ASSERT(cctx); - } - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } - { const size_t result = roundTripTest(rBuf, rBufSize, cBuf, cBufSize, generatedSrc, generatedSrcSize, - dictBuffer, dictSize, generatedSequences, nbSequences, - (int)wLog, cLevel, hasDict, mode); + hasDict, mode); FUZZ_ASSERT(result <= generatedSrcSize); /* can be 0 when no round-trip */ } free(rBuf); free(cBuf); FUZZ_dataProducer_free(producer); - if (hasDict) { - free(dictBuffer); - } #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 6ad3d975e..2797ed726 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -115,6 +115,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer); setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer); setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer); + setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer); if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ce0bea573..4a091c897 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -25,7 +25,8 @@ #include /* free */ #include /* fgets, sscanf */ #include /* strcmp */ -#undef NDEBUG +#include /* time(), time_t */ +#undef NDEBUG /* always enable assert() */ #include #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "debug.h" /* DEBUG_STATIC_ASSERT */ @@ -476,7 +477,7 @@ static void test_compressBound(unsigned tnb) CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w)); } } - // Ensure error if srcSize too big + /* Ensure error if srcSize too big */ { size_t const w = ZSTD_MAX_INPUT_SIZE + 1; CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */ CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0); @@ -489,7 +490,7 @@ static void test_decompressBound(unsigned tnb) { DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb); - // Simple compression, with size : should provide size; + /* Simple compression, with size : should provide size; */ { const char example[] = "abcd"; char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))]; size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0); @@ -497,7 +498,7 @@ static void test_decompressBound(unsigned tnb) CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example)); } - // Simple small compression without size : should provide 1 block size + /* Simple small compression without size : should provide 1 block size */ { char cBuffer[ZSTD_COMPRESSBOUND(0)]; ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 }; ZSTD_inBuffer in = { NULL, 0, 0 }; @@ -510,14 +511,14 @@ static void test_decompressBound(unsigned tnb) ZSTD_freeCCtx(cctx); } - // Attempt to overflow 32-bit intermediate multiplication result - // This requires dBound >= 4 GB, aka 2^32. - // This requires 2^32 / 2^17 = 2^15 blocks - // => create 2^15 blocks (can be empty, or just 1 byte). + /* Attempt to overflow 32-bit intermediate multiplication result + * This requires dBound >= 4 GB, aka 2^32. + * This requires 2^32 / 2^17 = 2^15 blocks + * => create 2^15 blocks (can be empty, or just 1 byte). */ { const char input[] = "a"; size_t const nbBlocks = (1 << 15) + 1; size_t blockNb; - size_t const outCapacity = 1 << 18; // large margin + size_t const outCapacity = 1 << 18; /* large margin */ char* const outBuffer = malloc (outCapacity); ZSTD_outBuffer out = { outBuffer, outCapacity, 0 }; ZSTD_CCtx* const cctx = ZSTD_createCCtx(); @@ -1779,6 +1780,94 @@ static int basicUnitTests(U32 const seed, double compressibility) if (!ZSTD_isError(r)) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++); + { + // Test ZSTD_estimateCCtxSize_usingCCtxParams + { + ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); + staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); + CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, 3)); + + { + size_t const r = ZSTD_decompressDCtx(staticDCtx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; + if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; + } + ZSTD_freeCCtxParams(params); + } + + // Test ZSTD_estimateCStreamSize_usingCCtxParams + { + ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); + staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); + CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, 3) ); + + { + size_t const r = ZSTD_decompressDCtx(staticDCtx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; + if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; + } + ZSTD_freeCCtxParams(params); + } + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++); + { + // Test ZSTD_estimateCCtxSize_usingCCtxParams + { + ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + size_t cctxSizeDefault; + CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); + cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); + staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); + CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, 3) ); + + { + size_t const r = ZSTD_decompressDCtx(staticDCtx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; + if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; + } + ZSTD_freeCCtxParams(params); + } + + // Test ZSTD_estimateCStreamSize_usingCCtxParams + { + ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + size_t cctxSizeDefault; + CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); + cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); + staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); + CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, 3) ); + + { + size_t const r = ZSTD_decompressDCtx(staticDCtx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; + if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; + } + ZSTD_freeCCtxParams(params); + } + } + DISPLAYLEVEL(3, "OK \n"); } free(staticCCtxBuffer); free(staticDCtxBuffer); @@ -3535,7 +3624,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++); { - U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed) + U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */ U32 rand32 = FUZ_rand(&seed_copy); U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy); U32 lowbit_only_32 = 1; @@ -3543,8 +3632,8 @@ static int basicUnitTests(U32 const seed, double compressibility) U32 highbit_only_32 = (U32)1 << 31; U64 highbit_only_64 = (U64)1 << 63; U32 i; - if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0 - if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0 + if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */ + if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */ /* Test ZSTD_countTrailingZeros32 */ CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u); diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 7117c1952..88b0578eb 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -29,8 +29,19 @@ test_dat_src = 'README.md' test_dat = 'test_dat' head = 'vdevel' dict_source = 'dict_source' -dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c ' -dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h' +dict_globs = [ + 'programs/*.c', + 'lib/common/*.c', + 'lib/compress/*.c', + 'lib/decompress/*.c', + 'lib/dictBuilder/*.c', + 'lib/legacy/*.c', + 'programs/*.h', + 'lib/common/*.h', + 'lib/compress/*.h', + 'lib/dictBuilder/*.h', + 'lib/legacy/*.h' +] def execute(command, print_output=False, print_error=True, param_shell=False): @@ -85,6 +96,7 @@ def create_dict(tag, dict_source_path): result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) if result == 0: print(dict_name + ' created') + assert os.path.isfile(dict_name) else: raise RuntimeError('ERROR: creating of ' + dict_name + ' failed') else: @@ -103,12 +115,15 @@ def zstd(tag, args, input_file, output_file): print("Running: '{}', input={}, output={}" .format( ' '.join(cmd), input_file, output_file )) - subprocess.check_call(cmd, stdin=i, stdout=o) + result = subprocess.run(cmd, stdin=i, stdout=o, stderr=subprocess.PIPE) + print("Stderr: {}".format(result.stderr.decode("ascii"))) + result.check_returncode() def dict_compress_sample(tag, sample): dict_name = 'dict.' + tag - zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst') + verbose = ['-v', '-v', '-v'] + zstd(tag, ['-D', dict_name, '-1'] + verbose, sample, sample + '_01_64_' + tag + '_dictio.zst') zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst') zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst') zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst') @@ -246,8 +261,12 @@ if __name__ == '__main__': # copy *.c and *.h to a temporary directory ("dict_source") if not os.path.isdir(dict_source_path): os.mkdir(dict_source_path) - print('cp ' + dict_files + ' ' + dict_source_path) - execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True) + for dict_glob in dict_globs: + files = glob.glob(dict_glob, root_dir=base_dir) + for file in files: + file = os.path.join(base_dir, file) + print("copying " + file + " to " + dict_source_path) + shutil.copy(file, dict_source_path) print('-----------------------------------------------') print('Compress test.dat by all released zstd') diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 10239d3f9..4a621692d 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1846,15 +1846,11 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed"); - ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters)); /* Reference external matchfinder outside the test loop to * check that the reference is preserved across compressions */ - ZSTD_registerExternalMatchFinder( - zc, - &externalMatchState, - zstreamExternalMatchFinder - ); + ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder); for (enableFallback = 0; enableFallback < 1; enableFallback++) { size_t testCaseId; @@ -1916,16 +1912,160 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests) } /* Test that reset clears the external matchfinder */ + CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters)); + externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */ + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0)); + CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize)); + + /* Test that registering mFinder == NULL clears the external matchfinder */ ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters); + ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder); externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */ CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0)); + ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */ CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize)); + /* Test that external matchfinder doesn't interact with older APIs */ + ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters); + ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder); + externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */ + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0)); + CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3)); + + /* Test that compression returns the correct error with LDM */ + CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters)); + { + size_t res; + ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); + res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize); + CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!"); + CHECK( + ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported, + "EMF: Wrong error code: %s", ZSTD_getErrorName(res) + ); + } + +#ifdef ZSTD_MULTITHREAD + /* Test that compression returns the correct error with nbWorkers > 0 */ + CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters)); + { + size_t res; + ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1)); + res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize); + CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!"); + CHECK( + ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported, + "EMF: Wrong error code: %s", ZSTD_getErrorName(res) + ); + } +#endif + free(dstBuf); free(checkBuf); } DISPLAYLEVEL(3, "OK \n"); + + /* Test maxBlockSize cctx param functionality */ + DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + + /* Quick test to make sure maxBlockSize bounds are enforced */ + assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1))); + assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1))); + + /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/ + { + size_t srcSize = 2 << 10; + void* const src = CNBuffer; + size_t dstSize = ZSTD_compressBound(srcSize); + void* const dst1 = compressedBuffer; + void* const dst2 = (BYTE*)compressedBuffer + dstSize; + size_t size1, size2; + void* const checkBuf = malloc(srcSize); + memset(src, 'x', srcSize); + + /* maxBlockSize = 1KB */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10)); + size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize); + + if (ZSTD_isError(size1)) goto _output_error; + CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1)); + CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!"); + + /* maxBlockSize = 3KB */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10)); + size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize); + + if (ZSTD_isError(size2)) goto _output_error; + CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2)); + CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!"); + + assert(size1 - size2 == 4); /* We add another RLE block with header + character */ + assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */ + + /* maxBlockSize = 1KB, windowLog = 10 */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10)); + size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize); + + if (ZSTD_isError(size1)) goto _output_error; + CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1)); + CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!"); + + /* maxBlockSize = 3KB, windowLog = 10 */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10)); + size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize); + + if (ZSTD_isError(size2)) goto _output_error; + CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2)); + CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!"); + + assert(size1 == size2); + assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */ + + free(checkBuf); + } + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + + /* Test maxBlockSize = 0 is valid */ + { size_t srcSize = 256 << 10; + void* const src = CNBuffer; + size_t dstSize = ZSTD_compressBound(srcSize); + void* const dst1 = compressedBuffer; + void* const dst2 = (BYTE*)compressedBuffer + dstSize; + size_t size1, size2; + void* const checkBuf = malloc(srcSize); + + /* maxBlockSize = 0 */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0)); + size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize); + + if (ZSTD_isError(size1)) goto _output_error; + CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1)); + CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!"); + + /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX)); + size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize); + + if (ZSTD_isError(size2)) goto _output_error; + CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2)); + CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!"); + + assert(size1 == size2); + assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */ + free(checkBuf); + } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc);