*.zstd
dictionary.
dictionary
+sequence_fuzz_dictionary
NUL
# Build artefacts
#endif
}
+/* Resolves maxBlockSize to the default if no value is present. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+ if (maxBlockSize == 0) {
+ return ZSTD_BLOCKSIZE_MAX;
+ } else {
+ return maxBlockSize;
+ }
+}
+
/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
* If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+ cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
assert(!ZSTD_checkCParams(cParams));
return cctxParams;
}
cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams);
cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams);
cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+ cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
}
bounds.upperBound = 1;
return bounds;
+ case ZSTD_c_maxBlockSize:
+ bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+ bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+ return bounds;
+
default:
bounds.error = ERROR(parameter_unsupported);
return bounds;
case ZSTD_c_deterministicRefPrefix:
case ZSTD_c_prefetchCDictTables:
case ZSTD_c_enableMatchFinderFallback:
+ case ZSTD_c_maxBlockSize:
default:
return 0;
}
case ZSTD_c_deterministicRefPrefix:
case ZSTD_c_prefetchCDictTables:
case ZSTD_c_enableMatchFinderFallback:
+ case ZSTD_c_maxBlockSize:
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
CCtxParams->enableMatchFinderFallback = value;
return CCtxParams->enableMatchFinderFallback;
+ case ZSTD_c_maxBlockSize:
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+ CCtxParams->maxBlockSize = value;
+ return CCtxParams->maxBlockSize;
+
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
}
case ZSTD_c_enableMatchFinderFallback:
*value = CCtxParams->enableMatchFinderFallback;
break;
+ case ZSTD_c_maxBlockSize:
+ *value = (int)CCtxParams->maxBlockSize;
+ break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
return 0;
const size_t buffInSize,
const size_t buffOutSize,
const U64 pledgedSrcSize,
- int useExternalMatchFinder)
+ int useExternalMatchFinder,
+ size_t maxBlockSize)
{
size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
- size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+ size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
size_t const externalSeqSpace = useExternalMatchFinder
- ? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+ ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
: 0;
size_t const neededSpace =
* be needed. However, we still allocate two 0-sized buffers, which can
* take space under ASAN. */
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
- &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+ &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
}
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
{ ZSTD_compressionParameters const cParams =
ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
- size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+ size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
? ((size_t)1 << cParams.windowLog) + blockSize
: 0;
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
- ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+ ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
}
}
assert(params->useRowMatchFinder != ZSTD_ps_auto);
assert(params->useBlockSplitter != ZSTD_ps_auto);
assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+ assert(params->maxBlockSize != 0);
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* Adjust long distance matching parameters */
ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams);
}
{ size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
- size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+ size_t const blockSize = MIN(params->maxBlockSize, windowSize);
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
? ZSTD_compressBound(blockSize) + 1
size_t const neededSpace =
ZSTD_estimateCCtxSize_usingCCtxParams_internal(
¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
- buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder);
+ buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder, params->maxBlockSize);
int resizeWorkspace;
FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
params.ldmParams = srcCCtx->appliedParams.ldmParams;
params.fParams = fParams;
+ params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize,
/* loadedDictSize */ 0,
ZSTDcrp_leaveDirty, zbuff);
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
zc->appliedParams.useRowMatchFinder,
dictMode);
- assert(zc->externalMatchCtx.mFinder == NULL);
ms->ldmSeqStore = NULL;
lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
}
{
ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
assert(!ZSTD_checkCParams(cParams));
- return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
+ return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
}
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams);
params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams);
params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+ params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
#ifdef ZSTD_MULTITHREAD
+ /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
+ RETURN_ERROR_IF(
+ params.useExternalMatchFinder == 1 && params.nbWorkers >= 1,
+ parameter_combination_unsupported,
+ "External matchfinder isn't supported with nbWorkers >= 1"
+ );
+
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
}
/* Reset to the original values. */
cctx->requestedParams.inBufferMode = originalInBufferMode;
cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
if (result != 0) { /* compression not completed, due to lack of output space */
assert(oPos == dstCapacity);
ZSTD_CCtx* zc, void* mState,
ZSTD_externalMatchFinder_F* mFinder
) {
- ZSTD_externalMatchCtx emctx = {
- mState,
- mFinder,
-
- /* seqBuffer is allocated later (from the cwskp) */
- NULL, /* seqBuffer */
- 0 /* seqBufferCapacity */
- };
- zc->externalMatchCtx = emctx;
- zc->requestedParams.useExternalMatchFinder = 1;
+ if (mFinder != NULL) {
+ ZSTD_externalMatchCtx emctx = {
+ mState,
+ mFinder,
+
+ /* seqBuffer is allocated later (from the cwskp) */
+ NULL, /* seqBuffer */
+ 0 /* seqBufferCapacity */
+ };
+ zc->externalMatchCtx = emctx;
+ zc->requestedParams.useExternalMatchFinder = 1;
+ } else {
+ ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+ zc->requestedParams.useExternalMatchFinder = 0;
+ }
}
* Users can't set this externally.
* It is set internally in ZSTD_registerExternalMatchFinder(). */
int useExternalMatchFinder;
+
+ /* Adjust the max block size*/
+ size_t maxBlockSize;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
* ZSTD_c_useRowMatchFinder
* ZSTD_c_prefetchCDictTables
* ZSTD_c_enableMatchFinderFallback
+ * ZSTD_c_maxBlockSize
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
ZSTD_c_experimentalParam14=1011,
ZSTD_c_experimentalParam15=1012,
ZSTD_c_experimentalParam16=1013,
- ZSTD_c_experimentalParam17=1014
+ ZSTD_c_experimentalParam17=1014,
+ ZSTD_c_experimentalParam18=1015
} ZSTD_cParameter;
typedef struct {
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
#define ZSTD_STRATEGY_MIN ZSTD_fast
#define ZSTD_STRATEGY_MAX ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
#define ZSTD_OVERLAPLOG_MIN 0
* documentation (below) before setting this parameter. */
#define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() innacurate). Only currently meant to be used for testing.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
* and store it into int* value.
* externalMatchState.
*
* *** LIMITATIONS ***
- * External matchfinders are compatible with all zstd compression APIs. There are
- * only two limitations.
+ * External matchfinders are compatible with all zstd compression APIs which respect
+ * advanced parameters. However, there are three limitations:
*
* First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
* COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
* APIs, work with the external matchfinder, but the external matchfinder won't
* receive any history from the previous block. Each block is an independent chunk.
*
- * Long-term, we plan to overcome both limitations. There is no technical blocker to
+ * Third, multi-threading within a single compression is not supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external matchfinder is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
* overcoming them. It is purely a question of engineering effort.
*/
* compressions. It will remain set until the user explicitly resets compression
* parameters.
*
+ * External matchfinder registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on
+ * compression APIs which respect advanced parameters, such as compress2() and
+ * compressStream(). Older compression APIs such as compressCCtx(), which predate
+ * the introduction of "advanced parameters", will ignore any external matchfinder
+ * setting.
+ *
+ * The external matchfinder can be "cleared" by registering a NULL external
+ * matchfinder function pointer. This removes all limitations described above in
+ * the "LIMITATIONS" section of the API docs.
+ *
* The user is strongly encouraged to read the full API documentation (above)
* before calling this function. */
ZSTDLIB_STATIC_API void
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
}
-#if defined(UTIL_TIME_USES_C90_CLOCK)
- if (adv->nbWorkers > 1) {
- OUTPUTLEVEL(2, "Warning : time measurements restricted to C90 clock_t. \n")
- OUTPUTLEVEL(2, "Warning : using C90 clock_t leads to incorrect measurements in multithreading mode. \n")
+ if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) {
+ OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n")
}
-#endif
/* Bench */
{ U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);
int n;
memset(&fs, 0, sizeof(fs));
- // We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX
+ /* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */
assert( chunkSize <= SAMPLESIZE_MAX );
for (n=0; n<nbFiles; n++) {
S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
- // TODO: is there a minimum sample size? What if the file is 1-byte?
+ /* TODO: is there a minimum sample size? What if the file is 1-byte? */
if (fileSize == 0) {
DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
continue;
}
/* the case where we are breaking up files in sample chunks */
- if (chunkSize > 0)
- {
- // TODO: is there a minimum sample size? Can we have a 1-byte sample?
+ if (chunkSize > 0) {
+ /* TODO: is there a minimum sample size? Can we have a 1-byte sample? */
fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);
fs.totalSizeToLoad += fileSize;
}
#if defined (__cplusplus)
}
#endif
-#endif //ZSTD_FILEIO_COMMON_H
+#endif /* ZSTD_FILEIO_COMMON_H */
/* === Dependencies === */
#include "timefn.h"
-
+#include "platform.h" /* set _POSIX_C_SOURCE */
+#include <time.h> /* CLOCK_MONOTONIC, TIME_UTC */
/*-****************************************
* Time functions
#if defined(_WIN32) /* Windows */
+#include <windows.h> /* LARGE_INTEGER */
#include <stdlib.h> /* abort */
#include <stdio.h> /* perror */
-UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
}
init = 1;
}
- return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+ { UTIL_time_t r;
+ LARGE_INTEGER x;
+ QueryPerformanceCounter(&x);
+ r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
+ return r;
+ }
}
-
#elif defined(__APPLE__) && defined(__MACH__)
-UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+#include <mach/mach_time.h> /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
{
static mach_timebase_info_data_t rate;
static int init = 0;
mach_timebase_info(&rate);
init = 1;
}
- return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
+ { UTIL_time_t r;
+ r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom;
+ return r;
+ }
}
+/* POSIX.1-2001 (optional) */
+#elif defined(CLOCK_MONOTONIC)
+
+#include <stdlib.h> /* abort */
+#include <stdio.h> /* perror */
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
- Android also lacks it but does define TIME_UTC. */
+UTIL_time_t UTIL_getTime(void)
+{
+ /* time must be initialized, othersize it may fail msan test.
+ * No good reason, likely a limitation of timespec_get() for some target */
+ struct timespec time = { 0, 0 };
+ if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
+ perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
+ abort();
+ }
+ { UTIL_time_t r;
+ r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+ return r;
+ }
+}
+
+
+/* C11 requires support of timespec_get().
+ * However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
+ * Double confirm timespec_get() support by checking the definition of TIME_UTC.
+ * However, some versions of Android manage to simultanously define TIME_UTC
+ * and lack timespec_get() support... */
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
&& defined(TIME_UTC) && !defined(__ANDROID__)
{
/* time must be initialized, othersize it may fail msan test.
* No good reason, likely a limitation of timespec_get() for some target */
- UTIL_time_t time = UTIL_TIME_INITIALIZER;
+ struct timespec time = { 0, 0 };
if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
- perror("timefn::timespec_get");
+ perror("timefn::timespec_get(TIME_UTC)");
abort();
}
- return time;
-}
-
-static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
-{
- UTIL_time_t diff;
- if (end.tv_nsec < begin.tv_nsec) {
- diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
- diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
- } else {
- diff.tv_sec = end.tv_sec - begin.tv_sec;
- diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
+ { UTIL_time_t r;
+ r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+ return r;
}
- return diff;
-}
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
-{
- UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
- PTime nano = 0;
- nano += 1000000000ULL * diff.tv_sec;
- nano += diff.tv_nsec;
- return nano;
}
#else /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
-UTIL_time_t UTIL_getTime(void) { return clock(); }
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+UTIL_time_t UTIL_getTime(void)
+{
+ UTIL_time_t r;
+ r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC;
+ return r;
+}
+
+#define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
#endif
/* ==== Common functions, valid for all time API ==== */
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+ return clockEnd.t - clockStart.t;
+}
+
PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}
+
+int UTIL_support_MT_measurements(void)
+{
+# if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED)
+ return 0;
+# else
+ return 1;
+# endif
+}
/*-****************************************
-* Local Types
+* Types
******************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
typedef unsigned long long PTime; /* does not support compilers without long long support */
#endif
-
-
-/*-****************************************
-* Time types (note: OS dependent)
-******************************************/
-#include <time.h> /* TIME_UTC, then struct timespec and clock_t */
-
-#if defined(_WIN32) /* Windows */
-
- #include <windows.h> /* LARGE_INTEGER */
- typedef LARGE_INTEGER UTIL_time_t;
- #define UTIL_TIME_INITIALIZER { { 0, 0 } }
-
-#elif defined(__APPLE__) && defined(__MACH__)
-
- #include <mach/mach_time.h>
- typedef PTime UTIL_time_t;
- #define UTIL_TIME_INITIALIZER 0
-
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
- Android also lacks it but does define TIME_UTC. */
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
- && defined(TIME_UTC) && !defined(__ANDROID__)
-
- typedef struct timespec UTIL_time_t;
- #define UTIL_TIME_INITIALIZER { 0, 0 }
-
-#else /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
-
- #define UTIL_TIME_USES_C90_CLOCK
- typedef clock_t UTIL_time_t;
- #define UTIL_TIME_INITIALIZER 0
-
-#endif
+/* UTIL_time_t contains a nanosecond time counter.
+ * The absolute value is not meaningful.
+ * It's only valid to compute the difference between 2 measurements. */
+typedef struct { PTime t; } UTIL_time_t;
+#define UTIL_TIME_INITIALIZER { 0 }
/*-****************************************
******************************************/
UTIL_time_t UTIL_getTime(void);
+
+/* Timer resolution can be low on some platforms.
+ * To improve accuracy, it's recommended to wait for a new tick
+ * before starting benchmark measurements */
void UTIL_waitForNextTick(void);
+/* tells if timefn will return correct time measurements
+ * in presence of multi-threaded workload.
+ * note : this is not the case if only C90 clock_t measurements are available */
+int UTIL_support_MT_measurements(void);
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
-#define SEC_TO_MICRO ((PTime)1000000)
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
-
+#define SEC_TO_MICRO ((PTime)1000000) /* nb of microseconds in a second */
#if defined (__cplusplus)
#include "zdict.h"
/* Direct access to internal compression functions is required */
-#include "zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
+#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
-#ifndef MIN
- #define MIN(a, b) ((a) < (b) ? (a) : (b))
-#endif
-
-#ifndef MAX_PATH
- #ifdef PATH_MAX
- #define MAX_PATH PATH_MAX
- #else
- #define MAX_PATH 256
- #endif
+#if !(defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */))
+# define inline /* disable */
#endif
/*-************************************
} \
} while (0)
+
/*-*******************************************************
* Random function
*********************************************************/
#define MIN_SEQ_LEN (3)
#define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
+#ifndef MAX_PATH
+ #ifdef PATH_MAX
+ #define MAX_PATH PATH_MAX
+ #else
+ #define MAX_PATH 256
+ #endif
+#endif
+
BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];
gt_block, /* generate compressed blocks without block/frame headers */
} genType_e;
+#ifndef MIN
+ #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
/*-*******************************************************
* Global variables (set from command line)
*********************************************************/
size_t windowSize
);
-#endif // EXTERNAL_MATCHFINDER
+#endif /* EXTERNAL_MATCHFINDER */
#include <stdio.h>
#include <string.h>
#include <time.h>
+#include <sys/mman.h>
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
static void* generatedSrc = NULL;
static ZSTD_Sequence* generatedSequences = NULL;
+static void* dictBuffer = NULL;
+static ZSTD_CDict* cdict = NULL;
+static ZSTD_DDict* ddict = NULL;
+
#define ZSTD_FUZZ_GENERATED_SRC_MAXSIZE (1 << 20) /* Allow up to 1MB generated data */
+#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 20) /* Fixed size 1MB literals buffer */
#define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */
-#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << 18) /* Allow up to a 256KB dict */
-#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 18) /* Fixed size 256KB literals buffer */
+#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << ZSTD_WINDOWLOG_MAX_32) /* Allow up to 1 << ZSTD_WINDOWLOG_MAX_32 dictionary */
#define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */
+#define ZSTD_FUZZ_DICT_FILE "sequence_fuzz_dictionary"
+
/* Deterministic random number generator */
#define FUZZ_RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
/* Make a pseudorandom string - this simple function exists to avoid
* taking a dependency on datagen.h to have RDG_genBuffer().
*/
-static char* generatePseudoRandomString(char* str, size_t size) {
+static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProducer_t* producer) {
const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
- uint32_t seed = 0;
+ uint32_t seed = FUZZ_dataProducer_uint32(producer);
if (size) {
for (size_t n = 0; n < size; n++) {
int key = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
return str;
}
+/*
+ * Create large dictionary file
+ */
+static void generateDictFile(size_t size, FUZZ_dataProducer_t* producer) {
+ char c;
+ FILE *dictFile;
+ const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
+ uint32_t seed = FUZZ_dataProducer_uint32(producer);
+
+ dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "w");
+ FUZZ_ASSERT(dictFile);
+
+ while (size) {
+ c = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
+ fputc(c, dictFile);
+ size--;
+ }
+ fclose(dictFile);
+}
+
/* Returns size of source buffer */
static size_t decodeSequences(void* dst, size_t nbSequences,
size_t literalsSize,
size_t j = 0;
size_t k = 0;
if (dictSize != 0) {
- if (generatedSequences[i].offset > bytesWritten) {
- /* Offset goes into the dictionary */
- size_t offsetFromEndOfDict = generatedSequences[i].offset - bytesWritten;
- for (; k < offsetFromEndOfDict && k < matchLength; ++k) {
- op[k] = dictPtr[dictSize - offsetFromEndOfDict + k];
+ if (generatedSequences[i].offset > bytesWritten) { /* Offset goes into the dictionary */
+ size_t dictOffset = generatedSequences[i].offset - bytesWritten;
+ size_t matchInDict = MIN(matchLength, dictOffset);
+ for (; k < matchInDict; ++k) {
+ op[k] = dictPtr[dictSize - dictOffset + k];
}
- matchLength -= k;
- op += k;
+ matchLength -= matchInDict;
+ op += matchInDict;
}
}
for (; j < matchLength; ++j) {
size_t literalsSizeLimit, size_t dictSize,
size_t windowLog, ZSTD_sequenceFormat_e mode)
{
- const uint32_t repCode = 0; /* not used by sequence ingestion api */
- const uint32_t windowSize = 1 << windowLog;
- const uint32_t blockSizeMax = MIN(128 << 10, 1 << windowLog);
+ const uint32_t repCode = 0; /* Not used by sequence ingestion api */
+ size_t windowSize = 1ULL << windowLog;
+ size_t blockSizeMax = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
uint32_t bytesGenerated = 0;
uint32_t nbSeqGenerated = 0;
uint32_t blockSize = 0;
if (mode == ZSTD_sf_explicitBlockDelimiters) {
- /* ensure that no sequence can be larger than one block */
+ /* Ensure that no sequence can be larger than one block */
literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2);
matchLengthMax = MIN(matchLengthMax, blockSizeMax/2);
}
- while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* extra room for explicit delimiters */
+ while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* Extra room for explicit delimiters */
&& bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE
&& !FUZZ_dataProducer_empty(producer)) {
uint32_t matchLength;
}
generatedSequences[nbSeqGenerated++] = seq;
isFirstSequence = 0;
- } }
+ }
+ }
+
if (mode == ZSTD_sf_explicitBlockDelimiters) {
/* always end sequences with a block delimiter */
const ZSTD_Sequence endBlock = {0, 0, 0, 0};
assert(nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ);
generatedSequences[nbSeqGenerated++] = endBlock;
}
-
return nbSeqGenerated;
}
static size_t roundTripTest(void* result, size_t resultCapacity,
void* compressed, size_t compressedCapacity,
const void* src, size_t srcSize,
- const void* dict, size_t dictSize,
const ZSTD_Sequence* seqs, size_t seqSize,
- int wLog, int cLevel, unsigned hasDict,
+ unsigned hasDict,
ZSTD_sequenceFormat_e mode)
{
size_t cSize;
size_t dSize;
- ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
- ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
- ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
- ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
- ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
- ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
- ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
if (hasDict) {
- FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize));
- FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize));
+ FUZZ_ZASSERT(ZSTD_CCtx_refCDict(cctx, cdict));
+ FUZZ_ZASSERT(ZSTD_DCtx_refDDict(dctx, ddict));
}
cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity,
size_t cBufSize;
size_t generatedSrcSize;
size_t nbSequences;
- void* dictBuffer = NULL;
size_t dictSize = 0;
unsigned hasDict;
unsigned wLog;
FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size);
FUZZ_ASSERT(producer);
- if (literalsBuffer == NULL) {
+
+ if (!cctx) {
+ cctx = ZSTD_createCCtx();
+ FUZZ_ASSERT(cctx);
+ }
+ if (!dctx) {
+ dctx = ZSTD_createDCtx();
+ FUZZ_ASSERT(dctx);
+ }
+
+ /* Generate window log first so we don't generate offsets too large */
+ wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+ cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
+ mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
+
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach);
+
+ if (!literalsBuffer) {
literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
FUZZ_ASSERT(literalsBuffer);
- literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
+ literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, producer);
+ }
+
+ if (!dictBuffer) { /* Generate global dictionary buffer */
+ FILE* dictFile;
+ ZSTD_compressionParameters cParams;
+
+ /* Generate a large dictionary file and mmap to buffer */
+ generateDictFile(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, producer);
+ dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "r");
+ dictBuffer = mmap(NULL, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, PROT_READ, MAP_PRIVATE, fileno(dictFile), 0);
+ FUZZ_ASSERT(dictBuffer);
+ fclose(dictFile);
+
+ /* Create global cdict and ddict*/
+ cParams = ZSTD_getCParams(1, ZSTD_FUZZ_GENERATED_SRC_MAXSIZE, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
+ cParams.minMatch = ZSTD_MINMATCH_MIN;
+ cParams.hashLog = ZSTD_HASHLOG_MIN;
+ cParams.chainLog = ZSTD_CHAINLOG_MIN;
+
+ cdict = ZSTD_createCDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
+ ddict = ZSTD_createDDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
+ FUZZ_ASSERT(cdict);
+ FUZZ_ASSERT(ddict);
}
+ FUZZ_ASSERT(cdict);
+ FUZZ_ASSERT(ddict);
+
hasDict = FUZZ_dataProducer_uint32Range(producer, 0, 1);
if (hasDict) {
- dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
- dictBuffer = FUZZ_malloc(dictSize);
- FUZZ_ASSERT(dictBuffer);
- dictBuffer = generatePseudoRandomString(dictBuffer, dictSize);
+ dictSize = ZSTD_FUZZ_GENERATED_DICT_MAXSIZE;
}
- /* Generate window log first so we don't generate offsets too large */
- wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32);
- cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
- mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
if (!generatedSequences) {
generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ);
if (!generatedSrc) {
generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE);
}
+
nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog, mode);
generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize, mode);
+
/* Note : in explicit block delimiters mode,
* the fuzzer might generate a lot of small blocks.
* In which case, the final compressed size might be > ZSTD_compressBound().
rBufSize = generatedSrcSize;
rBuf = FUZZ_malloc(rBufSize);
- if (!cctx) {
- cctx = ZSTD_createCCtx();
- FUZZ_ASSERT(cctx);
- }
- if (!dctx) {
- dctx = ZSTD_createDCtx();
- FUZZ_ASSERT(dctx);
- }
-
{ const size_t result = roundTripTest(rBuf, rBufSize,
cBuf, cBufSize,
generatedSrc, generatedSrcSize,
- dictBuffer, dictSize,
generatedSequences, nbSequences,
- (int)wLog, cLevel, hasDict, mode);
+ hasDict, mode);
FUZZ_ASSERT(result <= generatedSrcSize); /* can be 0 when no round-trip */
}
free(rBuf);
free(cBuf);
FUZZ_dataProducer_free(producer);
- if (hasDict) {
- free(dictBuffer);
- }
#ifndef STATEFUL_FUZZING
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
+ setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
}
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
-#undef NDEBUG
+#include <time.h> /* time(), time_t */
+#undef NDEBUG /* always enable assert() */
#include <assert.h>
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "debug.h" /* DEBUG_STATIC_ASSERT */
CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
} }
- // Ensure error if srcSize too big
+ /* Ensure error if srcSize too big */
{ size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
{
DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
- // Simple compression, with size : should provide size;
+ /* Simple compression, with size : should provide size; */
{ const char example[] = "abcd";
char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
}
- // Simple small compression without size : should provide 1 block size
+ /* Simple small compression without size : should provide 1 block size */
{ char cBuffer[ZSTD_COMPRESSBOUND(0)];
ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
ZSTD_inBuffer in = { NULL, 0, 0 };
ZSTD_freeCCtx(cctx);
}
- // Attempt to overflow 32-bit intermediate multiplication result
- // This requires dBound >= 4 GB, aka 2^32.
- // This requires 2^32 / 2^17 = 2^15 blocks
- // => create 2^15 blocks (can be empty, or just 1 byte).
+ /* Attempt to overflow 32-bit intermediate multiplication result
+ * This requires dBound >= 4 GB, aka 2^32.
+ * This requires 2^32 / 2^17 = 2^15 blocks
+ * => create 2^15 blocks (can be empty, or just 1 byte). */
{ const char input[] = "a";
size_t const nbBlocks = (1 << 15) + 1;
size_t blockNb;
- size_t const outCapacity = 1 << 18; // large margin
+ size_t const outCapacity = 1 << 18; /* large margin */
char* const outBuffer = malloc (outCapacity);
ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
if (!ZSTD_isError(r)) goto _output_error;
}
DISPLAYLEVEL(3, "OK \n");
+
+ DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
+ {
+ // Test ZSTD_estimateCCtxSize_usingCCtxParams
+ {
+ ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+ size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+ staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+ CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+ compressedBuffer, compressedBufferSize,
+ CNBuffer, CNBuffSize, 3));
+
+ {
+ size_t const r = ZSTD_decompressDCtx(staticDCtx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize);
+ if (r != CNBuffSize) goto _output_error;
+ if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+ }
+ ZSTD_freeCCtxParams(params);
+ }
+
+ // Test ZSTD_estimateCStreamSize_usingCCtxParams
+ {
+ ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+ size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+ staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+ CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+ compressedBuffer, compressedBufferSize,
+ CNBuffer, CNBuffSize, 3) );
+
+ {
+ size_t const r = ZSTD_decompressDCtx(staticDCtx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize);
+ if (r != CNBuffSize) goto _output_error;
+ if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+ }
+ ZSTD_freeCCtxParams(params);
+ }
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
+ DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
+ {
+ // Test ZSTD_estimateCCtxSize_usingCCtxParams
+ {
+ ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+ size_t cctxSizeDefault;
+ CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+ cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+ staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+ CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+ compressedBuffer, compressedBufferSize,
+ CNBuffer, CNBuffSize, 3) );
+
+ {
+ size_t const r = ZSTD_decompressDCtx(staticDCtx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize);
+ if (r != CNBuffSize) goto _output_error;
+ if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+ }
+ ZSTD_freeCCtxParams(params);
+ }
+
+ // Test ZSTD_estimateCStreamSize_usingCCtxParams
+ {
+ ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+ size_t cctxSizeDefault;
+ CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+ cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+ staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+ CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+ compressedBuffer, compressedBufferSize,
+ CNBuffer, CNBuffSize, 3) );
+
+ {
+ size_t const r = ZSTD_decompressDCtx(staticDCtx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize);
+ if (r != CNBuffSize) goto _output_error;
+ if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+ }
+ ZSTD_freeCCtxParams(params);
+ }
+ }
+ DISPLAYLEVEL(3, "OK \n");
}
free(staticCCtxBuffer);
free(staticDCtxBuffer);
DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
{
- U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed)
+ U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
U32 rand32 = FUZ_rand(&seed_copy);
U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
U32 lowbit_only_32 = 1;
U32 highbit_only_32 = (U32)1 << 31;
U64 highbit_only_64 = (U64)1 << 63;
U32 i;
- if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0
- if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0
+ if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
+ if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
/* Test ZSTD_countTrailingZeros32 */
CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
test_dat = 'test_dat'
head = 'vdevel'
dict_source = 'dict_source'
-dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
-dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+dict_globs = [
+ 'programs/*.c',
+ 'lib/common/*.c',
+ 'lib/compress/*.c',
+ 'lib/decompress/*.c',
+ 'lib/dictBuilder/*.c',
+ 'lib/legacy/*.c',
+ 'programs/*.h',
+ 'lib/common/*.h',
+ 'lib/compress/*.h',
+ 'lib/dictBuilder/*.h',
+ 'lib/legacy/*.h'
+]
def execute(command, print_output=False, print_error=True, param_shell=False):
result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
if result == 0:
print(dict_name + ' created')
+ assert os.path.isfile(dict_name)
else:
raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
else:
print("Running: '{}', input={}, output={}" .format(
' '.join(cmd), input_file, output_file
))
- subprocess.check_call(cmd, stdin=i, stdout=o)
+ result = subprocess.run(cmd, stdin=i, stdout=o, stderr=subprocess.PIPE)
+ print("Stderr: {}".format(result.stderr.decode("ascii")))
+ result.check_returncode()
def dict_compress_sample(tag, sample):
dict_name = 'dict.' + tag
- zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
+ verbose = ['-v', '-v', '-v']
+ zstd(tag, ['-D', dict_name, '-1'] + verbose, sample, sample + '_01_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
# copy *.c and *.h to a temporary directory ("dict_source")
if not os.path.isdir(dict_source_path):
os.mkdir(dict_source_path)
- print('cp ' + dict_files + ' ' + dict_source_path)
- execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+ for dict_glob in dict_globs:
+ files = glob.glob(dict_glob, root_dir=base_dir)
+ for file in files:
+ file = os.path.join(base_dir, file)
+ print("copying " + file + " to " + dict_source_path)
+ shutil.copy(file, dict_source_path)
print('-----------------------------------------------')
print('Compress test.dat by all released zstd')
CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
- ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+ CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
/* Reference external matchfinder outside the test loop to
* check that the reference is preserved across compressions */
- ZSTD_registerExternalMatchFinder(
- zc,
- &externalMatchState,
- zstreamExternalMatchFinder
- );
+ ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
for (enableFallback = 0; enableFallback < 1; enableFallback++) {
size_t testCaseId;
}
/* Test that reset clears the external matchfinder */
+ CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+ externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+ CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+ CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+ /* Test that registering mFinder == NULL clears the external matchfinder */
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+ ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+ ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+ /* Test that external matchfinder doesn't interact with older APIs */
+ ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+ ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+ externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
+ CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+ CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
+
+ /* Test that compression returns the correct error with LDM */
+ CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+ {
+ size_t res;
+ ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+ CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
+ res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+ CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+ CHECK(
+ ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+ "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+ );
+ }
+
+#ifdef ZSTD_MULTITHREAD
+ /* Test that compression returns the correct error with nbWorkers > 0 */
+ CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+ {
+ size_t res;
+ ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+ CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
+ res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+ CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+ CHECK(
+ ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+ "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+ );
+ }
+#endif
+
free(dstBuf);
free(checkBuf);
}
DISPLAYLEVEL(3, "OK \n");
+
+ /* Test maxBlockSize cctx param functionality */
+ DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
+ {
+ ZSTD_CCtx* cctx = ZSTD_createCCtx();
+
+ /* Quick test to make sure maxBlockSize bounds are enforced */
+ assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
+ assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
+
+ /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
+ {
+ size_t srcSize = 2 << 10;
+ void* const src = CNBuffer;
+ size_t dstSize = ZSTD_compressBound(srcSize);
+ void* const dst1 = compressedBuffer;
+ void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+ size_t size1, size2;
+ void* const checkBuf = malloc(srcSize);
+ memset(src, 'x', srcSize);
+
+ /* maxBlockSize = 1KB */
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+ size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+ if (ZSTD_isError(size1)) goto _output_error;
+ CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+ CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+ /* maxBlockSize = 3KB */
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+ size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+ if (ZSTD_isError(size2)) goto _output_error;
+ CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+ CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+ assert(size1 - size2 == 4); /* We add another RLE block with header + character */
+ assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
+
+ /* maxBlockSize = 1KB, windowLog = 10 */
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+ size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+ if (ZSTD_isError(size1)) goto _output_error;
+ CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+ CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+ /* maxBlockSize = 3KB, windowLog = 10 */
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+ size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+ if (ZSTD_isError(size2)) goto _output_error;
+ CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+ CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+ assert(size1 == size2);
+ assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+
+ free(checkBuf);
+ }
+
+ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+ /* Test maxBlockSize = 0 is valid */
+ { size_t srcSize = 256 << 10;
+ void* const src = CNBuffer;
+ size_t dstSize = ZSTD_compressBound(srcSize);
+ void* const dst1 = compressedBuffer;
+ void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+ size_t size1, size2;
+ void* const checkBuf = malloc(srcSize);
+
+ /* maxBlockSize = 0 */
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
+ size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+ if (ZSTD_isError(size1)) goto _output_error;
+ CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+ CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+ /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
+ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
+ size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+ if (ZSTD_isError(size2)) goto _output_error;
+ CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+ CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+ assert(size1 == size2);
+ assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+ free(checkBuf);
+ }
+ ZSTD_freeCCtx(cctx);
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
_end:
FUZ_freeDictionary(dictionary);
ZSTD_freeCStream(zc);
projects / thirdparty / zstd.git / commitdiff
