U32 repToConfirm[ZSTD_REP_NUM];
} seqStore_t;
-struct ZSTD_CCtx_params_s {
- ZSTD_compressionParameters cParams;
- ZSTD_frameParameters fParams;
-
- int compressionLevel;
- U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
-
- /* Dictionary */
- ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
- U32 dictContentByRef;
-
- /* Multithreading: used only to set mtctx parameters */
- U32 nbThreads;
- unsigned jobSize;
- unsigned overlapSizeLog;
-#if 0
- /* Test parameter */
- U32 testParam;
-#endif
-};
-
typedef struct {
U32 off;
U32 len;
FSE_repeat litlength_repeatMode;
} ZSTD_entropyCTables_t;
+struct ZSTD_CCtx_params_s {
+ ZSTD_compressionParameters cParams;
+ ZSTD_frameParameters fParams;
+
+ int compressionLevel;
+ U32 forceWindow; /* force back-references to respect limit of
+ * 1<<wLog, even for dictionary */
+ /* Dictionary */
+ U32 dictContentByRef;
+ ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent"
+ * or "fullDict" only */
+
+ /* Multithreading: used only to set mtctx parameters */
+ U32 nbThreads;
+ unsigned jobSize;
+ unsigned overlapSizeLog;
+}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
+
+
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
/* private API call, for dictBuilder only */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
-
-/* TODO: get rid of this function if possible*/
-static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
- ZSTD_compressionParameters cParams)
-{
- ZSTD_CCtx_params cctxParams;
- memset(&cctxParams, 0, sizeof(ZSTD_CCtx_params));
- cctxParams.cParams = cParams;
- return cctxParams;
-}
-
/* older variant; will be deprecated */
size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
{
}
}
-
#define ZSTD_CLEVEL_CUSTOM 999
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
{
cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
}
+static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
+ ZSTD_compressionParameters cParams)
+{
+ ZSTD_CCtx_params cctxParams;
+ memset(&cctxParams, 0, sizeof(ZSTD_CCtx_params));
+ cctxParams.cParams = cParams;
+ cctxParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
+ return cctxParams;
+}
+
ZSTD_CCtx_params* ZSTD_createCCtxParams(void)
{
ZSTD_CCtx_params* params =
static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* params)
{
if (params->compressionLevel == ZSTD_CLEVEL_CUSTOM) return;
- // TODO: src size, code duplication
params->cParams = ZSTD_getCParams(params->compressionLevel, 0, 0);
params->compressionLevel = ZSTD_CLEVEL_CUSTOM;
-
}
#define CLAMPCHECK(val,min,max) { \
switch(param)
{
case ZSTD_p_compressionLevel:
- if (value == 0) return 0;
+ if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_minMatch:
case ZSTD_p_targetLength:
case ZSTD_p_compressionStrategy:
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
+ if (value == 0) return 0;
if (cctx->cdict) return ERROR(stage_wrong);
- ZSTD_cLevelToCParams(cctx);
+ ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_contentSizeFlag:
if (cctx->requestedParams.nbThreads <= 1) return ERROR(parameter_unsupported);
assert(cctx->mtctx != NULL);
return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
-#if 0
- case ZSTD_p_test :
- DEBUGLOG(2, "Setting test parameter = %u", value);
- cctx->requestedParams.testParam = (value > 0);
- return 0;
-#endif
default: return ERROR(parameter_unsupported);
}
if (params->nbThreads <= 1) { return ERROR(parameter_unsupported); }
params->overlapSizeLog = value;
return 0;
-#if 0
- case ZSTD_p_test :
- DEBUGLOG(2, "setting opaque: ZSTD_p_test: %u", value);
- params->testParam = (value > 0);
- return 0;
-#endif
default: return ERROR(parameter_unsupported);
}
params->nbThreads,
params->jobSize,
params->overlapSizeLog);
-#if 0
- DEBUGLOG(2, "testParam: %u",
- params->testParam);
-#endif
}
#endif
* Parameters are copied manually before the dictionary is loaded.
* The multithreading parameters jobSize and overlapSizeLog are set only if
* nbThreads >= 1.
+ *
+ * Pledged srcSize is treated as unknown.
*/
size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params)
{
cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) );
}
-#if 0
- /* Copy test parameter */
- cctx->requestedParams.testParam = params->testParam;
-#endif
return 0;
}
}
}
-
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
size_t dictSize,
ZSTD_CCtx_params* params)
{
- size_t const cctxSize = ZSTD_estimateCCtxSize_advanced_opaque(params);
- size_t const neededSize = sizeof(ZSTD_CDict) + (params->dictContentByRef ? 0 : dictSize)
- + cctxSize;
- ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
- void* ptr;
- DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
- if ((size_t)workspace & 7) return NULL; /* 8-aligned */
- DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u",
- (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
- if (workspaceSize < neededSize) return NULL;
-
- if (!params->dictContentByRef) {
- memcpy(cdict+1, dict, dictSize);
- dict = cdict+1;
- ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize;
- } else {
- ptr = cdict+1;
- }
- cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize);
- params->dictContentByRef = 1;
+ if (params == NULL) { return NULL; }
+ { ZSTD_CCtx_params cctxParams = *params;
+ size_t const cctxSize = ZSTD_estimateCCtxSize_advanced_opaque(params);
+ size_t const neededSize = sizeof(ZSTD_CDict)
+ + (cctxParams.dictContentByRef ? 0 : dictSize)
+ + cctxSize;
+ ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
+ void* ptr;
+ DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
+ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
+ DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u",
+ (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
+ if (workspaceSize < neededSize) return NULL;
+
+ if (!cctxParams.dictContentByRef) {
+ memcpy(cdict+1, dict, dictSize);
+ dict = cdict+1;
+ ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize;
+ } else {
+ ptr = cdict+1;
+ }
+ cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize);
+ cctxParams.dictContentByRef = 1;
- /* What if nbThreads > 1? */
- if (ZSTD_isError( ZSTD_initCDict_internal_opaque(cdict, dict, dictSize, *params) ))
- return NULL;
+ /* What if nbThreads > 1? */
+ if (ZSTD_isError( ZSTD_initCDict_internal_opaque(cdict, dict, dictSize, cctxParams) ))
+ return NULL;
- return cdict;
+ return cdict;
+ }
}
/*! ZSTD_initStaticCDict_advanced() :
ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
{
if (cdict==NULL) return ERROR(dictionary_wrong);
- {
- ZSTD_CCtx_params params = cdict->refContext->appliedParams;
+ { ZSTD_CCtx_params params = cdict->refContext->appliedParams;
params.fParams = fParams;
params.dictMode = ZSTD_dm_auto;
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
/**
* TODO
- * Resets parameters to zero for jobs?
+ *
+ * Sets parameters to zero for jobs. Notably, nbThreads should be zero?
*/
static void ZSTDMT_zeroCCtxParams(ZSTD_CCtx_params* params)
{
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
if (nbChunks==1) { /* fallback to single-thread mode */
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
-
if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, cctxParams.fParams);
return ZSTD_compress_advanced_opaque(cctx, dst, dstCapacity, src, srcSize, NULL, 0, requestedParams);
}
if (!error) DEBUGLOG(4, "compressed size : %u ", (U32)dstPos);
return error ? error : dstPos;
}
-
}
size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
if (dict) {
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
ZSTD_freeCDict(zcs->cdictLocal);
+ /* TODO: This will need a cctxParam version? */
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */
params.cParams, zcs->cMem);
zcs->allJobsCompleted = 0;
if (params.fParams.checksumFlag) XXH64_reset(&zcs->xxhState, 0);
return 0;
-
}
-/** ZSTDMT_initCStream_internal() :
- * internal usage only */
-size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
+/** ZSTDMT_initCStream_internal() */
+static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- * TODO: ZSTD_estimateCCtxSize_advanced_opaque()
+ * ZSTD_estimateCCtxSize_advanced_opaque() can be used in tandem with ZSTD_CCtxParam_setParameter().
* Note : CCtx estimation is only correct for single-threaded compression */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
- * TODO: ZSTD_estimateCStreamSize_advanced_opaque()
+ * ZSTD_estimateCStreamSize_advanced_opaque() can be used in tandem with ZSTD_CCtxParam_setParameter().
* Note : CStream estimation is only correct for single-threaded compression.
* ZSTD_DStream memory budget depends on window Size.
* This information can be passed manually, using ZSTD_estimateDStreamSize,
/*! ZSTD_estimate?DictSize() :
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
* ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
- * TODO: ZSTD_estimateCDictSize_advanced_opaque()
+ * ZSTD_estimateCDictSize_advanced_opaque() allows further compression parameters. ByReference can be set with ZSTD_CCtxParam_setParameter.
* Note : dictionary created "byReference" are smaller */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
const void* dict, size_t dictSize,
ZSTD_CCtx_params* params);
+/* TODO */
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+/*! ZSTD_resetCCtxParams() :
+ * Reset params to default, with the default compression level. */
ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
/* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */
-#if 0
- ZSTD_p_test,
-#endif
} ZSTD_cParameter;
* @result : 0, or an error code (which can be tested with ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
-/* TODO */
+/*! ZSTD_CCtxParam_setParameter() :
+ * Similar to ZSTD_CCtx_setParameter.
+ * Set one compression parameter, selected by enum ZSTD_cParameter.
+ * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_applyCCtxParams().
+ * Note : when `value` is an enum, cast it to unsigned for proper type checking.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
-/* TODO */
+
+/*! ZSTD_CCtx_applyCCtxParams() :
+ * Apply a set of ZSTD_CCtx_params to the compression context.
+ * This must be done before the dictionary is loaded.
+ * The pledgedSrcSize is treated as unknown.
+ * Multithreading parameters are applied only if nbThreads > 1. */
ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params);
/*! ZSTD_CCtx_setPledgedSrcSize() :
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
g_overlapLog = overlapLog;
}
-#if 0
-static U32 g_testParamFlag = 0;
-void FIO_setTestParamFlag(unsigned testParamFlag) { g_testParamFlag = testParamFlag; }
-#endif
/*-*************************************
* Functions
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
-#if 0
- /* Test */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_test, g_testParamFlag) );
-#endif
}
#elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
void FIO_setNbThreads(unsigned nbThreads);
void FIO_setBlockSize(unsigned blockSize);
void FIO_setOverlapLog(unsigned overlapLog);
-#if 0
-void FIO_setTestParamFlag(unsigned testParamFlag);
-#endif
/*-*************************************
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
-#if 0
- if (!strcmp(argument, "--testParam")) { FIO_setTestParamFlag(1); continue; }
-#endif
#ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
#endif
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
zstreamtest$(EXT) zstreamtest32$(EXT) \
- datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
+ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) cctxParamRoundTrip$(EXT) longmatch$(EXT) \
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
decodecorpus$(EXT)
@echo Cleaning completed
*==========================================*/
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+static void crash(int errorCode){
+ /* abort if AFL/libfuzzer, exit otherwise */
+ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
+ abort();
+ #else
+ exit(errorCode);
+ #endif
+}
+
#define CHECK_Z(f) { \
size_t const err = f; \
if (ZSTD_isError(err)) { \
crash(1); \
} }
-
/** roundTripTest() :
* Compresses `srcBuff` into `compressedBuff`,
* then decompresses `compressedBuff` into `resultBuff`.
ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 };
ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 };
- ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, 1);
- ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, 1);
-
- ZSTD_CCtx_applyCCtxParams(cctx, cctxParams);
+ CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, 1) );
+ CHECK_Z( ZSTD_CCtx_applyCCtxParams(cctx, cctxParams) );
CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
return pos;
}
-static void crash(int errorCode){
- /* abort if AFL/libfuzzer, exit otherwise */
- #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
- abort();
- #else
- exit(errorCode);
- #endif
-}
-
static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
{
size_t const cBuffSize = ZSTD_compressBound(srcBuffSize);
}
if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1) );
-#if 0
- if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, FUZ_rand(&lseed) & 1) );
-#endif
/* Apply parameters */
-
CHECK_Z (ZSTD_CCtx_applyCCtxParams(zc, cctxParams) );
if (FUZ_rand(&lseed) & 1) {
goto _cleanup;
}
-
-
-
/*-*******************************************************
* Command line
*********************************************************/
projects / thirdparty / zstd.git / commitdiff
