/* Dictionary */
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
U32 dictContentByRef;
- U32 nbThreads;
/* Multithreading: used only to set mtctx parameters */
+ U32 nbThreads;
unsigned jobSize;
unsigned overlapSizeLog;
+
+ /* Test parameter */
+ U32 testParam;
};
typedef struct {
#define ZSTD_CLEVEL_CUSTOM 999
+#if 0
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
{
if (cctx->requestedParams.compressionLevel==ZSTD_CLEVEL_CUSTOM) return;
cctx->pledgedSrcSizePlusOne-1, 0);
cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
}
+#endif
ZSTD_CCtx_params* ZSTD_createCCtxParams(void)
{
return params;
}
+size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params)
+{
+ if (!params) { return ERROR(GENERIC); }
+ memset(params, 0, sizeof(ZSTD_CCtx_params));
+ params->compressionLevel = ZSTD_CLEVEL_DEFAULT;
+ return 0;
+}
+
size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params,
ZSTD_compressionParameters cParams)
{
switch(param)
{
- case ZSTD_p_compressionLevel :
- if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel(); /* cap max compression level */
+ case ZSTD_p_compressionLevel:
+ case ZSTD_p_windowLog:
+ case ZSTD_p_hashLog:
+ case ZSTD_p_chainLog:
+ case ZSTD_p_searchLog:
+ case ZSTD_p_minMatch:
+ case ZSTD_p_targetLength:
+ case ZSTD_p_compressionStrategy:
if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong);
- cctx->requestedParams.compressionLevel = value;
- return 0;
+ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_windowLog :
- DEBUGLOG(5, "setting ZSTD_p_windowLog = %u (cdict:%u)",
- value, (cctx->cdict!=NULL));
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
- if (cctx->cdict) return ERROR(stage_wrong);
- CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
- ZSTD_cLevelToCParams(cctx);
- cctx->requestedParams.cParams.windowLog = value;
- return 0;
+ case ZSTD_p_contentSizeFlag:
+ case ZSTD_p_checksumFlag:
+ case ZSTD_p_dictIDFlag:
+ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
- case ZSTD_p_hashLog :
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
- if (cctx->cdict) return ERROR(stage_wrong);
- CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
- ZSTD_cLevelToCParams(cctx);
- cctx->requestedParams.cParams.hashLog = value;
- return 0;
-
- case ZSTD_p_chainLog :
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
- if (cctx->cdict) return ERROR(stage_wrong);
- CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
- ZSTD_cLevelToCParams(cctx);
- cctx->requestedParams.cParams.chainLog = value;
- return 0;
-
- case ZSTD_p_searchLog :
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
- if (cctx->cdict) return ERROR(stage_wrong);
- CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
- ZSTD_cLevelToCParams(cctx);
- cctx->requestedParams.cParams.searchLog = value;
- return 0;
-
- case ZSTD_p_minMatch :
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
- if (cctx->cdict) return ERROR(stage_wrong);
- CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
- ZSTD_cLevelToCParams(cctx);
- cctx->requestedParams.cParams.searchLength = value;
- return 0;
-
- case ZSTD_p_targetLength :
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
- if (cctx->cdict) return ERROR(stage_wrong);
- CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
- ZSTD_cLevelToCParams(cctx);
- cctx->requestedParams.cParams.targetLength = value;
- return 0;
-
- case ZSTD_p_compressionStrategy :
- if (value == 0) return 0; /* special value : 0 means "don't change anything" */
- if (cctx->cdict) return ERROR(stage_wrong);
- CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
- ZSTD_cLevelToCParams(cctx);
- cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
- return 0;
-
- case ZSTD_p_contentSizeFlag :
- DEBUGLOG(5, "set content size flag = %u", (value>0));
- /* Content size written in frame header _when known_ (default:1) */
- cctx->requestedParams.fParams.contentSizeFlag = value>0;
- return 0;
-
- case ZSTD_p_checksumFlag :
- /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
- cctx->requestedParams.fParams.checksumFlag = value>0;
- return 0;
-
- case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
- DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
- cctx->requestedParams.fParams.noDictIDFlag = (value==0);
- return 0;
-
- /* Dictionary parameters */
- case ZSTD_p_dictMode :
+ case ZSTD_p_dictMode:
+ case ZSTD_p_refDictContent:
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
- /* restrict dictionary mode, to "rawContent" or "fullDict" only */
- ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
- if (value > (unsigned)ZSTD_dm_fullDict)
- return ERROR(parameter_outOfBound);
- cctx->requestedParams.dictMode = (ZSTD_dictMode_e)value;
- return 0;
-
- case ZSTD_p_refDictContent :
- if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
- /* dictionary content will be referenced, instead of copied */
- cctx->requestedParams.dictContentByRef = value>0;
- return 0;
+ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content
assert(cctx->mtctx != NULL);
return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
+ case ZSTD_p_test :
+ DEBUGLOG(2, "Setting test parameter = %u", value);
+ cctx->requestedParams.testParam = (value > 0);
+ return 0;
+
default: return ERROR(parameter_unsupported);
}
}
return 0;
case ZSTD_p_contentSizeFlag :
+ /* Content size written in frame header _when known_ (default:1) */
+ DEBUGLOG(5, "set content size flag = %u", (value>0));
params->fParams.contentSizeFlag = value > 0;
return 0;
case ZSTD_p_checksumFlag :
+ /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
params->fParams.checksumFlag = value > 0;
return 0;
case ZSTD_p_dictIDFlag :
+ DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
params->fParams.noDictIDFlag = (value == 0);
return 0;
#ifndef ZSTD_MULTITHREAD
if (value > 1) return ERROR(parameter_unsupported);
#endif
- // Do checks when applying parameters to cctx.
+ /* Do checks when applying params to cctx */
params->nbThreads = value;
return 0;
return 0;
case ZSTD_p_overlapSizeLog :
+ if (params->nbThreads <= 1) { return ERROR(parameter_unsupported); }
params->overlapSizeLog = value;
return 0;
+ case ZSTD_p_test :
+ DEBUGLOG(2, "setting opaque: ZSTD_p_test: %u", value);
+ params->testParam = (value > 0);
+ return 0;
+
default: return ERROR(parameter_unsupported);
}
}
+static void ZSTD_debugPrintCCtxParams(ZSTD_CCtx_params* params)
+{
+ DEBUGLOG(2, "======CCtxParams======");
+ DEBUGLOG(2, "cParams: %u %u %u %u %u %u %u",
+ params->cParams.windowLog,
+ params->cParams.chainLog,
+ params->cParams.hashLog,
+ params->cParams.searchLog,
+ params->cParams.searchLength,
+ params->cParams.targetLength,
+ params->cParams.strategy);
+ DEBUGLOG(2, "fParams: %u %u %u",
+ params->fParams.contentSizeFlag,
+ params->fParams.checksumFlag,
+ params->fParams.noDictIDFlag);
+ DEBUGLOG(2, "cLevel, forceWindow: %u %u",
+ params->compressionLevel,
+ params->forceWindow);
+ DEBUGLOG(2, "dictionary: %u %u",
+ params->dictMode,
+ params->dictContentByRef);
+ DEBUGLOG(2, "multithreading: %u %u %u",
+ params->nbThreads,
+ params->jobSize,
+ params->overlapSizeLog);
+ DEBUGLOG(2, "testParam: %u",
+ params->testParam);
+}
+
// This function should probably be updated whenever ZSTD_CCtx_params is updated.
ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params)
{
+ if (params == NULL) { return ERROR(GENERIC); }
if (cctx->cdict) { return ERROR(stage_wrong); }
+ DEBUGLOG(2, "Applying cctx params\n");
+ ZSTD_debugPrintCCtxParams(params);
+
/* Assume the compression and frame parameters are validated */
cctx->requestedParams.cParams = params->cParams;
cctx->requestedParams.fParams = params->fParams;
/* Set multithreading parameters explicitly */
CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) );
- CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) );
- CHECK_F( ZSTD_CCtx_setParameter(
+ if (params->nbThreads > 1) {
+ CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) );
+ CHECK_F( ZSTD_CCtx_setParameter(
cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) );
+
+ }
+ /* Copy test parameter */
+ cctx->requestedParams.testParam = params->testParam;
return 0;
}
size_t ZSTD_estimateCStreamSize_advanced_opaque(ZSTD_CCtx_params* params)
{
if (params == NULL) { return 0; }
- {
- size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_opaque(params);
+ { size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_opaque(params);
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ?
cctx->appliedParams.cParams :
ZSTD_getCParams(cLevel, 0, 0);
- DEBUGLOG(2, "ZSTD_getBlockSize: cLevel %u\n", cLevel);
return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
}
return ZSTD_compress_insertDictionary(cctx, dict, dictSize, params.dictMode);
}
-size_t ZSTD_compressBegin_advanced_opaque(ZSTD_CCtx* cctx,
+size_t ZSTD_compressBegin_advanced_opaque(
+ ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
}
/* Internal */
-size_t ZSTD_compress_advanced_opaque(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize,
- ZSTD_CCtx_params params)
+size_t ZSTD_compress_advanced_opaque(
+ ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize,
+ ZSTD_CCtx_params params)
{
CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
params, srcSize, ZSTDb_not_buffered) );
}
#endif
-ZSTD_CDict* ZSTD_createCDict_advanced_opaque(
+static ZSTD_CDict* ZSTD_createCDict_advanced_opaque(
const void* dictBuffer, size_t dictSize,
ZSTD_CCtx_params params, ZSTD_customMem customMem)
{
- DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
+ DEBUGLOG(5, "ZSTD_createCDict_advanced_opaque, mode %u", (U32)params.dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
cctxParams.dictMode = dictMode;
cctxParams.dictContentByRef = byReference;
return ZSTD_createCDict_advanced_opaque(dictBuffer, dictSize, cctxParams, customMem);
-#if 0
- DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
- if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
-
- { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
- ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
-
- if (!cdict || !cctx) {
- ZSTD_free(cdict, customMem);
- ZSTD_freeCCtx(cctx);
- return NULL;
- }
- cdict->refContext = cctx;
-
- if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
- dictBuffer, dictSize,
- byReference, dictMode,
- cParams) )) {
- ZSTD_freeCDict(cdict);
- return NULL;
- }
-
- return cdict;
- }
-#endif
}
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
{
- DEBUGLOG(5, "ZSTD_initCStream_internal");
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
{
ZSTD_CCtx_params cctxParams = zcs->requestedParams;
CHECK_F( ZSTD_checkCParams(params.cParams) );
- zcs->requestedParams.cParams = params.cParams;
- zcs->requestedParams.fParams = params.fParams;
- zcs->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
+ cctxParams.cParams = params.cParams;
+ cctxParams.fParams = params.fParams;
+ cctxParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
return ZSTD_initCStream_internal_opaque(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize);
}
DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
elapsedTime, #mutex); \
} } \
- } else pthread_mutex_lock(mutex); \
+ } else { pthread_mutex_lock(mutex); } \
}
#else
}
} /* for (chunkID=0; chunkID<nbChunks; chunkID++) */
- DEBUGLOG(4, "checksumFlag : %u ", params.fParams.checksumFlag);
+ DEBUGLOG(4, "checksumFlag : %u ", cctxParams.fParams.checksumFlag);
if (cctxParams.fParams.checksumFlag) {
U32 const checksum = (U32)XXH64_digest(&xxh64);
if (dstPos + 4 > dstCapacity) {
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ * TODO: ZSTD_estimateCCtxSize_advanced_opaque()
* Note : CCtx estimation is only correct for single-threaded compression */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ * TODO: ZSTD_estimateCStreamSize_advanced_opaque
* Note : CStream estimation is only correct for single-threaded compression.
* ZSTD_DStream memory budget depends on window Size.
* This information can be passed manually, using ZSTD_estimateDStreamSize,
/*! ZSTD_estimate?DictSize() :
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
* ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
+ * TODO: ZSTD_estimateCDictSize_advanced_opaque(), can set by reference
* Note : dictionary created "byReference" are smaller */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
-
-// By reference
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced_opaque(size_t dictSize, ZSTD_CCtx_params* params);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
ZSTD_CCtx_params* params);
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
+ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createAndInitCCtxParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
-ZSTDLIB_API
-
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
const void* dict,size_t dictSize,
ZSTD_parameters params);
+
/*! ZSTD_compress_usingCDict_advanced() :
* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
/* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */
+ ZSTD_p_test,
} ZSTD_cParameter;
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls
-CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+ZSTD_DEBUG_FLAGS = -g -DZSTD_DEBUG=2
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) $(ZSTD_DEBUG_FLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
g_overlapLog = overlapLog;
}
+static U32 g_testParamFlag = 0;
+void FIO_setTestParamFlag(unsigned testParamFlag) { g_testParamFlag = testParamFlag; }
/*-*************************************
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
+
+ /* Test */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_test, g_testParamFlag) );
}
#elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
+ if (!strcmp(argument, "--testParam")) { FIO_setTestParamFlag(1); continue; }
#ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
#endif
PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest
-DEBUGLEVEL= 1
+DEBUGLEVEL= 2
DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL)
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+OPAQUEFILES := $(ZSTD_FILES) $(ZDICT_FILES) roundTripCrashOpaque.c
+roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_CPP)
+roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_LD)
+roundTripCrashOpaque : $(OPAQUEFILES)
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
longmatch : $(ZSTD_FILES) longmatch.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
+
ZSTD_compressStream(g_cstream, &buffOut, &buffIn);
ZSTD_endStream(g_cstream, &buffOut);
+
return buffOut.pos;
}
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t frameHeaderSize, cBlockSize;
+
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */
+
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
}
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
ip += ZSTD_blockHeaderSize; /* skip block header */
+
ZSTD_decompressBegin(g_zdc);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip;
--- /dev/null
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*
+ This program takes a file in input,
+ performs a zstd round-trip test (compression - decompress)
+ compares the result with original
+ and generates a crash (double free) on corruption detection.
+*/
+
+/*===========================================
+* Dependencies
+*==========================================*/
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, free, exit */
+#include <stdio.h> /* fprintf */
+#include <sys/types.h> /* stat */
+#include <sys/stat.h> /* stat */
+#include "xxhash.h"
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+/*===========================================
+* Macros
+*==========================================*/
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+#define CHECK_Z(f) { \
+ size_t const err = f; \
+ if (ZSTD_isError(err)) { \
+ fprintf(stderr, \
+ "Error=> %s: %s", \
+ #f, ZSTD_getErrorName(err)); \
+ exit(1); \
+} }
+
+
+/** roundTripTest() :
+* Compresses `srcBuff` into `compressedBuff`,
+* then decompresses `compressedBuff` into `resultBuff`.
+* @return : result of decompression, which should be == `srcSize`
+* or an error code if either compression or decompression fails.
+* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
+* for compression to be guaranteed to work */
+static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
+ void* compressedBuff, size_t compressedBuffCapacity,
+ const void* srcBuff, size_t srcBuffSize)
+{
+ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
+ ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 };
+ ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 };
+
+ ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, 1);
+ ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, 1);
+
+ ZSTD_CCtx_applyCCtxParams(cctx, cctxParams);
+
+ CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
+
+ ZSTD_freeCCtxParams(cctxParams);
+ ZSTD_freeCCtx(cctx);
+
+ return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos);
+}
+
+
+static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
+{
+ const char* ip1 = (const char*)buff1;
+ const char* ip2 = (const char*)buff2;
+ size_t pos;
+
+ for (pos=0; pos<buffSize; pos++)
+ if (ip1[pos]!=ip2[pos])
+ break;
+
+ return pos;
+}
+
+static void crash(int errorCode){
+ /* abort if AFL/libfuzzer, exit otherwise */
+ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
+ abort();
+ #else
+ exit(errorCode);
+ #endif
+}
+
+static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
+{
+ size_t const cBuffSize = ZSTD_compressBound(srcBuffSize);
+ void* cBuff = malloc(cBuffSize);
+ void* rBuff = malloc(cBuffSize);
+
+ if (!cBuff || !rBuff) {
+ fprintf(stderr, "not enough memory ! \n");
+ exit (1);
+ }
+
+ { size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize);
+ if (ZSTD_isError(result)) {
+ fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result));
+ crash(1);
+ }
+ if (result != srcBuffSize) {
+ fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
+ crash(1);
+ }
+ if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
+ fprintf(stderr, "Silent decoding corruption !!!");
+ crash(1);
+ }
+ }
+
+ free(cBuff);
+ free(rBuff);
+}
+
+
+static size_t getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (size_t)statbuf.st_size;
+}
+
+
+static int isDirectory(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+/** loadFile() :
+* requirement : `buffer` size >= `fileSize` */
+static void loadFile(void* buffer, const char* fileName, size_t fileSize)
+{
+ FILE* const f = fopen(fileName, "rb");
+ if (isDirectory(fileName)) {
+ fprintf(stderr, "Ignoring %s directory \n", fileName);
+ exit(2);
+ }
+ if (f==NULL) {
+ fprintf(stderr, "Impossible to open %s \n", fileName);
+ exit(3);
+ }
+ { size_t const readSize = fread(buffer, 1, fileSize, f);
+ if (readSize != fileSize) {
+ fprintf(stderr, "Error reading %s \n", fileName);
+ exit(5);
+ } }
+ fclose(f);
+}
+
+
+static void fileCheck(const char* fileName)
+{
+ size_t const fileSize = getFileSize(fileName);
+ void* buffer = malloc(fileSize);
+ if (!buffer) {
+ fprintf(stderr, "not enough memory \n");
+ exit(4);
+ }
+ loadFile(buffer, fileName, fileSize);
+ roundTripCheck(buffer, fileSize);
+ free (buffer);
+}
+
+int main(int argCount, const char** argv) {
+ if (argCount < 2) {
+ fprintf(stderr, "Error : no argument : need input file \n");
+ exit(9);
+ }
+ fileCheck(argv[1]);
+ fprintf(stderr, "no pb detected\n");
+ return 0;
+}
goto _cleanup;
}
+static int fuzzerTests_newAPI_opaque(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
+{
+ U32 const maxSrcLog = bigTests ? 24 : 22;
+ static const U32 maxSampleLog = 19;
+ size_t const srcBufferSize = (size_t)1<<maxSrcLog;
+ BYTE* cNoiseBuffer[5];
+ size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
+ BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
+ size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
+ BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
+ size_t const dstBufferSize = srcBufferSize;
+ BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
+ U32 result = 0;
+ U32 testNb = 0;
+ U32 coreSeed = seed;
+ ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
+ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
+ ZSTD_DStream* const zd_noise = ZSTD_createDStream();
+ clock_t const startClock = clock();
+ const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
+ size_t dictSize = 0;
+ U32 oldTestLog = 0;
+ U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
+ U32 const nbThreadsMax = bigTests ? 5 : 1;
+ ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
+
+ /* allocations */
+ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
+ cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
+ cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
+ cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
+ cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
+ CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
+ !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
+ "Not enough memory, fuzzer tests cancelled");
+
+ /* Create initial samples */
+ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
+ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
+ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
+ RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
+ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
+ memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
+ CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
+
+ /* catch up testNb */
+ for (testNb=1; testNb < startTest; testNb++)
+ FUZ_rand(&coreSeed);
+
+ /* test loop */
+ for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+ U32 lseed;
+ const BYTE* srcBuffer;
+ size_t totalTestSize, totalGenSize, cSize;
+ XXH64_state_t xxhState;
+ U64 crcOrig;
+ U32 resetAllowed = 1;
+ size_t maxTestSize;
+
+ /* init */
+ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
+ else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
+ FUZ_rand(&coreSeed);
+ lseed = coreSeed ^ prime32;
+
+ /* states full reset (deliberately not synchronized) */
+ /* some issues can only happen when reusing states */
+ if ((FUZ_rand(&lseed) & 0xFF) == 131) {
+ DISPLAYLEVEL(5, "Creating new context \n");
+ ZSTD_freeCCtx(zc);
+ zc = ZSTD_createCCtx();
+ CHECK(zc==NULL, "ZSTD_createCCtx allocation error");
+ resetAllowed=0;
+ }
+ if ((FUZ_rand(&lseed) & 0xFF) == 132) {
+ ZSTD_freeDStream(zd);
+ zd = ZSTD_createDStream();
+ CHECK(zd==NULL, "ZSTD_createDStream allocation error");
+ ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
+ }
+
+ /* srcBuffer selection [0-4] */
+ { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
+ if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
+ else {
+ buffNb >>= 3;
+ if (buffNb & 7) {
+ const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
+ buffNb = tnb[buffNb >> 3];
+ } else {
+ const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
+ buffNb = tnb[buffNb >> 3];
+ } }
+ srcBuffer = cNoiseBuffer[buffNb];
+ }
+
+ /* compression init */
+ CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
+ if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
+ && oldTestLog /* at least one test happened */ && resetAllowed) {
+ maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
+ if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
+ { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
+ CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, compressionLevel));
+ }
+ } else {
+ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
+ U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+ U32 const cLevelCandidate = (FUZ_rand(&lseed) %
+ (ZSTD_maxCLevel() -
+ (MAX(testLog, dictLog) / 3))) +
+ 1;
+ U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
+ maxTestSize = FUZ_rLogLength(&lseed, testLog);
+ oldTestLog = testLog;
+ /* random dictionary selection */
+ dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
+ { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
+ dict = srcBuffer + dictStart;
+ if (!dictSize) dict=NULL;
+ }
+ { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
+ ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
+
+ /* mess with compression parameters */
+ cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
+ cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
+ cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
+ cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
+ cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
+ cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
+ cParams = ZSTD_adjustCParams(cParams, 0, 0);
+
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_windowLog, cParams.windowLog) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_hashLog, cParams.hashLog) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_chainLog, cParams.chainLog) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_searchLog, cParams.searchLog) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_minMatch, cParams.searchLength) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_targetLength, cParams.targetLength) );
+
+ /* unconditionally set, to be sync with decoder */
+ /* mess with frame parameters */
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
+ DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
+
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
+ /* multi-threading parameters */
+ { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
+ U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
+ CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbThreads, nbThreads) );
+ if (nbThreads > 1) {
+ U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
+ CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) );
+ CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog)) );
+ }
+ }
+
+ if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1) );
+
+ if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, FUZ_rand(&lseed) & 1) );
+
+
+ /* Apply parameters */
+
+ CHECK_Z (ZSTD_CCtx_applyCCtxParams(zc, cctxParams) );
+
+ if (FUZ_rand(&lseed) & 1) {
+ CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
+ if (dict && dictSize) {
+ /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */
+ size_t const setError = ZSTD_CCtx_applyCCtxParams(zc, cctxParams);
+ CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_applyCCtxParams should have failed");
+ } } else {
+ CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
+ }
+ } }
+
+ /* multi-segments compression test */
+ XXH64_reset(&xxhState, 0);
+ { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
+ for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
+ /* compress random chunks into randomly sized dst buffers */
+ size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+ size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
+ size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
+ size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
+ size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+ ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
+ ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
+ outBuff.size = outBuff.pos + dstBuffSize;
+
+ CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
+ DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n",
+ (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
+
+ XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
+ memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
+ totalTestSize += inBuff.pos;
+ }
+
+ /* final frame epilogue */
+ { size_t remainingToFlush = (size_t)(-1);
+ while (remainingToFlush) {
+ ZSTD_inBuffer inBuff = { NULL, 0, 0 };
+ size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
+ size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
+ outBuff.size = outBuff.pos + adjustedDstSize;
+ DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);
+ remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
+ CHECK(ZSTD_isError(remainingToFlush),
+ "ZSTD_compress_generic w/ ZSTD_e_end error : %s",
+ ZSTD_getErrorName(remainingToFlush) );
+ } }
+ crcOrig = XXH64_digest(&xxhState);
+ cSize = outBuff.pos;
+ DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize);
+ }
+
+ /* multi - fragments decompression test */
+ if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
+ DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict);
+ CHECK_Z( ZSTD_resetDStream(zd) );
+ } else {
+ DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize);
+ CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
+ }
+ { size_t decompressionResult = 1;
+ ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
+ ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
+ for (totalGenSize = 0 ; decompressionResult ; ) {
+ size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+ size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+ size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
+ inBuff.size = inBuff.pos + readCSrcSize;
+ outBuff.size = inBuff.pos + dstBuffSize;
+ DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n",
+ (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize);
+ decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+ CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
+ DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
+ }
+ CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
+ CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
+ { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
+ if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
+ CHECK (crcDest!=crcOrig, "decompressed data corrupted");
+ } }
+
+ /*===== noisy/erroneous src decompression test =====*/
+
+ /* add some noise */
+ { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
+ U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
+ size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
+ size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
+ size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
+ size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
+ memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
+ } }
+
+ /* try decompression on noisy data */
+ CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
+ { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
+ ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
+ while (outBuff.pos < dstBufferSize) {
+ size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+ size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+ size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
+ size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
+ outBuff.size = outBuff.pos + adjustedDstSize;
+ inBuff.size = inBuff.pos + adjustedCSrcSize;
+ { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+ if (ZSTD_isError(decompressError)) break; /* error correctly detected */
+ /* Good so far, but no more progress possible */
+ if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
+ } } } }
+ DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
+
+_cleanup:
+ ZSTD_freeCCtx(zc);
+ ZSTD_freeDStream(zd);
+ ZSTD_freeDStream(zd_noise);
+ ZSTD_freeCCtxParams(cctxParams);
+ free(cNoiseBuffer[0]);
+ free(cNoiseBuffer[1]);
+ free(cNoiseBuffer[2]);
+ free(cNoiseBuffer[3]);
+ free(cNoiseBuffer[4]);
+ free(copyBuffer);
+ free(cBuffer);
+ free(dstBuffer);
+ return result;
+
+_output_error:
+ result = 1;
+ goto _cleanup;
+}
+
+
+
/*-*******************************************************
* Command line
return 0;
}
-typedef enum { simple_api, mt_api, advanced_api } e_api;
+typedef enum { simple_api, mt_api, advanced_api, advanced_api_opaque } e_api;
int main(int argc, const char** argv)
{
if (!strcmp(argument, "--mt")) { selected_api=mt_api; continue; }
if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; continue; }
+ if (!strcmp(argument, "--opaqueapi")) { selected_api=advanced_api_opaque; continue; }
if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
argument++;
case advanced_api :
result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
break;
+ case advanced_api_opaque :
+ result = fuzzerTests_newAPI_opaque(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
+ break;
default :
assert(0); /* impossible */
}
projects / thirdparty / zstd.git / commitdiff
