From: Yann Collet Date: Mon, 30 May 2016 14:17:33 +0000 (+0200) Subject: fuzzer tests dictBuilder. X-Git-Tag: v0.7.0^2~49^2~46 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=30009521d7f988cb78da49934a9a3d6542e831f1;p=thirdparty%2Fzstd.git fuzzer tests dictBuilder. Added : ability to not store dictID during compression; decompression doesn't check dictID then --- diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index dfcc6e2d7..b0831617c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -362,10 +362,11 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t /** ZSTD_decodeFrameHeader() : * `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ -static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize) +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize) { - size_t const result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize); - if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); + size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); + if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); return result; } @@ -1046,7 +1047,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; - if (dctx->dictID != dctx->fParams.dictID) return ERROR(dictionary_wrong); dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; return 0; diff --git a/programs/Makefile b/programs/Makefile index 76367db45..377e49475 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -82,7 +82,6 @@ all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzze zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c - @echo $(ZSTD_FILES) $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) zstd32: $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ @@ -123,10 +122,12 @@ fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -fuzzer : $(ZSTD_FILES) datagen.c fuzzer.c +fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder +fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fuzzer32: $(ZSTD_FILES) datagen.c fuzzer.c +fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder +fuzzer32: $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 27d4bcf10..c5d5f46a7 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -41,6 +41,7 @@ #include /* strcmp */ #include /* clock_t */ #include "zstd_static.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */ +#include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" #define XXH_STATIC_LINKING_ONLY @@ -54,7 +55,6 @@ #define MB *(1U<<20) #define GB *(1U<<30) -static const size_t COMPRESSIBLE_NOISE_LENGTH = 10 MB; /* capital, used to be a macro */ static const U32 FUZ_compressibility_default = 50; static const U32 nbTestsDefault = 30000; @@ -113,7 +113,7 @@ static unsigned FUZ_highbit32(U32 v32) #define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; } static int basicUnitTests(U32 seed, double compressibility) { - size_t const CNBuffSize = COMPRESSIBLE_NOISE_LENGTH; + size_t const CNBuffSize = 5 MB; void* const CNBuffer = malloc(CNBuffSize); void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize)); void* const decodedBuffer = malloc(CNBuffSize); @@ -176,7 +176,7 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) ); DISPLAYLEVEL(4, "OK \n"); - DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); + DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++); cSize = 0; CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize), (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), @@ -185,7 +185,7 @@ static int basicUnitTests(U32 seed, double compressibility) cSize += r); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); + DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, @@ -234,6 +234,68 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeDCtx(dctx); } + /* Dictionary and dictBuilder tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t dictSize = 16 KB; + void* dictBuffer = malloc(dictSize); + size_t const totalSampleSize = 1 MB; + size_t const sampleUnitSize = 8 KB; + U32 const nbSamples = totalSampleSize / sampleUnitSize; + size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); + + if (dictBuffer==NULL || samplesSizes==NULL) { + free(dictBuffer); + free(samplesSizes); + goto _output_error; + } + + DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++); + { U32 u; for (u=0; u