From: Yann Collet Date: Fri, 8 Jan 2016 16:27:50 +0000 (+0100) Subject: added duplication tests X-Git-Tag: v0.4.6~2^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=600962711da2ede9322296a76937653338126558;p=thirdparty%2Fzstd.git added duplication tests --- diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c167d7824..6f94922f3 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -115,8 +115,8 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; - void* headerBuffer; size_t hbSize; + char headerBuffer[ZSTD_frameHeaderSize_max]; seqStore_t seqStore; /* sequences storage ptrs */ @@ -210,8 +210,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize; zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2); zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2); - zc->headerBuffer = (char*)zc->workSpace + zc->workSpaceSize - g_hbSize; zc->hbSize = 0; + zc->stage = 0; return 0; } @@ -1961,28 +1961,27 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src * @return : 0, or an error code */ size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { - void* dstWorkSpace = dstCCtx->workSpace; - size_t dstWorkSpaceSize = dstCCtx->workSpaceSize; const U32 contentLog = (srcCCtx->params.strategy == ZSTD_fast) ? 1 : srcCCtx->params.contentLog; const size_t tableSpace = ((1 << contentLog) + (1 << srcCCtx->params.hashLog)) * sizeof(U32); - const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << srcCCtx->params.windowLog); - const size_t neededSpace = tableSpace + (3*blockSize); if (srcCCtx->stage!=0) return ERROR(stage_wrong); - if (dstWorkSpaceSize < neededSpace) - { - free(dstWorkSpace); - dstWorkSpaceSize = neededSpace; - dstWorkSpace = malloc(dstWorkSpaceSize); - if (dstWorkSpace==NULL) return ERROR(memory_allocation); - } + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params); + + /* copy tables */ + memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); - memcpy(dstCCtx, srcCCtx, sizeof(*dstCCtx)); - dstCCtx->workSpace = dstWorkSpace; - dstCCtx->workSpaceSize = dstWorkSpaceSize; + /* copy frame header */ + dstCCtx->hbSize = srcCCtx->hbSize; + memcpy(dstCCtx->headerBuffer , srcCCtx->headerBuffer, srcCCtx->hbSize); - memcpy(dstWorkSpace, srcCCtx->workSpace, tableSpace); + /* copy dictionary pointers */ + dstCCtx->nextToUpdate= srcCCtx->nextToUpdate; + dstCCtx->nextSrc = srcCCtx->nextSrc; + dstCCtx->base = srcCCtx->base; + dstCCtx->dictBase = srcCCtx->dictBase; + dstCCtx->dictLimit = srcCCtx->dictLimit; + dstCCtx->lowLimit = srcCCtx->lowLimit; return 0; } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 801ae75bb..2d1d45151 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -194,6 +194,65 @@ static int basicUnitTests(U32 seed, double compressibility) if (result != ERROR(srcSize_wrong)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); + /* Dictionary and Duplication tests */ + { + ZSTD_CCtx* ctxOrig = ZSTD_createCCtx(); + ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + const size_t dictSize = 500; + size_t cSizeOrig; + + DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++); + result = ZSTD_compressBegin(ctxOrig, 2); + if (ZSTD_isError(result)) goto _output_error; + result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize); + if (ZSTD_isError(result)) goto _output_error; + result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig); + if (ZSTD_isError(result)) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); + cSize = 0; + result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); + result = ZSTD_decompress_usingDict(dctx, + decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, + compressedBuffer, cSize, + CNBuffer, dictSize); + if (ZSTD_isError(result)) goto _output_error; + ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */ + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++); + cSizeOrig = cSize; + cSize = 0; + result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); + if (ZSTD_isError(result)) goto _output_error; + cSize += result; + if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */ + ZSTD_freeCCtx(ctxDuplicated); + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); + result = ZSTD_decompress_usingDict(dctx, + decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, + compressedBuffer, cSize, + CNBuffer, dictSize); + if (ZSTD_isError(result)) goto _output_error; + ZSTD_freeDCtx(dctx); + DISPLAYLEVEL(4, "OK \n"); + } + /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);