From 5f2ec63852a6e7d91ba496aeda5230761bc64108 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Nov 2015 10:32:17 +0100 Subject: [PATCH] fixed decompression bug (buffered mode) --- lib/zstd_buffered.c | 4 ++-- programs/Makefile | 20 ++++++++++++++++---- programs/zbufftest.c | 28 +++++++++++++++++++--------- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index 242ddf24d..aa858c7f1 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -479,10 +479,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt case ZBUFFds_load: { size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); - size_t toLoad = neededInSize - zbc->inPos; + size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */ size_t loadedSize; if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, zbc->inBuffSize - zbc->inPos, ip, iend-ip); + loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip); ip += loadedSize; zbc->inPos += loadedSize; if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ diff --git a/programs/Makefile b/programs/Makefile index 4683a9468..a0cad24c1 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -53,10 +53,11 @@ EXT = VOID = /dev/null endif +.PHONY: default all clean install uninstall test test32 test-all default: zstd -all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen +all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ @@ -93,6 +94,11 @@ zbufftest : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/ datagen.c xxhash.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) +zbufftest32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ + $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ + datagen.c xxhash.c zbufftest.c + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ datagen.c xxhash.c paramgrill.c @@ -105,7 +111,7 @@ clean: @rm -f core *.o tmp* \ zstd$(EXT) zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) \ + fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) @echo Cleaning completed @@ -135,9 +141,9 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled -test: test-zstd test-fullbench test-fuzzer +test: test-zstd test-fullbench test-fuzzer test-zbuff -test32: test-zstd32 test-fullbench32 test-fuzzer32 +test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest @@ -233,6 +239,12 @@ test-fuzzer: fuzzer test-fuzzer32: fuzzer32 ./fuzzer32 +test-zbuff: zbufftest + ./zbufftest + +test-zbuff32: zbufftest32 + ./zbufftest32 + valgrindTest: zstd datagen fuzzer fullbench @echo "\n ---- valgrind tests : memory analyzer ----" valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 0a10e4de5..717c4bce3 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -64,7 +64,7 @@ #define MB *(1U<<20) #define GB *(1U<<30) -static const U32 nbTestsDefault = 30000; +static const U32 nbTestsDefault = 10000; #define COMPRESSIBLE_NOISE_LENGTH (10 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 static const U32 prime1 = 2654435761U; @@ -280,7 +280,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit { size_t sampleSize, sampleStart; size_t cSize; - size_t maxTestSize, totalTestSize, readSize, genSize; + size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; U64 crcOrig, crcDest; @@ -330,13 +330,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit genSize = cBufferSize - cSize; errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); - CHECK (readSize != sampleSize, "test condition not respected : input should be fully consumed") + CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed") cSize += genSize; totalTestSize += sampleSize; if (totalTestSize > maxTestSize) break; } - genSize = cBufferSize - cSize; errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); @@ -346,12 +345,23 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit /* multi - fragments decompression test */ ZBUFF_decompressInit(zd); - genSize = dstBufferSize; - readSize = cBufferSize; - errorCode = ZBUFF_decompressContinue(zd, dstBuffer, &genSize, cBuffer, &readSize); - CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); + totalCSize = 0; + totalGenSize = 0; + while (totalCSize < cSize) + { + sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; + sampleSize = (size_t)1 << sampleSizeLog; + sampleSize += FUZ_rand(&lseed) & (sampleSize-1); + readSize = sampleSize; + genSize = dstBufferSize - totalGenSize; + errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize); + CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); + totalGenSize += genSize; + totalCSize += readSize; + } CHECK (errorCode != 0, "frame not fully decoded"); - CHECK (genSize != totalTestSize, "decompressed data : wrong size") + CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") + CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); -- 2.47.2