fixed decompression bug (buffered mode)

diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c
index 242ddf24d772b59f40cf7e03003e1d283d89ef2a..aa858c7f167cc7b69ef587d055139befb9d73f1f 100644 (file)
--- a/lib/zstd_buffered.c
+++ b/lib/zstd_buffered.c
@@ -479,10 +479,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
         case ZBUFFds_load:
             {
                 size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
-                size_t toLoad = neededInSize - zbc->inPos;
+                size_t toLoad = neededInSize - zbc->inPos;   /* should always be <= remaining space within inBuff */
                 size_t loadedSize;
                 if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected);   /* should never happen */
-                loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, zbc->inBuffSize - zbc->inPos, ip, iend-ip);
+                loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
                 ip += loadedSize;
                 zbc->inPos += loadedSize;
                 if (loadedSize < toLoad) { notDone = 0; break; }   /* not enough input, wait for more */

diff --git a/programs/Makefile b/programs/Makefile
index 4683a9468935357e767dea569b21a343cabf8cf7..a0cad24c13ea4dfe1ced4fc70de68c8be0936510 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -53,10 +53,11 @@ EXT =
 VOID = /dev/null
 endif
 
+.PHONY: default all clean install uninstall test test32 test-all
 
 default: zstd
 
-all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
+all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen
 
 zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
       $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
@@ -93,6 +94,11 @@ zbufftest  : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/
       datagen.c xxhash.c zbufftest.c
        $(CC)      $(FLAGS) $^ -o $@$(EXT)
 
+zbufftest32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
+      $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
+      datagen.c xxhash.c zbufftest.c
+       $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+
 paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
              $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
              datagen.c xxhash.c paramgrill.c
@@ -105,7 +111,7 @@ clean:
        @rm -f core *.o tmp* \
         zstd$(EXT) zstd32$(EXT) \
         fullbench$(EXT) fullbench32$(EXT) \
-        fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) \
+        fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
        datagen$(EXT) paramgrill$(EXT)
        @echo Cleaning completed
 
@@ -135,9 +141,9 @@ uninstall:
        [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
        @echo zstd programs successfully uninstalled
 
-test: test-zstd test-fullbench test-fuzzer
+test: test-zstd test-fullbench test-fuzzer test-zbuff
 
-test32: test-zstd32 test-fullbench32 test-fuzzer32
+test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
 
 test-all: test test32 valgrindTest
 
@@ -233,6 +239,12 @@ test-fuzzer: fuzzer
 test-fuzzer32: fuzzer32
        ./fuzzer32
 
+test-zbuff: zbufftest
+       ./zbufftest
+
+test-zbuff32: zbufftest32
+       ./zbufftest32
+
 valgrindTest: zstd datagen fuzzer fullbench
        @echo "\n ---- valgrind tests : memory analyzer ----"
        valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)

diff --git a/programs/zbufftest.c b/programs/zbufftest.c
index 0a10e4de5ee9b49be0e649584e8c66b97c1d5160..717c4bce30af352f12755c7a923f4fe36cdbb801 100644 (file)
--- a/programs/zbufftest.c
+++ b/programs/zbufftest.c
@@ -64,7 +64,7 @@
 #define MB *(1U<<20)
 #define GB *(1U<<30)
 
-static const U32 nbTestsDefault = 30000;
+static const U32 nbTestsDefault = 10000;
 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
 #define FUZ_COMPRESSIBILITY_DEFAULT 50
 static const U32 prime1 = 2654435761U;
@@ -280,7 +280,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
     {
         size_t sampleSize, sampleStart;
         size_t cSize;
-        size_t maxTestSize, totalTestSize, readSize, genSize;
+        size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
         size_t errorCode;
         U32 sampleSizeLog, buffNb, n, nbChunks;
         U64 crcOrig, crcDest;
@@ -330,13 +330,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
             genSize = cBufferSize - cSize;
             errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
             CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
-            CHECK (readSize != sampleSize, "test condition not respected : input should be fully consumed")
+            CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed")
 
             cSize += genSize;
             totalTestSize += sampleSize;
             if (totalTestSize > maxTestSize) break;
         }
-
         genSize = cBufferSize - cSize;
         errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
         CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
@@ -346,12 +345,23 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
 
         /* multi - fragments decompression test */
         ZBUFF_decompressInit(zd);
-        genSize = dstBufferSize;
-        readSize = cBufferSize;
-        errorCode = ZBUFF_decompressContinue(zd, dstBuffer, &genSize, cBuffer, &readSize);
-        CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
+        totalCSize = 0;
+        totalGenSize = 0;
+        while (totalCSize < cSize)
+        {
+            sampleSizeLog  = FUZ_rand(&lseed) % maxSampleLog;
+            sampleSize  = (size_t)1 << sampleSizeLog;
+            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+            readSize = sampleSize;
+            genSize = dstBufferSize - totalGenSize;
+            errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
+            CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
+            totalGenSize += genSize;
+            totalCSize += readSize;
+        }
         CHECK (errorCode != 0, "frame not fully decoded");
-        CHECK (genSize != totalTestSize, "decompressed data : wrong size")
+        CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
+        CHECK (totalCSize != cSize, "compressed data should be fully read")
         crcDest = XXH64(dstBuffer, totalTestSize, 0);
         if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
         CHECK (crcDest!=crcOrig, "decompressed data corrupted");