zbuff properly supports small dst buffers

diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c
index b7937f1d8f927555399dedd77158f731b51632d6..844c50405a5424e729c0298c9065edace634e4ef 100644 (file)
--- a/lib/zstd_buffered.c
+++ b/lib/zstd_buffered.c
@@ -204,11 +204,12 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
                 void* cDst;
                 size_t cSize;
                 size_t iSize = zbc->inBuffPos - zbc->inToCompress;
-                if ((size_t)(oend-op) > ZSTD_compressBound(iSize))
+                size_t oSize = oend-op;
+                if (oSize >= ZSTD_compressBound(iSize))
                     cDst = op;   /* compress directly into output buffer (avoid flush stage) */
                 else
-                    cDst = zbc->outBuff;
-                cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize);
+                    cDst = zbc->outBuff, oSize = zbc->outBuffSize;
+                cSize = ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
                 if (ZSTD_isError(cSize)) return cSize;
                 /* prepare next block */
                 zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;

diff --git a/programs/Makefile b/programs/Makefile
index 3259de07db41234f3ae32ca7f4c1c8b4a267049a..65ba9e06458ea0b79403c67c85365fb6355cadc8 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -171,60 +171,60 @@ test-zstd: zstd datagen
        @diff tmp1 tmp2   
        ./datagen              | ./zstd -6 -v | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       @./datagen -g257MB     | md5sum > tmp1
-       ./datagen -g257MB      | ./zstd -v    | ./zstd -d  | md5sum > tmp2
+       @./datagen -g270000000 | md5sum > tmp1
+       ./datagen -g270000000  | ./zstd -v    | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g257MB      | ./zstd -v2   | ./zstd -d  | md5sum > tmp2
+       ./datagen -g270000000  | ./zstd -v2   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g257MB      | ./zstd -v3   | ./zstd -d  | md5sum > tmp2
+       ./datagen -g270000000  | ./zstd -v3   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       @./datagen -g129MB -P60| md5sum > tmp1
-       ./datagen -g129MB -P60 | ./zstd -v4   | ./zstd -d  | md5sum > tmp2
+       @./datagen -g140000000 -P60| md5sum > tmp1
+       ./datagen -g140000000 -P60 | ./zstd -v4   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g129MB -P60 | ./zstd -v5   | ./zstd -d  | md5sum > tmp2
+       ./datagen -g140000000 -P60 | ./zstd -v5   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g129MB -P60 | ./zstd -v6   | ./zstd -d  | md5sum > tmp2
+       ./datagen -g140000000 -P60 | ./zstd -v6   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       @./datagen -g65MB -P70 | md5sum > tmp1
-       ./datagen -g65MB -P70  | ./zstd -v7   | ./zstd -d  | md5sum > tmp2
+       @./datagen -g70000000 -P70 | md5sum > tmp1
+       ./datagen -g70000000 -P70  | ./zstd -v7   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g65MB -P70  | ./zstd -v8   | ./zstd -d  | md5sum > tmp2
+       ./datagen -g70000000 -P70  | ./zstd -v8   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g65MB -P70  | ./zstd -v9   | ./zstd -d  | md5sum > tmp2
+       ./datagen -g70000000 -P70  | ./zstd -v9   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       @./datagen -g33MB -P75 | md5sum > tmp1
-       ./datagen -g33MB -P75  | ./zstd -v10  | ./zstd -d  | md5sum > tmp2
+       @./datagen -g35000000 -P75 | md5sum > tmp1
+       ./datagen -g35000000 -P75  | ./zstd -v10  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g33MB -P75  | ./zstd -v11  | ./zstd -d  | md5sum > tmp2
+       ./datagen -g35000000 -P75  | ./zstd -v11  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g33MB -P75  | ./zstd -v12  | ./zstd -d  | md5sum > tmp2
+       ./datagen -g35000000 -P75  | ./zstd -v12  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       @./datagen -g17MB -P80 | md5sum > tmp1
-       ./datagen -g17MB -P80  | ./zstd -v13  | ./zstd -d  | md5sum > tmp2
+       @./datagen -g18000000 -P80 | md5sum > tmp1
+       ./datagen -g18000000 -P80  | ./zstd -v13  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g17MB -P80  | ./zstd -v14  | ./zstd -d  | md5sum > tmp2
+       ./datagen -g18000000 -P80  | ./zstd -v14  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g17MB -P80  | ./zstd -v15  | ./zstd -d  | md5sum > tmp2
+       ./datagen -g18000000 -P80  | ./zstd -v15  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g17MB -P80  | ./zstd -v16  | ./zstd -d  | md5sum > tmp2
+       ./datagen -g18000000 -P80  | ./zstd -v16  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g17MB -P80  | ./zstd -v17  | ./zstd -d  | md5sum > tmp2
+       ./datagen -g18000000 -P80  | ./zstd -v17  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       @./datagen -g49MB -P93 | md5sum > tmp1
-       ./datagen -g49MB -P93  | ./zstd -v18  | ./zstd -d  | md5sum > tmp2
+       @./datagen -g50000000 -P94 | md5sum > tmp1
+       ./datagen -g50000000 -P94  | ./zstd -v18  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g49MB -P93  | ./zstd -v19  | ./zstd -d  | md5sum > tmp2
+       ./datagen -g50000000 -P94  | ./zstd -v19  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       @./datagen -g97MB -P99 | md5sum > tmp1
-       ./datagen -g97MB -P99  | ./zstd -v20  | ./zstd -d  | md5sum > tmp2
+       @./datagen -g99000000 -P99 | md5sum > tmp1
+       ./datagen -g99000000 -P99  | ./zstd -v20  | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
-       ./datagen -g6GB -P99   | md5sum > tmp1
-       ./datagen -g6GB -P99   | ./zstd -vq   | ./zstd -d  | md5sum > tmp2
+       ./datagen -g6000000000 -P99| md5sum > tmp1
+       ./datagen -g6000000000 -P99| ./zstd -vq   | ./zstd -d  | md5sum > tmp2
        @diff tmp1 tmp2   
 
 test-zstd32: zstd32 datagen
-       ./datagen          | ./zstd32 -v  | ./zstd32 -d > $(VOID)
-       ./datagen -g256MB  | ./zstd32 -v  | ./zstd32 -d > $(VOID)
+       ./datagen            | ./zstd32 -v  | ./zstd32 -d > $(VOID)
+       ./datagen -g256MB    | ./zstd32 -v  | ./zstd32 -d > $(VOID)
        ./datagen -g6GB -P99 | ./zstd32 -vq | ./zstd32 -d > $(VOID)
 
 test-fullbench: fullbench datagen

diff --git a/programs/fileio.c b/programs/fileio.c
index 267ddb5620cd8c2e176d30de5eab4a41887557e2..7e5f80da683fec66ff73ab40597bdf482d944f49 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -337,7 +337,6 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
         size_t inStart=0;
         size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
         if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
-        if (toRead==0) break;   /* end of Frame */
         readSize -= inSize;
         inStart += inSize;
 
@@ -347,6 +346,7 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
         frameSize += decodedSize;
         DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
 
+        if (toRead == 0) break;
         if (readSize) continue;   /* still some data left within inBuff */
 
         /* Fill input buffer */

diff --git a/programs/zbufftest.c b/programs/zbufftest.c
index f7d2f9833f3fae6fefd7f19dc4f5710624b7612a..f1fb5e38747960407a251d06ef259b16a8ed2ee7 100644 (file)
--- a/programs/zbufftest.c
+++ b/programs/zbufftest.c
@@ -324,22 +324,29 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
             sampleSize = (size_t)1 << sampleSizeLog;
             sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
             sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+            readSize = sampleSize;
 
-            XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
-            memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, sampleSize);
+            /* random size output buffer */
+            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+            sampleSize = (size_t)1 << sampleSizeLog;
+            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+            genSize = MIN (cBufferSize - cSize, sampleSize);
 
-            readSize = sampleSize;
-            genSize = cBufferSize - cSize;
             errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
             CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
-            CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed")
+
+            XXH64_update(&crc64, srcBuffer+sampleStart, readSize);
+            memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
             cSize += genSize;
-            totalTestSize += sampleSize;
+            totalTestSize += readSize;
 
             if ((FUZ_rand(&lseed) & 15) == 0)
             {
                 /* add a few random flushes operations, to mess around */
-                genSize = cBufferSize - cSize;
+                sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
+                sampleSize = (size_t)1 << sampleSizeLog;
+                sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
+                genSize = MIN (cBufferSize - cSize, sampleSize);
                 errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
                 CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
                 cSize += genSize;