Fixed : ZSTD_compress() can attempt compression on a too small buffer

diff --git a/Makefile b/Makefile
index 8049649d697f40434347442ad609607f44a960c0..6591e26df6422df61a86c4b0b670d6546f398920 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version number
-export VERSION=0.0.2
+export VERSION=0.1.0
 export RELEASE=r$(VERSION)
 
 DESTDIR?=

diff --git a/lib/fse.c b/lib/fse.c
index 63684fc04bdd9a2b77179a013ebc361ed2c20ea3..a82baa4177f4786d8f7bbe38b99ccb0bf811a999 100644 (file)
--- a/lib/fse.c
+++ b/lib/fse.c
@@ -1132,7 +1132,7 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
 
 size_t FSE_initCStream(FSE_CStream_t* bitC, void* start, size_t maxSize)
 {
-    if (maxSize < 8) return (size_t)-FSE_ERROR_dstSize_tooSmall;
+    if (maxSize < sizeof(bitC->ptr)) return (size_t)-FSE_ERROR_dstSize_tooSmall;
     bitC->bitContainer = 0;
     bitC->bitPos = 0;
     bitC->startPtr = (char*)start;
@@ -1186,12 +1186,9 @@ void FSE_flushBits(FSE_CStream_t* bitC)
     size_t nbBytes = bitC->bitPos >> 3;
     FSE_writeLEST(bitC->ptr, bitC->bitContainer);
     bitC->ptr += nbBytes;
-    if (bitC->ptr <= bitC->endPtr)
-    {
-        bitC->bitPos &= 7;
-        bitC->bitContainer >>= nbBytes*8;
-        return;
-    }
+    if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
+    bitC->bitPos &= 7;
+    bitC->bitContainer >>= nbBytes*8;
 }
 
 void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* statePtr)
@@ -1208,7 +1205,7 @@ size_t FSE_closeCStream(FSE_CStream_t* bitC)
     FSE_addBitsFast(bitC, 1, 1);
     FSE_flushBits(bitC);
 
-    if (bitC->bitPos > 7)   /* still some data to flush => too close to buffer's end */
+    if (bitC->ptr >= bitC->endPtr)   /* too close to buffer's end */
         return 0;   /* not compressible */
 
     endPtr = bitC->ptr;
@@ -1887,7 +1884,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
     U16 nodeNb = STARTNODE;
     U32 nodeRoot;
 
-    // check
+    /* safety checks */
     if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
     if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
        memset(huffNode0, 0, sizeof(huffNode0));
@@ -1976,7 +1973,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
 
     /* init */
     op += 6;   /* jump Table -- could be optimized by delta / deviation */
-    errorCode = FSE_initCStream(&bitC, op, dstSize);
+    errorCode = FSE_initCStream(&bitC, op, oend-op);
     if (FSE_isError(errorCode)) return 0;
 
     n = srcSize & ~15;  // mod 16
@@ -2124,7 +2121,10 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
     op += errorCode;
 
     /* Compress */
-    op += HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable);
+    errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable);
+    if (FSE_isError(errorCode)) return errorCode;
+    if (errorCode==0) return 0;
+    op += errorCode;
 
     /* check compressibility */
     if ((size_t)(op-ostart) >= srcSize-1)

diff --git a/lib/zstd.c b/lib/zstd.c
index 181eaf8fab61da2629c20ca7aebe20fab01d44c0..35474b12f5de9f97bdd7c70bfc7cb02fcb28c826 100644 (file)
--- a/lib/zstd.c
+++ b/lib/zstd.c
@@ -912,7 +912,7 @@ static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const
 }
 
 
-size_t ZSTD_compressBegin(ZSTD_Cctx*  ctx, void* dst, size_t maxDstSize)
+size_t ZSTD_compressBegin(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize)
 {
     /* Sanity check */
     if (maxDstSize < ZSTD_frameHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
@@ -1081,7 +1081,6 @@ size_t ZSTD_compressEnd(ZSTD_Cctx*  ctx, void* dst, size_t maxDstSize)
 static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
 {
     BYTE* const ostart = (BYTE* const)dst;
-    BYTE* const oend = ostart + maxDstSize;
     BYTE* op = ostart;
 
     /* Header */
@@ -1094,7 +1093,7 @@ static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, co
 
     /* Compression */
     {
-        size_t cSize = ZSTD_compressContinue(ctx, op, oend-op, src, srcSize);
+        size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
         if (ZSTD_isError(cSize)) return cSize;
         op += cSize;
         maxDstSize -= cSize;
@@ -1102,7 +1101,7 @@ static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, co
 
     /* Close frame */
     {
-        size_t endSize = ZSTD_compressEnd(ctx, op, oend-op);
+        size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
         if(ZSTD_isError(endSize)) return endSize;
         op += endSize;
     }

diff --git a/programs/Makefile b/programs/Makefile
index 20205b90a2bca7d4975f792200d569e147423464..1f7bdd5f54f541ee85c9ba7b1fee22bc491f89bb 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -30,7 +30,7 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-RELEASE?= v0.0.2
+RELEASE?= v0.1.0
 
 DESTDIR?=
 PREFIX ?= /usr
@@ -61,7 +61,7 @@ default: zstd
 
 all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
 
-zstd: $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c
+zstd  : $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c
        $(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 zstd32: $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c

diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index db4bc65fa1d4c94332bfee039c718f4baa6b1c4c..658a0cc6efe5c863e968fe40930d20e1715cb5b0 100644 (file)
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -323,7 +323,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
     U32 result = 0;
     U32 testNb = 0;
     U32 coreSeed = seed, lseed = 0;
-    (void)startTest; (void)compressibility;
 
     /* allocation */
     srcBuffer = (BYTE*)malloc (srcBufferSize);
@@ -332,7 +331,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
     CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled");
 
     /* Create initial sample */
-    FUZ_generateSynthetic(srcBuffer, srcBufferSize, 0.50, &coreSeed);
+    FUZ_generateSynthetic(srcBuffer, srcBufferSize, compressibility, &coreSeed);
 
     /* catch up testNb */
     for (testNb=0; testNb < startTest; testNb++)
@@ -356,10 +355,20 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
         sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
         crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
 
-        /* compression tests*/
+        /* compression test */
         cSize = ZSTD_compress(cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize);
         CHECK(ZSTD_isError(cSize), "ZSTD_compress failed");
 
+        /* compression failure test */
+        {
+            size_t errorCode;
+            void* dBufferTooSmall = malloc(cSize-1);   /* valgrind should catch overflows */
+            if (dBufferTooSmall==NULL) { DISPLAY("not enough memory !"); exit(1); }
+            errorCode = ZSTD_compress(dBufferTooSmall, cSize-1, srcBuffer + sampleStart, sampleSize);
+            CHECK(!ZSTD_isError(errorCode), "ZSTD_compress should have failed ! (buffer too small)");
+            free(dBufferTooSmall);
+        }
+
         /* decompression tests*/
         dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
         dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
@@ -393,8 +402,9 @@ int FUZ_usage(char* programName)
     DISPLAY( " -i#    : Nb of tests (default:%u) \n", nbTestsDefault);
     DISPLAY( " -s#    : Select seed (default:prompt user)\n");
     DISPLAY( " -t#    : Select starting test number (default:0)\n");
-    DISPLAY( " -p#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
     DISPLAY( " -v     : verbose\n");
+    DISPLAY( " -p     : pause at the end\n");
     DISPLAY( " -h     : display help and exit\n");
     return 0;
 }