fixed OSSfuzz 11849

The problem was already masked,
due to no longer accepting tiny blocks for statistics.

But in case it could still happen with not-so-tiny blocks,
there is a stricter control which ensures that
nothing was already loaded prior to statistics collection.

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9b047e3ab32296b868749485148bf3fbe563d767..527765c6e99cd198ed7b5551b827184b2d7f4a12 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3882,7 +3882,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 /* shortcut to compression pass directly into output buffer */
                 size_t const cSize = ZSTD_compressEnd(zcs,
                                                 op, oend-op, ip, iend-ip);
-                DEBUGLOG(4, "ZSTD_compressEnd : %u", (U32)cSize);
+                DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (U32)cSize);
                 if (ZSTD_isError(cSize)) return cSize;
                 ip = iend;
                 op += cSize;

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 351c7d01cde238a81c8656d5ff83a600f3099ce9..98079822f8c75b936cfe75c744828dabcade7784 100644 (file)
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -1124,8 +1124,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
     U32 tmpRep[ZSTD_REP_NUM];  /* updated rep codes will sink here */
     memcpy(tmpRep, rep, sizeof(tmpRep));
 
-    DEBUGLOG(5, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
-    DEBUGLOG(5, "repCodes: %u, %u, %u", tmpRep[0], tmpRep[1], tmpRep[2]);
+    DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
     assert(ms->opt.litLengthSum == 0);    /* first block */
     assert(seqStore->sequences == seqStore->sequencesStart);   /* no ldm */
     assert(ms->window.dictLimit == ms->window.lowLimit);   /* no dictionary */
@@ -1157,6 +1156,7 @@ size_t ZSTD_compressBlock_btultra2(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         const void* src, size_t srcSize)
 {
+    U32 const current = (U32)((const BYTE*)src - ms->window.base);
     DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
 
     /* 2-pass strategy:
@@ -1171,7 +1171,7 @@ size_t ZSTD_compressBlock_btultra2(
     if ( (ms->opt.litLengthSum==0)   /* first block */
       && (seqStore->sequences == seqStore->sequencesStart)  /* no ldm */
       && (ms->window.dictLimit == ms->window.lowLimit)   /* no dictionary */
-      && (ms->window.dictLimit - ms->nextToUpdate <= 1)  /* no prefix (note: intentional overflow, defined as 2-complement) */
+      && (current == ms->window.dictLimit)   /* start of frame, nothing already loaded nor skipped */
       && (srcSize > ZSTD_PREDEF_THRESHOLD)
       ) {
         ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ff85f46230e890aea31851ea77a4dac98b7264b8..d3be8f2aeb208831e7a2271f276671e92a6edbd2 100644 (file)
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -538,12 +538,51 @@ static int basicUnitTests(U32 seed, double compressibility)
                 CHECK_EQ(size1, outb.pos);
             }
 
-
             ZSTD_freeCCtx(cctx);
         }
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
+    {   size_t const sampleSize = 1024;
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        ZSTD_inBuffer inb;
+        ZSTD_outBuffer outb;
+        inb.src = CNBuffer;
+        inb.pos = 0;
+        inb.size = 0;
+        outb.dst = compressedBuffer;
+        outb.pos = 0;
+        outb.size = compressedBufferSize;
+        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
+
+        inb.size = sampleSize;   /* start with something, so that context is already used */
+        CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );   /* will break internal assert if stats_init is not disabled */
+        assert(inb.pos == inb.size);
+        outb.pos = 0;     /* cancel output */
+
+        CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
+        inb.size = 4;   /* too small size : compression will be skipped */
+        inb.pos = 0;
+        CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
+        assert(inb.pos == inb.size);
+
+        inb.size += 5;   /* too small size : compression will be skipped */
+        CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
+        assert(inb.pos == inb.size);
+
+        inb.size += 11;   /* small enough to attempt compression */
+        CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
+        assert(inb.pos == inb.size);
+
+        assert(inb.pos < sampleSize);
+        inb.size = sampleSize;   /* large enough to trigger stats_init, but no longer at beginning */
+        CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );   /* will break internal assert if stats_init is not disabled */
+        assert(inb.pos == inb.size);
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
         ZSTD_outBuffer out = {NULL, 0, 0};