From: Yann Collet Date: Tue, 5 Jun 2018 21:53:28 +0000 (-0700) Subject: added test case X-Git-Tag: v1.3.5~3^2~28^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f7392f3dc95e26922763bc5817ab98b0ee4fa696;p=thirdparty%2Fzstd.git added test case --- diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index aae6d0f37..3d107746b 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -82,7 +82,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);

`src` should point to the start of a ZSTD encoded frame. `srcSize` must be at least as large as the frame header. hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - @return : - decompressed size of the frame in `src`, if known + @return : - decompressed size of `src` frame content, if known - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) note 1 : a 0 return value means the frame is valid but "empty". @@ -92,7 +92,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); Optionally, application can rely on some implicit limit, as ZSTD_decompress() only needs an upper bound of decompressed size. (For example, data could be necessarily cut into blocks <= 16 KB). - note 3 : decompressed size is always present when compression is done with ZSTD_compress() + note 3 : decompressed size is always present when compression is completed using single-pass functions, + such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). note 4 : decompressed size can be very large (64-bits value), potentially larger than what local system can handle as a single memory segment. In which case, it's necessary to use streaming mode to decompress data. @@ -107,8 +108,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); Both functions work the same way, but ZSTD_getDecompressedSize() blends "empty", "unknown" and "error" results to the same return value (0), while ZSTD_getFrameContentSize() gives them separate return values. - `src` is the start of a zstd compressed frame. - @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. + @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise.


Helper functions

#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b87ae3819..d9e3f8143 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1079,6 +1079,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                 zbuff, pledgedSrcSize)) {
             DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
                         zc->appliedParams.cParams.windowLog, zc->blockSize);
+            zc->workSpaceTooLarge += (zc->workSpaceTooLarge > 0);   /* if it was too large, it still is */
             return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
     }   }
     DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
@@ -1116,12 +1117,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
             int const workSpaceTooSmall = zc->workSpaceSize < neededSpace;
             int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace;
-            int const workSpaceWasteful = 0 && workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX);
+            int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX);
+            zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0;
+
             DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers",
                         neededSpace>>10, matchStateSize>>10, bufferSpace>>10);
             DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize);
 
-            zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0;
             if (workSpaceTooSmall || workSpaceWasteful) {
                 DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB",
                             zc->workSpaceSize >> 10,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8dabcddbc..6e60b74cb 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -450,6 +450,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    /* this test is really too long, and should be made faster */
     DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
         ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0);
@@ -467,6 +468,40 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
+    {   ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
+        assert(largeCCtx != NULL);
+        CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) );   /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
+        CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
+        {   size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx);   /* size of context must be measured after compression */
+            {   ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
+                assert(smallCCtx != NULL);
+                CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
+                {   size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
+                    DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
+                                largeCCtxSize>>10, smallCCtxSize>>10);
+                    assert(largeCCtxSize > 32* smallCCtxSize);  /* note : "too large" definition is handled within zstd_compress.c .
+                                                                 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
+                }
+                ZSTD_freeCCtx(smallCCtx);
+            }
+            {   U32 const maxNbAttempts = 1100;   /* nb of usages before triggering size down is handled within zstd_compress.c.
+                                                   * currently defined as 128x, but could be adjusted in the future.
+                                                   * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
+                U32 u;
+                for (u=0; u