From: Sen Huang <senhuang96@fb.com>
Date: Fri, 19 Mar 2021 23:56:58 +0000 (-0700)
Subject: Make ZSTD_estimateCCtxSize_internal() loop through all srcSize parameter sets as... 
X-Git-Tag: v1.4.10~17^2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F2538%2Fhead;p=thirdparty%2Fzstd.git

Make ZSTD_estimateCCtxSize_internal() loop through all srcSize parameter sets as well
---

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 93c4075c5..84a5a146a 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1390,8 +1390,15 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
 
 static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
 {
-    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-    return ZSTD_estimateCCtxSize_usingCParams(cParams);
+    int tier = 0;
+    size_t largestSize = 0;
+    static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN};
+    for (; tier < 4; ++tier) {
+        /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */
+        ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict);
+        largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize);
+    }
+    return largestSize;
 }
 
 size_t ZSTD_estimateCCtxSize(int compressionLevel)
@@ -1399,6 +1406,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel)
     int level;
     size_t memBudget = 0;
     for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
+        /* Ensure monotonically increasing memory usage as compression level increases */
         size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
         if (newMB > memBudget) memBudget = newMB;
     }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 4fa57c8ef..1bc7f9c7e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -3181,7 +3181,23 @@ static int basicUnitTests(U32 const seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
-    DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over srcSize : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
+    {
+        int level = 1;
+        size_t prevSize = 0;
+        for (; level < ZSTD_maxCLevel(); ++level) {
+            size_t const currSize = ZSTD_estimateCCtxSize(level);
+            if (prevSize > currSize) {
+                DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
+                             prevSize, level-1, currSize, level);
+                goto _output_error;
+            }
+            prevSize = currSize;
+        }
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
     {
         size_t const kSizeIncrement = 2 KB;
         int level = -3;
@@ -3190,16 +3206,17 @@ static int basicUnitTests(U32 const seed, double compressibility)
             size_t dictSize = 0;
             for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
                 size_t srcSize = 2 KB;
-                size_t prevCCtxSize = 0;
                 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
                     ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
-                    size_t const cctxSize = ZSTD_estimateCCtxSize_usingCParams(cParams);
-                    if (cctxSize < prevCCtxSize || ZSTD_isError(cctxSize)) {
-                        DISPLAYLEVEL(3, "error! level: %d dictSize: %zu srcSize: %zu cctx size: %zu, prevsize: %zu\n",
-                                     level, dictSize, srcSize, cctxSize, prevCCtxSize);
+                    size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
+                    size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
+                    if (cctxSizeUsingLevel < cctxSizeUsingCParams
+                     || ZSTD_isError(cctxSizeUsingCParams)
+                     || ZSTD_isError(cctxSizeUsingLevel)) {
+                        DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
+                                     level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
                         goto _output_error;
                     }
-                    prevCCtxSize = cctxSize;
                 }
             }
         }