]> git.ipfire.org Git - thirdparty/zstd.git/commitdiff
huf log speed optimization: unidirectional scan of logs + break when regressing
authorDanielle Rozenblit <drozenblit@fb.com>
Tue, 20 Dec 2022 20:27:38 +0000 (12:27 -0800)
committerDanielle Rozenblit <drozenblit@fb.com>
Tue, 20 Dec 2022 20:27:38 +0000 (12:27 -0800)
21 files changed:
.github/dependabot.yml [new file with mode: 0644]
.github/workflows/dev-long-tests.yml
.github/workflows/dev-short-tests.yml
.github/workflows/publish-release-artifacts.yml
.github/workflows/scorecards.yml
build/meson/meson.build
contrib/linux-kernel/test/test.c
lib/common/huf.h
lib/compress/huf_compress.c
lib/compress/zstd_compress.c
lib/compress/zstd_lazy.c
programs/fileio.c
programs/util.c
programs/zstdcli.c
tests/cli-tests/compression/window-resize.sh [new file with mode: 0755]
tests/cli-tests/compression/window-resize.sh.stderr.ignore [new file with mode: 0644]
tests/cli-tests/compression/window-resize.sh.stdout.glob [new file with mode: 0644]
tests/fuzz/zstd_helpers.c
tests/fuzzer.c
tests/regression/config.c
tests/zstreamtest.c

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644 (file)
index 0000000..5ace460
--- /dev/null
@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
index aadc0ab549967bd9732d484bd652f8821fc81034..ac34c7165e5dbecec4412e4989fe54010e74fd71 100644 (file)
@@ -71,7 +71,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: gcc-8 + ASan + UBSan + Test Zstd
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -qqq update
         make gcc8install
         CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
@@ -102,7 +104,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: gcc-8 + ASan + UBSan + Fuzz Test
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -qqq update
         make gcc8install
         CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
index 37186b5c9199c6645cba7165548e6217e002103a..08df1c07cd885129d014516349a5b56f5b6de381 100644 (file)
@@ -52,7 +52,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: gcc-7 + libzstdmt compilation
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -qqq update
         make gcc7install
         CC=gcc-7 CFLAGS=-Werror make -j all
@@ -158,7 +160,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: gcc-8 build
+        # See https://askubuntu.com/a/1428822
         run: |
+          echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
           sudo apt-get -qqq update
           make gcc8install
           CC=gcc-8 CFLAGS="-Werror" make -j all
@@ -187,7 +191,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
+      uses: microsoft/setup-msbuild@v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
@@ -206,7 +210,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
+      uses: microsoft/setup-msbuild@v1.1.3
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@@ -225,7 +229,7 @@ jobs:
 #    steps:
 #    - uses: actions/checkout@v3
 #    - name: Add MSBuild to PATH
-#      uses: microsoft/setup-msbuild@v1.0.2
+#      uses: microsoft/setup-msbuild@v1.1.3
 #    - name: Build
 #      working-directory: ${{env.GITHUB_WORKSPACE}}
 #      run: >
@@ -271,7 +275,7 @@ jobs:
 
   qemu-consistency:
     name: QEMU ${{ matrix.name }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
       matrix:
@@ -376,7 +380,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1.0.2
+      uses: microsoft/setup-msbuild@v1.1.3
     - name: Build and run tests
       working-directory: ${{env.GITHUB_WORKSPACE}}
       env:
index b5a3ac689bf12b937702040b4ffde9fb67312b82..4a70fb8c63e9a998868f20a221ae326d505d17db 100644 (file)
@@ -68,7 +68,7 @@ jobs:
           fi
 
       - name: Publish
-        uses: skx/github-action-publish-binaries@release-1.3
+        uses: skx/github-action-publish-binaries@release-2.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
index 2eae4749437fefe8732afc94d34512dc0d455f58..1c19b48b2ccd744ec574dede27b1e7bece9f6788 100644 (file)
@@ -32,7 +32,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3
+        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6
         with:
           results_file: results.sarif
           results_format: sarif
index f264760a34c30e38668b26a306347d709898fc71..98d556aa3bf0a4e588947ad15a27072cfb950457 100644 (file)
@@ -23,7 +23,9 @@ project('zstd',
     # so this isn't safe
     #'werror=true'
   ],
-  version: 'DUMMY',
+  version: run_command(
+    find_program('GetZstdLibraryVersion.py'), '../../lib/zstd.h',
+    check: true).stdout().strip(),
   meson_version: '>=0.48.0')
 
 cc = meson.get_compiler('c')
@@ -45,16 +47,6 @@ compiler_msvc = 'msvc'
 
 zstd_version = meson.project_version()
 
-zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
-GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true)
-r = run_command(GetZstdLibraryVersion_py, zstd_h_file)
-if r.returncode() == 0
-  zstd_version = r.stdout().strip()
-  message('Project version is now: @0@'.format(zstd_version))
-else
-  error('Cannot find project version in @0@'.format(zstd_h_file))
-endif
-
 zstd_libversion = zstd_version
 
 # =============================================================================
index 6cd1730bb3a23293cdc9749be0e5f0384ae2988a..67d248e0cbf34c30f3731f189209612b4c14f1bc 100644 (file)
@@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) {
   asm volatile("" : "+r"(x));
 }
 
+static void __attribute__((noinline)) fill_stack(void) {
+  memset(g_stack, 0x33, 8192);
+}
+
 static void __attribute__((noinline)) set_stack(void) {
 
   char stack[8192];
   g_stack = stack;
-  memset(g_stack, 0x33, 8192);
   use(g_stack);
 }
 
@@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) {
 
 static void test_stack_usage(test_data_t const *data) {
   set_stack();
+  fill_stack();
   test_f2fs();
   test_btrfs(data);
   test_decompress_unzstd(data);
index dee99da772ab54d0969a7872a972d85e18e3b827..595b2f6db5d2cfb9a5ddde0c07fbb0626e2ef158 100644 (file)
@@ -173,7 +173,7 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
-#define HUF_OPTIMAL_DEPTH_THRESHOLD 3
+#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
 typedef enum {
    HUF_depth_fast, /** Use heuristic to find the table depth**/
    HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
index a6a5a257c4c2ea3a10977f399c39e9447a3ea2ca..eb2cb7b94a5a265f90d39b8a07a0327d41e7de19 100644 (file)
@@ -1253,14 +1253,6 @@ unsigned HUF_minTableLog(unsigned symbolCardinality)
     return minBitsSymbols;
 }
 
-#define ESTIMATE_TOTAL_SIZE(huffLog) {\
-            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, huffLog, workSpace, wkspSize);\
-            if (ERR_isError(maxBits)) continue;\
-            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);\
-            if (ERR_isError(hSize)) continue;\
-            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;\
-            }\
-
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
 {
     unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
@@ -1273,28 +1265,20 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
         const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
         const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
         size_t optSize = ((size_t) ~0);
-        unsigned optLogGuess = optLog;
 
         if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/
 
-        /* Search below estimate log until size increases */
-        for (; optLogGuess >= minTableLog; optLogGuess--) {
-            ESTIMATE_TOTAL_SIZE(optLogGuess);
+        /* Search until size increases */
+        for (unsigned optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
 
-            if (newSize > optSize) {
-                break;
-            }
-            optSize = newSize;
-            optLog = optLogGuess;
-        }
+            if (ERR_isError(maxBits)) continue;
 
-        if (optSize < ((size_t) ~0)) {
-            return optLog;
-        }
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
+
+            if (ERR_isError(hSize)) continue;
 
-        /* Search above estimate log until size increases */
-        for (; optLogGuess <= maxTableLog; optLogGuess++) {
-            ESTIMATE_TOTAL_SIZE(optLogGuess);
+            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
 
             if (newSize > optSize) {
                 break;
index adf1f6e7afc6332da0f967f5f8e6e4411fa9bc37..0069a7b1bee3e0eb66b6e5c78d7cb938b9f61843 100644 (file)
@@ -265,9 +265,9 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
     return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
 }
 
-/* Returns 1 if compression parameters are such that we should
+/* Returns ZSTD_ps_enable if compression parameters are such that we should
  * enable long distance matching (wlog >= 27, strategy >= btopt).
- * Returns 0 otherwise.
+ * Returns ZSTD_ps_disable otherwise.
  */
 static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
                                  const ZSTD_compressionParameters* const cParams) {
@@ -482,8 +482,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         return bounds;
 
     case ZSTD_c_enableLongDistanceMatching:
-        bounds.lowerBound = 0;
-        bounds.upperBound = 1;
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
     case ZSTD_c_ldmHashLog:
@@ -854,6 +854,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         return (size_t)CCtxParams->enableDedicatedDictSearch;
 
     case ZSTD_c_enableLongDistanceMatching :
+        BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
         CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
         return CCtxParams->ldmParams.enableLdm;
 
@@ -1096,7 +1097,7 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
 
 size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
+    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
     RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                     "Can't set pledgedSrcSize when not in init stage.");
     cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
@@ -1369,8 +1370,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     }
 
     /* resize windowLog if input is small enough, to use less memory */
-    if ( (srcSize < maxWindowResize)
-      && (dictSize < maxWindowResize) )  {
+    if ( (srcSize <= maxWindowResize)
+      && (dictSize <= maxWindowResize) )  {
         U32 const tSize = (U32)(srcSize + dictSize);
         static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
         U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
index 2ee9cf6302560895966d688173b5c8fef827b39e..d5a7b5cbf954a73bff2b63b0e2e27b0f84933a3d 100644 (file)
@@ -1078,7 +1078,7 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
     }
 # endif /* ZSTD_ARCH_ARM_NEON */
     /* SWAR */
-    {   const size_t chunkSize = sizeof(size_t);
+    {   const int chunkSize = sizeof(size_t);
         const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
         const size_t xFF = ~((size_t)0);
         const size_t x01 = xFF / 0xFF;
@@ -2123,7 +2123,6 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
 size_t ZSTD_compressBlock_lazy2_extDict_row(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize)
-
 {
     return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
 }
index 96cf602a3006ff26d05cc893b797958e5e01a130..e80d370110a13ddd9db60bf90364a0bfd91af6ba 100644 (file)
@@ -2749,6 +2749,7 @@ typedef struct {
     int numSkippableFrames;
     int decompUnavailable;
     int usesCheck;
+    BYTE checksum[4];
     U32 nbFiles;
     unsigned dictID;
 } fileInfo_t;
@@ -2843,8 +2844,8 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
                     int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
                     if (contentChecksumFlag) {
                         info->usesCheck = 1;
-                        ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0,
-                                info_frame_error, "Error: could not skip past checksum");
+                        ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4,
+                                info_frame_error, "Error: could not read checksum");
                 }   }
                 info->numActualFrames++;
             }
@@ -2936,7 +2937,16 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel)
                     (unsigned long long)info->decompressedSize);
             DISPLAYOUT("Ratio: %.4f\n", ratio);
         }
-        DISPLAYOUT("Check: %s\n", checkString);
+
+        if (info->usesCheck && info->numActualFrames == 1) {
+            DISPLAYOUT("Check: %s %02x%02x%02x%02x\n", checkString,
+                info->checksum[3], info->checksum[2],
+                info->checksum[1], info->checksum[0]
+            );
+        } else {
+            DISPLAYOUT("Check: %s\n", checkString);
+        }
+
         DISPLAYOUT("\n");
     }
 }
index a3af262114365f3cf8767a27c27b00716cb10359..63b3ae1767f5a391f927628362f05850c7a6abf8 100644 (file)
@@ -569,7 +569,7 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
         for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
             size_t const curLen = strlen(table2->fileNames[idx2]);
             memcpy(buf+pos, table2->fileNames[idx2], curLen);
-            assert(newTableIdx <= newTable->tableSize);
+            assert(newTableIdx < newTable->tableSize);
             newTable->fileNames[newTableIdx] = buf+pos;
             pos += curLen+1;
     }   }
@@ -693,8 +693,11 @@ static int UTIL_prepareFileList(const char *dirName,
                 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
                 assert(newListSize >= 0);
                 *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
-                *bufEnd = *bufStart + newListSize;
-                if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+                if (*bufStart != NULL) {
+                    *bufEnd = *bufStart + newListSize;
+                } else {
+                    free(path); closedir(dir); return 0;
+                }
             }
             if (*bufStart + *pos + pathLength < *bufEnd) {
                 memcpy(*bufStart + *pos, path, pathLength + 1);  /* with final \0 */
index 583c8a5919b1afe032d24e676d24851036795de7..362f320a9982b3e04385d55c0918b17926888f4d 100644 (file)
@@ -216,7 +216,6 @@ static void usage_advanced(const char* programName)
     DISPLAYOUT("\n");
     DISPLAYOUT("Advanced compression options :\n");
     DISPLAYOUT("     --ultra           enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
-    DISPLAYOUT("     --long[=#]        enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
     DISPLAYOUT("     --fast[=#]        switch to very fast compression levels (default: %u)\n", 1);
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
@@ -224,9 +223,9 @@ static void usage_advanced(const char* programName)
         DISPLAYOUT("     --no-name         do not store original filename when compressing\n");
     }
 #endif
-    DISPLAYOUT("     --adapt           dynamically adapt compression level to I/O conditions\n");
-    DISPLAYOUT("     --[no-]row-match-finder :  force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
+    DISPLAYOUT("     --long[=#]        enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
     DISPLAYOUT("     --patch-from=FILE :  specify the file to be used as a reference point for zstd's diff engine. \n");
+    DISPLAYOUT("     --adapt           dynamically adapt compression level to I/O conditions\n");
 # ifdef ZSTD_MULTITHREAD
     DISPLAYOUT("  -T#                  spawn # compression threads (default: 1, 0==# cores) \n");
     DISPLAYOUT("  -B#                  select size of each job (default: 0==automatic) \n");
@@ -240,6 +239,7 @@ static void usage_advanced(const char* programName)
     DISPLAYOUT("     --target-compressed-block-size=# :  generate compressed block of approximately targeted size \n");
     DISPLAYOUT("     --no-dictID       don't write dictID into header (dictionary compression only)\n");
     DISPLAYOUT("     --[no-]compress-literals :  force (un)compressed literals\n");
+    DISPLAYOUT("     --[no-]row-match-finder :  force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
 
     DISPLAYOUT("     --format=zstd     compress files to the .zst format (default)\n");
 #ifdef ZSTD_GZCOMPRESS
diff --git a/tests/cli-tests/compression/window-resize.sh b/tests/cli-tests/compression/window-resize.sh
new file mode 100755 (executable)
index 0000000..3b5e6fe
--- /dev/null
@@ -0,0 +1,9 @@
+#!/bin/sh
+datagen -g1G > file
+zstd --long=31 -1 --single-thread --no-content-size -f file
+zstd -l -v file.zst
+
+# We want to ignore stderr (its outputting "*** zstd command line interface
+# 64-bits v1.5.3, by Yann Collet ***")
+
+rm file file.zst
diff --git a/tests/cli-tests/compression/window-resize.sh.stderr.ignore b/tests/cli-tests/compression/window-resize.sh.stderr.ignore
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/tests/cli-tests/compression/window-resize.sh.stdout.glob b/tests/cli-tests/compression/window-resize.sh.stdout.glob
new file mode 100644 (file)
index 0000000..313d216
--- /dev/null
@@ -0,0 +1,3 @@
+...
+Window Size: 1.000 GiB (1073741824 B)
+...
index b4a6509db1854485e1804e2c97238980b850a15f..08ce70dd7a031e6eac75de99669e7bedaefb49f5 100644 (file)
@@ -80,7 +80,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer);
     setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer);
     /* Select long distance matching parameters */
-    setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer);
+    setRand(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_auto, ZSTD_ps_disable, producer);
     setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer);
     setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
             ZSTD_LDM_MINMATCH_MAX, producer);
index 81c2d9dba22e70a4ecb1ab483da4698639fec65c..879e537bc90101313692f99d26e02fc8275b8a67 100644 (file)
@@ -851,7 +851,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         RDG_genBuffer(dict, size, 0.5, 0.5, seed);
         RDG_genBuffer(src, size, 0.5, 0.5, seed);
 
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
 
         ZSTD_freeCCtx(cctx);
@@ -875,7 +875,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
-            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
             CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
             cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
             CHECK_Z(cSize);
@@ -900,7 +900,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
         memcpy(dict + testSize, CNBuffer, testSize);
         for (level = 1; level <= 5; ++level) {
-            for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) {
+            for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
                 size_t cSize0;
                 XXH64_hash_t compressedChecksum0;
 
@@ -956,7 +956,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* Enable MT, LDM, and opt parser */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
 
@@ -995,7 +995,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         /* Disable content size to skip single-pass decompression. */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
@@ -1092,7 +1092,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* compress on level 1 using refPrefix and ldm */
         ZSTD_CCtx_refPrefix(cctx, dict, size);;
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1))
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
         refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
         assert(!ZSTD_isError(refPrefixLdmCompressedSize));
 
@@ -2820,7 +2820,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
             assert(cctx != NULL);
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
+            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
             {   size_t const compressedSize = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
@@ -2836,7 +2836,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
             DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
-            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
+            CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
             CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
             {   size_t const result = ZSTD_compress2(cctx,
                                 compressedBuffer, ZSTD_compressBound(inputSize),
@@ -3492,7 +3492,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* Enable MT, LDM, and use refPrefix() for a small dict */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
 
         CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
@@ -3686,7 +3686,7 @@ static int longUnitTests(U32 const seed, double compressibility)
 
         /* Enable checksum to verify round trip. */
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
 
         /* Round trip once with ldm. */
@@ -3696,7 +3696,7 @@ static int longUnitTests(U32 const seed, double compressibility)
 
         ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
-        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
 
         /* Round trip once without ldm. */
index 57cd110c6bbe9dc97d05692b5fe2aaaeab596dca..30d0ca5e212f66b064e1991700eb2bd8489fea34 100644 (file)
@@ -183,7 +183,7 @@ static config_t no_pledged_src_size_with_dict = {
 };
 
 static param_value_t const ldm_param_values[] = {
-    {.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
+    {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
 };
 
 static config_t ldm = {
@@ -204,7 +204,7 @@ static config_t mt = {
 
 static param_value_t const mt_ldm_param_values[] = {
     {.param = ZSTD_c_nbWorkers, .value = 2},
-    {.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
+    {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
 };
 
 static config_t mt_ldm = {
index ce9020f128c0228b0f7229305abb7abded7499c4..348f72ed42679b32572d7650a459513efeef3328 100644 (file)
@@ -2281,7 +2281,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
 
                 /* mess with long distance matching parameters */
                 if (bigTests) {
-                    if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
+                    if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );