From: Danielle Rozenblit Date: Tue, 20 Dec 2022 20:27:38 +0000 (-0800) Subject: huf log speed optimization: unidirectional scan of logs + break when regressing X-Git-Tag: v1.5.4^2~66^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=482689b995bd6afb4a7b335bb80b9e60f501e3c2;p=thirdparty%2Fzstd.git huf log speed optimization: unidirectional scan of logs + break when regressing --- diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..5ace4600a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index aadc0ab54..ac34c7165 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -71,7 +71,9 @@ jobs: steps: - uses: actions/checkout@v3 - name: gcc-8 + ASan + UBSan + Test Zstd + # See https://askubuntu.com/a/1428822 run: | + echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list sudo apt-get -qqq update make gcc8install CC=gcc-8 make -j uasan-test-zstd @@ -271,7 +275,7 @@ jobs: qemu-consistency: name: QEMU ${{ matrix.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed. matrix: @@ -376,7 +380,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Add MSBuild to PATH - uses: microsoft/setup-msbuild@v1.0.2 + uses: microsoft/setup-msbuild@v1.1.3 - name: Build and run tests working-directory: ${{env.GITHUB_WORKSPACE}} env: diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml index b5a3ac689..4a70fb8c6 100644 --- a/.github/workflows/publish-release-artifacts.yml +++ b/.github/workflows/publish-release-artifacts.yml @@ -68,7 +68,7 @@ jobs: fi - name: Publish - uses: skx/github-action-publish-binaries@release-1.3 + uses: skx/github-action-publish-binaries@release-2.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 2eae47494..1c19b48b2 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -32,7 +32,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3 + uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6 with: results_file: results.sarif results_format: sarif diff --git a/build/meson/meson.build b/build/meson/meson.build index f264760a3..98d556aa3 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -23,7 +23,9 @@ project('zstd', # so this isn't safe #'werror=true' ], - version: 'DUMMY', + version: run_command( + find_program('GetZstdLibraryVersion.py'), '../../lib/zstd.h', + check: true).stdout().strip(), meson_version: '>=0.48.0') cc = meson.get_compiler('c') @@ -45,16 +47,6 @@ compiler_msvc = 'msvc' zstd_version = meson.project_version() -zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h') -GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true) -r = run_command(GetZstdLibraryVersion_py, zstd_h_file) -if r.returncode() == 0 - zstd_version = r.stdout().strip() - message('Project version is now: @0@'.format(zstd_version)) -else - error('Cannot find project version in @0@'.format(zstd_h_file)) -endif - zstd_libversion = zstd_version # ============================================================================= diff --git a/contrib/linux-kernel/test/test.c b/contrib/linux-kernel/test/test.c index 6cd1730bb..67d248e0c 100644 --- a/contrib/linux-kernel/test/test.c +++ b/contrib/linux-kernel/test/test.c @@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) { asm volatile("" : "+r"(x)); } +static void __attribute__((noinline)) fill_stack(void) { + memset(g_stack, 0x33, 8192); +} + static void __attribute__((noinline)) set_stack(void) { char stack[8192]; g_stack = stack; - memset(g_stack, 0x33, 8192); use(g_stack); } @@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) { static void test_stack_usage(test_data_t const *data) { set_stack(); + fill_stack(); test_f2fs(); test_btrfs(data); test_decompress_unzstd(data); diff --git a/lib/common/huf.h b/lib/common/huf.h index dee99da77..595b2f6db 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -173,7 +173,7 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, /* **************************************** * HUF detailed API * ****************************************/ -#define HUF_OPTIMAL_DEPTH_THRESHOLD 3 +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra typedef enum { HUF_depth_fast, /** Use heuristic to find the table depth**/ HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/ diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index a6a5a257c..eb2cb7b94 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -1253,14 +1253,6 @@ unsigned HUF_minTableLog(unsigned symbolCardinality) return minBitsSymbols; } -#define ESTIMATE_TOTAL_SIZE(huffLog) {\ - maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, huffLog, workSpace, wkspSize);\ - if (ERR_isError(maxBits)) continue;\ - hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);\ - if (ERR_isError(hSize)) continue;\ - newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;\ - }\ - unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode) { unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); @@ -1273,28 +1265,20 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); const unsigned minTableLog = HUF_minTableLog(symbolCardinality); size_t optSize = ((size_t) ~0); - unsigned optLogGuess = optLog; if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/ - /* Search below estimate log until size increases */ - for (; optLogGuess >= minTableLog; optLogGuess--) { - ESTIMATE_TOTAL_SIZE(optLogGuess); + /* Search until size increases */ + for (unsigned optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); - if (newSize > optSize) { - break; - } - optSize = newSize; - optLog = optLogGuess; - } + if (ERR_isError(maxBits)) continue; - if (optSize < ((size_t) ~0)) { - return optLog; - } + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); + + if (ERR_isError(hSize)) continue; - /* Search above estimate log until size increases */ - for (; optLogGuess <= maxTableLog; optLogGuess++) { - ESTIMATE_TOTAL_SIZE(optLogGuess); + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; if (newSize > optSize) { break; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index adf1f6e7a..0069a7b1b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -265,9 +265,9 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); } -/* Returns 1 if compression parameters are such that we should +/* Returns ZSTD_ps_enable if compression parameters are such that we should * enable long distance matching (wlog >= 27, strategy >= btopt). - * Returns 0 otherwise. + * Returns ZSTD_ps_disable otherwise. */ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, const ZSTD_compressionParameters* const cParams) { @@ -482,8 +482,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_c_enableLongDistanceMatching: - bounds.lowerBound = 0; - bounds.upperBound = 1; + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; return bounds; case ZSTD_c_ldmHashLog: @@ -854,6 +854,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, return (size_t)CCtxParams->enableDedicatedDictSearch; case ZSTD_c_enableLongDistanceMatching : + BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; return CCtxParams->ldmParams.enableLdm; @@ -1096,7 +1097,7 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) { - DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, "Can't set pledgedSrcSize when not in init stage."); cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; @@ -1369,8 +1370,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, } /* resize windowLog if input is small enough, to use less memory */ - if ( (srcSize < maxWindowResize) - && (dictSize < maxWindowResize) ) { + if ( (srcSize <= maxWindowResize) + && (dictSize <= maxWindowResize) ) { U32 const tSize = (U32)(srcSize + dictSize); static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 2ee9cf630..d5a7b5cbf 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -1078,7 +1078,7 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr } # endif /* ZSTD_ARCH_ARM_NEON */ /* SWAR */ - { const size_t chunkSize = sizeof(size_t); + { const int chunkSize = sizeof(size_t); const size_t shiftAmount = ((chunkSize * 8) - chunkSize); const size_t xFF = ~((size_t)0); const size_t x01 = xFF / 0xFF; @@ -2123,7 +2123,6 @@ size_t ZSTD_compressBlock_lazy_extDict_row( size_t ZSTD_compressBlock_lazy2_extDict_row( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) - { return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); } diff --git a/programs/fileio.c b/programs/fileio.c index 96cf602a3..e80d37011 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2749,6 +2749,7 @@ typedef struct { int numSkippableFrames; int decompUnavailable; int usesCheck; + BYTE checksum[4]; U32 nbFiles; unsigned dictID; } fileInfo_t; @@ -2843,8 +2844,8 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; if (contentChecksumFlag) { info->usesCheck = 1; - ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, - info_frame_error, "Error: could not skip past checksum"); + ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4, + info_frame_error, "Error: could not read checksum"); } } info->numActualFrames++; } @@ -2936,7 +2937,16 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) (unsigned long long)info->decompressedSize); DISPLAYOUT("Ratio: %.4f\n", ratio); } - DISPLAYOUT("Check: %s\n", checkString); + + if (info->usesCheck && info->numActualFrames == 1) { + DISPLAYOUT("Check: %s %02x%02x%02x%02x\n", checkString, + info->checksum[3], info->checksum[2], + info->checksum[1], info->checksum[0] + ); + } else { + DISPLAYOUT("Check: %s\n", checkString); + } + DISPLAYOUT("\n"); } } diff --git a/programs/util.c b/programs/util.c index a3af26211..63b3ae176 100644 --- a/programs/util.c +++ b/programs/util.c @@ -569,7 +569,7 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2) for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { size_t const curLen = strlen(table2->fileNames[idx2]); memcpy(buf+pos, table2->fileNames[idx2], curLen); - assert(newTableIdx <= newTable->tableSize); + assert(newTableIdx < newTable->tableSize); newTable->fileNames[newTableIdx] = buf+pos; pos += curLen+1; } } @@ -693,8 +693,11 @@ static int UTIL_prepareFileList(const char *dirName, ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; assert(newListSize >= 0); *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); - *bufEnd = *bufStart + newListSize; - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } + if (*bufStart != NULL) { + *bufEnd = *bufStart + newListSize; + } else { + free(path); closedir(dir); return 0; + } } if (*bufStart + *pos + pathLength < *bufEnd) { memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 583c8a591..362f320a9 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -216,7 +216,6 @@ static void usage_advanced(const char* programName) DISPLAYOUT("\n"); DISPLAYOUT("Advanced compression options :\n"); DISPLAYOUT(" --ultra enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); - DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); DISPLAYOUT(" --fast[=#] switch to very fast compression levels (default: %u)\n", 1); #ifdef ZSTD_GZCOMPRESS if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */ @@ -224,9 +223,9 @@ static void usage_advanced(const char* programName) DISPLAYOUT(" --no-name do not store original filename when compressing\n"); } #endif - DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n"); - DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n"); + DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); DISPLAYOUT(" --patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine. \n"); + DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n"); # ifdef ZSTD_MULTITHREAD DISPLAYOUT(" -T# spawn # compression threads (default: 1, 0==# cores) \n"); DISPLAYOUT(" -B# select size of each job (default: 0==automatic) \n"); @@ -240,6 +239,7 @@ static void usage_advanced(const char* programName) DISPLAYOUT(" --target-compressed-block-size=# : generate compressed block of approximately targeted size \n"); DISPLAYOUT(" --no-dictID don't write dictID into header (dictionary compression only)\n"); DISPLAYOUT(" --[no-]compress-literals : force (un)compressed literals\n"); + DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n"); DISPLAYOUT(" --format=zstd compress files to the .zst format (default)\n"); #ifdef ZSTD_GZCOMPRESS diff --git a/tests/cli-tests/compression/window-resize.sh b/tests/cli-tests/compression/window-resize.sh new file mode 100755 index 000000000..3b5e6fe24 --- /dev/null +++ b/tests/cli-tests/compression/window-resize.sh @@ -0,0 +1,9 @@ +#!/bin/sh +datagen -g1G > file +zstd --long=31 -1 --single-thread --no-content-size -f file +zstd -l -v file.zst + +# We want to ignore stderr (its outputting "*** zstd command line interface +# 64-bits v1.5.3, by Yann Collet ***") + +rm file file.zst diff --git a/tests/cli-tests/compression/window-resize.sh.stderr.ignore b/tests/cli-tests/compression/window-resize.sh.stderr.ignore new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli-tests/compression/window-resize.sh.stdout.glob b/tests/cli-tests/compression/window-resize.sh.stdout.glob new file mode 100644 index 000000000..313d216e1 --- /dev/null +++ b/tests/cli-tests/compression/window-resize.sh.stdout.glob @@ -0,0 +1,3 @@ +... +Window Size: 1.000 GiB (1073741824 B) +... diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index b4a6509db..08ce70dd7 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -80,7 +80,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer); setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer); /* Select long distance matching parameters */ - setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer); + setRand(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_auto, ZSTD_ps_disable, producer); setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer); setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX, producer); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 81c2d9dba..879e537bc 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -851,7 +851,7 @@ static int basicUnitTests(U32 const seed, double compressibility) RDG_genBuffer(dict, size, 0.5, 0.5, seed); RDG_genBuffer(src, size, 0.5, 0.5, seed); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3))); ZSTD_freeCCtx(cctx); @@ -875,7 +875,7 @@ static int basicUnitTests(U32 const seed, double compressibility) CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize)); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); CHECK_Z(cSize); @@ -900,7 +900,7 @@ static int basicUnitTests(U32 const seed, double compressibility) RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); memcpy(dict + testSize, CNBuffer, testSize); for (level = 1; level <= 5; ++level) { - for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) { + for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) { size_t cSize0; XXH64_hash_t compressedChecksum0; @@ -956,7 +956,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* Enable MT, LDM, and opt parser */ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); @@ -995,7 +995,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* Disable content size to skip single-pass decompression. */ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); @@ -1092,7 +1092,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* compress on level 1 using refPrefix and ldm */ ZSTD_CCtx_refPrefix(cctx, dict, size);; - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)) + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)) refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); assert(!ZSTD_isError(refPrefixLdmCompressedSize)); @@ -2820,7 +2820,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); assert(cctx != NULL); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); { size_t const compressedSize = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(inputSize), @@ -2836,7 +2836,7 @@ static int basicUnitTests(U32 const seed, double compressibility) { ZSTD_CCtx* cctx = ZSTD_createCCtx(); DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); { size_t const result = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(inputSize), @@ -3492,7 +3492,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* Enable MT, LDM, and use refPrefix() for a small dict */ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); @@ -3686,7 +3686,7 @@ static int longUnitTests(U32 const seed, double compressibility) /* Enable checksum to verify round trip. */ CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); /* Round trip once with ldm. */ @@ -3696,7 +3696,7 @@ static int longUnitTests(U32 const seed, double compressibility) ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); /* Round trip once without ldm. */ diff --git a/tests/regression/config.c b/tests/regression/config.c index 57cd110c6..30d0ca5e2 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -183,7 +183,7 @@ static config_t no_pledged_src_size_with_dict = { }; static param_value_t const ldm_param_values[] = { - {.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, + {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable}, }; static config_t ldm = { @@ -204,7 +204,7 @@ static config_t mt = { static param_value_t const mt_ldm_param_values[] = { {.param = ZSTD_c_nbWorkers, .value = 2}, - {.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, + {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable}, }; static config_t mt_ldm = { diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index ce9020f12..348f72ed4 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -2281,7 +2281,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, /* mess with long distance matching parameters */ if (bigTests) { - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );