]> git.ipfire.org Git - thirdparty/zstd.git/commitdiff
Use new paramSwitch enum for LCM, row matchfinder, and block splitter
authorsenhuang42 <senhuang96@fb.com>
Mon, 20 Sep 2021 13:04:07 +0000 (09:04 -0400)
committersenhuang42 <senhuang96@fb.com>
Tue, 21 Sep 2021 18:22:02 +0000 (14:22 -0400)
16 files changed:
lib/compress/zstd_compress.c
lib/compress/zstd_compress_internal.h
lib/compress/zstd_ldm.c
lib/compress/zstd_ldm.h
lib/compress/zstd_opt.c
lib/zstd.h
programs/benchzstd.c
programs/benchzstd.h
programs/fileio.c
programs/fileio.h
programs/zstdcli.c
tests/fuzz/zstd_helpers.c
tests/fuzzer.c
tests/regression/config.c
tests/regression/levels.h
tests/regression/results.csv

index e68f7352ac7f173c2ae685f0dc99d93ae629b7cc..700f4567f677ba830bc3a6359f6fc4c6e3b4725b 100644 (file)
@@ -84,10 +84,10 @@ struct ZSTD_CDict_s {
     ZSTD_customMem customMem;
     U32 dictID;
     int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */
-    ZSTD_useRowMatchFinderMode_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use
-                                                     * row-based matchfinder. Unless the cdict is reloaded, we will use
-                                                     * the same greedy/lazy matchfinder at compression time.
-                                                     */
+    ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use
+                                           * row-based matchfinder. Unless the cdict is reloaded, we will use
+                                           * the same greedy/lazy matchfinder at compression time.
+                                           */
 };  /* typedef'd to ZSTD_CDict within "zstd.h" */
 
 ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -226,35 +226,42 @@ static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
 /* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder
  * for this compression.
  */
-static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_useRowMatchFinderMode_e mode) {
-    assert(mode != ZSTD_urm_auto);
-    return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_urm_enableRowMatchFinder);
+static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) {
+    assert(mode != ZSTD_ps_auto);
+    return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable);
 }
 
-/* Returns row matchfinder usage enum given an initial mode and cParams */
-static ZSTD_useRowMatchFinderMode_e ZSTD_resolveRowMatchFinderMode(ZSTD_useRowMatchFinderMode_e mode,
-                                                                   const ZSTD_compressionParameters* const cParams) {
+/* Returns row matchfinder usage given an initial mode and cParams */
+static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode,
+                                                         const ZSTD_compressionParameters* const cParams) {
 #if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON)
     int const kHasSIMD128 = 1;
 #else
     int const kHasSIMD128 = 0;
 #endif
-    if (mode != ZSTD_urm_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */
-    mode = ZSTD_urm_disableRowMatchFinder;
+    if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */
+    mode = ZSTD_ps_disable;
     if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode;
     if (kHasSIMD128) {
-        if (cParams->windowLog > 14) mode = ZSTD_urm_enableRowMatchFinder;
+        if (cParams->windowLog > 14) mode = ZSTD_ps_enable;
     } else {
-        if (cParams->windowLog > 17) mode = ZSTD_urm_enableRowMatchFinder;
+        if (cParams->windowLog > 17) mode = ZSTD_ps_enable;
     }
     return mode;
 }
 
+/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */
+static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode,
+                                                        const ZSTD_compressionParameters* const cParams) {
+    if (mode != ZSTD_ps_auto) return mode;
+    return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable;
+}
+
 /* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */
 static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
-                                   const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
+                                   const ZSTD_paramSwitch_e useRowMatchFinder,
                                    const U32 forDDSDict) {
-    assert(useRowMatchFinder != ZSTD_urm_auto);
+    assert(useRowMatchFinder != ZSTD_ps_auto);
     /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate.
      * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder.
      */
@@ -269,14 +276,6 @@ static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const
     return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27;
 }
 
-/* Returns 1 if compression parameters are such that we should
- * enable blockSplitter (wlog >= 17, strategy >= btopt).
- * Returns 0 otherwise.
- */
-static U32 ZSTD_CParams_useBlockSplitter(const ZSTD_compressionParameters* const cParams) {
-    return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17;
-}
-
 static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
         ZSTD_compressionParameters cParams)
 {
@@ -295,11 +294,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
         assert(cctxParams.ldmParams.hashRateLog < 32);
     }
 
-    if (ZSTD_CParams_useBlockSplitter(&cParams)) {
-        DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including block splitting into cctx params");
-        cctxParams.splitBlocks = 1;
-    }
-
+    cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
@@ -360,18 +355,14 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par
      */
     cctxParams->compressionLevel = compressionLevel;
     cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
-    DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d", cctxParams->useRowMatchFinder);
+    cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
+    DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d", cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter);
 
     if (ZSTD_CParams_shouldEnableLdm(&params->cParams)) {
         /* Enable LDM by default for optimal parser and window size >= 128MB */
         DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)");
         cctxParams->ldmParams.enableLdm = 1;
     }
-
-    if (ZSTD_CParams_useBlockSplitter(&params->cParams)) {
-        DEBUGLOG(4, "Block splitter enabled by default (window size >= 128K, strategy >= btopt)");
-        cctxParams->splitBlocks = 1;
-    }
 }
 
 size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
@@ -541,9 +532,9 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         return bounds;
 
     case ZSTD_c_literalCompressionMode:
-        ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed);
-        bounds.lowerBound = ZSTD_lcm_auto;
-        bounds.upperBound = ZSTD_lcm_uncompressed;
+        ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable);
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
     case ZSTD_c_targetCBlockSize:
@@ -572,14 +563,14 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = 1;
         return bounds;
 
-    case ZSTD_c_splitBlocks:
-        bounds.lowerBound = 0;
-        bounds.upperBound = 1;
+    case ZSTD_c_useBlockSplitter:
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
     case ZSTD_c_useRowMatchFinder:
-        bounds.lowerBound = (int)ZSTD_urm_auto;
-        bounds.upperBound = (int)ZSTD_urm_enableRowMatchFinder;
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
     case ZSTD_c_deterministicRefPrefix:
@@ -648,7 +639,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_stableOutBuffer:
     case ZSTD_c_blockDelimiters:
     case ZSTD_c_validateSequences:
-    case ZSTD_c_splitBlocks:
+    case ZSTD_c_useBlockSplitter:
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
     default:
@@ -703,7 +694,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_stableOutBuffer:
     case ZSTD_c_blockDelimiters:
     case ZSTD_c_validateSequences:
-    case ZSTD_c_splitBlocks:
+    case ZSTD_c_useBlockSplitter:
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
         break;
@@ -803,7 +794,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
     }
 
     case ZSTD_c_literalCompressionMode : {
-        const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value;
+        const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
         BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
         CCtxParams->literalCompressionMode = lcm;
         return CCtxParams->literalCompressionMode;
@@ -917,14 +908,14 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->validateSequences = value;
         return CCtxParams->validateSequences;
 
-    case ZSTD_c_splitBlocks:
-        BOUNDCHECK(ZSTD_c_splitBlocks, value);
-        CCtxParams->splitBlocks = value;
-        return CCtxParams->splitBlocks;
+    case ZSTD_c_useBlockSplitter:
+        BOUNDCHECK(ZSTD_c_useBlockSplitter, value);
+        CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value;
+        return CCtxParams->useBlockSplitter;
 
     case ZSTD_c_useRowMatchFinder:
         BOUNDCHECK(ZSTD_c_useRowMatchFinder, value);
-        CCtxParams->useRowMatchFinder = (ZSTD_useRowMatchFinderMode_e)value;
+        CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value;
         return CCtxParams->useRowMatchFinder;
 
     case ZSTD_c_deterministicRefPrefix:
@@ -1055,8 +1046,8 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_validateSequences :
         *value = (int)CCtxParams->validateSequences;
         break;
-    case ZSTD_c_splitBlocks :
-        *value = (int)CCtxParams->splitBlocks;
+    case ZSTD_c_useBlockSplitter :
+        *value = (int)CCtxParams->useBlockSplitter;
         break;
     case ZSTD_c_useRowMatchFinder :
         *value = (int)CCtxParams->useRowMatchFinder;
@@ -1430,7 +1421,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
 
 static size_t
 ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
-                       const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
+                       const ZSTD_paramSwitch_e useRowMatchFinder,
                        const U32 enableDedicatedDictSearch,
                        const U32 forCCtx)
 {
@@ -1463,7 +1454,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
 
     /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */
     ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4);
-    assert(useRowMatchFinder != ZSTD_urm_auto);
+    assert(useRowMatchFinder != ZSTD_ps_auto);
 
     DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u",
                 (U32)chainSize, (U32)hSize, (U32)h3Size);
@@ -1474,7 +1465,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_compressionParameters* cParams,
         const ldmParams_t* ldmParams,
         const int isStatic,
-        const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
+        const ZSTD_paramSwitch_e useRowMatchFinder,
         const size_t buffInSize,
         const size_t buffOutSize,
         const U64 pledgedSrcSize)
@@ -1519,7 +1510,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 {
     ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-    ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder,
+    ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder,
                                                                                          &cParams);
 
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
@@ -1537,9 +1528,9 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
         /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
         size_t noRowCCtxSize;
         size_t rowCCtxSize;
-        initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder;
+        initialParams.useRowMatchFinder = ZSTD_ps_disable;
         noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
-        initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder;
+        initialParams.useRowMatchFinder = ZSTD_ps_enable;
         rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
         return MAX(noRowCCtxSize, rowCCtxSize);
     } else {
@@ -1584,7 +1575,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
         size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
                 : 0;
-        ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &params->cParams);
+        ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &params->cParams);
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
@@ -1599,9 +1590,9 @@ size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams)
         /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
         size_t noRowCCtxSize;
         size_t rowCCtxSize;
-        initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder;
+        initialParams.useRowMatchFinder = ZSTD_ps_disable;
         noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
-        initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder;
+        initialParams.useRowMatchFinder = ZSTD_ps_enable;
         rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
         return MAX(noRowCCtxSize, rowCCtxSize);
     } else {
@@ -1736,7 +1727,7 @@ static size_t
 ZSTD_reset_matchState(ZSTD_matchState_t* ms,
                       ZSTD_cwksp* ws,
                 const ZSTD_compressionParameters* cParams,
-                const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
+                const ZSTD_paramSwitch_e useRowMatchFinder,
                 const ZSTD_compResetPolicy_e crp,
                 const ZSTD_indexResetPolicy_e forceResetIndex,
                 const ZSTD_resetTarget_e forWho)
@@ -1751,7 +1742,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
     size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
 
     DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
-    assert(useRowMatchFinder != ZSTD_urm_auto);
+    assert(useRowMatchFinder != ZSTD_ps_auto);
     if (forceResetIndex == ZSTDirp_reset) {
         ZSTD_window_init(&ms->window);
         ZSTD_cwksp_mark_tables_dirty(ws);
@@ -1847,8 +1838,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                       ZSTD_buffered_policy_e const zbuff)
 {
     ZSTD_cwksp* const ws = &zc->workspace;
-    DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d",
-                (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder);
+    DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d",
+                (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter);
     assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
 
     zc->isFirstBlock = 1;
@@ -1859,7 +1850,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     zc->appliedParams = *params;
     params = &zc->appliedParams;
 
-    assert(params->useRowMatchFinder != ZSTD_urm_auto);
+    assert(params->useRowMatchFinder != ZSTD_ps_auto);
+    assert(params->useBlockSplitter != ZSTD_ps_auto);
     if (params->ldmParams.enableLdm) {
         /* Adjust long distance matching parameters */
         ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
@@ -2138,7 +2130,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
     }
 
     ZSTD_cwksp_mark_tables_dirty(&cctx->workspace);
-    assert(params.useRowMatchFinder != ZSTD_urm_auto);
+    assert(params.useRowMatchFinder != ZSTD_ps_auto);
 
     /* copy tables */
     {   size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */)
@@ -2232,8 +2224,10 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
     {   ZSTD_CCtx_params params = dstCCtx->requestedParams;
         /* Copy only compression parameters related to tables. */
         params.cParams = srcCCtx->appliedParams.cParams;
-        assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_urm_auto);
+        assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto);
+        assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto);
         params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder;
+        params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
         params.fParams = fParams;
         ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
                                 /* loadedDictSize */ 0,
@@ -2422,11 +2416,13 @@ static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams)
 /* ZSTD_blockSplitterEnabled():
  * Returns if block splitting param is being used
  * If used, compression will do best effort to split a block in order to improve compression ratio.
+ * At the time this function is called, the parameter must be finalized.
  * Returns 1 if true, 0 otherwise. */
 static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams)
 {
-    DEBUGLOG(5, "ZSTD_blockSplitterEnabled(splitBlocks=%d)", cctxParams->splitBlocks);
-    return (cctxParams->splitBlocks != 0);
+    DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter);
+    assert(cctxParams->useBlockSplitter != ZSTD_ps_auto);
+    return (cctxParams->useBlockSplitter == ZSTD_ps_enable);
 }
 
 /* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types
@@ -2612,7 +2608,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
         size_t const cSize = ZSTD_compressLiterals(
                                     &prevEntropy->huf, &nextEntropy->huf,
                                     cctxParams->cParams.strategy,
-                                    ZSTD_disableLiteralsCompression(cctxParams),
+                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
                                     op, dstCapacity,
                                     literals, litSize,
                                     entropyWorkspace, entropyWkspSize,
@@ -2721,7 +2717,7 @@ ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
 /* ZSTD_selectBlockCompressor() :
  * Not static, but internal use only (used by long distance matcher)
  * assumption : strat is a valid strategy */
-ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
+ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
 {
     static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
         { ZSTD_compressBlock_fast  /* default for 0 */,
@@ -2786,7 +2782,7 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRow
             ZSTD_compressBlock_lazy2_dedicatedDictSearch_row }
         };
         DEBUGLOG(4, "Selecting a row-based matchfinder");
-        assert(useRowMatchFinder != ZSTD_urm_auto);
+        assert(useRowMatchFinder != ZSTD_ps_auto);
         selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy];
     } else {
         selectedCompressor = blockCompressor[(int)dictMode][(int)strat];
@@ -3055,7 +3051,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
                                             const ZSTD_hufCTables_t* prevHuf,
                                                   ZSTD_hufCTables_t* nextHuf,
                                                   ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                  const int disableLiteralsCompression,
+                                                  const int literalsCompressionIsDisabled,
                                                   void* workspace, size_t wkspSize)
 {
     BYTE* const wkspStart = (BYTE*)workspace;
@@ -3073,7 +3069,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
     /* Prepare nextEntropy assuming reusing the existing table */
     ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
 
-    if (disableLiteralsCompression) {
+    if (literalsCompressionIsDisabled) {
         DEBUGLOG(5, "set_basic - disabled");
         hufMetadata->hType = set_basic;
         return 0;
@@ -3220,7 +3216,7 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
         ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
                                             &prevEntropy->huf, &nextEntropy->huf,
                                             &entropyMetadata->hufMetadata,
-                                            ZSTD_disableLiteralsCompression(cctxParams),
+                                            ZSTD_literalsCompressionIsDisabled(cctxParams),
                                             workspace, wkspSize);
     FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
     entropyMetadata->fseMetadata.fseTablesSize =
@@ -3723,6 +3719,7 @@ static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
     U32 nbSeq;
     size_t cSize;
     DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
+    assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable);
 
     {   const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
         FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
@@ -3737,7 +3734,6 @@ static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
         nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart);
     }
 
-    assert(zc->appliedParams.splitBlocks == 1);
     cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq);
     FORWARD_IF_ERROR(cSize, "Splitting blocks failed!");
     return cSize;
@@ -4252,8 +4248,8 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
             assert(ms->chainTable != NULL);
             ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE);
         } else {
-            assert(params->useRowMatchFinder != ZSTD_urm_auto);
-            if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) {
+            assert(params->useRowMatchFinder != ZSTD_ps_auto);
+            if (params->useRowMatchFinder == ZSTD_ps_enable) {
                 size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
                 ZSTD_memset(ms->tagTable, 0, tagTableSize);
                 ZSTD_row_update(ms, iend-HASH_READ_SIZE);
@@ -4753,7 +4749,7 @@ size_t ZSTD_estimateCDictSize_advanced(
          + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
          /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small
           * in case we are using DDS with row-hash. */
-         + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams),
+         + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams),
                                   /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0)
          + (dictLoadMethod == ZSTD_dlm_byRef ? 0
             : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
@@ -4830,7 +4826,7 @@ static size_t ZSTD_initCDict_internal(
 static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
                                       ZSTD_dictLoadMethod_e dictLoadMethod,
                                       ZSTD_compressionParameters cParams,
-                                      ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
+                                      ZSTD_paramSwitch_e useRowMatchFinder,
                                       U32 enableDedicatedDictSearch,
                                       ZSTD_customMem customMem)
 {
@@ -4985,7 +4981,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
                                  ZSTD_dictContentType_e dictContentType,
                                  ZSTD_compressionParameters cParams)
 {
-    ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams);
+    ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
     /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */
     size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0);
     size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
@@ -5568,11 +5564,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
         params.ldmParams.enableLdm = 1;
     }
 
-    if (ZSTD_CParams_useBlockSplitter(&params.cParams)) {
-        DEBUGLOG(4, "Block splitter enabled by default (window size >= 128K, strategy >= btopt)");
-        params.splitBlocks = 1;
-    }
-
+    params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, &params.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
 
 #ifdef ZSTD_MULTITHREAD
index c032d7d18c7ce4e5721ce4d8a292c02a89fac0e7..c168c67e75934799d8ef30445e056a85bdc43fa3 100644 (file)
@@ -179,7 +179,7 @@ typedef struct {
     U32  offCodeSumBasePrice;    /* to compare to log2(offreq)  */
     ZSTD_OptPrice_e priceType;   /* prices can be determined dynamically, or follow a pre-defined cost structure */
     const ZSTD_entropyCTables_t* symbolCosts;  /* pre-calculated dictionary statistics */
-    ZSTD_literalCompressionMode_e literalCompressionMode;
+    ZSTD_paramSwitch_e literalCompressionMode;
 } optState_t;
 
 typedef struct {
@@ -297,7 +297,7 @@ struct ZSTD_CCtx_params_s {
                                 * There is no guarantee that hint is close to actual source size */
 
     ZSTD_dictAttachPref_e attachDictPref;
-    ZSTD_literalCompressionMode_e literalCompressionMode;
+    ZSTD_paramSwitch_e literalCompressionMode;
 
     /* Multithreading: used to pass parameters to mtctx */
     int nbWorkers;
@@ -320,10 +320,10 @@ struct ZSTD_CCtx_params_s {
     int validateSequences;
 
     /* Block splitting */
-    int splitBlocks;
+    ZSTD_paramSwitch_e useBlockSplitter;
 
     /* Param for deciding whether to use row-based matchfinder */
-    ZSTD_useRowMatchFinderMode_e useRowMatchFinder;
+    ZSTD_paramSwitch_e useRowMatchFinder;
 
     /* Always load a dictionary in ext-dict mode (not prefix mode)? */
     int deterministicRefPrefix;
@@ -444,7 +444,7 @@ typedef enum {
 typedef size_t (*ZSTD_blockCompressor) (
         ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize);
-ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
+ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
 
 
 MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@@ -551,17 +551,17 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
     return (srcSize >> minlog) + 2;
 }
 
-MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams)
+MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams)
 {
     switch (cctxParams->literalCompressionMode) {
-    case ZSTD_lcm_huffman:
+    case ZSTD_ps_enable:
         return 0;
-    case ZSTD_lcm_uncompressed:
+    case ZSTD_ps_disable:
         return 1;
     default:
         assert(0 /* impossible: pre-validated */);
         /* fall-through */
-    case ZSTD_lcm_auto:
+    case ZSTD_ps_auto:
         return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
     }
 }
index fa4ebeabd70fc203d6e1d29791727c1114387c05..c97b97a03bd708155cc27fe3a48aaae519d912d0 100644 (file)
@@ -657,7 +657,7 @@ void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
 
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-    ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
+    ZSTD_paramSwitch_e useRowMatchFinder,
     void const* src, size_t srcSize)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
index 393466fa9f518f43f0af4734f76976d12873c1e5..4e68dbf52e38804cc4adbfc87182519a700a21c0 100644 (file)
@@ -66,7 +66,7 @@ size_t ZSTD_ldm_generateSequences(
  */
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
             ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
-            ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
+            ZSTD_paramSwitch_e useRowMatchFinder,
             void const* src, size_t srcSize);
 
 /**
index 9b4b236c0bc74db0c7cc23242632d70974b42eac..40c25daa1f063743a6a3224d1831dd0a81d911ae 100644 (file)
@@ -65,7 +65,7 @@ MEM_STATIC double ZSTD_fCost(U32 price)
 
 static int ZSTD_compressedLiterals(optState_t const* const optPtr)
 {
-    return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed;
+    return optPtr->literalCompressionMode != ZSTD_ps_disable;
 }
 
 static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
index ebe2e04b5b76108253a09a17e983fe401edc5505..2cbb1ddf78dba2148a25ffb5eb3404968c451d44 100644 (file)
@@ -417,7 +417,7 @@ typedef enum {
      * ZSTD_c_stableOutBuffer
      * ZSTD_c_blockDelimiters
      * ZSTD_c_validateSequences
-     * ZSTD_c_splitBlocks
+     * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
@@ -1299,10 +1299,14 @@ typedef enum {
 } ZSTD_literalCompressionMode_e;
 
 typedef enum {
-  ZSTD_urm_auto = 0,                   /* Automatically determine whether or not we use row matchfinder */
-  ZSTD_urm_disableRowMatchFinder = 1,  /* Never use row matchfinder */
-  ZSTD_urm_enableRowMatchFinder = 2    /* Always use row matchfinder when applicable */
-} ZSTD_useRowMatchFinderMode_e;
+  /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
+   * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
+   * or ZSTD_ps_disable allow for a force enable/disable the feature.
+   */
+  ZSTD_ps_auto = 0,         /* Let the library automatically determine whether the feature shall be enabled */
+  ZSTD_ps_enable = 1,       /* Force-enable the feature */
+  ZSTD_ps_disable = 2       /* Do not use the feature */
+} ZSTD_paramSwitch_e;
 
 /***************************************
 *  Frame size functions
@@ -1729,9 +1733,15 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
  * See the comments on that enum for an explanation of the feature. */
 #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
 
-/* Controls how the literals are compressed (default is auto).
- * The value must be of type ZSTD_literalCompressionMode_e.
- * See ZSTD_literalCompressionMode_e enum definition for details.
+/* Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never compress literals.
+ * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
+ * may still be emitted if huffman is not beneficial to use.)
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * literals compression based on the compression parameters - specifically,
+ * negative compression levels do not use literal compression.
  */
 #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
 
@@ -1883,23 +1893,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
  */
 #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
 
-/* ZSTD_c_splitBlocks
- * Default is 0 == disabled. Set to 1 to enable block splitting.
+/* ZSTD_c_useBlockSplitter
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use block splitter.
+ * Set to ZSTD_ps_enable to always use block splitter.
  *
- * Will attempt to split blocks in order to improve compression ratio at the cost of speed.
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * block splitting based on the compression parameters.
  */
-#define ZSTD_c_splitBlocks ZSTD_c_experimentalParam13
+#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
 
 /* ZSTD_c_useRowMatchFinder
- * Default is ZSTD_urm_auto.
- * Controlled with ZSTD_useRowMatchFinderMode_e enum.
- *
- * By default, in ZSTD_urm_auto, when finalizing the compression parameters, the library
- * will decide at runtime whether to use the row-based matchfinder based on support for SIMD
- * instructions as well as the windowLog.
- *
- * Set to ZSTD_urm_disableRowMatchFinder to never use row-based matchfinder.
- * Set to ZSTD_urm_enableRowMatchFinder to force usage of row-based matchfinder.
+ * Controlled with ZSTD_paramSwitch_e enum.
+ * Default is ZSTD_ps_auto.
+ * Set to ZSTD_ps_disable to never use row-based matchfinder.
+ * Set to ZSTD_ps_enable to force usage of row-based matchfinder.
+ *
+ * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
+ * the row-based matchfinder based on support for SIMD instructions and the window log.
+ * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
  */
 #define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
 
index 5cfe624b072ea9b50002525e8943f89e0b304234..09035ea458ee0904aa3eefaa7e4fdfbd159645f0 100644 (file)
@@ -131,7 +131,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
         0, /* ldmHashLog */
         0, /* ldmBuckSizeLog */
         0,  /* ldmHashRateLog */
-        ZSTD_lcm_auto, /* literalCompressionMode */
+        ZSTD_ps_auto, /* literalCompressionMode */
         0 /* useRowMatchFinder */
     };
     return res;
index 9b40dcc29bcbf5f894cd1f82ea14323e11b3532a..11ac85da7f9441206f7743f735f9cd5dd8715a0f 100644 (file)
@@ -116,7 +116,7 @@ typedef struct {
     int ldmHashLog;
     int ldmBucketSizeLog;
     int ldmHashRateLog;
-    ZSTD_literalCompressionMode_e literalCompressionMode;
+    ZSTD_paramSwitch_e literalCompressionMode;
     int useRowMatchFinder;  /* use row-based matchfinder if possible */
 } BMK_advancedParams_t;
 
index 317ec87e05e9d13ffbbb7de9d31a40311c40c934..db64ecb645f6f139bd1a9e8289d2cfb1476391e8 100644 (file)
@@ -320,7 +320,7 @@ struct FIO_prefs_s {
     size_t targetCBlockSize;
     int srcSizeHint;
     int testMode;
-    ZSTD_literalCompressionMode_e literalCompressionMode;
+    ZSTD_paramSwitch_e literalCompressionMode;
 
     /* IO preferences */
     U32 removeSrcFile;
@@ -392,7 +392,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->targetCBlockSize = 0;
     ret->srcSizeHint = 0;
     ret->testMode = 0;
-    ret->literalCompressionMode = ZSTD_lcm_auto;
+    ret->literalCompressionMode = ZSTD_ps_auto;
     ret->excludeCompressedFiles = 0;
     ret->allowBlockDevices = 0;
     return ret;
@@ -510,7 +510,7 @@ void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
 
 void FIO_setLiteralCompressionMode(
         FIO_prefs_t* const prefs,
-        ZSTD_literalCompressionMode_e mode) {
+        ZSTD_paramSwitch_e mode) {
     prefs->literalCompressionMode = mode;
 }
 
index 9d97ec8bfb9cf20741bbb29b39ada72397b36ddc..82d5bc84929feed1ab0669fb956e47063f431cf2 100644 (file)
@@ -100,7 +100,7 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
 void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
 void FIO_setLiteralCompressionMode(
         FIO_prefs_t* const prefs,
-        ZSTD_literalCompressionMode_e mode);
+        ZSTD_paramSwitch_e mode);
 
 void FIO_setProgressSetting(FIO_progressSetting_e progressSetting);
 void FIO_setNotificationLevel(int level);
index 9178a4bef4f9867e1b74ad23708c5a2428cb755b..7e4f530f87039064895def0c0fe680a5fc115b90 100644 (file)
@@ -804,7 +804,7 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NOBENCH
     BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
 #endif
-    ZSTD_literalCompressionMode_e literalCompressionMode = ZSTD_lcm_auto;
+    ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;
 
 
     /* init */
@@ -900,8 +900,8 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression);  continue; }
 #endif
                 if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
-                if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
-                if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
+                if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_ps_enable; continue; }
+                if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_ps_disable; continue; }
                 if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
                 if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
                 if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
index 4d889deeb4550e13c0f5df3438909312428cb370..0fbf3bed7ab0bbd3d33ffb9e716bcdb94718c641 100644 (file)
@@ -96,7 +96,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer);
     setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer);
     setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer);
-    setRand(cctx, ZSTD_c_splitBlocks, 0, 1, producer);
+    setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
     setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
index c4de54247bf2158e7b95af48c7fea45785d471b4..9be7a38ab035ebf34b8eabcf0a95f6c744111271 100644 (file)
@@ -1718,7 +1718,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
     {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitBlocks, 1) );
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
         cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
         CHECK(cSize);
         ZSTD_freeCCtx(cctx);
@@ -1732,7 +1732,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
         cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
         CHECK(cSize1);
-        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_lcm_uncompressed) );
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
         cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
         CHECK(cSize2);
         CHECK_LT(cSize1, cSize2);
@@ -2019,7 +2019,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
                 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
                 size_t nodict_cSize;
                 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
-                ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_urm_enableRowMatchFinder);
+                ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
                 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
                                                            contentStart, contentSize);
                 if (nodict_cSize > target_nodict_cSize[l]) {
index 4c66dd15064613ee70c03492be7e887b1888793f..57cd110c6bbe9dc97d05692b5fe2aaaeab596dca 100644 (file)
@@ -215,7 +215,7 @@ static config_t mt_ldm = {
 
 static param_value_t mt_advanced_param_values[] = {
     {.param = ZSTD_c_nbWorkers, .value = 2},
-    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable},
 };
 
 static config_t mt_advanced = {
@@ -258,7 +258,7 @@ static config_t small_clog = {
 
 static param_value_t const uncompressed_literals_param_values[] = {
     {.param = ZSTD_c_compressionLevel, .value = 3},
-    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable},
 };
 
 static config_t uncompressed_literals = {
@@ -269,7 +269,7 @@ static config_t uncompressed_literals = {
 
 static param_value_t const uncompressed_literals_opt_param_values[] = {
     {.param = ZSTD_c_compressionLevel, .value = 19},
-    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable},
 };
 
 static config_t uncompressed_literals_opt = {
@@ -280,7 +280,7 @@ static config_t uncompressed_literals_opt = {
 
 static param_value_t const huffman_literals_param_values[] = {
     {.param = ZSTD_c_compressionLevel, .value = -1},
-    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_huffman},
+    {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_enable},
 };
 
 static config_t huffman_literals = {
index 5a933c1339536a465eff26de7ae8ffe1f5529976..e98209d8015d7e8a7f5f5063a3a3a36ba79ec0aa 100644 (file)
@@ -36,7 +36,7 @@ LEVEL(3)
 LEVEL(4)
 /* ROW_LEVEL triggers the row hash (force enabled and disabled) with different
  * dictionary strategies, and 16/32/64 row entries based on the level/searchLog.
- * 1 == disabled, 2 == enabled.
+ * 1 == enabled, 2 == disabled.
  */
 ROW_LEVEL(5, 1)
 ROW_LEVEL(5, 2) /* 16-entry rows */
index 9909a15a01490377bb44e4b711d9675352bb697d..923b4498b281e372c4e62bc5c811cbcc3ce51895 100644 (file)
@@ -235,18 +235,18 @@ silesia,                            level 0,                            advanced
 silesia,                            level 1,                            advanced one pass,                  5309098
 silesia,                            level 3,                            advanced one pass,                  4849553
 silesia,                            level 4,                            advanced one pass,                  4786968
-silesia,                            level 5 row 1,                      advanced one pass,                  4640752
-silesia,                            level 5 row 2,                      advanced one pass,                  4638961
+silesia,                            level 5 row 1,                      advanced one pass,                  4638961
+silesia,                            level 5 row 2,                      advanced one pass,                  4640752
 silesia,                            level 5,                            advanced one pass,                  4638961
 silesia,                            level 6,                            advanced one pass,                  4605369
-silesia,                            level 7 row 1,                      advanced one pass,                  4564868
-silesia,                            level 7 row 2,                      advanced one pass,                  4567204
+silesia,                            level 7 row 1,                      advanced one pass,                  4567204
+silesia,                            level 7 row 2,                      advanced one pass,                  4564868
 silesia,                            level 7,                            advanced one pass,                  4567204
 silesia,                            level 9,                            advanced one pass,                  4543310
-silesia,                            level 11 row 1,                     advanced one pass,                  4519288
-silesia,                            level 11 row 2,                     advanced one pass,                  4521399
-silesia,                            level 12 row 1,                     advanced one pass,                  4503116
-silesia,                            level 12 row 2,                     advanced one pass,                  4505153
+silesia,                            level 11 row 1,                     advanced one pass,                  4521399
+silesia,                            level 11 row 2,                     advanced one pass,                  4519288
+silesia,                            level 12 row 1,                     advanced one pass,                  4505153
+silesia,                            level 12 row 2,                     advanced one pass,                  4503116
 silesia,                            level 13,                           advanced one pass,                  4493990
 silesia,                            level 16,                           advanced one pass,                  4359864
 silesia,                            level 19,                           advanced one pass,                  4296880
@@ -269,18 +269,18 @@ silesia.tar,                        level 0,                            advanced
 silesia.tar,                        level 1,                            advanced one pass,                  5331946
 silesia.tar,                        level 3,                            advanced one pass,                  4861424
 silesia.tar,                        level 4,                            advanced one pass,                  4799632
-silesia.tar,                        level 5 row 1,                      advanced one pass,                  4652862
-silesia.tar,                        level 5 row 2,                      advanced one pass,                  4650202
+silesia.tar,                        level 5 row 1,                      advanced one pass,                  4650202
+silesia.tar,                        level 5 row 2,                      advanced one pass,                  4652862
 silesia.tar,                        level 5,                            advanced one pass,                  4650202
 silesia.tar,                        level 6,                            advanced one pass,                  4616811
-silesia.tar,                        level 7 row 1,                      advanced one pass,                  4575393
-silesia.tar,                        level 7 row 2,                      advanced one pass,                  4576829
+silesia.tar,                        level 7 row 1,                      advanced one pass,                  4576829
+silesia.tar,                        level 7 row 2,                      advanced one pass,                  4575393
 silesia.tar,                        level 7,                            advanced one pass,                  4576829
 silesia.tar,                        level 9,                            advanced one pass,                  4552584
-silesia.tar,                        level 11 row 1,                     advanced one pass,                  4529461
-silesia.tar,                        level 11 row 2,                     advanced one pass,                  4530256
-silesia.tar,                        level 12 row 1,                     advanced one pass,                  4513604
-silesia.tar,                        level 12 row 2,                     advanced one pass,                  4514568
+silesia.tar,                        level 11 row 1,                     advanced one pass,                  4530256
+silesia.tar,                        level 11 row 2,                     advanced one pass,                  4529461
+silesia.tar,                        level 12 row 1,                     advanced one pass,                  4514568
+silesia.tar,                        level 12 row 2,                     advanced one pass,                  4513604
 silesia.tar,                        level 13,                           advanced one pass,                  4502956
 silesia.tar,                        level 16,                           advanced one pass,                  4360527
 silesia.tar,                        level 19,                           advanced one pass,                  4267266
@@ -326,16 +326,16 @@ github,                             level 4 with dict dms,              advanced
 github,                             level 4 with dict dds,              advanced one pass,                  41251
 github,                             level 4 with dict copy,             advanced one pass,                  41216
 github,                             level 4 with dict load,             advanced one pass,                  41159
-github,                             level 5 row 1,                      advanced one pass,                  135121
-github,                             level 5 row 1 with dict dms,        advanced one pass,                  38938
-github,                             level 5 row 1 with dict dds,        advanced one pass,                  38732
-github,                             level 5 row 1 with dict copy,       advanced one pass,                  38934
-github,                             level 5 row 1 with dict load,       advanced one pass,                  40725
-github,                             level 5 row 2,                      advanced one pass,                  134584
-github,                             level 5 row 2 with dict dms,        advanced one pass,                  38758
-github,                             level 5 row 2 with dict dds,        advanced one pass,                  38728
-github,                             level 5 row 2 with dict copy,       advanced one pass,                  38759
-github,                             level 5 row 2 with dict load,       advanced one pass,                  41518
+github,                             level 5 row 1,                      advanced one pass,                  134584
+github,                             level 5 row 1 with dict dms,        advanced one pass,                  38758
+github,                             level 5 row 1 with dict dds,        advanced one pass,                  38728
+github,                             level 5 row 1 with dict copy,       advanced one pass,                  38759
+github,                             level 5 row 1 with dict load,       advanced one pass,                  41518
+github,                             level 5 row 2,                      advanced one pass,                  135121
+github,                             level 5 row 2 with dict dms,        advanced one pass,                  38938
+github,                             level 5 row 2 with dict dds,        advanced one pass,                  38732
+github,                             level 5 row 2 with dict copy,       advanced one pass,                  38934
+github,                             level 5 row 2 with dict load,       advanced one pass,                  40725
 github,                             level 5,                            advanced one pass,                  135121
 github,                             level 5 with dict,                  advanced one pass,                  38758
 github,                             level 5 with dict dms,              advanced one pass,                  38758
@@ -348,16 +348,16 @@ github,                             level 6 with dict dms,              advanced
 github,                             level 6 with dict dds,              advanced one pass,                  38636
 github,                             level 6 with dict copy,             advanced one pass,                  38669
 github,                             level 6 with dict load,             advanced one pass,                  40695
-github,                             level 7 row 1,                      advanced one pass,                  135122
-github,                             level 7 row 1 with dict dms,        advanced one pass,                  38860
-github,                             level 7 row 1 with dict dds,        advanced one pass,                  38766
-github,                             level 7 row 1 with dict copy,       advanced one pass,                  38834
-github,                             level 7 row 1 with dict load,       advanced one pass,                  40695
-github,                             level 7 row 2,                      advanced one pass,                  134584
-github,                             level 7 row 2 with dict dms,        advanced one pass,                  38758
-github,                             level 7 row 2 with dict dds,        advanced one pass,                  38745
-github,                             level 7 row 2 with dict copy,       advanced one pass,                  38755
-github,                             level 7 row 2 with dict load,       advanced one pass,                  43154
+github,                             level 7 row 1,                      advanced one pass,                  134584
+github,                             level 7 row 1 with dict dms,        advanced one pass,                  38758
+github,                             level 7 row 1 with dict dds,        advanced one pass,                  38745
+github,                             level 7 row 1 with dict copy,       advanced one pass,                  38755
+github,                             level 7 row 1 with dict load,       advanced one pass,                  43154
+github,                             level 7 row 2,                      advanced one pass,                  135122
+github,                             level 7 row 2 with dict dms,        advanced one pass,                  38860
+github,                             level 7 row 2 with dict dds,        advanced one pass,                  38766
+github,                             level 7 row 2 with dict copy,       advanced one pass,                  38834
+github,                             level 7 row 2 with dict load,       advanced one pass,                  40695
 github,                             level 7,                            advanced one pass,                  135122
 github,                             level 7 with dict,                  advanced one pass,                  38758
 github,                             level 7 with dict dms,              advanced one pass,                  38758
@@ -451,16 +451,16 @@ github.tar,                         level 4 with dict dms,              advanced
 github.tar,                         level 4 with dict dds,              advanced one pass,                  37954
 github.tar,                         level 4 with dict copy,             advanced one pass,                  37948
 github.tar,                         level 4 with dict load,             advanced one pass,                  37927
-github.tar,                         level 5 row 1,                      advanced one pass,                  38534
-github.tar,                         level 5 row 1 with dict dms,        advanced one pass,                  39365
-github.tar,                         level 5 row 1 with dict dds,        advanced one pass,                  39233
-github.tar,                         level 5 row 1 with dict copy,       advanced one pass,                  39715
-github.tar,                         level 5 row 1 with dict load,       advanced one pass,                  38019
-github.tar,                         level 5 row 2,                      advanced one pass,                  38376
-github.tar,                         level 5 row 2 with dict dms,        advanced one pass,                  39024
-github.tar,                         level 5 row 2 with dict dds,        advanced one pass,                  39028
-github.tar,                         level 5 row 2 with dict copy,       advanced one pass,                  39040
-github.tar,                         level 5 row 2 with dict load,       advanced one pass,                  37600
+github.tar,                         level 5 row 1,                      advanced one pass,                  38376
+github.tar,                         level 5 row 1 with dict dms,        advanced one pass,                  39024
+github.tar,                         level 5 row 1 with dict dds,        advanced one pass,                  39028
+github.tar,                         level 5 row 1 with dict copy,       advanced one pass,                  39040
+github.tar,                         level 5 row 1 with dict load,       advanced one pass,                  37600
+github.tar,                         level 5 row 2,                      advanced one pass,                  38534
+github.tar,                         level 5 row 2 with dict dms,        advanced one pass,                  39365
+github.tar,                         level 5 row 2 with dict dds,        advanced one pass,                  39233
+github.tar,                         level 5 row 2 with dict copy,       advanced one pass,                  39715
+github.tar,                         level 5 row 2 with dict load,       advanced one pass,                  38019
 github.tar,                         level 5,                            advanced one pass,                  38376
 github.tar,                         level 5 with dict,                  advanced one pass,                  39040
 github.tar,                         level 5 with dict dms,              advanced one pass,                  39024
@@ -473,16 +473,16 @@ github.tar,                         level 6 with dict dms,              advanced
 github.tar,                         level 6 with dict dds,              advanced one pass,                  38610
 github.tar,                         level 6 with dict copy,             advanced one pass,                  38622
 github.tar,                         level 6 with dict load,             advanced one pass,                  37829
-github.tar,                         level 7 row 1,                      advanced one pass,                  38077
-github.tar,                         level 7 row 1 with dict dms,        advanced one pass,                  38012
-github.tar,                         level 7 row 1 with dict dds,        advanced one pass,                  38014
-github.tar,                         level 7 row 1 with dict copy,       advanced one pass,                  38101
-github.tar,                         level 7 row 1 with dict load,       advanced one pass,                  37402
-github.tar,                         level 7 row 2,                      advanced one pass,                  38073
-github.tar,                         level 7 row 2 with dict dms,        advanced one pass,                  37848
-github.tar,                         level 7 row 2 with dict dds,        advanced one pass,                  37869
-github.tar,                         level 7 row 2 with dict copy,       advanced one pass,                  37848
-github.tar,                         level 7 row 2 with dict load,       advanced one pass,                  37371
+github.tar,                         level 7 row 1,                      advanced one pass,                  38073
+github.tar,                         level 7 row 1 with dict dms,        advanced one pass,                  37848
+github.tar,                         level 7 row 1 with dict dds,        advanced one pass,                  37869
+github.tar,                         level 7 row 1 with dict copy,       advanced one pass,                  37848
+github.tar,                         level 7 row 1 with dict load,       advanced one pass,                  37371
+github.tar,                         level 7 row 2,                      advanced one pass,                  38077
+github.tar,                         level 7 row 2 with dict dms,        advanced one pass,                  38012
+github.tar,                         level 7 row 2 with dict dds,        advanced one pass,                  38014
+github.tar,                         level 7 row 2 with dict copy,       advanced one pass,                  38101
+github.tar,                         level 7 row 2 with dict load,       advanced one pass,                  37402
 github.tar,                         level 7,                            advanced one pass,                  38073
 github.tar,                         level 7 with dict,                  advanced one pass,                  37848
 github.tar,                         level 7 with dict dms,              advanced one pass,                  37848
@@ -499,22 +499,22 @@ github.tar,                         level 11 row 1,                     advanced
 github.tar,                         level 11 row 1 with dict dms,       advanced one pass,                  36963
 github.tar,                         level 11 row 1 with dict dds,       advanced one pass,                  36963
 github.tar,                         level 11 row 1 with dict copy,      advanced one pass,                  36557
-github.tar,                         level 11 row 1 with dict load,      advanced one pass,                  36419
+github.tar,                         level 11 row 1 with dict load,      advanced one pass,                  36424
 github.tar,                         level 11 row 2,                     advanced one pass,                  36435
 github.tar,                         level 11 row 2 with dict dms,       advanced one pass,                  36963
 github.tar,                         level 11 row 2 with dict dds,       advanced one pass,                  36963
 github.tar,                         level 11 row 2 with dict copy,      advanced one pass,                  36557
-github.tar,                         level 11 row 2 with dict load,      advanced one pass,                  36424
-github.tar,                         level 12 row 1,                     advanced one pass,                  36110
+github.tar,                         level 11 row 2 with dict load,      advanced one pass,                  36419
+github.tar,                         level 12 row 1,                     advanced one pass,                  36105
 github.tar,                         level 12 row 1 with dict dms,       advanced one pass,                  36986
 github.tar,                         level 12 row 1 with dict dds,       advanced one pass,                  36986
 github.tar,                         level 12 row 1 with dict copy,      advanced one pass,                  36609
-github.tar,                         level 12 row 1 with dict load,      advanced one pass,                  36459
-github.tar,                         level 12 row 2,                     advanced one pass,                  36105
+github.tar,                         level 12 row 1 with dict load,      advanced one pass,                  36460
+github.tar,                         level 12 row 2,                     advanced one pass,                  36110
 github.tar,                         level 12 row 2 with dict dms,       advanced one pass,                  36986
 github.tar,                         level 12 row 2 with dict dds,       advanced one pass,                  36986
 github.tar,                         level 12 row 2 with dict copy,      advanced one pass,                  36609
-github.tar,                         level 12 row 2 with dict load,      advanced one pass,                  36460
+github.tar,                         level 12 row 2 with dict load,      advanced one pass,                  36459
 github.tar,                         level 13,                           advanced one pass,                  35501
 github.tar,                         level 13 with dict,                 advanced one pass,                  37130
 github.tar,                         level 13 with dict dms,             advanced one pass,                  37267
@@ -553,18 +553,18 @@ silesia,                            level 0,                            advanced
 silesia,                            level 1,                            advanced one pass small out,        5309098
 silesia,                            level 3,                            advanced one pass small out,        4849553
 silesia,                            level 4,                            advanced one pass small out,        4786968
-silesia,                            level 5 row 1,                      advanced one pass small out,        4640752
-silesia,                            level 5 row 2,                      advanced one pass small out,        4638961
+silesia,                            level 5 row 1,                      advanced one pass small out,        4638961
+silesia,                            level 5 row 2,                      advanced one pass small out,        4640752
 silesia,                            level 5,                            advanced one pass small out,        4638961
 silesia,                            level 6,                            advanced one pass small out,        4605369
-silesia,                            level 7 row 1,                      advanced one pass small out,        4564868
-silesia,                            level 7 row 2,                      advanced one pass small out,        4567204
+silesia,                            level 7 row 1,                      advanced one pass small out,        4567204
+silesia,                            level 7 row 2,                      advanced one pass small out,        4564868
 silesia,                            level 7,                            advanced one pass small out,        4567204
 silesia,                            level 9,                            advanced one pass small out,        4543310
-silesia,                            level 11 row 1,                     advanced one pass small out,        4519288
-silesia,                            level 11 row 2,                     advanced one pass small out,        4521399
-silesia,                            level 12 row 1,                     advanced one pass small out,        4503116
-silesia,                            level 12 row 2,                     advanced one pass small out,        4505153
+silesia,                            level 11 row 1,                     advanced one pass small out,        4521399
+silesia,                            level 11 row 2,                     advanced one pass small out,        4519288
+silesia,                            level 12 row 1,                     advanced one pass small out,        4505153
+silesia,                            level 12 row 2,                     advanced one pass small out,        4503116
 silesia,                            level 13,                           advanced one pass small out,        4493990
 silesia,                            level 16,                           advanced one pass small out,        4359864
 silesia,                            level 19,                           advanced one pass small out,        4296880
@@ -587,18 +587,18 @@ silesia.tar,                        level 0,                            advanced
 silesia.tar,                        level 1,                            advanced one pass small out,        5331946
 silesia.tar,                        level 3,                            advanced one pass small out,        4861424
 silesia.tar,                        level 4,                            advanced one pass small out,        4799632
-silesia.tar,                        level 5 row 1,                      advanced one pass small out,        4652862
-silesia.tar,                        level 5 row 2,                      advanced one pass small out,        4650202
+silesia.tar,                        level 5 row 1,                      advanced one pass small out,        4650202
+silesia.tar,                        level 5 row 2,                      advanced one pass small out,        4652862
 silesia.tar,                        level 5,                            advanced one pass small out,        4650202
 silesia.tar,                        level 6,                            advanced one pass small out,        4616811
-silesia.tar,                        level 7 row 1,                      advanced one pass small out,        4575393
-silesia.tar,                        level 7 row 2,                      advanced one pass small out,        4576829
+silesia.tar,                        level 7 row 1,                      advanced one pass small out,        4576829
+silesia.tar,                        level 7 row 2,                      advanced one pass small out,        4575393
 silesia.tar,                        level 7,                            advanced one pass small out,        4576829
 silesia.tar,                        level 9,                            advanced one pass small out,        4552584
-silesia.tar,                        level 11 row 1,                     advanced one pass small out,        4529461
-silesia.tar,                        level 11 row 2,                     advanced one pass small out,        4530256
-silesia.tar,                        level 12 row 1,                     advanced one pass small out,        4513604
-silesia.tar,                        level 12 row 2,                     advanced one pass small out,        4514568
+silesia.tar,                        level 11 row 1,                     advanced one pass small out,        4530256
+silesia.tar,                        level 11 row 2,                     advanced one pass small out,        4529461
+silesia.tar,                        level 12 row 1,                     advanced one pass small out,        4514568
+silesia.tar,                        level 12 row 2,                     advanced one pass small out,        4513604
 silesia.tar,                        level 13,                           advanced one pass small out,        4502956
 silesia.tar,                        level 16,                           advanced one pass small out,        4360527
 silesia.tar,                        level 19,                           advanced one pass small out,        4267266
@@ -644,16 +644,16 @@ github,                             level 4 with dict dms,              advanced
 github,                             level 4 with dict dds,              advanced one pass small out,        41251
 github,                             level 4 with dict copy,             advanced one pass small out,        41216
 github,                             level 4 with dict load,             advanced one pass small out,        41159
-github,                             level 5 row 1,                      advanced one pass small out,        135121
-github,                             level 5 row 1 with dict dms,        advanced one pass small out,        38938
-github,                             level 5 row 1 with dict dds,        advanced one pass small out,        38732
-github,                             level 5 row 1 with dict copy,       advanced one pass small out,        38934
-github,                             level 5 row 1 with dict load,       advanced one pass small out,        40725
-github,                             level 5 row 2,                      advanced one pass small out,        134584
-github,                             level 5 row 2 with dict dms,        advanced one pass small out,        38758
-github,                             level 5 row 2 with dict dds,        advanced one pass small out,        38728
-github,                             level 5 row 2 with dict copy,       advanced one pass small out,        38759
-github,                             level 5 row 2 with dict load,       advanced one pass small out,        41518
+github,                             level 5 row 1,                      advanced one pass small out,        134584
+github,                             level 5 row 1 with dict dms,        advanced one pass small out,        38758
+github,                             level 5 row 1 with dict dds,        advanced one pass small out,        38728
+github,                             level 5 row 1 with dict copy,       advanced one pass small out,        38759
+github,                             level 5 row 1 with dict load,       advanced one pass small out,        41518
+github,                             level 5 row 2,                      advanced one pass small out,        135121
+github,                             level 5 row 2 with dict dms,        advanced one pass small out,        38938
+github,                             level 5 row 2 with dict dds,        advanced one pass small out,        38732
+github,                             level 5 row 2 with dict copy,       advanced one pass small out,        38934
+github,                             level 5 row 2 with dict load,       advanced one pass small out,        40725
 github,                             level 5,                            advanced one pass small out,        135121
 github,                             level 5 with dict,                  advanced one pass small out,        38758
 github,                             level 5 with dict dms,              advanced one pass small out,        38758
@@ -666,16 +666,16 @@ github,                             level 6 with dict dms,              advanced
 github,                             level 6 with dict dds,              advanced one pass small out,        38636
 github,                             level 6 with dict copy,             advanced one pass small out,        38669
 github,                             level 6 with dict load,             advanced one pass small out,        40695
-github,                             level 7 row 1,                      advanced one pass small out,        135122
-github,                             level 7 row 1 with dict dms,        advanced one pass small out,        38860
-github,                             level 7 row 1 with dict dds,        advanced one pass small out,        38766
-github,                             level 7 row 1 with dict copy,       advanced one pass small out,        38834
-github,                             level 7 row 1 with dict load,       advanced one pass small out,        40695
-github,                             level 7 row 2,                      advanced one pass small out,        134584
-github,                             level 7 row 2 with dict dms,        advanced one pass small out,        38758
-github,                             level 7 row 2 with dict dds,        advanced one pass small out,        38745
-github,                             level 7 row 2 with dict copy,       advanced one pass small out,        38755
-github,                             level 7 row 2 with dict load,       advanced one pass small out,        43154
+github,                             level 7 row 1,                      advanced one pass small out,        134584
+github,                             level 7 row 1 with dict dms,        advanced one pass small out,        38758
+github,                             level 7 row 1 with dict dds,        advanced one pass small out,        38745
+github,                             level 7 row 1 with dict copy,       advanced one pass small out,        38755
+github,                             level 7 row 1 with dict load,       advanced one pass small out,        43154
+github,                             level 7 row 2,                      advanced one pass small out,        135122
+github,                             level 7 row 2 with dict dms,        advanced one pass small out,        38860
+github,                             level 7 row 2 with dict dds,        advanced one pass small out,        38766
+github,                             level 7 row 2 with dict copy,       advanced one pass small out,        38834
+github,                             level 7 row 2 with dict load,       advanced one pass small out,        40695
 github,                             level 7,                            advanced one pass small out,        135122
 github,                             level 7 with dict,                  advanced one pass small out,        38758
 github,                             level 7 with dict dms,              advanced one pass small out,        38758
@@ -769,16 +769,16 @@ github.tar,                         level 4 with dict dms,              advanced
 github.tar,                         level 4 with dict dds,              advanced one pass small out,        37954
 github.tar,                         level 4 with dict copy,             advanced one pass small out,        37948
 github.tar,                         level 4 with dict load,             advanced one pass small out,        37927
-github.tar,                         level 5 row 1,                      advanced one pass small out,        38534
-github.tar,                         level 5 row 1 with dict dms,        advanced one pass small out,        39365
-github.tar,                         level 5 row 1 with dict dds,        advanced one pass small out,        39233
-github.tar,                         level 5 row 1 with dict copy,       advanced one pass small out,        39715
-github.tar,                         level 5 row 1 with dict load,       advanced one pass small out,        38019
-github.tar,                         level 5 row 2,                      advanced one pass small out,        38376
-github.tar,                         level 5 row 2 with dict dms,        advanced one pass small out,        39024
-github.tar,                         level 5 row 2 with dict dds,        advanced one pass small out,        39028
-github.tar,                         level 5 row 2 with dict copy,       advanced one pass small out,        39040
-github.tar,                         level 5 row 2 with dict load,       advanced one pass small out,        37600
+github.tar,                         level 5 row 1,                      advanced one pass small out,        38376
+github.tar,                         level 5 row 1 with dict dms,        advanced one pass small out,        39024
+github.tar,                         level 5 row 1 with dict dds,        advanced one pass small out,        39028
+github.tar,                         level 5 row 1 with dict copy,       advanced one pass small out,        39040
+github.tar,                         level 5 row 1 with dict load,       advanced one pass small out,        37600
+github.tar,                         level 5 row 2,                      advanced one pass small out,        38534
+github.tar,                         level 5 row 2 with dict dms,        advanced one pass small out,        39365
+github.tar,                         level 5 row 2 with dict dds,        advanced one pass small out,        39233
+github.tar,                         level 5 row 2 with dict copy,       advanced one pass small out,        39715
+github.tar,                         level 5 row 2 with dict load,       advanced one pass small out,        38019
 github.tar,                         level 5,                            advanced one pass small out,        38376
 github.tar,                         level 5 with dict,                  advanced one pass small out,        39040
 github.tar,                         level 5 with dict dms,              advanced one pass small out,        39024
@@ -791,16 +791,16 @@ github.tar,                         level 6 with dict dms,              advanced
 github.tar,                         level 6 with dict dds,              advanced one pass small out,        38610
 github.tar,                         level 6 with dict copy,             advanced one pass small out,        38622
 github.tar,                         level 6 with dict load,             advanced one pass small out,        37829
-github.tar,                         level 7 row 1,                      advanced one pass small out,        38077
-github.tar,                         level 7 row 1 with dict dms,        advanced one pass small out,        38012
-github.tar,                         level 7 row 1 with dict dds,        advanced one pass small out,        38014
-github.tar,                         level 7 row 1 with dict copy,       advanced one pass small out,        38101
-github.tar,                         level 7 row 1 with dict load,       advanced one pass small out,        37402
-github.tar,                         level 7 row 2,                      advanced one pass small out,        38073
-github.tar,                         level 7 row 2 with dict dms,        advanced one pass small out,        37848
-github.tar,                         level 7 row 2 with dict dds,        advanced one pass small out,        37869
-github.tar,                         level 7 row 2 with dict copy,       advanced one pass small out,        37848
-github.tar,                         level 7 row 2 with dict load,       advanced one pass small out,        37371
+github.tar,                         level 7 row 1,                      advanced one pass small out,        38073
+github.tar,                         level 7 row 1 with dict dms,        advanced one pass small out,        37848
+github.tar,                         level 7 row 1 with dict dds,        advanced one pass small out,        37869
+github.tar,                         level 7 row 1 with dict copy,       advanced one pass small out,        37848
+github.tar,                         level 7 row 1 with dict load,       advanced one pass small out,        37371
+github.tar,                         level 7 row 2,                      advanced one pass small out,        38077
+github.tar,                         level 7 row 2 with dict dms,        advanced one pass small out,        38012
+github.tar,                         level 7 row 2 with dict dds,        advanced one pass small out,        38014
+github.tar,                         level 7 row 2 with dict copy,       advanced one pass small out,        38101
+github.tar,                         level 7 row 2 with dict load,       advanced one pass small out,        37402
 github.tar,                         level 7,                            advanced one pass small out,        38073
 github.tar,                         level 7 with dict,                  advanced one pass small out,        37848
 github.tar,                         level 7 with dict dms,              advanced one pass small out,        37848
@@ -817,22 +817,22 @@ github.tar,                         level 11 row 1,                     advanced
 github.tar,                         level 11 row 1 with dict dms,       advanced one pass small out,        36963
 github.tar,                         level 11 row 1 with dict dds,       advanced one pass small out,        36963
 github.tar,                         level 11 row 1 with dict copy,      advanced one pass small out,        36557
-github.tar,                         level 11 row 1 with dict load,      advanced one pass small out,        36419
+github.tar,                         level 11 row 1 with dict load,      advanced one pass small out,        36424
 github.tar,                         level 11 row 2,                     advanced one pass small out,        36435
 github.tar,                         level 11 row 2 with dict dms,       advanced one pass small out,        36963
 github.tar,                         level 11 row 2 with dict dds,       advanced one pass small out,        36963
 github.tar,                         level 11 row 2 with dict copy,      advanced one pass small out,        36557
-github.tar,                         level 11 row 2 with dict load,      advanced one pass small out,        36424
-github.tar,                         level 12 row 1,                     advanced one pass small out,        36110
+github.tar,                         level 11 row 2 with dict load,      advanced one pass small out,        36419
+github.tar,                         level 12 row 1,                     advanced one pass small out,        36105
 github.tar,                         level 12 row 1 with dict dms,       advanced one pass small out,        36986
 github.tar,                         level 12 row 1 with dict dds,       advanced one pass small out,        36986
 github.tar,                         level 12 row 1 with dict copy,      advanced one pass small out,        36609
-github.tar,                         level 12 row 1 with dict load,      advanced one pass small out,        36459
-github.tar,                         level 12 row 2,                     advanced one pass small out,        36105
+github.tar,                         level 12 row 1 with dict load,      advanced one pass small out,        36460
+github.tar,                         level 12 row 2,                     advanced one pass small out,        36110
 github.tar,                         level 12 row 2 with dict dms,       advanced one pass small out,        36986
 github.tar,                         level 12 row 2 with dict dds,       advanced one pass small out,        36986
 github.tar,                         level 12 row 2 with dict copy,      advanced one pass small out,        36609
-github.tar,                         level 12 row 2 with dict load,      advanced one pass small out,        36460
+github.tar,                         level 12 row 2 with dict load,      advanced one pass small out,        36459
 github.tar,                         level 13,                           advanced one pass small out,        35501
 github.tar,                         level 13 with dict,                 advanced one pass small out,        37130
 github.tar,                         level 13 with dict dms,             advanced one pass small out,        37267
@@ -871,18 +871,18 @@ silesia,                            level 0,                            advanced
 silesia,                            level 1,                            advanced streaming,                 5312694
 silesia,                            level 3,                            advanced streaming,                 4849553
 silesia,                            level 4,                            advanced streaming,                 4786968
-silesia,                            level 5 row 1,                      advanced streaming,                 4640752
-silesia,                            level 5 row 2,                      advanced streaming,                 4638961
+silesia,                            level 5 row 1,                      advanced streaming,                 4638961
+silesia,                            level 5 row 2,                      advanced streaming,                 4640752
 silesia,                            level 5,                            advanced streaming,                 4638961
 silesia,                            level 6,                            advanced streaming,                 4605369
-silesia,                            level 7 row 1,                      advanced streaming,                 4564868
-silesia,                            level 7 row 2,                      advanced streaming,                 4567204
+silesia,                            level 7 row 1,                      advanced streaming,                 4567204
+silesia,                            level 7 row 2,                      advanced streaming,                 4564868
 silesia,                            level 7,                            advanced streaming,                 4567204
 silesia,                            level 9,                            advanced streaming,                 4543310
-silesia,                            level 11 row 1,                     advanced streaming,                 4519288
-silesia,                            level 11 row 2,                     advanced streaming,                 4521399
-silesia,                            level 12 row 1,                     advanced streaming,                 4503116
-silesia,                            level 12 row 2,                     advanced streaming,                 4505153
+silesia,                            level 11 row 1,                     advanced streaming,                 4521399
+silesia,                            level 11 row 2,                     advanced streaming,                 4519288
+silesia,                            level 12 row 1,                     advanced streaming,                 4505153
+silesia,                            level 12 row 2,                     advanced streaming,                 4503116
 silesia,                            level 13,                           advanced streaming,                 4493990
 silesia,                            level 16,                           advanced streaming,                 4359864
 silesia,                            level 19,                           advanced streaming,                 4296880
@@ -905,18 +905,18 @@ silesia.tar,                        level 0,                            advanced
 silesia.tar,                        level 1,                            advanced streaming,                 5334890
 silesia.tar,                        level 3,                            advanced streaming,                 4861426
 silesia.tar,                        level 4,                            advanced streaming,                 4799632
-silesia.tar,                        level 5 row 1,                      advanced streaming,                 4652866
-silesia.tar,                        level 5 row 2,                      advanced streaming,                 4650207
+silesia.tar,                        level 5 row 1,                      advanced streaming,                 4650207
+silesia.tar,                        level 5 row 2,                      advanced streaming,                 4652866
 silesia.tar,                        level 5,                            advanced streaming,                 4650207
 silesia.tar,                        level 6,                            advanced streaming,                 4616816
-silesia.tar,                        level 7 row 1,                      advanced streaming,                 4575394
-silesia.tar,                        level 7 row 2,                      advanced streaming,                 4576831
+silesia.tar,                        level 7 row 1,                      advanced streaming,                 4576831
+silesia.tar,                        level 7 row 2,                      advanced streaming,                 4575394
 silesia.tar,                        level 7,                            advanced streaming,                 4576831
 silesia.tar,                        level 9,                            advanced streaming,                 4552590
-silesia.tar,                        level 11 row 1,                     advanced streaming,                 4529461
-silesia.tar,                        level 11 row 2,                     advanced streaming,                 4530258
-silesia.tar,                        level 12 row 1,                     advanced streaming,                 4513604
-silesia.tar,                        level 12 row 2,                     advanced streaming,                 4514569
+silesia.tar,                        level 11 row 1,                     advanced streaming,                 4530258
+silesia.tar,                        level 11 row 2,                     advanced streaming,                 4529461
+silesia.tar,                        level 12 row 1,                     advanced streaming,                 4514569
+silesia.tar,                        level 12 row 2,                     advanced streaming,                 4513604
 silesia.tar,                        level 13,                           advanced streaming,                 4502956
 silesia.tar,                        level 16,                           advanced streaming,                 4360527
 silesia.tar,                        level 19,                           advanced streaming,                 4267266
@@ -962,16 +962,16 @@ github,                             level 4 with dict dms,              advanced
 github,                             level 4 with dict dds,              advanced streaming,                 41251
 github,                             level 4 with dict copy,             advanced streaming,                 41216
 github,                             level 4 with dict load,             advanced streaming,                 41159
-github,                             level 5 row 1,                      advanced streaming,                 135121
-github,                             level 5 row 1 with dict dms,        advanced streaming,                 38938
-github,                             level 5 row 1 with dict dds,        advanced streaming,                 38732
-github,                             level 5 row 1 with dict copy,       advanced streaming,                 38934
-github,                             level 5 row 1 with dict load,       advanced streaming,                 40725
-github,                             level 5 row 2,                      advanced streaming,                 134584
-github,                             level 5 row 2 with dict dms,        advanced streaming,                 38758
-github,                             level 5 row 2 with dict dds,        advanced streaming,                 38728
-github,                             level 5 row 2 with dict copy,       advanced streaming,                 38759
-github,                             level 5 row 2 with dict load,       advanced streaming,                 41518
+github,                             level 5 row 1,                      advanced streaming,                 134584
+github,                             level 5 row 1 with dict dms,        advanced streaming,                 38758
+github,                             level 5 row 1 with dict dds,        advanced streaming,                 38728
+github,                             level 5 row 1 with dict copy,       advanced streaming,                 38759
+github,                             level 5 row 1 with dict load,       advanced streaming,                 41518
+github,                             level 5 row 2,                      advanced streaming,                 135121
+github,                             level 5 row 2 with dict dms,        advanced streaming,                 38938
+github,                             level 5 row 2 with dict dds,        advanced streaming,                 38732
+github,                             level 5 row 2 with dict copy,       advanced streaming,                 38934
+github,                             level 5 row 2 with dict load,       advanced streaming,                 40725
 github,                             level 5,                            advanced streaming,                 135121
 github,                             level 5 with dict,                  advanced streaming,                 38758
 github,                             level 5 with dict dms,              advanced streaming,                 38758
@@ -984,16 +984,16 @@ github,                             level 6 with dict dms,              advanced
 github,                             level 6 with dict dds,              advanced streaming,                 38636
 github,                             level 6 with dict copy,             advanced streaming,                 38669
 github,                             level 6 with dict load,             advanced streaming,                 40695
-github,                             level 7 row 1,                      advanced streaming,                 135122
-github,                             level 7 row 1 with dict dms,        advanced streaming,                 38860
-github,                             level 7 row 1 with dict dds,        advanced streaming,                 38766
-github,                             level 7 row 1 with dict copy,       advanced streaming,                 38834
-github,                             level 7 row 1 with dict load,       advanced streaming,                 40695
-github,                             level 7 row 2,                      advanced streaming,                 134584
-github,                             level 7 row 2 with dict dms,        advanced streaming,                 38758
-github,                             level 7 row 2 with dict dds,        advanced streaming,                 38745
-github,                             level 7 row 2 with dict copy,       advanced streaming,                 38755
-github,                             level 7 row 2 with dict load,       advanced streaming,                 43154
+github,                             level 7 row 1,                      advanced streaming,                 134584
+github,                             level 7 row 1 with dict dms,        advanced streaming,                 38758
+github,                             level 7 row 1 with dict dds,        advanced streaming,                 38745
+github,                             level 7 row 1 with dict copy,       advanced streaming,                 38755
+github,                             level 7 row 1 with dict load,       advanced streaming,                 43154
+github,                             level 7 row 2,                      advanced streaming,                 135122
+github,                             level 7 row 2 with dict dms,        advanced streaming,                 38860
+github,                             level 7 row 2 with dict dds,        advanced streaming,                 38766
+github,                             level 7 row 2 with dict copy,       advanced streaming,                 38834
+github,                             level 7 row 2 with dict load,       advanced streaming,                 40695
 github,                             level 7,                            advanced streaming,                 135122
 github,                             level 7 with dict,                  advanced streaming,                 38758
 github,                             level 7 with dict dms,              advanced streaming,                 38758
@@ -1087,16 +1087,16 @@ github.tar,                         level 4 with dict dms,              advanced
 github.tar,                         level 4 with dict dds,              advanced streaming,                 37954
 github.tar,                         level 4 with dict copy,             advanced streaming,                 37948
 github.tar,                         level 4 with dict load,             advanced streaming,                 37927
-github.tar,                         level 5 row 1,                      advanced streaming,                 38534
-github.tar,                         level 5 row 1 with dict dms,        advanced streaming,                 39365
-github.tar,                         level 5 row 1 with dict dds,        advanced streaming,                 39233
-github.tar,                         level 5 row 1 with dict copy,       advanced streaming,                 39715
-github.tar,                         level 5 row 1 with dict load,       advanced streaming,                 38019
-github.tar,                         level 5 row 2,                      advanced streaming,                 38376
-github.tar,                         level 5 row 2 with dict dms,        advanced streaming,                 39024
-github.tar,                         level 5 row 2 with dict dds,        advanced streaming,                 39028
-github.tar,                         level 5 row 2 with dict copy,       advanced streaming,                 39040
-github.tar,                         level 5 row 2 with dict load,       advanced streaming,                 37600
+github.tar,                         level 5 row 1,                      advanced streaming,                 38376
+github.tar,                         level 5 row 1 with dict dms,        advanced streaming,                 39024
+github.tar,                         level 5 row 1 with dict dds,        advanced streaming,                 39028
+github.tar,                         level 5 row 1 with dict copy,       advanced streaming,                 39040
+github.tar,                         level 5 row 1 with dict load,       advanced streaming,                 37600
+github.tar,                         level 5 row 2,                      advanced streaming,                 38534
+github.tar,                         level 5 row 2 with dict dms,        advanced streaming,                 39365
+github.tar,                         level 5 row 2 with dict dds,        advanced streaming,                 39233
+github.tar,                         level 5 row 2 with dict copy,       advanced streaming,                 39715
+github.tar,                         level 5 row 2 with dict load,       advanced streaming,                 38019
 github.tar,                         level 5,                            advanced streaming,                 38376
 github.tar,                         level 5 with dict,                  advanced streaming,                 39040
 github.tar,                         level 5 with dict dms,              advanced streaming,                 39024
@@ -1109,16 +1109,16 @@ github.tar,                         level 6 with dict dms,              advanced
 github.tar,                         level 6 with dict dds,              advanced streaming,                 38610
 github.tar,                         level 6 with dict copy,             advanced streaming,                 38622
 github.tar,                         level 6 with dict load,             advanced streaming,                 37829
-github.tar,                         level 7 row 1,                      advanced streaming,                 38077
-github.tar,                         level 7 row 1 with dict dms,        advanced streaming,                 38012
-github.tar,                         level 7 row 1 with dict dds,        advanced streaming,                 38014
-github.tar,                         level 7 row 1 with dict copy,       advanced streaming,                 38101
-github.tar,                         level 7 row 1 with dict load,       advanced streaming,                 37402
-github.tar,                         level 7 row 2,                      advanced streaming,                 38073
-github.tar,                         level 7 row 2 with dict dms,        advanced streaming,                 37848
-github.tar,                         level 7 row 2 with dict dds,        advanced streaming,                 37869
-github.tar,                         level 7 row 2 with dict copy,       advanced streaming,                 37848
-github.tar,                         level 7 row 2 with dict load,       advanced streaming,                 37371
+github.tar,                         level 7 row 1,                      advanced streaming,                 38073
+github.tar,                         level 7 row 1 with dict dms,        advanced streaming,                 37848
+github.tar,                         level 7 row 1 with dict dds,        advanced streaming,                 37869
+github.tar,                         level 7 row 1 with dict copy,       advanced streaming,                 37848
+github.tar,                         level 7 row 1 with dict load,       advanced streaming,                 37371
+github.tar,                         level 7 row 2,                      advanced streaming,                 38077
+github.tar,                         level 7 row 2 with dict dms,        advanced streaming,                 38012
+github.tar,                         level 7 row 2 with dict dds,        advanced streaming,                 38014
+github.tar,                         level 7 row 2 with dict copy,       advanced streaming,                 38101
+github.tar,                         level 7 row 2 with dict load,       advanced streaming,                 37402
 github.tar,                         level 7,                            advanced streaming,                 38073
 github.tar,                         level 7 with dict,                  advanced streaming,                 37848
 github.tar,                         level 7 with dict dms,              advanced streaming,                 37848
@@ -1135,22 +1135,22 @@ github.tar,                         level 11 row 1,                     advanced
 github.tar,                         level 11 row 1 with dict dms,       advanced streaming,                 36963
 github.tar,                         level 11 row 1 with dict dds,       advanced streaming,                 36963
 github.tar,                         level 11 row 1 with dict copy,      advanced streaming,                 36557
-github.tar,                         level 11 row 1 with dict load,      advanced streaming,                 36419
+github.tar,                         level 11 row 1 with dict load,      advanced streaming,                 36424
 github.tar,                         level 11 row 2,                     advanced streaming,                 36435
 github.tar,                         level 11 row 2 with dict dms,       advanced streaming,                 36963
 github.tar,                         level 11 row 2 with dict dds,       advanced streaming,                 36963
 github.tar,                         level 11 row 2 with dict copy,      advanced streaming,                 36557
-github.tar,                         level 11 row 2 with dict load,      advanced streaming,                 36424
-github.tar,                         level 12 row 1,                     advanced streaming,                 36110
+github.tar,                         level 11 row 2 with dict load,      advanced streaming,                 36419
+github.tar,                         level 12 row 1,                     advanced streaming,                 36105
 github.tar,                         level 12 row 1 with dict dms,       advanced streaming,                 36986
 github.tar,                         level 12 row 1 with dict dds,       advanced streaming,                 36986
 github.tar,                         level 12 row 1 with dict copy,      advanced streaming,                 36609
-github.tar,                         level 12 row 1 with dict load,      advanced streaming,                 36459
-github.tar,                         level 12 row 2,                     advanced streaming,                 36105
+github.tar,                         level 12 row 1 with dict load,      advanced streaming,                 36460
+github.tar,                         level 12 row 2,                     advanced streaming,                 36110
 github.tar,                         level 12 row 2 with dict dms,       advanced streaming,                 36986
 github.tar,                         level 12 row 2 with dict dds,       advanced streaming,                 36986
 github.tar,                         level 12 row 2 with dict copy,      advanced streaming,                 36609
-github.tar,                         level 12 row 2 with dict load,      advanced streaming,                 36460
+github.tar,                         level 12 row 2 with dict load,      advanced streaming,                 36459
 github.tar,                         level 13,                           advanced streaming,                 35501
 github.tar,                         level 13 with dict,                 advanced streaming,                 37130
 github.tar,                         level 13 with dict dms,             advanced streaming,                 37267