]>
Commit | Line | Data |
---|---|---|
32fb407c | 1 | /* |
a494308a | 2 | * Copyright (c) Yann Collet, Facebook, Inc. |
4ded9e59 YC |
3 | * All rights reserved. |
4 | * | |
32fb407c YC |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found | |
7 | * in the COPYING file in the root directory of this source tree). | |
3128e03b | 8 | * You may select, at your option, one of the above-listed licenses. |
4ded9e59 | 9 | */ |
f3eca253 | 10 | |
7d360280 | 11 | /*-************************************* |
ae7aa066 | 12 | * Dependencies |
f3eca253 | 13 | ***************************************/ |
c465f244 | 14 | #include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ |
6028827f FH |
15 | #include "../common/cpu.h" |
16 | #include "../common/mem.h" | |
608f1bfc | 17 | #include "hist.h" /* HIST_countFast_wksp */ |
5a0c8e24 | 18 | #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ |
6028827f | 19 | #include "../common/fse.h" |
130fe113 | 20 | #define HUF_STATIC_LINKING_ONLY |
6028827f | 21 | #include "../common/huf.h" |
ee441d5d | 22 | #include "zstd_compress_internal.h" |
1dc98de2 EP |
23 | #include "zstd_compress_sequences.h" |
24 | #include "zstd_compress_literals.h" | |
721726d6 NT |
25 | #include "zstd_fast.h" |
26 | #include "zstd_double_fast.h" | |
27 | #include "zstd_lazy.h" | |
28 | #include "zstd_opt.h" | |
360428c5 | 29 | #include "zstd_ldm.h" |
c26d32c9 | 30 | #include "zstd_compress_superblock.h" |
31533bac | 31 | |
6a1e526e NT |
32 | /* *************************************************************** |
33 | * Tuning parameters | |
34 | *****************************************************************/ | |
35 | /*! | |
36 | * COMPRESS_HEAPMODE : | |
37 | * Select how default decompression function ZSTD_compress() allocates its context, | |
38 | * on stack (0, default), or into heap (1). | |
39 | * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. | |
40 | */ | |
41 | #ifndef ZSTD_COMPRESS_HEAPMODE | |
42 | # define ZSTD_COMPRESS_HEAPMODE 0 | |
43 | #endif | |
44 | ||
f3eca253 | 45 | |
7d360280 | 46 | /*-************************************* |
59d1f797 YC |
47 | * Helper functions |
48 | ***************************************/ | |
7ce89187 SH |
49 | /* ZSTD_compressBound() |
50 | * Note that the result from this function is only compatible with the "normal" | |
51 | * full-block strategy. | |
52 | * When there are a lot of small blocks due to frequent flush in streaming mode | |
036b30b5 NT |
53 | * the overhead of headers can make the compressed data to be larger than the |
54 | * return value of ZSTD_compressBound(). | |
7ce89187 | 55 | */ |
3f75d525 | 56 | size_t ZSTD_compressBound(size_t srcSize) { |
5b10345b | 57 | return ZSTD_COMPRESSBOUND(srcSize); |
3f75d525 | 58 | } |
59d1f797 YC |
59 | |
60 | ||
7d360280 | 61 | /*-************************************* |
14983e7a | 62 | * Context memory management |
f3eca253 | 63 | ***************************************/ |
1880337c | 64 | struct ZSTD_CDict_s { |
1880337c YC |
65 | const void* dictContent; |
66 | size_t dictContentSize; | |
7083f790 | 67 | ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ |
786f2266 | 68 | U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ |
077a2d7d | 69 | ZSTD_cwksp workspace; |
16bd0fd4 NT |
70 | ZSTD_matchState_t matchState; |
71 | ZSTD_compressedBlockState_t cBlockState; | |
16bd0fd4 NT |
72 | ZSTD_customMem customMem; |
73 | U32 dictID; | |
4f7d26b0 | 74 | int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ |
4694423c NT |
75 | ZSTD_useRowMatchFinderMode_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use |
76 | * row-based matchfinder. Unless the cdict is reloaded, we will use | |
77 | * the same greedy/lazy matchfinder at compression time. | |
78 | */ | |
8c910d20 | 79 | }; /* typedef'd to ZSTD_CDict within "zstd.h" */ |
0be6fd34 | 80 | |
5be2dd25 | 81 | ZSTD_CCtx* ZSTD_createCCtx(void) |
f3eca253 | 82 | { |
ae728a43 | 83 | return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); |
50e82c01 | 84 | } |
85 | ||
e3c42c73 YC |
86 | static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) |
87 | { | |
88 | assert(cctx != NULL); | |
c465f244 | 89 | ZSTD_memset(cctx, 0, sizeof(*cctx)); |
e3c42c73 YC |
90 | cctx->customMem = memManager; |
91 | cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); | |
5c686391 | 92 | { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); |
e3c42c73 YC |
93 | assert(!ZSTD_isError(err)); |
94 | (void)err; | |
95 | } | |
96 | } | |
97 | ||
50e82c01 | 98 | ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) |
99 | { | |
a0ba849f | 100 | ZSTD_STATIC_ASSERT(zcss_init==0); |
d3de3d51 | 101 | ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); |
9261476b | 102 | if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; |
a686d306 | 103 | { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); |
f36da5b4 | 104 | if (!cctx) return NULL; |
e3c42c73 | 105 | ZSTD_initCCtx(cctx, customMem); |
f36da5b4 YC |
106 | return cctx; |
107 | } | |
f3eca253 YC |
108 | } |
109 | ||
608f1bfc | 110 | ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) |
c7fe262d | 111 | { |
077a2d7d | 112 | ZSTD_cwksp ws; |
65057cf0 | 113 | ZSTD_CCtx* cctx; |
c7fe262d YC |
114 | if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ |
115 | if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ | |
9dab03db | 116 | ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); |
65057cf0 | 117 | |
077a2d7d | 118 | cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); |
608f1bfc YC |
119 | if (cctx == NULL) return NULL; |
120 | ||
c465f244 | 121 | ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); |
8549ae9f | 122 | ZSTD_cwksp_move(&cctx->workspace, &ws); |
c7fe262d | 123 | cctx->staticSize = workspaceSize; |
c7fe262d | 124 | |
887cd4e3 | 125 | /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ |
a9077939 | 126 | if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; |
077a2d7d FH |
127 | cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); |
128 | cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); | |
a9077939 | 129 | cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); |
b58f0153 | 130 | cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); |
69c2cdb4 | 131 | return cctx; |
f3eca253 YC |
132 | } |
133 | ||
6b053b9f NT |
134 | /** |
135 | * Clears and frees all of the dictionaries in the CCtx. | |
136 | */ | |
137 | static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) | |
138 | { | |
a686d306 | 139 | ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); |
6b053b9f | 140 | ZSTD_freeCDict(cctx->localDict.cdict); |
c465f244 NT |
141 | ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); |
142 | ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); | |
6b053b9f NT |
143 | cctx->cdict = NULL; |
144 | } | |
145 | ||
146 | static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) | |
147 | { | |
148 | size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; | |
149 | size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); | |
150 | return bufferSize + cdictSize; | |
151 | } | |
152 | ||
e3c42c73 | 153 | static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) |
083fcc82 | 154 | { |
e3c42c73 YC |
155 | assert(cctx != NULL); |
156 | assert(cctx->staticSize == 0); | |
6b053b9f | 157 | ZSTD_clearAllDicts(cctx); |
0d6ecc72 | 158 | #ifdef ZSTD_MULTITHREAD |
f36da5b4 | 159 | ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; |
0d6ecc72 | 160 | #endif |
ef0b5707 | 161 | ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); |
e3c42c73 YC |
162 | } |
163 | ||
164 | size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) | |
165 | { | |
166 | if (cctx==NULL) return 0; /* support free on NULL */ | |
cafc3b1b FH |
167 | RETURN_ERROR_IF(cctx->staticSize, memory_allocation, |
168 | "not compatible with static CCtx"); | |
2c80a9f8 FH |
169 | { |
170 | int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); | |
171 | ZSTD_freeCCtxContent(cctx); | |
172 | if (!cctxInWorkspace) { | |
a686d306 | 173 | ZSTD_customFree(cctx, cctx->customMem); |
2c80a9f8 | 174 | } |
ef0b5707 | 175 | } |
e3c42c73 | 176 | return 0; |
083fcc82 | 177 | } |
f3eca253 | 178 | |
0d6ecc72 YC |
179 | |
180 | static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) | |
181 | { | |
182 | #ifdef ZSTD_MULTITHREAD | |
183 | return ZSTDMT_sizeof_CCtx(cctx->mtctx); | |
184 | #else | |
aec945f0 | 185 | (void)cctx; |
0d6ecc72 YC |
186 | return 0; |
187 | #endif | |
188 | } | |
189 | ||
190 | ||
70e3b313 | 191 | size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) |
3ae543ce | 192 | { |
d7c6589d | 193 | if (cctx==NULL) return 0; /* support sizeof on NULL */ |
eb6f69d9 FH |
194 | /* cctx may be in the workspace */ |
195 | return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) | |
196 | + ZSTD_cwksp_sizeof(&cctx->workspace) | |
6b053b9f | 197 | + ZSTD_sizeof_localDict(cctx->localDict) |
0d6ecc72 | 198 | + ZSTD_sizeof_mtctx(cctx); |
3ae543ce YC |
199 | } |
200 | ||
009d604e YC |
201 | size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) |
202 | { | |
203 | return ZSTD_sizeof_CCtx(zcs); /* same object */ | |
204 | } | |
205 | ||
b0edb7fb YC |
206 | /* private API call, for dictBuilder only */ |
207 | const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } | |
208 | ||
4694423c NT |
209 | /* Returns true if the strategy supports using a row based matchfinder */ |
210 | static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { | |
211 | return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); | |
212 | } | |
213 | ||
214 | /* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder | |
215 | * for this compression. | |
216 | */ | |
217 | static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_useRowMatchFinderMode_e mode) { | |
218 | assert(mode != ZSTD_urm_auto); | |
219 | return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_urm_enableRowMatchFinder); | |
220 | } | |
221 | ||
222 | /* Returns row matchfinder usage enum given an initial mode and cParams */ | |
223 | static ZSTD_useRowMatchFinderMode_e ZSTD_resolveRowMatchFinderMode(ZSTD_useRowMatchFinderMode_e mode, | |
224 | const ZSTD_compressionParameters* const cParams) { | |
225 | #if !defined(ZSTD_NO_INTRINSICS) && (defined(__SSE2__) || defined(__ARM_NEON)) | |
226 | int const kHasSIMD128 = 1; | |
227 | #else | |
228 | int const kHasSIMD128 = 0; | |
229 | #endif | |
230 | if (mode != ZSTD_urm_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ | |
231 | mode = ZSTD_urm_disableRowMatchFinder; | |
232 | if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; | |
233 | if (kHasSIMD128) { | |
234 | if (cParams->windowLog > 14) mode = ZSTD_urm_enableRowMatchFinder; | |
235 | } else { | |
236 | if (cParams->windowLog > 17) mode = ZSTD_urm_enableRowMatchFinder; | |
237 | } | |
238 | return mode; | |
239 | } | |
240 | ||
241 | /* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ | |
242 | static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, | |
243 | const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, | |
244 | const U32 forDDSDict) { | |
245 | assert(useRowMatchFinder != ZSTD_urm_auto); | |
246 | /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. | |
247 | * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. | |
248 | */ | |
249 | return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); | |
250 | } | |
251 | ||
8bdb32ae | 252 | /* Returns 1 if compression parameters are such that we should |
253 | * enable long distance matching (wlog >= 27, strategy >= btopt). | |
254 | * Returns 0 otherwise. | |
255 | */ | |
256 | static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const cParams) { | |
257 | return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27; | |
258 | } | |
259 | ||
c90e81a6 SH |
260 | /* Returns 1 if compression parameters are such that we should |
261 | * enable blockSplitter (wlog >= 17, strategy >= btopt). | |
262 | * Returns 0 otherwise. | |
263 | */ | |
264 | static U32 ZSTD_CParams_useBlockSplitter(const ZSTD_compressionParameters* const cParams) { | |
265 | return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17; | |
266 | } | |
267 | ||
91b30dbe SL |
268 | static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( |
269 | ZSTD_compressionParameters cParams) | |
270 | { | |
271 | ZSTD_CCtx_params cctxParams; | |
8930c6e5 FH |
272 | /* should not matter, as all cParams are presumed properly defined */ |
273 | ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); | |
91b30dbe | 274 | cctxParams.cParams = cParams; |
d28d8a1d | 275 | |
c90e81a6 | 276 | /* Adjust advanced params according to cParams */ |
8bdb32ae | 277 | if (ZSTD_CParams_shouldEnableLdm(&cParams)) { |
d28d8a1d | 278 | DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including LDM into cctx params"); |
279 | cctxParams.ldmParams.enableLdm = 1; | |
280 | /* LDM is enabled by default for optimal parser and window size >= 128MB */ | |
281 | ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); | |
282 | assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); | |
283 | assert(cctxParams.ldmParams.hashRateLog < 32); | |
284 | } | |
285 | ||
c90e81a6 SH |
286 | if (ZSTD_CParams_useBlockSplitter(&cParams)) { |
287 | DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including block splitting into cctx params"); | |
288 | cctxParams.splitBlocks = 1; | |
289 | } | |
290 | ||
4694423c | 291 | cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); |
6f4d0778 | 292 | assert(!ZSTD_checkCParams(cParams)); |
91b30dbe SL |
293 | return cctxParams; |
294 | } | |
295 | ||
5bc2c1e9 SL |
296 | static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( |
297 | ZSTD_customMem customMem) | |
4169f491 | 298 | { |
5bc2c1e9 | 299 | ZSTD_CCtx_params* params; |
9261476b | 300 | if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; |
a686d306 | 301 | params = (ZSTD_CCtx_params*)ZSTD_customCalloc( |
5bc2c1e9 | 302 | sizeof(ZSTD_CCtx_params), customMem); |
4169f491 | 303 | if (!params) { return NULL; } |
8930c6e5 | 304 | ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); |
5bc2c1e9 | 305 | params->customMem = customMem; |
4169f491 SL |
306 | return params; |
307 | } | |
308 | ||
5bc2c1e9 SL |
309 | ZSTD_CCtx_params* ZSTD_createCCtxParams(void) |
310 | { | |
311 | return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); | |
312 | } | |
313 | ||
314 | size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) | |
315 | { | |
316 | if (params == NULL) { return 0; } | |
a686d306 | 317 | ZSTD_customFree(params, params->customMem); |
5bc2c1e9 SL |
318 | return 0; |
319 | } | |
320 | ||
c8b3d389 | 321 | size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) |
023b24e6 | 322 | { |
c8b3d389 | 323 | return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); |
023b24e6 SL |
324 | } |
325 | ||
c8b3d389 | 326 | size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { |
5e5f2626 | 327 | RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); |
c465f244 | 328 | ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); |
0e56a84a | 329 | cctxParams->compressionLevel = compressionLevel; |
6f4d0778 | 330 | cctxParams->fParams.contentSizeFlag = 1; |
0e56a84a | 331 | return 0; |
0744592d SL |
332 | } |
333 | ||
c62eb059 NT |
334 | #define ZSTD_NO_CLEVEL 0 |
335 | ||
7736549b NT |
336 | /** |
337 | * Initializes the cctxParams from params and compressionLevel. | |
338 | * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. | |
339 | */ | |
340 | static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel) | |
341 | { | |
342 | assert(!ZSTD_checkCParams(params->cParams)); | |
343 | ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); | |
344 | cctxParams->cParams = params->cParams; | |
345 | cctxParams->fParams = params->fParams; | |
346 | /* Should not matter, as all cParams are presumed properly defined. | |
347 | * But, set it for tracing anyway. | |
348 | */ | |
349 | cctxParams->compressionLevel = compressionLevel; | |
4694423c NT |
350 | cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); |
351 | DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d", cctxParams->useRowMatchFinder); | |
7736549b NT |
352 | } |
353 | ||
c8b3d389 | 354 | size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) |
4169f491 | 355 | { |
5e5f2626 FH |
356 | RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); |
357 | FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); | |
7736549b | 358 | ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); |
4169f491 SL |
359 | return 0; |
360 | } | |
361 | ||
7736549b NT |
362 | /** |
363 | * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. | |
364 | * @param param Validated zstd parameters. | |
c62eb059 | 365 | */ |
7736549b NT |
366 | static void ZSTD_CCtxParams_setZstdParams( |
367 | ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) | |
11303778 | 368 | { |
eda7946a | 369 | assert(!ZSTD_checkCParams(params->cParams)); |
7736549b NT |
370 | cctxParams->cParams = params->cParams; |
371 | cctxParams->fParams = params->fParams; | |
c62eb059 NT |
372 | /* Should not matter, as all cParams are presumed properly defined. |
373 | * But, set it for tracing anyway. | |
374 | */ | |
7736549b | 375 | cctxParams->compressionLevel = ZSTD_NO_CLEVEL; |
add66f81 YC |
376 | } |
377 | ||
e874dacc YC |
378 | ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) |
379 | { | |
380 | ZSTD_bounds bounds = { 0, 0, 0 }; | |
381 | ||
382 | switch(param) | |
383 | { | |
3583d19c | 384 | case ZSTD_c_compressionLevel: |
e874dacc YC |
385 | bounds.lowerBound = ZSTD_minCLevel(); |
386 | bounds.upperBound = ZSTD_maxCLevel(); | |
387 | return bounds; | |
388 | ||
3583d19c | 389 | case ZSTD_c_windowLog: |
e874dacc YC |
390 | bounds.lowerBound = ZSTD_WINDOWLOG_MIN; |
391 | bounds.upperBound = ZSTD_WINDOWLOG_MAX; | |
392 | return bounds; | |
393 | ||
3583d19c | 394 | case ZSTD_c_hashLog: |
e874dacc YC |
395 | bounds.lowerBound = ZSTD_HASHLOG_MIN; |
396 | bounds.upperBound = ZSTD_HASHLOG_MAX; | |
397 | return bounds; | |
398 | ||
3583d19c | 399 | case ZSTD_c_chainLog: |
e874dacc YC |
400 | bounds.lowerBound = ZSTD_CHAINLOG_MIN; |
401 | bounds.upperBound = ZSTD_CHAINLOG_MAX; | |
402 | return bounds; | |
403 | ||
3583d19c | 404 | case ZSTD_c_searchLog: |
e874dacc YC |
405 | bounds.lowerBound = ZSTD_SEARCHLOG_MIN; |
406 | bounds.upperBound = ZSTD_SEARCHLOG_MAX; | |
407 | return bounds; | |
408 | ||
3583d19c | 409 | case ZSTD_c_minMatch: |
e874dacc YC |
410 | bounds.lowerBound = ZSTD_MINMATCH_MIN; |
411 | bounds.upperBound = ZSTD_MINMATCH_MAX; | |
412 | return bounds; | |
413 | ||
3583d19c | 414 | case ZSTD_c_targetLength: |
2e7fd6a2 YC |
415 | bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; |
416 | bounds.upperBound = ZSTD_TARGETLENGTH_MAX; | |
417 | return bounds; | |
418 | ||
be9e561d | 419 | case ZSTD_c_strategy: |
ae370b0e YC |
420 | bounds.lowerBound = ZSTD_STRATEGY_MIN; |
421 | bounds.upperBound = ZSTD_STRATEGY_MAX; | |
2e7fd6a2 YC |
422 | return bounds; |
423 | ||
3583d19c | 424 | case ZSTD_c_contentSizeFlag: |
2e7fd6a2 YC |
425 | bounds.lowerBound = 0; |
426 | bounds.upperBound = 1; | |
427 | return bounds; | |
428 | ||
3583d19c | 429 | case ZSTD_c_checksumFlag: |
2e7fd6a2 YC |
430 | bounds.lowerBound = 0; |
431 | bounds.upperBound = 1; | |
432 | return bounds; | |
433 | ||
3583d19c | 434 | case ZSTD_c_dictIDFlag: |
2e7fd6a2 YC |
435 | bounds.lowerBound = 0; |
436 | bounds.upperBound = 1; | |
437 | return bounds; | |
438 | ||
3583d19c | 439 | case ZSTD_c_nbWorkers: |
2e7fd6a2 YC |
440 | bounds.lowerBound = 0; |
441 | #ifdef ZSTD_MULTITHREAD | |
442 | bounds.upperBound = ZSTDMT_NBWORKERS_MAX; | |
443 | #else | |
444 | bounds.upperBound = 0; | |
445 | #endif | |
446 | return bounds; | |
447 | ||
3583d19c | 448 | case ZSTD_c_jobSize: |
5c6d4b18 YC |
449 | bounds.lowerBound = 0; |
450 | #ifdef ZSTD_MULTITHREAD | |
451 | bounds.upperBound = ZSTDMT_JOBSIZE_MAX; | |
452 | #else | |
453 | bounds.upperBound = 0; | |
454 | #endif | |
5d359239 | 455 | return bounds; |
5c6d4b18 | 456 | |
9b784dec | 457 | case ZSTD_c_overlapLog: |
c13f8190 | 458 | #ifdef ZSTD_MULTITHREAD |
5c6d4b18 YC |
459 | bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; |
460 | bounds.upperBound = ZSTD_OVERLAPLOG_MAX; | |
c13f8190 FH |
461 | #else |
462 | bounds.lowerBound = 0; | |
463 | bounds.upperBound = 0; | |
464 | #endif | |
5c6d4b18 YC |
465 | return bounds; |
466 | ||
c497cb67 BS |
467 | case ZSTD_c_enableDedicatedDictSearch: |
468 | bounds.lowerBound = 0; | |
469 | bounds.upperBound = 1; | |
470 | return bounds; | |
471 | ||
3583d19c | 472 | case ZSTD_c_enableLongDistanceMatching: |
5c6d4b18 YC |
473 | bounds.lowerBound = 0; |
474 | bounds.upperBound = 1; | |
475 | return bounds; | |
476 | ||
3583d19c | 477 | case ZSTD_c_ldmHashLog: |
5c6d4b18 YC |
478 | bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; |
479 | bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; | |
480 | return bounds; | |
481 | ||
3583d19c | 482 | case ZSTD_c_ldmMinMatch: |
5c6d4b18 YC |
483 | bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; |
484 | bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; | |
485 | return bounds; | |
486 | ||
3583d19c | 487 | case ZSTD_c_ldmBucketSizeLog: |
5c6d4b18 YC |
488 | bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; |
489 | bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; | |
490 | return bounds; | |
491 | ||
3583d19c | 492 | case ZSTD_c_ldmHashRateLog: |
41c7d0b1 YC |
493 | bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; |
494 | bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; | |
5c6d4b18 | 495 | return bounds; |
2e7fd6a2 YC |
496 | |
497 | /* experimental parameters */ | |
3583d19c | 498 | case ZSTD_c_rsyncable: |
5c6d4b18 YC |
499 | bounds.lowerBound = 0; |
500 | bounds.upperBound = 1; | |
501 | return bounds; | |
502 | ||
3583d19c | 503 | case ZSTD_c_forceMaxWindow : |
2e7fd6a2 YC |
504 | bounds.lowerBound = 0; |
505 | bounds.upperBound = 1; | |
506 | return bounds; | |
5c6d4b18 | 507 | |
3583d19c | 508 | case ZSTD_c_format: |
ae370b0e YC |
509 | ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); |
510 | bounds.lowerBound = ZSTD_f_zstd1; | |
511 | bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ | |
5c6d4b18 YC |
512 | return bounds; |
513 | ||
3583d19c | 514 | case ZSTD_c_forceAttachDict: |
beefdb0d | 515 | ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); |
be9e561d | 516 | bounds.lowerBound = ZSTD_dictDefaultAttach; |
676f8990 | 517 | bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ |
5c6d4b18 YC |
518 | return bounds; |
519 | ||
f9513115 NT |
520 | case ZSTD_c_literalCompressionMode: |
521 | ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed); | |
522 | bounds.lowerBound = ZSTD_lcm_auto; | |
523 | bounds.upperBound = ZSTD_lcm_uncompressed; | |
524 | return bounds; | |
525 | ||
90077016 EP |
526 | case ZSTD_c_targetCBlockSize: |
527 | bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; | |
528 | bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; | |
529 | return bounds; | |
530 | ||
dffbac5f | 531 | case ZSTD_c_srcSizeHint: |
c7a24d7a | 532 | bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; |
dffbac5f NM |
533 | bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; |
534 | return bounds; | |
535 | ||
e3e0775c NT |
536 | case ZSTD_c_stableInBuffer: |
537 | case ZSTD_c_stableOutBuffer: | |
538 | bounds.lowerBound = (int)ZSTD_bm_buffered; | |
539 | bounds.upperBound = (int)ZSTD_bm_stable; | |
540 | return bounds; | |
db9e73cb | 541 | |
7f563b05 | 542 | case ZSTD_c_blockDelimiters: |
543 | bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; | |
544 | bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; | |
545 | return bounds; | |
db9e73cb | 546 | |
7742f076 | 547 | case ZSTD_c_validateSequences: |
548 | bounds.lowerBound = 0; | |
549 | bounds.upperBound = 1; | |
550 | return bounds; | |
4694423c | 551 | |
c56d6e49 | 552 | case ZSTD_c_splitBlocks: |
553 | bounds.lowerBound = 0; | |
554 | bounds.upperBound = 1; | |
555 | return bounds; | |
e3e0775c | 556 | |
4694423c NT |
557 | case ZSTD_c_useRowMatchFinder: |
558 | bounds.lowerBound = (int)ZSTD_urm_auto; | |
559 | bounds.upperBound = (int)ZSTD_urm_enableRowMatchFinder; | |
560 | return bounds; | |
561 | ||
e874dacc | 562 | default: |
608f1bfc YC |
563 | bounds.error = ERROR(parameter_unsupported); |
564 | return bounds; | |
e874dacc YC |
565 | } |
566 | } | |
567 | ||
f4abba02 NT |
568 | /* ZSTD_cParam_clampBounds: |
569 | * Clamps the value into the bounded range. | |
570 | */ | |
571 | static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) | |
572 | { | |
573 | ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); | |
574 | if (ZSTD_isError(bounds.error)) return bounds.error; | |
575 | if (*value < bounds.lowerBound) *value = bounds.lowerBound; | |
576 | if (*value > bounds.upperBound) *value = bounds.upperBound; | |
577 | return 0; | |
578 | } | |
579 | ||
cafc3b1b FH |
580 | #define BOUNDCHECK(cParam, val) { \ |
581 | RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ | |
5e5f2626 | 582 | parameter_outOfBound, "Param out of bounds"); \ |
cafc3b1b | 583 | } |
c3bce24e | 584 | |
90eca318 YC |
585 | |
586 | static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) | |
587 | { | |
588 | switch(param) | |
589 | { | |
3583d19c YC |
590 | case ZSTD_c_compressionLevel: |
591 | case ZSTD_c_hashLog: | |
592 | case ZSTD_c_chainLog: | |
593 | case ZSTD_c_searchLog: | |
594 | case ZSTD_c_minMatch: | |
595 | case ZSTD_c_targetLength: | |
be9e561d | 596 | case ZSTD_c_strategy: |
90eca318 YC |
597 | return 1; |
598 | ||
3583d19c YC |
599 | case ZSTD_c_format: |
600 | case ZSTD_c_windowLog: | |
601 | case ZSTD_c_contentSizeFlag: | |
602 | case ZSTD_c_checksumFlag: | |
603 | case ZSTD_c_dictIDFlag: | |
604 | case ZSTD_c_forceMaxWindow : | |
605 | case ZSTD_c_nbWorkers: | |
606 | case ZSTD_c_jobSize: | |
9b784dec | 607 | case ZSTD_c_overlapLog: |
3583d19c | 608 | case ZSTD_c_rsyncable: |
c497cb67 | 609 | case ZSTD_c_enableDedicatedDictSearch: |
3583d19c YC |
610 | case ZSTD_c_enableLongDistanceMatching: |
611 | case ZSTD_c_ldmHashLog: | |
612 | case ZSTD_c_ldmMinMatch: | |
613 | case ZSTD_c_ldmBucketSizeLog: | |
614 | case ZSTD_c_ldmHashRateLog: | |
615 | case ZSTD_c_forceAttachDict: | |
3d7377b8 | 616 | case ZSTD_c_literalCompressionMode: |
90077016 | 617 | case ZSTD_c_targetCBlockSize: |
dffbac5f | 618 | case ZSTD_c_srcSizeHint: |
e3e0775c NT |
619 | case ZSTD_c_stableInBuffer: |
620 | case ZSTD_c_stableOutBuffer: | |
7f563b05 | 621 | case ZSTD_c_blockDelimiters: |
7742f076 | 622 | case ZSTD_c_validateSequences: |
c56d6e49 | 623 | case ZSTD_c_splitBlocks: |
4694423c | 624 | case ZSTD_c_useRowMatchFinder: |
90eca318 | 625 | default: |
90eca318 YC |
626 | return 0; |
627 | } | |
628 | } | |
629 | ||
3b838abf | 630 | size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) |
b0edb7fb | 631 | { |
3b838abf | 632 | DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); |
90eca318 YC |
633 | if (cctx->streamStage != zcss_init) { |
634 | if (ZSTD_isUpdateAuthorized(param)) { | |
635 | cctx->cParamsChanged = 1; | |
636 | } else { | |
5e5f2626 | 637 | RETURN_ERROR(stage_wrong, "can only set params in ctx init stage"); |
90eca318 | 638 | } } |
b0edb7fb YC |
639 | |
640 | switch(param) | |
641 | { | |
787b7690 NT |
642 | case ZSTD_c_nbWorkers: |
643 | RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, | |
644 | "MT not compatible with static alloc"); | |
f4abba02 | 645 | break; |
f181f33b | 646 | |
787b7690 | 647 | case ZSTD_c_compressionLevel: |
3583d19c YC |
648 | case ZSTD_c_windowLog: |
649 | case ZSTD_c_hashLog: | |
650 | case ZSTD_c_chainLog: | |
651 | case ZSTD_c_searchLog: | |
652 | case ZSTD_c_minMatch: | |
653 | case ZSTD_c_targetLength: | |
be9e561d | 654 | case ZSTD_c_strategy: |
f4abba02 | 655 | case ZSTD_c_ldmHashRateLog: |
f4abba02 | 656 | case ZSTD_c_format: |
3583d19c YC |
657 | case ZSTD_c_contentSizeFlag: |
658 | case ZSTD_c_checksumFlag: | |
659 | case ZSTD_c_dictIDFlag: | |
f4abba02 | 660 | case ZSTD_c_forceMaxWindow: |
3583d19c | 661 | case ZSTD_c_forceAttachDict: |
f9513115 | 662 | case ZSTD_c_literalCompressionMode: |
3583d19c | 663 | case ZSTD_c_jobSize: |
9b784dec | 664 | case ZSTD_c_overlapLog: |
3583d19c | 665 | case ZSTD_c_rsyncable: |
c497cb67 | 666 | case ZSTD_c_enableDedicatedDictSearch: |
3583d19c YC |
667 | case ZSTD_c_enableLongDistanceMatching: |
668 | case ZSTD_c_ldmHashLog: | |
669 | case ZSTD_c_ldmMinMatch: | |
670 | case ZSTD_c_ldmBucketSizeLog: | |
90077016 | 671 | case ZSTD_c_targetCBlockSize: |
dffbac5f | 672 | case ZSTD_c_srcSizeHint: |
e3e0775c NT |
673 | case ZSTD_c_stableInBuffer: |
674 | case ZSTD_c_stableOutBuffer: | |
7f563b05 | 675 | case ZSTD_c_blockDelimiters: |
7742f076 | 676 | case ZSTD_c_validateSequences: |
c56d6e49 | 677 | case ZSTD_c_splitBlocks: |
4694423c | 678 | case ZSTD_c_useRowMatchFinder: |
f4abba02 | 679 | break; |
a1f04d51 | 680 | |
5e5f2626 | 681 | default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); |
bb002740 | 682 | } |
7ad7ba31 | 683 | return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); |
bb002740 YC |
684 | } |
685 | ||
7ad7ba31 NT |
686 | size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, |
687 | ZSTD_cParameter param, int value) | |
c0221124 | 688 | { |
7ad7ba31 | 689 | DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); |
c0221124 SL |
690 | switch(param) |
691 | { | |
3583d19c | 692 | case ZSTD_c_format : |
ae370b0e | 693 | BOUNDCHECK(ZSTD_c_format, value); |
05dffe43 YC |
694 | CCtxParams->format = (ZSTD_format_e)value; |
695 | return (size_t)CCtxParams->format; | |
7c3dea42 | 696 | |
3583d19c | 697 | case ZSTD_c_compressionLevel : { |
5e5f2626 | 698 | FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); |
d514281e | 699 | if (value == 0) |
700 | CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ | |
701 | else | |
f4abba02 | 702 | CCtxParams->compressionLevel = value; |
5d5c895b | 703 | if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; |
6a9b41b7 YC |
704 | return 0; /* return type (size_t) cannot represent negative values */ |
705 | } | |
b0edb7fb | 706 | |
3583d19c | 707 | case ZSTD_c_windowLog : |
5c6d4b18 | 708 | if (value!=0) /* 0 => use default */ |
c3c34889 | 709 | BOUNDCHECK(ZSTD_c_windowLog, value); |
5d5c895b | 710 | CCtxParams->cParams.windowLog = (U32)value; |
05dffe43 | 711 | return CCtxParams->cParams.windowLog; |
b0edb7fb | 712 | |
3583d19c | 713 | case ZSTD_c_hashLog : |
5c6d4b18 | 714 | if (value!=0) /* 0 => use default */ |
c3c34889 | 715 | BOUNDCHECK(ZSTD_c_hashLog, value); |
5d5c895b | 716 | CCtxParams->cParams.hashLog = (U32)value; |
05dffe43 | 717 | return CCtxParams->cParams.hashLog; |
b0edb7fb | 718 | |
3583d19c | 719 | case ZSTD_c_chainLog : |
5c6d4b18 | 720 | if (value!=0) /* 0 => use default */ |
c3c34889 | 721 | BOUNDCHECK(ZSTD_c_chainLog, value); |
5d5c895b | 722 | CCtxParams->cParams.chainLog = (U32)value; |
05dffe43 | 723 | return CCtxParams->cParams.chainLog; |
b0edb7fb | 724 | |
3583d19c | 725 | case ZSTD_c_searchLog : |
5c6d4b18 | 726 | if (value!=0) /* 0 => use default */ |
c3c34889 | 727 | BOUNDCHECK(ZSTD_c_searchLog, value); |
5d5c895b YC |
728 | CCtxParams->cParams.searchLog = (U32)value; |
729 | return (size_t)value; | |
b0edb7fb | 730 | |
3583d19c | 731 | case ZSTD_c_minMatch : |
5c6d4b18 | 732 | if (value!=0) /* 0 => use default */ |
c3c34889 | 733 | BOUNDCHECK(ZSTD_c_minMatch, value); |
e874dacc YC |
734 | CCtxParams->cParams.minMatch = value; |
735 | return CCtxParams->cParams.minMatch; | |
b0edb7fb | 736 | |
3583d19c | 737 | case ZSTD_c_targetLength : |
c3c34889 | 738 | BOUNDCHECK(ZSTD_c_targetLength, value); |
6f4d0778 | 739 | CCtxParams->cParams.targetLength = value; |
05dffe43 | 740 | return CCtxParams->cParams.targetLength; |
b0edb7fb | 741 | |
be9e561d | 742 | case ZSTD_c_strategy : |
5c6d4b18 | 743 | if (value!=0) /* 0 => use default */ |
c3c34889 | 744 | BOUNDCHECK(ZSTD_c_strategy, value); |
6f4d0778 | 745 | CCtxParams->cParams.strategy = (ZSTD_strategy)value; |
05dffe43 | 746 | return (size_t)CCtxParams->cParams.strategy; |
b0edb7fb | 747 | |
3583d19c | 748 | case ZSTD_c_contentSizeFlag : |
cd2892fd | 749 | /* Content size written in frame header _when known_ (default:1) */ |
5c6d4b18 YC |
750 | DEBUGLOG(4, "set content size flag = %u", (value!=0)); |
751 | CCtxParams->fParams.contentSizeFlag = value != 0; | |
05dffe43 | 752 | return CCtxParams->fParams.contentSizeFlag; |
b0edb7fb | 753 | |
3583d19c | 754 | case ZSTD_c_checksumFlag : |
cd2892fd | 755 | /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ |
5c6d4b18 | 756 | CCtxParams->fParams.checksumFlag = value != 0; |
05dffe43 | 757 | return CCtxParams->fParams.checksumFlag; |
b0edb7fb | 758 | |
3583d19c | 759 | case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ |
5c6d4b18 | 760 | DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); |
6f4d0778 | 761 | CCtxParams->fParams.noDictIDFlag = !value; |
05dffe43 | 762 | return !CCtxParams->fParams.noDictIDFlag; |
b0edb7fb | 763 | |
3583d19c | 764 | case ZSTD_c_forceMaxWindow : |
5c6d4b18 | 765 | CCtxParams->forceWindow = (value != 0); |
05dffe43 | 766 | return CCtxParams->forceWindow; |
7bd1a290 | 767 | |
3583d19c | 768 | case ZSTD_c_forceAttachDict : { |
5faef4d3 | 769 | const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; |
c3c34889 | 770 | BOUNDCHECK(ZSTD_c_forceAttachDict, pref); |
2d9332eb | 771 | CCtxParams->attachDictPref = pref; |
01bb1c10 | 772 | return CCtxParams->attachDictPref; |
596f7d12 | 773 | } |
01bb1c10 | 774 | |
f9513115 NT |
775 | case ZSTD_c_literalCompressionMode : { |
776 | const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value; | |
777 | BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); | |
778 | CCtxParams->literalCompressionMode = lcm; | |
779 | return CCtxParams->literalCompressionMode; | |
780 | } | |
781 | ||
3583d19c | 782 | case ZSTD_c_nbWorkers : |
c35e5350 | 783 | #ifndef ZSTD_MULTITHREAD |
cafc3b1b | 784 | RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); |
209df52b | 785 | return 0; |
1dba98d5 | 786 | #else |
5e5f2626 | 787 | FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); |
f4abba02 NT |
788 | CCtxParams->nbWorkers = value; |
789 | return CCtxParams->nbWorkers; | |
1dba98d5 YC |
790 | #endif |
791 | ||
3583d19c | 792 | case ZSTD_c_jobSize : |
0d6ecc72 | 793 | #ifndef ZSTD_MULTITHREAD |
f4abba02 NT |
794 | RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); |
795 | return 0; | |
0d6ecc72 | 796 | #else |
f4abba02 NT |
797 | /* Adjust to the minimum non-default value. */ |
798 | if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) | |
799 | value = ZSTDMT_JOBSIZE_MIN; | |
5e5f2626 | 800 | FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); |
f4abba02 NT |
801 | assert(value >= 0); |
802 | CCtxParams->jobSize = value; | |
803 | return CCtxParams->jobSize; | |
0d6ecc72 | 804 | #endif |
c35e5350 | 805 | |
9b784dec | 806 | case ZSTD_c_overlapLog : |
0d6ecc72 | 807 | #ifndef ZSTD_MULTITHREAD |
f4abba02 NT |
808 | RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); |
809 | return 0; | |
0d6ecc72 | 810 | #else |
5e5f2626 | 811 | FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); |
f4abba02 NT |
812 | CCtxParams->overlapLog = value; |
813 | return CCtxParams->overlapLog; | |
0d6ecc72 | 814 | #endif |
b0edb7fb | 815 | |
3583d19c | 816 | case ZSTD_c_rsyncable : |
b9693d3a | 817 | #ifndef ZSTD_MULTITHREAD |
f4abba02 NT |
818 | RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); |
819 | return 0; | |
b9693d3a | 820 | #else |
5e5f2626 | 821 | FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); |
f4abba02 NT |
822 | CCtxParams->rsyncable = value; |
823 | return CCtxParams->rsyncable; | |
b9693d3a NT |
824 | #endif |
825 | ||
c497cb67 BS |
826 | case ZSTD_c_enableDedicatedDictSearch : |
827 | CCtxParams->enableDedicatedDictSearch = (value!=0); | |
828 | return CCtxParams->enableDedicatedDictSearch; | |
829 | ||
3583d19c | 830 | case ZSTD_c_enableLongDistanceMatching : |
5c6d4b18 | 831 | CCtxParams->ldmParams.enableLdm = (value!=0); |
6f4d0778 | 832 | return CCtxParams->ldmParams.enableLdm; |
767a0b3b | 833 | |
3583d19c | 834 | case ZSTD_c_ldmHashLog : |
5c6d4b18 | 835 | if (value!=0) /* 0 ==> auto */ |
c3c34889 | 836 | BOUNDCHECK(ZSTD_c_ldmHashLog, value); |
9618c0c8 | 837 | CCtxParams->ldmParams.hashLog = value; |
05dffe43 | 838 | return CCtxParams->ldmParams.hashLog; |
767a0b3b | 839 | |
3583d19c | 840 | case ZSTD_c_ldmMinMatch : |
5c6d4b18 | 841 | if (value!=0) /* 0 ==> default */ |
c3c34889 | 842 | BOUNDCHECK(ZSTD_c_ldmMinMatch, value); |
9618c0c8 | 843 | CCtxParams->ldmParams.minMatchLength = value; |
05dffe43 | 844 | return CCtxParams->ldmParams.minMatchLength; |
6a546efb | 845 | |
3583d19c | 846 | case ZSTD_c_ldmBucketSizeLog : |
5c6d4b18 | 847 | if (value!=0) /* 0 ==> default */ |
c3c34889 | 848 | BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); |
05dffe43 | 849 | CCtxParams->ldmParams.bucketSizeLog = value; |
6f4d0778 | 850 | return CCtxParams->ldmParams.bucketSizeLog; |
67d4a616 | 851 | |
3583d19c | 852 | case ZSTD_c_ldmHashRateLog : |
cafc3b1b | 853 | RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, |
5e5f2626 | 854 | parameter_outOfBound, "Param out of bounds!"); |
41c7d0b1 YC |
855 | CCtxParams->ldmParams.hashRateLog = value; |
856 | return CCtxParams->ldmParams.hashRateLog; | |
a1f04d51 | 857 | |
90077016 EP |
858 | case ZSTD_c_targetCBlockSize : |
859 | if (value!=0) /* 0 ==> default */ | |
860 | BOUNDCHECK(ZSTD_c_targetCBlockSize, value); | |
861 | CCtxParams->targetCBlockSize = value; | |
862 | return CCtxParams->targetCBlockSize; | |
863 | ||
dffbac5f NM |
864 | case ZSTD_c_srcSizeHint : |
865 | if (value!=0) /* 0 ==> default */ | |
866 | BOUNDCHECK(ZSTD_c_srcSizeHint, value); | |
867 | CCtxParams->srcSizeHint = value; | |
868 | return CCtxParams->srcSizeHint; | |
869 | ||
e3e0775c NT |
870 | case ZSTD_c_stableInBuffer: |
871 | BOUNDCHECK(ZSTD_c_stableInBuffer, value); | |
872 | CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; | |
873 | return CCtxParams->inBufferMode; | |
874 | ||
875 | case ZSTD_c_stableOutBuffer: | |
876 | BOUNDCHECK(ZSTD_c_stableOutBuffer, value); | |
877 | CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; | |
878 | return CCtxParams->outBufferMode; | |
db9e73cb | 879 | |
7f563b05 | 880 | case ZSTD_c_blockDelimiters: |
881 | BOUNDCHECK(ZSTD_c_blockDelimiters, value); | |
882 | CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; | |
883 | return CCtxParams->blockDelimiters; | |
db9e73cb | 884 | |
7742f076 | 885 | case ZSTD_c_validateSequences: |
886 | BOUNDCHECK(ZSTD_c_validateSequences, value); | |
887 | CCtxParams->validateSequences = value; | |
888 | return CCtxParams->validateSequences; | |
e3e0775c | 889 | |
c56d6e49 | 890 | case ZSTD_c_splitBlocks: |
891 | BOUNDCHECK(ZSTD_c_splitBlocks, value); | |
892 | CCtxParams->splitBlocks = value; | |
893 | return CCtxParams->splitBlocks; | |
894 | ||
4694423c NT |
895 | case ZSTD_c_useRowMatchFinder: |
896 | BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); | |
897 | CCtxParams->useRowMatchFinder = (ZSTD_useRowMatchFinderMode_e)value; | |
898 | return CCtxParams->useRowMatchFinder; | |
899 | ||
cafc3b1b | 900 | default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); |
bb002740 YC |
901 | } |
902 | } | |
903 | ||
f520f6df | 904 | size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) |
280a236e | 905 | { |
7ad7ba31 | 906 | return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); |
280a236e NT |
907 | } |
908 | ||
7ad7ba31 | 909 | size_t ZSTD_CCtxParams_getParameter( |
54a4998a | 910 | ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) |
280a236e NT |
911 | { |
912 | switch(param) | |
913 | { | |
3583d19c | 914 | case ZSTD_c_format : |
280a236e NT |
915 | *value = CCtxParams->format; |
916 | break; | |
3583d19c | 917 | case ZSTD_c_compressionLevel : |
280a236e NT |
918 | *value = CCtxParams->compressionLevel; |
919 | break; | |
3583d19c | 920 | case ZSTD_c_windowLog : |
45c9fbd6 | 921 | *value = (int)CCtxParams->cParams.windowLog; |
280a236e | 922 | break; |
3583d19c | 923 | case ZSTD_c_hashLog : |
45c9fbd6 | 924 | *value = (int)CCtxParams->cParams.hashLog; |
280a236e | 925 | break; |
3583d19c | 926 | case ZSTD_c_chainLog : |
45c9fbd6 | 927 | *value = (int)CCtxParams->cParams.chainLog; |
280a236e | 928 | break; |
3583d19c | 929 | case ZSTD_c_searchLog : |
280a236e NT |
930 | *value = CCtxParams->cParams.searchLog; |
931 | break; | |
3583d19c | 932 | case ZSTD_c_minMatch : |
e874dacc | 933 | *value = CCtxParams->cParams.minMatch; |
280a236e | 934 | break; |
3583d19c | 935 | case ZSTD_c_targetLength : |
280a236e NT |
936 | *value = CCtxParams->cParams.targetLength; |
937 | break; | |
be9e561d | 938 | case ZSTD_c_strategy : |
280a236e NT |
939 | *value = (unsigned)CCtxParams->cParams.strategy; |
940 | break; | |
3583d19c | 941 | case ZSTD_c_contentSizeFlag : |
280a236e NT |
942 | *value = CCtxParams->fParams.contentSizeFlag; |
943 | break; | |
3583d19c | 944 | case ZSTD_c_checksumFlag : |
280a236e NT |
945 | *value = CCtxParams->fParams.checksumFlag; |
946 | break; | |
3583d19c | 947 | case ZSTD_c_dictIDFlag : |
280a236e NT |
948 | *value = !CCtxParams->fParams.noDictIDFlag; |
949 | break; | |
3583d19c | 950 | case ZSTD_c_forceMaxWindow : |
280a236e NT |
951 | *value = CCtxParams->forceWindow; |
952 | break; | |
3583d19c | 953 | case ZSTD_c_forceAttachDict : |
01bb1c10 FH |
954 | *value = CCtxParams->attachDictPref; |
955 | break; | |
f9513115 NT |
956 | case ZSTD_c_literalCompressionMode : |
957 | *value = CCtxParams->literalCompressionMode; | |
958 | break; | |
3583d19c | 959 | case ZSTD_c_nbWorkers : |
280a236e NT |
960 | #ifndef ZSTD_MULTITHREAD |
961 | assert(CCtxParams->nbWorkers == 0); | |
962 | #endif | |
963 | *value = CCtxParams->nbWorkers; | |
964 | break; | |
3583d19c | 965 | case ZSTD_c_jobSize : |
280a236e | 966 | #ifndef ZSTD_MULTITHREAD |
cafc3b1b | 967 | RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); |
280a236e | 968 | #else |
34f01e60 YC |
969 | assert(CCtxParams->jobSize <= INT_MAX); |
970 | *value = (int)CCtxParams->jobSize; | |
280a236e NT |
971 | break; |
972 | #endif | |
9b784dec | 973 | case ZSTD_c_overlapLog : |
280a236e | 974 | #ifndef ZSTD_MULTITHREAD |
cafc3b1b | 975 | RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); |
280a236e | 976 | #else |
eee789b7 | 977 | *value = CCtxParams->overlapLog; |
280a236e | 978 | break; |
b9693d3a | 979 | #endif |
3583d19c | 980 | case ZSTD_c_rsyncable : |
b9693d3a | 981 | #ifndef ZSTD_MULTITHREAD |
cafc3b1b | 982 | RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); |
b9693d3a NT |
983 | #else |
984 | *value = CCtxParams->rsyncable; | |
985 | break; | |
280a236e | 986 | #endif |
c497cb67 BS |
987 | case ZSTD_c_enableDedicatedDictSearch : |
988 | *value = CCtxParams->enableDedicatedDictSearch; | |
989 | break; | |
3583d19c | 990 | case ZSTD_c_enableLongDistanceMatching : |
280a236e NT |
991 | *value = CCtxParams->ldmParams.enableLdm; |
992 | break; | |
3583d19c | 993 | case ZSTD_c_ldmHashLog : |
280a236e NT |
994 | *value = CCtxParams->ldmParams.hashLog; |
995 | break; | |
3583d19c | 996 | case ZSTD_c_ldmMinMatch : |
280a236e NT |
997 | *value = CCtxParams->ldmParams.minMatchLength; |
998 | break; | |
3583d19c | 999 | case ZSTD_c_ldmBucketSizeLog : |
280a236e NT |
1000 | *value = CCtxParams->ldmParams.bucketSizeLog; |
1001 | break; | |
3583d19c | 1002 | case ZSTD_c_ldmHashRateLog : |
41c7d0b1 | 1003 | *value = CCtxParams->ldmParams.hashRateLog; |
280a236e | 1004 | break; |
90077016 EP |
1005 | case ZSTD_c_targetCBlockSize : |
1006 | *value = (int)CCtxParams->targetCBlockSize; | |
1007 | break; | |
dffbac5f NM |
1008 | case ZSTD_c_srcSizeHint : |
1009 | *value = (int)CCtxParams->srcSizeHint; | |
edf2abf1 | 1010 | break; |
e3e0775c NT |
1011 | case ZSTD_c_stableInBuffer : |
1012 | *value = (int)CCtxParams->inBufferMode; | |
1013 | break; | |
1014 | case ZSTD_c_stableOutBuffer : | |
1015 | *value = (int)CCtxParams->outBufferMode; | |
1016 | break; | |
7f563b05 | 1017 | case ZSTD_c_blockDelimiters : |
1018 | *value = (int)CCtxParams->blockDelimiters; | |
1019 | break; | |
7742f076 | 1020 | case ZSTD_c_validateSequences : |
1021 | *value = (int)CCtxParams->validateSequences; | |
1022 | break; | |
c56d6e49 | 1023 | case ZSTD_c_splitBlocks : |
1024 | *value = (int)CCtxParams->splitBlocks; | |
1025 | break; | |
4694423c NT |
1026 | case ZSTD_c_useRowMatchFinder : |
1027 | *value = (int)CCtxParams->useRowMatchFinder; | |
1028 | break; | |
cafc3b1b | 1029 | default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); |
280a236e NT |
1030 | } |
1031 | return 0; | |
1032 | } | |
1033 | ||
5e1f34b7 YC |
1034 | /** ZSTD_CCtx_setParametersUsingCCtxParams() : |
1035 | * just applies `params` into `cctx` | |
1036 | * no action is performed, parameters are merely stored. | |
60fa90b6 YC |
1037 | * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. |
1038 | * This is possible even if a compression is ongoing. | |
1039 | * In which case, new parameters will be applied on the fly, starting with next compression job. | |
f181f33b | 1040 | */ |
82d636b7 | 1041 | size_t ZSTD_CCtx_setParametersUsingCCtxParams( |
78290874 | 1042 | ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) |
97e27aff | 1043 | { |
2108decb | 1044 | DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); |
5e5f2626 FH |
1045 | RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, |
1046 | "The context is in the wrong stage!"); | |
1047 | RETURN_ERROR_IF(cctx->cdict, stage_wrong, | |
1048 | "Can't override parameters with cdict attached (some must " | |
1049 | "be inherited from the cdict)."); | |
399ae013 | 1050 | |
998a93b7 | 1051 | cctx->requestedParams = *params; |
97e27aff SL |
1052 | return 0; |
1053 | } | |
1054 | ||
b0edb7fb YC |
1055 | ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) |
1056 | { | |
03832b7a | 1057 | DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); |
5e5f2626 FH |
1058 | RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, |
1059 | "Can't set pledgedSrcSize when not in init stage."); | |
a0ba849f | 1060 | cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; |
b0edb7fb YC |
1061 | return 0; |
1062 | } | |
1063 | ||
a3659fe1 FH |
1064 | static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( |
1065 | int const compressionLevel, | |
a3659fe1 FH |
1066 | size_t const dictSize); |
1067 | static int ZSTD_dedicatedDictSearch_isSupported( | |
eee51a66 | 1068 | const ZSTD_compressionParameters* cParams); |
7b5d2f72 FH |
1069 | static void ZSTD_dedicatedDictSearch_revertCParams( |
1070 | ZSTD_compressionParameters* cParams); | |
e36a373d | 1071 | |
6b053b9f NT |
1072 | /** |
1073 | * Initializes the local dict using the requested parameters. | |
1074 | * NOTE: This does not use the pledged src size, because it may be used for more | |
1075 | * than one compression. | |
1076 | */ | |
1077 | static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) | |
1078 | { | |
1079 | ZSTD_localDict* const dl = &cctx->localDict; | |
6b053b9f NT |
1080 | if (dl->dict == NULL) { |
1081 | /* No local dictionary. */ | |
1082 | assert(dl->dictBuffer == NULL); | |
1083 | assert(dl->cdict == NULL); | |
1084 | assert(dl->dictSize == 0); | |
1085 | return 0; | |
1086 | } | |
1087 | if (dl->cdict != NULL) { | |
1088 | assert(cctx->cdict == dl->cdict); | |
1089 | /* Local dictionary already initialized. */ | |
1090 | return 0; | |
1091 | } | |
1092 | assert(dl->dictSize > 0); | |
1093 | assert(cctx->cdict == NULL); | |
1094 | assert(cctx->prefixDict.dict == NULL); | |
1095 | ||
914bfe7e | 1096 | dl->cdict = ZSTD_createCDict_advanced2( |
6b053b9f NT |
1097 | dl->dict, |
1098 | dl->dictSize, | |
1099 | ZSTD_dlm_byRef, | |
1100 | dl->dictContentType, | |
914bfe7e | 1101 | &cctx->requestedParams, |
6b053b9f | 1102 | cctx->customMem); |
5e5f2626 | 1103 | RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); |
6b053b9f NT |
1104 | cctx->cdict = dl->cdict; |
1105 | return 0; | |
1106 | } | |
1107 | ||
c7a18b7c SL |
1108 | size_t ZSTD_CCtx_loadDictionary_advanced( |
1109 | ZSTD_CCtx* cctx, const void* dict, size_t dictSize, | |
6873fec6 | 1110 | ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) |
6d4fef36 | 1111 | { |
5e5f2626 FH |
1112 | RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, |
1113 | "Can't load a dictionary when ctx is not in init stage."); | |
03832b7a | 1114 | DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); |
6b053b9f NT |
1115 | ZSTD_clearAllDicts(cctx); /* in case one already exists */ |
1116 | if (dict == NULL || dictSize == 0) /* no dictionary mode */ | |
1117 | return 0; | |
1118 | if (dictLoadMethod == ZSTD_dlm_byRef) { | |
1119 | cctx->localDict.dict = dict; | |
6d4fef36 | 1120 | } else { |
046aca19 NT |
1121 | void* dictBuffer; |
1122 | RETURN_ERROR_IF(cctx->staticSize, memory_allocation, | |
1123 | "no malloc for static CCtx"); | |
1124 | dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); | |
5e5f2626 | 1125 | RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); |
c465f244 | 1126 | ZSTD_memcpy(dictBuffer, dict, dictSize); |
6b053b9f NT |
1127 | cctx->localDict.dictBuffer = dictBuffer; |
1128 | cctx->localDict.dict = dictBuffer; | |
6d4fef36 | 1129 | } |
6b053b9f NT |
1130 | cctx->localDict.dictSize = dictSize; |
1131 | cctx->localDict.dictContentType = dictContentType; | |
6d4fef36 YC |
1132 | return 0; |
1133 | } | |
1134 | ||
eb7bbab3 SL |
1135 | ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference( |
1136 | ZSTD_CCtx* cctx, const void* dict, size_t dictSize) | |
1137 | { | |
c7a18b7c | 1138 | return ZSTD_CCtx_loadDictionary_advanced( |
6873fec6 | 1139 | cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); |
eb7bbab3 SL |
1140 | } |
1141 | ||
1142 | ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) | |
1143 | { | |
c7a18b7c | 1144 | return ZSTD_CCtx_loadDictionary_advanced( |
6873fec6 | 1145 | cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); |
eb7bbab3 SL |
1146 | } |
1147 | ||
1148 | ||
b7372933 | 1149 | size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) |
7d360280 | 1150 | { |
5e5f2626 FH |
1151 | RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, |
1152 | "Can't ref a dict when ctx not in init stage."); | |
0594e813 | 1153 | /* Free the existing local cdict (if any) to save memory. */ |
6b053b9f | 1154 | ZSTD_clearAllDicts(cctx); |
b7372933 | 1155 | cctx->cdict = cdict; |
b7372933 | 1156 | return 0; |
7d360280 YC |
1157 | } |
1158 | ||
b684900a ML |
1159 | size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) |
1160 | { | |
1161 | RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, | |
1162 | "Can't ref a pool when ctx not in init stage."); | |
1163 | cctx->pool = pool; | |
1164 | return 0; | |
1165 | } | |
1166 | ||
b7372933 | 1167 | size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) |
c7a18b7c | 1168 | { |
6873fec6 | 1169 | return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); |
c7a18b7c SL |
1170 | } |
1171 | ||
1172 | size_t ZSTD_CCtx_refPrefix_advanced( | |
6873fec6 | 1173 | ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) |
7d360280 | 1174 | { |
5e5f2626 FH |
1175 | RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, |
1176 | "Can't ref a prefix when ctx not in init stage."); | |
6b053b9f | 1177 | ZSTD_clearAllDicts(cctx); |
1658ae75 BS |
1178 | if (prefix != NULL && prefixSize > 0) { |
1179 | cctx->prefixDict.dict = prefix; | |
1180 | cctx->prefixDict.dictSize = prefixSize; | |
1181 | cctx->prefixDict.dictContentType = dictContentType; | |
1182 | } | |
b7372933 | 1183 | return 0; |
7d360280 YC |
1184 | } |
1185 | ||
b26728c9 YC |
1186 | /*! ZSTD_CCtx_reset() : |
1187 | * Also dumps dictionary */ | |
114bd434 | 1188 | size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) |
b26728c9 | 1189 | { |
5c686391 YC |
1190 | if ( (reset == ZSTD_reset_session_only) |
1191 | || (reset == ZSTD_reset_session_and_parameters) ) { | |
ff8d3717 YC |
1192 | cctx->streamStage = zcss_init; |
1193 | cctx->pledgedSrcSizePlusOne = 0; | |
1194 | } | |
5c686391 YC |
1195 | if ( (reset == ZSTD_reset_parameters) |
1196 | || (reset == ZSTD_reset_session_and_parameters) ) { | |
5e5f2626 FH |
1197 | RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, |
1198 | "Can't reset parameters only when not in init stage."); | |
6b053b9f | 1199 | ZSTD_clearAllDicts(cctx); |
ff8d3717 YC |
1200 | return ZSTD_CCtxParams_reset(&cctx->requestedParams); |
1201 | } | |
1202 | return 0; | |
bd18c885 | 1203 | } |
59d7063f | 1204 | |
9f76eebd | 1205 | |
381e66cf YC |
1206 | /** ZSTD_checkCParams() : |
1207 | control CParam values remain within authorized range. | |
21588e37 | 1208 | @return : 0, or an error code if one value is beyond authorized range */ |
3b71925c | 1209 | size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) |
21588e37 | 1210 | { |
ed38b645 YC |
1211 | BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); |
1212 | BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); | |
1213 | BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); | |
1214 | BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); | |
1215 | BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); | |
1216 | BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); | |
c3c34889 | 1217 | BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); |
21588e37 YC |
1218 | return 0; |
1219 | } | |
1220 | ||
381e66cf YC |
1221 | /** ZSTD_clampCParams() : |
1222 | * make CParam values within valid range. | |
1223 | * @return : valid CParams */ | |
3d523c74 YC |
1224 | static ZSTD_compressionParameters |
1225 | ZSTD_clampCParams(ZSTD_compressionParameters cParams) | |
381e66cf | 1226 | { |
c3c34889 YC |
1227 | # define CLAMP_TYPE(cParam, val, type) { \ |
1228 | ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ | |
1229 | if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \ | |
1230 | else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \ | |
381e66cf | 1231 | } |
ed38b645 | 1232 | # define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) |
e9448cdf YC |
1233 | CLAMP(ZSTD_c_windowLog, cParams.windowLog); |
1234 | CLAMP(ZSTD_c_chainLog, cParams.chainLog); | |
1235 | CLAMP(ZSTD_c_hashLog, cParams.hashLog); | |
1236 | CLAMP(ZSTD_c_searchLog, cParams.searchLog); | |
1237 | CLAMP(ZSTD_c_minMatch, cParams.minMatch); | |
be9e561d | 1238 | CLAMP(ZSTD_c_targetLength,cParams.targetLength); |
eee789b7 | 1239 | CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); |
381e66cf YC |
1240 | return cParams; |
1241 | } | |
70e8c389 | 1242 | |
c3a5c4be YC |
1243 | /** ZSTD_cycleLog() : |
1244 | * condition for correct operation : hashLog > 1 */ | |
5b0a452c | 1245 | U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) |
c3a5c4be YC |
1246 | { |
1247 | U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); | |
1248 | return hashLog - btScale; | |
1249 | } | |
1250 | ||
d5c688e8 | 1251 | /** ZSTD_dictAndWindowLog() : |
7e6f91ed NT |
1252 | * Returns an adjusted window log that is large enough to fit the source and the dictionary. |
1253 | * The zstd format says that the entire dictionary is valid if one byte of the dictionary | |
1254 | * is within the window. So the hashLog and chainLog should be large enough to reference both | |
1255 | * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing | |
1256 | * the hashLog and windowLog. | |
1257 | * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. | |
d5c688e8 NT |
1258 | */ |
1259 | static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) | |
1260 | { | |
1261 | const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; | |
1262 | /* No dictionary ==> No change */ | |
1263 | if (dictSize == 0) { | |
1264 | return windowLog; | |
1265 | } | |
7e6f91ed | 1266 | assert(windowLog <= ZSTD_WINDOWLOG_MAX); |
d5c688e8 NT |
1267 | assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ |
1268 | { | |
1269 | U64 const windowSize = 1ULL << windowLog; | |
1270 | U64 const dictAndWindowSize = dictSize + windowSize; | |
1271 | /* If the window size is already large enough to fit both the source and the dictionary | |
1272 | * then just use the window size. Otherwise adjust so that it fits the dictionary and | |
1273 | * the window. | |
1274 | */ | |
1275 | if (windowSize >= dictSize + srcSize) { | |
1276 | return windowLog; /* Window size large enough already */ | |
1277 | } else if (dictAndWindowSize >= maxWindowSize) { | |
1278 | return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ | |
1279 | } else { | |
1280 | return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; | |
1281 | } | |
1282 | } | |
1283 | } | |
1284 | ||
381e66cf | 1285 | /** ZSTD_adjustCParams_internal() : |
f9e4f892 YC |
1286 | * optimize `cPar` for a specified input (`srcSize` and `dictSize`). |
1287 | * mostly downsize to reduce memory consumption and initialization latency. | |
1288 | * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. | |
d5c688e8 | 1289 | * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. |
8c474f98 | 1290 | * note : `srcSize==0` means 0! |
f9e4f892 | 1291 | * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ |
3d523c74 YC |
1292 | static ZSTD_compressionParameters |
1293 | ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, | |
1294 | unsigned long long srcSize, | |
d5c688e8 NT |
1295 | size_t dictSize, |
1296 | ZSTD_cParamMode_e mode) | |
21588e37 | 1297 | { |
d5c688e8 NT |
1298 | const U64 minSrcSize = 513; /* (1<<9) + 1 */ |
1299 | const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); | |
381e66cf | 1300 | assert(ZSTD_checkCParams(cPar)==0); |
59d7063f | 1301 | |
d5c688e8 | 1302 | switch (mode) { |
58476bcf | 1303 | case ZSTD_cpm_unknown: |
d5c688e8 | 1304 | case ZSTD_cpm_noAttachDict: |
9d31c704 NT |
1305 | /* If we don't know the source size, don't make any |
1306 | * assumptions about it. We will already have selected | |
1307 | * smaller parameters if a dictionary is in use. | |
1308 | */ | |
1309 | break; | |
d5c688e8 | 1310 | case ZSTD_cpm_createCDict: |
9d31c704 NT |
1311 | /* Assume a small source size when creating a dictionary |
1312 | * with an unkown source size. | |
1313 | */ | |
1314 | if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) | |
1315 | srcSize = minSrcSize; | |
d5c688e8 NT |
1316 | break; |
1317 | case ZSTD_cpm_attachDict: | |
9d31c704 NT |
1318 | /* Dictionary has its own dedicated parameters which have |
1319 | * already been selected. We are selecting parameters | |
1320 | * for only the source. | |
1321 | */ | |
d5c688e8 NT |
1322 | dictSize = 0; |
1323 | break; | |
1324 | default: | |
1325 | assert(0); | |
1326 | break; | |
1327 | } | |
1328 | ||
86b4fe5b YC |
1329 | /* resize windowLog if input is small enough, to use less memory */ |
1330 | if ( (srcSize < maxWindowResize) | |
1331 | && (dictSize < maxWindowResize) ) { | |
1332 | U32 const tSize = (U32)(srcSize + dictSize); | |
1333 | static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; | |
1334 | U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : | |
1335 | ZSTD_highbit32(tSize-1) + 1; | |
1336 | if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; | |
1337 | } | |
9d31c704 NT |
1338 | if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { |
1339 | U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); | |
d5c688e8 NT |
1340 | U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); |
1341 | if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; | |
1342 | if (cycleLog > dictAndWindowLog) | |
1343 | cPar.chainLog -= (cycleLog - dictAndWindowLog); | |
c3a5c4be | 1344 | } |
c6eea2b2 | 1345 | |
9b5b47ac | 1346 | if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) |
f9e4f892 | 1347 | cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ |
70d1301d YC |
1348 | |
1349 | return cPar; | |
59d7063f YC |
1350 | } |
1351 | ||
3d523c74 YC |
1352 | ZSTD_compressionParameters |
1353 | ZSTD_adjustCParams(ZSTD_compressionParameters cPar, | |
1354 | unsigned long long srcSize, | |
1355 | size_t dictSize) | |
381e66cf | 1356 | { |
f9e4f892 | 1357 | cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ |
8c474f98 | 1358 | if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; |
d5c688e8 | 1359 | return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown); |
381e66cf YC |
1360 | } |
1361 | ||
d5c688e8 NT |
1362 | static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); |
1363 | static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); | |
8c474f98 | 1364 | |
a2af8041 FH |
1365 | static void ZSTD_overrideCParams( |
1366 | ZSTD_compressionParameters* cParams, | |
1367 | const ZSTD_compressionParameters* overrides) | |
1368 | { | |
1369 | if (overrides->windowLog) cParams->windowLog = overrides->windowLog; | |
1370 | if (overrides->hashLog) cParams->hashLog = overrides->hashLog; | |
1371 | if (overrides->chainLog) cParams->chainLog = overrides->chainLog; | |
1372 | if (overrides->searchLog) cParams->searchLog = overrides->searchLog; | |
1373 | if (overrides->minMatch) cParams->minMatch = overrides->minMatch; | |
1374 | if (overrides->targetLength) cParams->targetLength = overrides->targetLength; | |
1375 | if (overrides->strategy) cParams->strategy = overrides->strategy; | |
1376 | } | |
1377 | ||
3841dbac | 1378 | ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( |
d5c688e8 | 1379 | const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) |
3841dbac | 1380 | { |
dffbac5f NM |
1381 | ZSTD_compressionParameters cParams; |
1382 | if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { | |
1383 | srcSizeHint = CCtxParams->srcSizeHint; | |
1384 | } | |
d5c688e8 | 1385 | cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); |
3841dbac | 1386 | if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; |
a2af8041 | 1387 | ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); |
3841dbac | 1388 | assert(!ZSTD_checkCParams(cParams)); |
8c474f98 | 1389 | /* srcSizeHint == 0 means 0 */ |
d5c688e8 | 1390 | return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode); |
3841dbac NT |
1391 | } |
1392 | ||
3d523c74 YC |
1393 | static size_t |
1394 | ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, | |
4694423c NT |
1395 | const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, |
1396 | const U32 enableDedicatedDictSearch, | |
3d523c74 | 1397 | const U32 forCCtx) |
ab3346af | 1398 | { |
4694423c NT |
1399 | /* chain table size should be 0 for fast or row-hash strategies */ |
1400 | size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) | |
1401 | ? ((size_t)1 << cParams->chainLog) | |
1402 | : 0; | |
ab3346af | 1403 | size_t const hSize = ((size_t)1) << cParams->hashLog; |
e874dacc | 1404 | U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; |
b6c0a02a | 1405 | size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; |
2c80a9f8 FH |
1406 | /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't |
1407 | * surrounded by redzones in ASAN. */ | |
0ffae7e4 FH |
1408 | size_t const tableSpace = chainSize * sizeof(U32) |
1409 | + hSize * sizeof(U32) | |
1410 | + h3Size * sizeof(U32); | |
19a0955e | 1411 | size_t const optPotentialSpace = |
980f3bbf | 1412 | ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32)) |
1413 | + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32)) | |
1414 | + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32)) | |
1415 | + ZSTD_cwksp_aligned_alloc_size((1<<Litbits) * sizeof(U32)) | |
1416 | + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)) | |
1417 | + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); | |
4694423c NT |
1418 | size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder) |
1419 | ? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16)) | |
1420 | : 0; | |
e9448cdf | 1421 | size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) |
ab3346af NT |
1422 | ? optPotentialSpace |
1423 | : 0; | |
980f3bbf | 1424 | size_t const slackSpace = ZSTD_cwksp_slack_space_required(); |
1425 | ||
1426 | /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ | |
1427 | ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); | |
4694423c | 1428 | assert(useRowMatchFinder != ZSTD_urm_auto); |
980f3bbf | 1429 | |
ab3346af NT |
1430 | DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", |
1431 | (U32)chainSize, (U32)hSize, (U32)h3Size); | |
4694423c | 1432 | return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; |
ab3346af NT |
1433 | } |
1434 | ||
7ed996f5 FH |
1435 | static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( |
1436 | const ZSTD_compressionParameters* cParams, | |
1437 | const ldmParams_t* ldmParams, | |
afc24889 | 1438 | const int isStatic, |
4694423c | 1439 | const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, |
7ed996f5 FH |
1440 | const size_t buffInSize, |
1441 | const size_t buffOutSize, | |
d9a1e37a | 1442 | const U64 pledgedSrcSize) |
7ed996f5 FH |
1443 | { |
1444 | size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << cParams->windowLog), pledgedSrcSize)); | |
1445 | size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); | |
1446 | U32 const divider = (cParams->minMatch==3) ? 3 : 4; | |
1447 | size_t const maxNbSeq = blockSize / divider; | |
1448 | size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) | |
980f3bbf | 1449 | + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) |
7ed996f5 | 1450 | + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); |
a9077939 | 1451 | size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); |
7ed996f5 | 1452 | size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); |
4694423c | 1453 | size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); |
7ed996f5 FH |
1454 | |
1455 | size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); | |
1456 | size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); | |
3bb79923 | 1457 | size_t const ldmSeqSpace = ldmParams->enableLdm ? |
980f3bbf | 1458 | ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; |
7ed996f5 FH |
1459 | |
1460 | ||
1461 | size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) | |
1462 | + ZSTD_cwksp_alloc_size(buffOutSize); | |
1463 | ||
afc24889 | 1464 | size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; |
7ed996f5 FH |
1465 | |
1466 | size_t const neededSpace = | |
1467 | cctxSpace + | |
1468 | entropySpace + | |
1469 | blockStateSpace + | |
1470 | ldmSpace + | |
1471 | ldmSeqSpace + | |
1472 | matchStateSize + | |
1473 | tokenSpace + | |
1474 | bufferSpace; | |
1475 | ||
1476 | DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); | |
1477 | return neededSpace; | |
1478 | } | |
1479 | ||
96f0cde3 | 1480 | size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) |
e74215e3 | 1481 | { |
7ed996f5 | 1482 | ZSTD_compressionParameters const cParams = |
d5c688e8 | 1483 | ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); |
4694423c NT |
1484 | ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, |
1485 | &cParams); | |
c6636afb | 1486 | |
7ed996f5 FH |
1487 | RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); |
1488 | /* estimateCCtxSize is for one-shot compression. So no buffers should | |
1489 | * be needed. However, we still allocate two 0-sized buffers, which can | |
1490 | * take space under ASAN. */ | |
1491 | return ZSTD_estimateCCtxSize_usingCCtxParams_internal( | |
4694423c | 1492 | &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN); |
ade95b8b | 1493 | } |
3ae543ce | 1494 | |
96f0cde3 | 1495 | size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) |
ade95b8b | 1496 | { |
4694423c NT |
1497 | ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); |
1498 | if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { | |
1499 | /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ | |
1500 | size_t noRowCCtxSize; | |
1501 | size_t rowCCtxSize; | |
1502 | initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder; | |
1503 | noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); | |
1504 | initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder; | |
1505 | rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); | |
1506 | return MAX(noRowCCtxSize, rowCCtxSize); | |
1507 | } else { | |
1508 | return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); | |
1509 | } | |
2e91dde4 YC |
1510 | } |
1511 | ||
a68b76af | 1512 | static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) |
31af8290 | 1513 | { |
dff4a0e8 SH |
1514 | int tier = 0; |
1515 | size_t largestSize = 0; | |
1516 | static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; | |
1517 | for (; tier < 4; ++tier) { | |
1518 | /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ | |
1519 | ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); | |
1520 | largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); | |
1521 | } | |
1522 | return largestSize; | |
31af8290 YC |
1523 | } |
1524 | ||
a68b76af YC |
1525 | size_t ZSTD_estimateCCtxSize(int compressionLevel) |
1526 | { | |
1527 | int level; | |
1528 | size_t memBudget = 0; | |
d7def456 | 1529 | for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { |
dff4a0e8 | 1530 | /* Ensure monotonically increasing memory usage as compression level increases */ |
a68b76af YC |
1531 | size_t const newMB = ZSTD_estimateCCtxSize_internal(level); |
1532 | if (newMB > memBudget) memBudget = newMB; | |
1533 | } | |
1534 | return memBudget; | |
1535 | } | |
1536 | ||
96f0cde3 | 1537 | size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) |
c7fe262d | 1538 | { |
cafc3b1b | 1539 | RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); |
56682a77 | 1540 | { ZSTD_compressionParameters const cParams = |
d5c688e8 | 1541 | ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); |
56682a77 | 1542 | size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); |
d4e021fe NT |
1543 | size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) |
1544 | ? ((size_t)1 << cParams.windowLog) + blockSize | |
1545 | : 0; | |
fcf81cee NT |
1546 | size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) |
1547 | ? ZSTD_compressBound(blockSize) + 1 | |
1548 | : 0; | |
4694423c | 1549 | ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); |
a7737f6a | 1550 | |
7ed996f5 | 1551 | return ZSTD_estimateCCtxSize_usingCCtxParams_internal( |
4694423c | 1552 | &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, |
7ed996f5 | 1553 | ZSTD_CONTENTSIZE_UNKNOWN); |
15fdeb9e | 1554 | } |
ade95b8b | 1555 | } |
a7737f6a | 1556 | |
96f0cde3 | 1557 | size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) |
ade95b8b | 1558 | { |
4694423c NT |
1559 | ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); |
1560 | if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { | |
1561 | /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ | |
1562 | size_t noRowCCtxSize; | |
1563 | size_t rowCCtxSize; | |
1564 | initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder; | |
1565 | noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); | |
1566 | initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder; | |
1567 | rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); | |
1568 | return MAX(noRowCCtxSize, rowCCtxSize); | |
1569 | } else { | |
1570 | return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); | |
1571 | } | |
c7fe262d YC |
1572 | } |
1573 | ||
3d523c74 YC |
1574 | static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) |
1575 | { | |
d5c688e8 | 1576 | ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); |
96f0cde3 | 1577 | return ZSTD_estimateCStreamSize_usingCParams(cParams); |
0c9a915a YC |
1578 | } |
1579 | ||
3d523c74 YC |
1580 | size_t ZSTD_estimateCStreamSize(int compressionLevel) |
1581 | { | |
a68b76af YC |
1582 | int level; |
1583 | size_t memBudget = 0; | |
d7def456 | 1584 | for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { |
a68b76af YC |
1585 | size_t const newMB = ZSTD_estimateCStreamSize_internal(level); |
1586 | if (newMB > memBudget) memBudget = newMB; | |
1587 | } | |
1588 | return memBudget; | |
1589 | } | |
1590 | ||
394eec69 YC |
1591 | /* ZSTD_getFrameProgression(): |
1592 | * tells how much data has been consumed (input) and produced (output) for current frame. | |
1593 | * able to count progression inside worker threads (non-blocking mode). | |
1594 | */ | |
1595 | ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) | |
1596 | { | |
1597 | #ifdef ZSTD_MULTITHREAD | |
209df52b | 1598 | if (cctx->appliedParams.nbWorkers > 0) { |
394eec69 YC |
1599 | return ZSTDMT_getFrameProgression(cctx->mtctx); |
1600 | } | |
1601 | #endif | |
1602 | { ZSTD_frameProgression fp; | |
1603 | size_t const buffered = (cctx->inBuff == NULL) ? 0 : | |
1604 | cctx->inBuffPos - cctx->inToCompress; | |
1605 | if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); | |
1606 | assert(buffered <= ZSTD_BLOCKSIZE_MAX); | |
1607 | fp.ingested = cctx->consumedSrcSize + buffered; | |
1608 | fp.consumed = cctx->consumedSrcSize; | |
1609 | fp.produced = cctx->producedCSize; | |
3e4617ef | 1610 | fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ |
2dd76037 | 1611 | fp.currentJobID = 0; |
3e4617ef | 1612 | fp.nbActiveWorkers = 0; |
394eec69 YC |
1613 | return fp; |
1614 | } } | |
1615 | ||
105677c6 YC |
1616 | /*! ZSTD_toFlushNow() |
1617 | * Only useful for multithreading scenarios currently (nbWorkers >= 1). | |
1618 | */ | |
1619 | size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) | |
1620 | { | |
1621 | #ifdef ZSTD_MULTITHREAD | |
1622 | if (cctx->appliedParams.nbWorkers > 0) { | |
1623 | return ZSTDMT_toFlushNow(cctx->mtctx); | |
1624 | } | |
1625 | #endif | |
c71c4f23 | 1626 | (void)cctx; |
105677c6 YC |
1627 | return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ |
1628 | } | |
1629 | ||
1f188ae6 FH |
1630 | static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, |
1631 | ZSTD_compressionParameters cParams2) | |
01e34d36 | 1632 | { |
b7fba599 FH |
1633 | (void)cParams1; |
1634 | (void)cParams2; | |
1f188ae6 FH |
1635 | assert(cParams1.windowLog == cParams2.windowLog); |
1636 | assert(cParams1.chainLog == cParams2.chainLog); | |
1637 | assert(cParams1.hashLog == cParams2.hashLog); | |
1638 | assert(cParams1.searchLog == cParams2.searchLog); | |
e874dacc | 1639 | assert(cParams1.minMatch == cParams2.minMatch); |
1f188ae6 FH |
1640 | assert(cParams1.targetLength == cParams2.targetLength); |
1641 | assert(cParams1.strategy == cParams2.strategy); | |
01e34d36 FH |
1642 | } |
1643 | ||
b39149e1 | 1644 | void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) |
887cd4e3 NT |
1645 | { |
1646 | int i; | |
1647 | for (i = 0; i < ZSTD_REP_NUM; ++i) | |
1648 | bs->rep[i] = repStartValue[i]; | |
e3959d5e NT |
1649 | bs->entropy.huf.repeatMode = HUF_repeat_none; |
1650 | bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; | |
1651 | bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; | |
1652 | bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; | |
887cd4e3 NT |
1653 | } |
1654 | ||
1655 | /*! ZSTD_invalidateMatchState() | |
4baecdf7 YC |
1656 | * Invalidate all the matches in the match finder tables. |
1657 | * Requires nextSrc and base to be set (can be NULL). | |
887cd4e3 NT |
1658 | */ |
1659 | static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) | |
1660 | { | |
7e5e226c | 1661 | ZSTD_window_clear(&ms->window); |
887cd4e3 | 1662 | |
8e0e495c | 1663 | ms->nextToUpdate = ms->window.dictLimit; |
887cd4e3 NT |
1664 | ms->loadedDictEnd = 0; |
1665 | ms->opt.litLengthSum = 0; /* force reset of btopt stats */ | |
d18a4057 | 1666 | ms->dictMatchState = NULL; |
887cd4e3 NT |
1667 | } |
1668 | ||
5b10bb5e FH |
1669 | /** |
1670 | * Controls, for this matchState reset, whether the tables need to be cleared / | |
1671 | * prepared for the coming compression (ZSTDcrp_makeClean), or whether the | |
1672 | * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a | |
1673 | * subsequent operation will overwrite the table space anyways (e.g., copying | |
1674 | * the matchState contents in from a CDict). | |
1675 | */ | |
14c5471d | 1676 | typedef enum { |
5b10bb5e FH |
1677 | ZSTDcrp_makeClean, |
1678 | ZSTDcrp_leaveDirty | |
14c5471d | 1679 | } ZSTD_compResetPolicy_e; |
a7737f6a | 1680 | |
14c5471d FH |
1681 | /** |
1682 | * Controls, for this matchState reset, whether indexing can continue where it | |
1683 | * left off (ZSTDirp_continue), or whether it needs to be restarted from zero | |
1684 | * (ZSTDirp_reset). | |
1685 | */ | |
1686 | typedef enum { | |
1687 | ZSTDirp_continue, | |
1688 | ZSTDirp_reset | |
1689 | } ZSTD_indexResetPolicy_e; | |
a7737f6a | 1690 | |
14c5471d FH |
1691 | typedef enum { |
1692 | ZSTD_resetTarget_CDict, | |
1693 | ZSTD_resetTarget_CCtx | |
1694 | } ZSTD_resetTarget_e; | |
621adde3 | 1695 | |
4694423c | 1696 | |
786f2266 | 1697 | static size_t |
a1550613 | 1698 | ZSTD_reset_matchState(ZSTD_matchState_t* ms, |
077a2d7d | 1699 | ZSTD_cwksp* ws, |
a1550613 | 1700 | const ZSTD_compressionParameters* cParams, |
4694423c | 1701 | const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, |
14c5471d | 1702 | const ZSTD_compResetPolicy_e crp, |
0492b9a9 | 1703 | const ZSTD_indexResetPolicy_e forceResetIndex, |
14c5471d | 1704 | const ZSTD_resetTarget_e forWho) |
16bd0fd4 | 1705 | { |
4694423c NT |
1706 | /* disable chain table allocation for fast or row-based strategies */ |
1707 | size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, | |
1708 | ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) | |
1709 | ? ((size_t)1 << cParams->chainLog) | |
1710 | : 0; | |
16bd0fd4 | 1711 | size_t const hSize = ((size_t)1) << cParams->hashLog; |
621adde3 | 1712 | U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; |
7c57e2b9 | 1713 | size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; |
16bd0fd4 | 1714 | |
1999b2ed | 1715 | DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); |
4694423c | 1716 | assert(useRowMatchFinder != ZSTD_urm_auto); |
ad16eda5 | 1717 | if (forceResetIndex == ZSTDirp_reset) { |
659e9f05 | 1718 | ZSTD_window_init(&ms->window); |
ad16eda5 FH |
1719 | ZSTD_cwksp_mark_tables_dirty(ws); |
1720 | } | |
16bd0fd4 | 1721 | |
16bd0fd4 | 1722 | ms->hashLog3 = hashLog3; |
ad16eda5 | 1723 | |
16bd0fd4 NT |
1724 | ZSTD_invalidateMatchState(ms); |
1725 | ||
077a2d7d | 1726 | assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ |
786f2266 | 1727 | |
077a2d7d | 1728 | ZSTD_cwksp_clear_tables(ws); |
e69b67e3 FH |
1729 | |
1730 | DEBUGLOG(5, "reserving table space"); | |
1731 | /* table Space */ | |
077a2d7d FH |
1732 | ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); |
1733 | ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); | |
1734 | ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); | |
1735 | RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, | |
e69b67e3 FH |
1736 | "failed a workspace allocation in ZSTD_reset_matchState"); |
1737 | ||
5b10bb5e FH |
1738 | DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); |
1739 | if (crp!=ZSTDcrp_leaveDirty) { | |
e69b67e3 | 1740 | /* reset tables only */ |
ad16eda5 | 1741 | ZSTD_cwksp_clean_tables(ws); |
786f2266 FH |
1742 | } |
1743 | ||
16bd0fd4 | 1744 | /* opt parser space */ |
621adde3 | 1745 | if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { |
16bd0fd4 | 1746 | DEBUGLOG(4, "reserving optimal parser space"); |
077a2d7d FH |
1747 | ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<<Litbits) * sizeof(unsigned)); |
1748 | ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); | |
1749 | ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); | |
1750 | ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); | |
1751 | ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); | |
1752 | ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); | |
16bd0fd4 NT |
1753 | } |
1754 | ||
4694423c NT |
1755 | if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { |
1756 | { /* Row match finder needs an additional table of hashes ("tags") */ | |
1757 | size_t const tagTableSize = hSize*sizeof(U16); | |
1758 | ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize); | |
1759 | if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize); | |
1760 | } | |
1761 | { /* Switch to 32-entry rows if searchLog is 5 (or more) */ | |
1762 | U32 const rowLog = cParams->searchLog < 5 ? 4 : 5; | |
1763 | assert(cParams->hashLog > rowLog); | |
1ffa80a0 | 1764 | ms->rowHashLog = cParams->hashLog - rowLog; |
4694423c NT |
1765 | } |
1766 | } | |
1767 | ||
76ef87ed FH |
1768 | ms->cParams = *cParams; |
1769 | ||
077a2d7d | 1770 | RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, |
786f2266 | 1771 | "failed a workspace allocation in ZSTD_reset_matchState"); |
786f2266 | 1772 | return 0; |
16bd0fd4 NT |
1773 | } |
1774 | ||
621adde3 YC |
1775 | /* ZSTD_indexTooCloseToMax() : |
1776 | * minor optimization : prefer memset() rather than reduceIndex() | |
1777 | * which is measurably slow in some circumstances (reported for Visual Studio). | |
45c9fbd6 YC |
1778 | * Works when re-using a context for a lot of smallish inputs : |
1779 | * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, | |
1780 | * memset() will be triggered before reduceIndex(). | |
1781 | */ | |
1782 | #define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) | |
621adde3 | 1783 | static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) |
45c9fbd6 | 1784 | { |
621adde3 | 1785 | return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); |
45c9fbd6 YC |
1786 | } |
1787 | ||
94db4398 NT |
1788 | /** ZSTD_dictTooBig(): |
1789 | * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in | |
1790 | * one go generically. So we ensure that in that case we reset the tables to zero, | |
1791 | * so that we can load as much of the dictionary as possible. | |
1792 | */ | |
1793 | static int ZSTD_dictTooBig(size_t const loadedDictSize) | |
1794 | { | |
1795 | return loadedDictSize > ZSTD_CHUNKSIZE_MAX; | |
1796 | } | |
1797 | ||
30fb4992 | 1798 | /*! ZSTD_resetCCtx_internal() : |
94db4398 NT |
1799 | * @param loadedDictSize The size of the dictionary to be loaded |
1800 | * into the context, if any. If no dictionary is used, or the | |
1801 | * dictionary is being attached / copied, then pass 0. | |
1802 | * note : `params` are assumed fully validated at this stage. | |
1803 | */ | |
5ac72b41 | 1804 | static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, |
c2183d7c | 1805 | ZSTD_CCtx_params const* params, |
bc601bdc | 1806 | U64 const pledgedSrcSize, |
94db4398 | 1807 | size_t const loadedDictSize, |
5ac72b41 YC |
1808 | ZSTD_compResetPolicy_e const crp, |
1809 | ZSTD_buffered_policy_e const zbuff) | |
a7737f6a | 1810 | { |
0a65a679 | 1811 | ZSTD_cwksp* const ws = &zc->workspace; |
4694423c | 1812 | DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d", |
c2183d7c NT |
1813 | (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder); |
1814 | assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); | |
0be6fd34 | 1815 | |
96201d97 | 1816 | zc->isFirstBlock = 1; |
a7737f6a | 1817 | |
c2183d7c NT |
1818 | /* Set applied params early so we can modify them for LDM, |
1819 | * and point params at the applied params. | |
1820 | */ | |
1821 | zc->appliedParams = *params; | |
1822 | params = &zc->appliedParams; | |
1823 | ||
1824 | assert(params->useRowMatchFinder != ZSTD_urm_auto); | |
1825 | if (params->ldmParams.enableLdm) { | |
67d4a616 | 1826 | /* Adjust long distance matching parameters */ |
c2183d7c NT |
1827 | ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); |
1828 | assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); | |
1829 | assert(params->ldmParams.hashRateLog < 32); | |
6a546efb SL |
1830 | } |
1831 | ||
c2183d7c | 1832 | { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); |
15768cab | 1833 | size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); |
c2183d7c | 1834 | U32 const divider = (params->cParams.minMatch==3) ? 3 : 4; |
a7737f6a | 1835 | size_t const maxNbSeq = blockSize / divider; |
c2183d7c | 1836 | size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) |
fcf81cee NT |
1837 | ? ZSTD_compressBound(blockSize) + 1 |
1838 | : 0; | |
c2183d7c | 1839 | size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) |
d4e021fe NT |
1840 | ? windowSize + blockSize |
1841 | : 0; | |
c2183d7c | 1842 | size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); |
a7737f6a | 1843 | |
27e24822 | 1844 | int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); |
94db4398 | 1845 | int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); |
27e24822 | 1846 | ZSTD_indexResetPolicy_e needsIndexReset = |
94db4398 | 1847 | (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; |
f31ef28f | 1848 | |
27e24822 FH |
1849 | size_t const neededSpace = |
1850 | ZSTD_estimateCCtxSize_usingCCtxParams_internal( | |
c2183d7c | 1851 | ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, |
27e24822 | 1852 | buffInSize, buffOutSize, pledgedSrcSize); |
980f3bbf | 1853 | int resizeWorkspace; |
1854 | ||
27e24822 | 1855 | FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); |
f31ef28f | 1856 | |
608f1bfc | 1857 | if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); |
f31ef28f | 1858 | |
980f3bbf | 1859 | { /* Check if workspace is large enough, alloc a new one if needed */ |
e3703825 | 1860 | int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; |
0a65a679 | 1861 | int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); |
980f3bbf | 1862 | resizeWorkspace = workspaceTooSmall || workspaceWasteful; |
7ed996f5 | 1863 | DEBUGLOG(4, "Need %zu B workspace", neededSpace); |
3d523c74 YC |
1864 | DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); |
1865 | ||
980f3bbf | 1866 | if (resizeWorkspace) { |
ccaac852 | 1867 | DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", |
0a65a679 | 1868 | ZSTD_cwksp_sizeof(ws) >> 10, |
3d523c74 | 1869 | neededSpace >> 10); |
cafc3b1b FH |
1870 | |
1871 | RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); | |
c7fe262d | 1872 | |
f31ef28f FH |
1873 | needsIndexReset = ZSTDirp_reset; |
1874 | ||
0a65a679 | 1875 | ZSTD_cwksp_free(ws, zc->customMem); |
5e5f2626 | 1876 | FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); |
7bb60b17 | 1877 | |
786f2266 | 1878 | DEBUGLOG(5, "reserving object space"); |
3d523c74 YC |
1879 | /* Statically sized space. |
1880 | * entropyWorkspace never moves, | |
1881 | * though prev/next block swap places */ | |
0a65a679 FH |
1882 | assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); |
1883 | zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); | |
786f2266 | 1884 | RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); |
0a65a679 | 1885 | zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); |
786f2266 | 1886 | RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); |
a9077939 | 1887 | zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); |
786f2266 | 1888 | RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); |
a7737f6a | 1889 | } } |
e74215e3 | 1890 | |
0a65a679 | 1891 | ZSTD_cwksp_clear(ws); |
786f2266 | 1892 | |
e6fa70a0 | 1893 | /* init params */ |
c2183d7c | 1894 | zc->blockState.matchState.cParams = params->cParams; |
a0ba849f | 1895 | zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; |
e6fa70a0 | 1896 | zc->consumedSrcSize = 0; |
394eec69 | 1897 | zc->producedCSize = 0; |
a0ba849f | 1898 | if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) |
cc9f9b7f | 1899 | zc->appliedParams.fParams.contentSizeFlag = 0; |
15768cab | 1900 | DEBUGLOG(4, "pledged content size : %u ; flag : %u", |
ededcfca | 1901 | (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); |
cc9f9b7f | 1902 | zc->blockSize = blockSize; |
a7737f6a | 1903 | |
a7737f6a | 1904 | XXH64_reset(&zc->xxhState, 0); |
e6fa70a0 YC |
1905 | zc->stage = ZSTDcs_init; |
1906 | zc->dictID = 0; | |
54a4998a | 1907 | zc->dictContentSize = 0; |
887cd4e3 | 1908 | |
16bd0fd4 | 1909 | ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); |
887cd4e3 | 1910 | |
5e580de6 NT |
1911 | /* ZSTD_wildcopy() is used to copy into the literals buffer, |
1912 | * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. | |
1913 | */ | |
0a65a679 | 1914 | zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); |
5e580de6 | 1915 | zc->seqStore.maxNbLit = blockSize; |
6177354b FH |
1916 | |
1917 | /* buffers */ | |
d4e021fe | 1918 | zc->bufferedPolicy = zbuff; |
6177354b | 1919 | zc->inBuffSize = buffInSize; |
0a65a679 | 1920 | zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); |
6177354b | 1921 | zc->outBuffSize = buffOutSize; |
0a65a679 | 1922 | zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); |
5ac72b41 | 1923 | |
9e406020 | 1924 | /* ldm bucketOffsets table */ |
c2183d7c | 1925 | if (params->ldmParams.enableLdm) { |
edb3ad05 | 1926 | /* TODO: avoid memset? */ |
1e65711c | 1927 | size_t const numBuckets = |
c2183d7c NT |
1928 | ((size_t)1) << (params->ldmParams.hashLog - |
1929 | params->ldmParams.bucketSizeLog); | |
1e65711c QC |
1930 | zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); |
1931 | ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); | |
9e406020 | 1932 | } |
6177354b FH |
1933 | |
1934 | /* sequences storage */ | |
136b9e23 | 1935 | ZSTD_referenceExternalSequences(zc, NULL, 0); |
6177354b | 1936 | zc->seqStore.maxNbSeq = maxNbSeq; |
0a65a679 FH |
1937 | zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); |
1938 | zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); | |
1939 | zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); | |
1940 | zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); | |
9e406020 | 1941 | |
786f2266 FH |
1942 | FORWARD_IF_ERROR(ZSTD_reset_matchState( |
1943 | &zc->blockState.matchState, | |
0a65a679 | 1944 | ws, |
c2183d7c NT |
1945 | ¶ms->cParams, |
1946 | params->useRowMatchFinder, | |
0492b9a9 | 1947 | crp, |
f31ef28f | 1948 | needsIndexReset, |
5e5f2626 | 1949 | ZSTD_resetTarget_CCtx), ""); |
9e406020 | 1950 | |
9e406020 | 1951 | /* ldm hash table */ |
c2183d7c | 1952 | if (params->ldmParams.enableLdm) { |
edb3ad05 | 1953 | /* TODO: avoid memset? */ |
c2183d7c | 1954 | size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; |
0a65a679 | 1955 | zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); |
c465f244 | 1956 | ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); |
0a65a679 | 1957 | zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); |
a9a6dcba | 1958 | zc->maxNbLdmSequences = maxNbLdmSeq; |
0a0e64c6 | 1959 | |
659e9f05 | 1960 | ZSTD_window_init(&zc->ldmState.window); |
b2092c6d | 1961 | zc->ldmState.loadedDictEnd = 0; |
9e406020 | 1962 | } |
a7737f6a | 1963 | |
980f3bbf | 1964 | assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace)); |
0a65a679 | 1965 | DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); |
980f3bbf | 1966 | |
659e9f05 | 1967 | zc->initialized = 1; |
a7737f6a | 1968 | |
a7737f6a | 1969 | return 0; |
4ab9c913 | 1970 | } |
f3eca253 YC |
1971 | } |
1972 | ||
32dfae6f YC |
1973 | /* ZSTD_invalidateRepCodes() : |
1974 | * ensures next compression will not use repcodes from previous block. | |
1975 | * Note : only works with regular variant; | |
1976 | * do not use with extDict variant ! */ | |
1977 | void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { | |
1978 | int i; | |
aae267a2 | 1979 | for (i=0; i<ZSTD_REP_NUM; i++) cctx->blockState.prevCBlock->rep[i] = 0; |
7e5e226c | 1980 | assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); |
32dfae6f | 1981 | } |
083fcc82 | 1982 | |
a6d6bbea FH |
1983 | /* These are the approximate sizes for each strategy past which copying the |
1984 | * dictionary tables into the working context is faster than using them | |
1985 | * in-place. | |
1986 | */ | |
39e28982 | 1987 | static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { |
e9448cdf YC |
1988 | 8 KB, /* unused */ |
1989 | 8 KB, /* ZSTD_fast */ | |
a6d6bbea FH |
1990 | 16 KB, /* ZSTD_dfast */ |
1991 | 32 KB, /* ZSTD_greedy */ | |
1992 | 32 KB, /* ZSTD_lazy */ | |
1993 | 32 KB, /* ZSTD_lazy2 */ | |
1994 | 32 KB, /* ZSTD_btlazy2 */ | |
1995 | 32 KB, /* ZSTD_btopt */ | |
e9448cdf YC |
1996 | 8 KB, /* ZSTD_btultra */ |
1997 | 8 KB /* ZSTD_btultra2 */ | |
a6d6bbea | 1998 | }; |
2108decb | 1999 | |
77fd17d9 | 2000 | static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, |
77164547 | 2001 | const ZSTD_CCtx_params* params, |
77fd17d9 | 2002 | U64 pledgedSrcSize) |
a6d6bbea FH |
2003 | { |
2004 | size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; | |
db2aa252 | 2005 | int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; |
0faefbf1 FH |
2006 | return dedicatedDictSearch |
2007 | || ( ( pledgedSrcSize <= cutoff | |
2008 | || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN | |
2009 | || params->attachDictPref == ZSTD_dictForceAttach ) | |
2010 | && params->attachDictPref != ZSTD_dictForceCopy | |
2011 | && !params->forceWindow ); /* dictMatchState isn't correctly | |
2012 | * handled in _enforceMaxDist */ | |
a6d6bbea | 2013 | } |
d18a4057 | 2014 | |
4baecdf7 YC |
2015 | static size_t |
2016 | ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, | |
2017 | const ZSTD_CDict* cdict, | |
2018 | ZSTD_CCtx_params params, | |
2019 | U64 pledgedSrcSize, | |
2020 | ZSTD_buffered_policy_e zbuff) | |
16bd0fd4 | 2021 | { |
eb7e74cc NT |
2022 | DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", |
2023 | (unsigned long long)pledgedSrcSize); | |
7b5d2f72 FH |
2024 | { |
2025 | ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; | |
01ff945e | 2026 | unsigned const windowLog = params.cParams.windowLog; |
ca77822d | 2027 | assert(windowLog != 0); |
c38acff9 FH |
2028 | /* Resize working context table params for input only, since the dict |
2029 | * has its own tables. */ | |
7ed996f5 | 2030 | /* pledgedSrcSize == 0 means 0! */ |
7b5d2f72 FH |
2031 | |
2032 | if (cdict->matchState.dedicatedDictSearch) { | |
2033 | ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); | |
2034 | } | |
2035 | ||
d5c688e8 NT |
2036 | params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, |
2037 | cdict->dictContentSize, ZSTD_cpm_attachDict); | |
ca77822d | 2038 | params.cParams.windowLog = windowLog; |
4694423c | 2039 | params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ |
c2183d7c | 2040 | FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, |
94db4398 | 2041 | /* loadedDictSize */ 0, |
5e5f2626 | 2042 | ZSTDcrp_makeClean, zbuff), ""); |
7b5d2f72 | 2043 | assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); |
16bd0fd4 NT |
2044 | } |
2045 | ||
4baecdf7 | 2046 | { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc |
7ef85e06 | 2047 | - cdict->matchState.window.base); |
2ca7c691 | 2048 | const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; |
7ef85e06 | 2049 | if (cdictLen == 0) { |
582b7f85 FH |
2050 | /* don't even attach dictionaries with no contents */ |
2051 | DEBUGLOG(4, "skipping attaching empty dictionary"); | |
2052 | } else { | |
2053 | DEBUGLOG(4, "attaching dictionary into context"); | |
2054 | cctx->blockState.matchState.dictMatchState = &cdict->matchState; | |
2055 | ||
2056 | /* prep working match state so dict matches never have negative indices | |
2057 | * when they are translated to the working context's index space. */ | |
2ca7c691 | 2058 | if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { |
582b7f85 | 2059 | cctx->blockState.matchState.window.nextSrc = |
2ca7c691 | 2060 | cctx->blockState.matchState.window.base + cdictEnd; |
582b7f85 FH |
2061 | ZSTD_window_clear(&cctx->blockState.matchState.window); |
2062 | } | |
4baecdf7 | 2063 | /* loadedDictEnd is expressed within the referential of the active context */ |
298d24fa | 2064 | cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; |
4baecdf7 | 2065 | } } |
16bd0fd4 | 2066 | |
01ff945e | 2067 | cctx->dictID = cdict->dictID; |
54a4998a | 2068 | cctx->dictContentSize = cdict->dictContentSize; |
d18a4057 | 2069 | |
01ff945e | 2070 | /* copy block state */ |
c465f244 | 2071 | ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); |
01ff945e FH |
2072 | |
2073 | return 0; | |
2074 | } | |
2075 | ||
2076 | static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, | |
2077 | const ZSTD_CDict* cdict, | |
2078 | ZSTD_CCtx_params params, | |
2079 | U64 pledgedSrcSize, | |
2080 | ZSTD_buffered_policy_e zbuff) | |
2081 | { | |
2082 | const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; | |
2083 | ||
db2aa252 | 2084 | assert(!cdict->matchState.dedicatedDictSearch); |
eb7e74cc NT |
2085 | DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", |
2086 | (unsigned long long)pledgedSrcSize); | |
01ff945e FH |
2087 | |
2088 | { unsigned const windowLog = params.cParams.windowLog; | |
2089 | assert(windowLog != 0); | |
2090 | /* Copy only compression parameters related to tables. */ | |
2091 | params.cParams = *cdict_cParams; | |
2092 | params.cParams.windowLog = windowLog; | |
4694423c | 2093 | params.useRowMatchFinder = cdict->useRowMatchFinder; |
c2183d7c | 2094 | FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, |
94db4398 | 2095 | /* loadedDictSize */ 0, |
5e5f2626 | 2096 | ZSTDcrp_leaveDirty, zbuff), ""); |
01ff945e FH |
2097 | assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); |
2098 | assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); | |
2099 | assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); | |
2100 | } | |
2101 | ||
13e29a56 | 2102 | ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); |
4694423c | 2103 | assert(params.useRowMatchFinder != ZSTD_urm_auto); |
13e29a56 | 2104 | |
01ff945e | 2105 | /* copy tables */ |
4694423c NT |
2106 | { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) |
2107 | ? ((size_t)1 << cdict_cParams->chainLog) | |
2108 | : 0; | |
01ff945e | 2109 | size_t const hSize = (size_t)1 << cdict_cParams->hashLog; |
da88c35d | 2110 | |
c465f244 | 2111 | ZSTD_memcpy(cctx->blockState.matchState.hashTable, |
da88c35d FH |
2112 | cdict->matchState.hashTable, |
2113 | hSize * sizeof(U32)); | |
4694423c NT |
2114 | /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ |
2115 | if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { | |
2116 | ZSTD_memcpy(cctx->blockState.matchState.chainTable, | |
da88c35d FH |
2117 | cdict->matchState.chainTable, |
2118 | chainSize * sizeof(U32)); | |
4694423c NT |
2119 | } |
2120 | /* copy tag table */ | |
2121 | if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { | |
2122 | size_t const tagTableSize = hSize*sizeof(U16); | |
2123 | ZSTD_memcpy(cctx->blockState.matchState.tagTable, | |
2124 | cdict->matchState.tagTable, | |
2125 | tagTableSize); | |
2126 | } | |
01ff945e FH |
2127 | } |
2128 | ||
2129 | /* Zero the hashTable3, since the cdict never fills it */ | |
7c57e2b9 FH |
2130 | { int const h3log = cctx->blockState.matchState.hashLog3; |
2131 | size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; | |
01ff945e | 2132 | assert(cdict->matchState.hashLog3 == 0); |
c465f244 | 2133 | ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); |
01ff945e FH |
2134 | } |
2135 | ||
13e29a56 FH |
2136 | ZSTD_cwksp_mark_tables_clean(&cctx->workspace); |
2137 | ||
01ff945e FH |
2138 | /* copy dictionary offsets */ |
2139 | { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; | |
2140 | ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; | |
2141 | dstMatchState->window = srcMatchState->window; | |
2142 | dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; | |
01ff945e | 2143 | dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; |
16bd0fd4 | 2144 | } |
d18a4057 | 2145 | |
16bd0fd4 | 2146 | cctx->dictID = cdict->dictID; |
54a4998a | 2147 | cctx->dictContentSize = cdict->dictContentSize; |
16bd0fd4 NT |
2148 | |
2149 | /* copy block state */ | |
c465f244 | 2150 | ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); |
16bd0fd4 NT |
2151 | |
2152 | return 0; | |
2153 | } | |
a4cab801 | 2154 | |
01ff945e FH |
2155 | /* We have a choice between copying the dictionary context into the working |
2156 | * context, or referencing the dictionary context from the working context | |
2157 | * in-place. We decide here which strategy to use. */ | |
2158 | static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, | |
2159 | const ZSTD_CDict* cdict, | |
77164547 | 2160 | const ZSTD_CCtx_params* params, |
01ff945e FH |
2161 | U64 pledgedSrcSize, |
2162 | ZSTD_buffered_policy_e zbuff) | |
2163 | { | |
2164 | ||
ededcfca YC |
2165 | DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", |
2166 | (unsigned)pledgedSrcSize); | |
01ff945e | 2167 | |
77fd17d9 | 2168 | if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { |
01ff945e | 2169 | return ZSTD_resetCCtx_byAttachingCDict( |
77164547 | 2170 | cctx, cdict, *params, pledgedSrcSize, zbuff); |
01ff945e FH |
2171 | } else { |
2172 | return ZSTD_resetCCtx_byCopyingCDict( | |
77164547 | 2173 | cctx, cdict, *params, pledgedSrcSize, zbuff); |
01ff945e FH |
2174 | } |
2175 | } | |
2176 | ||
a4cab801 YC |
2177 | /*! ZSTD_copyCCtx_internal() : |
2178 | * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. | |
2179 | * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). | |
e28305fc YC |
2180 | * The "context", in this case, refers to the hash and chain tables, |
2181 | * entropy tables, and dictionary references. | |
2182 | * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. | |
2183 | * @return : 0, or an error code */ | |
1ad7c82e YC |
2184 | static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, |
2185 | const ZSTD_CCtx* srcCCtx, | |
2186 | ZSTD_frameParameters fParams, | |
e28305fc | 2187 | U64 pledgedSrcSize, |
204b6b7e | 2188 | ZSTD_buffered_policy_e zbuff) |
7b51a294 | 2189 | { |
608f1bfc | 2190 | RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, |
5e5f2626 | 2191 | "Can't copy a ctx that's not in init stage."); |
4694423c | 2192 | DEBUGLOG(5, "ZSTD_copyCCtx_internal"); |
c465f244 | 2193 | ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); |
fd0071da SL |
2194 | { ZSTD_CCtx_params params = dstCCtx->requestedParams; |
2195 | /* Copy only compression parameters related to tables. */ | |
2196 | params.cParams = srcCCtx->appliedParams.cParams; | |
4694423c NT |
2197 | assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_urm_auto); |
2198 | params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; | |
a4cab801 | 2199 | params.fParams = fParams; |
c2183d7c | 2200 | ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, |
94db4398 | 2201 | /* loadedDictSize */ 0, |
5b10bb5e | 2202 | ZSTDcrp_leaveDirty, zbuff); |
ab3346af NT |
2203 | assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); |
2204 | assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); | |
2205 | assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); | |
2206 | assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); | |
2207 | assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); | |
2db72492 | 2208 | } |
7b51a294 | 2209 | |
13e29a56 FH |
2210 | ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); |
2211 | ||
7b51a294 | 2212 | /* copy tables */ |
4694423c NT |
2213 | { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, |
2214 | srcCCtx->appliedParams.useRowMatchFinder, | |
2215 | 0 /* forDDSDict */) | |
2216 | ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) | |
2217 | : 0; | |
1ad7c82e | 2218 | size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; |
7c57e2b9 FH |
2219 | int const h3log = srcCCtx->blockState.matchState.hashLog3; |
2220 | size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; | |
da88c35d | 2221 | |
c465f244 | 2222 | ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, |
da88c35d FH |
2223 | srcCCtx->blockState.matchState.hashTable, |
2224 | hSize * sizeof(U32)); | |
c465f244 | 2225 | ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, |
da88c35d FH |
2226 | srcCCtx->blockState.matchState.chainTable, |
2227 | chainSize * sizeof(U32)); | |
c465f244 | 2228 | ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, |
da88c35d FH |
2229 | srcCCtx->blockState.matchState.hashTable3, |
2230 | h3Size * sizeof(U32)); | |
c6eea2b2 | 2231 | } |
7b51a294 | 2232 | |
13e29a56 FH |
2233 | ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); |
2234 | ||
c46fb924 | 2235 | /* copy dictionary offsets */ |
22727a74 | 2236 | { |
a1550613 | 2237 | const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; |
aae267a2 | 2238 | ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; |
7e5e226c | 2239 | dstMatchState->window = srcMatchState->window; |
887cd4e3 | 2240 | dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; |
887cd4e3 | 2241 | dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; |
22727a74 | 2242 | } |
887cd4e3 | 2243 | dstCCtx->dictID = srcCCtx->dictID; |
54a4998a | 2244 | dstCCtx->dictContentSize = srcCCtx->dictContentSize; |
887cd4e3 NT |
2245 | |
2246 | /* copy block state */ | |
c465f244 | 2247 | ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); |
7b51a294 YC |
2248 | |
2249 | return 0; | |
2250 | } | |
2251 | ||
a4cab801 YC |
2252 | /*! ZSTD_copyCCtx() : |
2253 | * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. | |
2254 | * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). | |
2255 | * pledgedSrcSize==0 means "unknown". | |
2256 | * @return : 0, or an error code */ | |
2257 | size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) | |
2258 | { | |
2259 | ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; | |
d4e021fe | 2260 | ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; |
204b6b7e | 2261 | ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); |
fb445166 YC |
2262 | if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; |
2263 | fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); | |
a4cab801 | 2264 | |
e28305fc | 2265 | return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, |
ab3346af | 2266 | fParams, pledgedSrcSize, |
e28305fc | 2267 | zbuff); |
a4cab801 YC |
2268 | } |
2269 | ||
7b51a294 | 2270 | |
f36da5b4 | 2271 | #define ZSTD_ROWSIZE 16 |
de68c2ff YC |
2272 | /*! ZSTD_reduceTable() : |
2273 | * reduce table indexes by `reducerValue`, or squash to zero. | |
2274 | * PreserveMark preserves "unsorted mark" for btlazy2 strategy. | |
2275 | * It must be set to a clear 0/1 value, to remove branch during inlining. | |
2276 | * Presume table size is a multiple of ZSTD_ROWSIZE | |
2277 | * to help auto-vectorization */ | |
2278 | FORCE_INLINE_TEMPLATE void | |
2279 | ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) | |
f36da5b4 | 2280 | { |
de68c2ff | 2281 | int const nbRows = (int)size / ZSTD_ROWSIZE; |
f36da5b4 YC |
2282 | int cellNb = 0; |
2283 | int rowNb; | |
de68c2ff YC |
2284 | assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ |
2285 | assert(size < (1U<<31)); /* can be casted to int */ | |
a10c1916 | 2286 | |
b8413872 | 2287 | #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) |
a10c1916 FH |
2288 | /* To validate that the table re-use logic is sound, and that we don't |
2289 | * access table space that we haven't cleaned, we re-"poison" the table | |
2290 | * space every time we mark it dirty. | |
2291 | * | |
2292 | * This function however is intended to operate on those dirty tables and | |
2293 | * re-clean them. So when this function is used correctly, we can unpoison | |
2294 | * the memory it operated on. This introduces a blind spot though, since | |
2295 | * if we now try to operate on __actually__ poisoned memory, we will not | |
2296 | * detect that. */ | |
2297 | __msan_unpoison(table, size * sizeof(U32)); | |
2298 | #endif | |
2299 | ||
f36da5b4 YC |
2300 | for (rowNb=0 ; rowNb < nbRows ; rowNb++) { |
2301 | int column; | |
2302 | for (column=0; column<ZSTD_ROWSIZE; column++) { | |
de68c2ff YC |
2303 | if (preserveMark) { |
2304 | U32 const adder = (table[cellNb] == ZSTD_DUBT_UNSORTED_MARK) ? reducerValue : 0; | |
2305 | table[cellNb] += adder; | |
2306 | } | |
f36da5b4 YC |
2307 | if (table[cellNb] < reducerValue) table[cellNb] = 0; |
2308 | else table[cellNb] -= reducerValue; | |
2309 | cellNb++; | |
a7e34ff6 | 2310 | } } |
f36da5b4 YC |
2311 | } |
2312 | ||
de68c2ff | 2313 | static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue) |
89db5e00 | 2314 | { |
de68c2ff | 2315 | ZSTD_reduceTable_internal(table, size, reducerValue, 0); |
89db5e00 YC |
2316 | } |
2317 | ||
de68c2ff YC |
2318 | static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue) |
2319 | { | |
2320 | ZSTD_reduceTable_internal(table, size, reducerValue, 1); | |
2321 | } | |
2322 | ||
ecabfe37 YC |
2323 | /*! ZSTD_reduceIndex() : |
2324 | * rescale all indexes to avoid future overflow (indexes are U32) */ | |
674534a7 | 2325 | static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) |
ecabfe37 | 2326 | { |
674534a7 | 2327 | { U32 const hSize = (U32)1 << params->cParams.hashLog; |
887cd4e3 | 2328 | ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); |
f36da5b4 | 2329 | } |
ecabfe37 | 2330 | |
61fe571a | 2331 | if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { |
674534a7 NT |
2332 | U32 const chainSize = (U32)1 << params->cParams.chainLog; |
2333 | if (params->cParams.strategy == ZSTD_btlazy2) | |
de68c2ff YC |
2334 | ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); |
2335 | else | |
2336 | ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); | |
f36da5b4 | 2337 | } |
ecabfe37 | 2338 | |
887cd4e3 NT |
2339 | if (ms->hashLog3) { |
2340 | U32 const h3Size = (U32)1 << ms->hashLog3; | |
2341 | ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); | |
f36da5b4 | 2342 | } |
ecabfe37 YC |
2343 | } |
2344 | ||
89db5e00 | 2345 | |
863ec40f | 2346 | /*-******************************************************* |
14983e7a YC |
2347 | * Block entropic compression |
2348 | *********************************************************/ | |
14983e7a | 2349 | |
3ee94a76 | 2350 | /* See doc/zstd_compression_format.md for detailed format description */ |
14983e7a | 2351 | |
ed57d853 | 2352 | void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) |
b44be742 | 2353 | { |
c0ce4f12 | 2354 | const seqDef* const sequences = seqStorePtr->sequencesStart; |
ed57d853 YC |
2355 | BYTE* const llCodeTable = seqStorePtr->llCode; |
2356 | BYTE* const ofCodeTable = seqStorePtr->ofCode; | |
2357 | BYTE* const mlCodeTable = seqStorePtr->mlCode; | |
c0ce4f12 | 2358 | U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); |
ed57d853 | 2359 | U32 u; |
924944e4 | 2360 | assert(nbSeq <= seqStorePtr->maxNbSeq); |
ed57d853 YC |
2361 | for (u=0; u<nbSeq; u++) { |
2362 | U32 const llv = sequences[u].litLength; | |
2363 | U32 const mlv = sequences[u].matchLength; | |
100d8ad6 | 2364 | llCodeTable[u] = (BYTE)ZSTD_LLcode(llv); |
ed57d853 | 2365 | ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); |
100d8ad6 | 2366 | mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv); |
5d39357b | 2367 | } |
b1a43455 | 2368 | if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) |
ed57d853 | 2369 | llCodeTable[seqStorePtr->longLengthPos] = MaxLL; |
b1a43455 | 2370 | if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) |
ed57d853 | 2371 | mlCodeTable[seqStorePtr->longLengthPos] = MaxML; |
b44be742 YC |
2372 | } |
2373 | ||
7ce89187 SH |
2374 | /* ZSTD_useTargetCBlockSize(): |
2375 | * Returns if target compressed block size param is being used. | |
2376 | * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. | |
2377 | * Returns 1 if true, 0 otherwise. */ | |
2378 | static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) | |
2379 | { | |
2380 | DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); | |
2381 | return (cctxParams->targetCBlockSize != 0); | |
2382 | } | |
2383 | ||
c90e81a6 | 2384 | /* ZSTD_blockSplitterEnabled(): |
e2bb2151 | 2385 | * Returns if block splitting param is being used |
2386 | * If used, compression will do best effort to split a block in order to improve compression ratio. | |
2387 | * Returns 1 if true, 0 otherwise. */ | |
c90e81a6 | 2388 | static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) |
e2bb2151 | 2389 | { |
c90e81a6 | 2390 | DEBUGLOG(5, "ZSTD_blockSplitterEnabled(splitBlocks=%d)", cctxParams->splitBlocks); |
e2bb2151 | 2391 | return (cctxParams->splitBlocks != 0); |
2392 | } | |
2393 | ||
e3433283 SH |
2394 | /* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types |
2395 | * and size of the sequences statistics | |
2396 | */ | |
2397 | typedef struct { | |
2398 | U32 LLtype; | |
2399 | U32 Offtype; | |
2400 | U32 MLtype; | |
2401 | size_t size; | |
2a907bf4 | 2402 | size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ |
e3433283 SH |
2403 | } ZSTD_symbolEncodingTypeStats_t; |
2404 | ||
f06f6626 | 2405 | /* ZSTD_buildSequencesStatistics(): |
e2bb2151 | 2406 | * Returns the size of the statistics for a given set of sequences, or a ZSTD error code, |
eb1ee868 | 2407 | * Also modifies LLtype, Offtype, MLtype, and lastNCount to the appropriate values. |
4694423c | 2408 | * |
e3433283 | 2409 | * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) |
f06f6626 | 2410 | */ |
e3433283 | 2411 | static ZSTD_symbolEncodingTypeStats_t |
2a907bf4 | 2412 | ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, |
2949a952 | 2413 | const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, |
2414 | BYTE* dst, const BYTE* const dstEnd, | |
41c3eae6 | 2415 | ZSTD_strategy strategy, unsigned* countWorkspace, |
e2bb2151 | 2416 | void* entropyWorkspace, size_t entropyWkspSize) { |
c05c090c | 2417 | BYTE* const ostart = dst; |
2418 | const BYTE* const oend = dstEnd; | |
2419 | BYTE* op = ostart; | |
e2bb2151 | 2420 | FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; |
2421 | FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; | |
2422 | FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; | |
2423 | const BYTE* const ofCodeTable = seqStorePtr->ofCode; | |
2424 | const BYTE* const llCodeTable = seqStorePtr->llCode; | |
2425 | const BYTE* const mlCodeTable = seqStorePtr->mlCode; | |
e3433283 | 2426 | ZSTD_symbolEncodingTypeStats_t stats; |
e2bb2151 | 2427 | |
2a907bf4 | 2428 | stats.lastCountSize = 0; |
e2bb2151 | 2429 | /* convert length/distances into codes */ |
2430 | ZSTD_seqToCodes(seqStorePtr); | |
c05c090c | 2431 | assert(op <= oend); |
2432 | /* build CTable for Literal Lengths */ | |
2433 | { unsigned max = MaxLL; | |
2434 | size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ | |
2435 | DEBUGLOG(5, "Building LL table"); | |
2436 | nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; | |
e3433283 | 2437 | stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, |
c05c090c | 2438 | countWorkspace, max, mostFrequent, nbSeq, |
2439 | LLFSELog, prevEntropy->litlengthCTable, | |
2440 | LL_defaultNorm, LL_defaultNormLog, | |
2441 | ZSTD_defaultAllowed, strategy); | |
2442 | assert(set_basic < set_compressed && set_rle < set_compressed); | |
e3433283 | 2443 | assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ |
c05c090c | 2444 | { size_t const countSize = ZSTD_buildCTable( |
2445 | op, (size_t)(oend - op), | |
e3433283 | 2446 | CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype, |
c05c090c | 2447 | countWorkspace, max, llCodeTable, nbSeq, |
2448 | LL_defaultNorm, LL_defaultNormLog, MaxLL, | |
2449 | prevEntropy->litlengthCTable, | |
2450 | sizeof(prevEntropy->litlengthCTable), | |
2451 | entropyWorkspace, entropyWkspSize); | |
e3433283 SH |
2452 | if (ZSTD_isError(countSize)) { |
2453 | DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); | |
2454 | stats.size = countSize; | |
2455 | return stats; | |
2456 | } | |
2457 | if (stats.LLtype == set_compressed) | |
2a907bf4 | 2458 | stats.lastCountSize = countSize; |
c05c090c | 2459 | op += countSize; |
c05c090c | 2460 | assert(op <= oend); |
2461 | } } | |
2462 | /* build CTable for Offsets */ | |
2463 | { unsigned max = MaxOff; | |
2464 | size_t const mostFrequent = HIST_countFast_wksp( | |
2465 | countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ | |
2466 | /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ | |
2467 | ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; | |
2468 | DEBUGLOG(5, "Building OF table"); | |
2469 | nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; | |
e3433283 | 2470 | stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, |
c05c090c | 2471 | countWorkspace, max, mostFrequent, nbSeq, |
2472 | OffFSELog, prevEntropy->offcodeCTable, | |
2473 | OF_defaultNorm, OF_defaultNormLog, | |
2474 | defaultPolicy, strategy); | |
e3433283 | 2475 | assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ |
c05c090c | 2476 | { size_t const countSize = ZSTD_buildCTable( |
2477 | op, (size_t)(oend - op), | |
e3433283 | 2478 | CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype, |
c05c090c | 2479 | countWorkspace, max, ofCodeTable, nbSeq, |
2480 | OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, | |
2481 | prevEntropy->offcodeCTable, | |
2482 | sizeof(prevEntropy->offcodeCTable), | |
2483 | entropyWorkspace, entropyWkspSize); | |
e3433283 SH |
2484 | if (ZSTD_isError(countSize)) { |
2485 | DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); | |
2486 | stats.size = countSize; | |
2487 | return stats; | |
2488 | } | |
2489 | if (stats.Offtype == set_compressed) | |
2a907bf4 | 2490 | stats.lastCountSize = countSize; |
c05c090c | 2491 | op += countSize; |
c05c090c | 2492 | assert(op <= oend); |
2493 | } } | |
2494 | /* build CTable for MatchLengths */ | |
2495 | { unsigned max = MaxML; | |
2496 | size_t const mostFrequent = HIST_countFast_wksp( | |
2497 | countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ | |
2498 | DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); | |
2499 | nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; | |
e3433283 | 2500 | stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, |
c05c090c | 2501 | countWorkspace, max, mostFrequent, nbSeq, |
2502 | MLFSELog, prevEntropy->matchlengthCTable, | |
2503 | ML_defaultNorm, ML_defaultNormLog, | |
2504 | ZSTD_defaultAllowed, strategy); | |
e3433283 | 2505 | assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ |
c05c090c | 2506 | { size_t const countSize = ZSTD_buildCTable( |
2507 | op, (size_t)(oend - op), | |
e3433283 | 2508 | CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype, |
c05c090c | 2509 | countWorkspace, max, mlCodeTable, nbSeq, |
2510 | ML_defaultNorm, ML_defaultNormLog, MaxML, | |
2511 | prevEntropy->matchlengthCTable, | |
2512 | sizeof(prevEntropy->matchlengthCTable), | |
2513 | entropyWorkspace, entropyWkspSize); | |
e3433283 SH |
2514 | if (ZSTD_isError(countSize)) { |
2515 | DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); | |
2516 | stats.size = countSize; | |
2517 | return stats; | |
2518 | } | |
2519 | if (stats.MLtype == set_compressed) | |
2a907bf4 | 2520 | stats.lastCountSize = countSize; |
c05c090c | 2521 | op += countSize; |
c05c090c | 2522 | assert(op <= oend); |
2523 | } } | |
e3433283 SH |
2524 | stats.size = (size_t)(op-ostart); |
2525 | return stats; | |
c05c090c | 2526 | } |
2527 | ||
5b566ebe SH |
2528 | /* ZSTD_entropyCompressSeqStore_internal(): |
2529 | * compresses both literals and sequences | |
2530 | * Returns compressed size of block, or a zstd error. | |
2531 | */ | |
7b744051 | 2532 | MEM_STATIC size_t |
5b566ebe | 2533 | ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, |
8e0e495c YC |
2534 | const ZSTD_entropyCTables_t* prevEntropy, |
2535 | ZSTD_entropyCTables_t* nextEntropy, | |
2536 | const ZSTD_CCtx_params* cctxParams, | |
2537 | void* dst, size_t dstCapacity, | |
c25283cf | 2538 | void* entropyWorkspace, size_t entropyWkspSize, |
8e0e495c | 2539 | const int bmi2) |
14983e7a | 2540 | { |
a146ee04 | 2541 | const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; |
49cf8805 | 2542 | ZSTD_strategy const strategy = cctxParams->cParams.strategy; |
a9077939 | 2543 | unsigned* count = (unsigned*)entropyWorkspace; |
e3959d5e NT |
2544 | FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; |
2545 | FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; | |
2546 | FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; | |
c0ce4f12 | 2547 | const seqDef* const sequences = seqStorePtr->sequencesStart; |
386111ad | 2548 | const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; |
ed57d853 YC |
2549 | const BYTE* const ofCodeTable = seqStorePtr->ofCode; |
2550 | const BYTE* const llCodeTable = seqStorePtr->llCode; | |
2551 | const BYTE* const mlCodeTable = seqStorePtr->mlCode; | |
5054ee0c | 2552 | BYTE* const ostart = (BYTE*)dst; |
d1b26849 | 2553 | BYTE* const oend = ostart + dstCapacity; |
a910dc82 | 2554 | BYTE* op = ostart; |
2a907bf4 | 2555 | size_t lastCountSize; |
634f0124 | 2556 | |
a9077939 NT |
2557 | entropyWorkspace = count + (MaxSeq + 1); |
2558 | entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); | |
2559 | ||
5b566ebe | 2560 | DEBUGLOG(4, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq); |
887cd4e3 | 2561 | ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); |
a9077939 | 2562 | assert(entropyWkspSize >= HUF_WORKSPACE_SIZE); |
14983e7a | 2563 | |
14983e7a | 2564 | /* Compress literals */ |
a5c2c08c | 2565 | { const BYTE* const literals = seqStorePtr->litStart; |
0b0b83e8 | 2566 | size_t const litSize = (size_t)(seqStorePtr->lit - literals); |
a146ee04 | 2567 | size_t const cSize = ZSTD_compressLiterals( |
e3959d5e | 2568 | &prevEntropy->huf, &nextEntropy->huf, |
f9513115 NT |
2569 | cctxParams->cParams.strategy, |
2570 | ZSTD_disableLiteralsCompression(cctxParams), | |
a146ee04 YC |
2571 | op, dstCapacity, |
2572 | literals, litSize, | |
c25283cf | 2573 | entropyWorkspace, entropyWkspSize, |
7b744051 | 2574 | bmi2); |
5e5f2626 | 2575 | FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); |
150354c5 | 2576 | assert(cSize <= dstCapacity); |
14983e7a YC |
2577 | op += cSize; |
2578 | } | |
2579 | ||
2580 | /* Sequences Header */ | |
cafc3b1b | 2581 | RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, |
5e5f2626 | 2582 | dstSize_tooSmall, "Can't fit seq hdr in output buf!"); |
782bfb85 | 2583 | if (nbSeq < 128) { |
150354c5 | 2584 | *op++ = (BYTE)nbSeq; |
782bfb85 YC |
2585 | } else if (nbSeq < LONGNBSEQ) { |
2586 | op[0] = (BYTE)((nbSeq>>8) + 0x80); | |
2587 | op[1] = (BYTE)nbSeq; | |
2588 | op+=2; | |
2589 | } else { | |
2590 | op[0]=0xFF; | |
2591 | MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); | |
2592 | op+=3; | |
2593 | } | |
6c92ba77 | 2594 | assert(op <= oend); |
887cd4e3 | 2595 | if (nbSeq==0) { |
e3959d5e | 2596 | /* Copy the old tables over as if we repeated them */ |
c465f244 | 2597 | ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); |
0b0b83e8 | 2598 | return (size_t)(op - ostart); |
887cd4e3 | 2599 | } |
eb1ee868 | 2600 | { |
e3433283 | 2601 | ZSTD_symbolEncodingTypeStats_t stats; |
0633bf17 | 2602 | BYTE* seqHead = op++; |
eb1ee868 | 2603 | /* build stats for sequences */ |
2a907bf4 SH |
2604 | stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, |
2605 | &prevEntropy->fse, &nextEntropy->fse, | |
2606 | op, oend, | |
2607 | strategy, count, | |
2608 | entropyWorkspace, entropyWkspSize); | |
e3433283 SH |
2609 | FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); |
2610 | *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); | |
2a907bf4 | 2611 | lastCountSize = stats.lastCountSize; |
e3433283 | 2612 | op += stats.size; |
eb1ee868 | 2613 | } |
14983e7a | 2614 | |
a9e57050 | 2615 | { size_t const bitstreamSize = ZSTD_encodeSequences( |
0b0b83e8 | 2616 | op, (size_t)(oend - op), |
a9e57050 YC |
2617 | CTable_MatchLength, mlCodeTable, |
2618 | CTable_OffsetBits, ofCodeTable, | |
2619 | CTable_LitLength, llCodeTable, | |
2620 | sequences, nbSeq, | |
b58f0153 | 2621 | longOffsets, bmi2); |
5e5f2626 | 2622 | FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); |
a9e57050 | 2623 | op += bitstreamSize; |
6c92ba77 | 2624 | assert(op <= oend); |
06b70179 | 2625 | /* zstd versions <= 1.3.4 mistakenly report corruption when |
a880ca23 | 2626 | * FSE_readNCount() receives a buffer < 4 bytes. |
06b70179 NT |
2627 | * Fixed by https://github.com/facebook/zstd/pull/1146. |
2628 | * This can happen when the last set_compressed table present is 2 | |
2629 | * bytes and the bitstream is only one byte. | |
2630 | * In this exceedingly rare case, we will simply emit an uncompressed | |
2631 | * block, since it isn't worth optimizing. | |
2632 | */ | |
0633bf17 | 2633 | if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { |
2634 | /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ | |
2635 | assert(lastCountSize + bitstreamSize == 3); | |
06b70179 NT |
2636 | DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " |
2637 | "emitting an uncompressed block."); | |
2638 | return 0; | |
2639 | } | |
634f0124 | 2640 | } |
14983e7a | 2641 | |
8e0e495c | 2642 | DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); |
0b0b83e8 | 2643 | return (size_t)(op - ostart); |
308047eb | 2644 | } |
14983e7a | 2645 | |
7b744051 | 2646 | MEM_STATIC size_t |
5b566ebe | 2647 | ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr, |
7b744051 YC |
2648 | const ZSTD_entropyCTables_t* prevEntropy, |
2649 | ZSTD_entropyCTables_t* nextEntropy, | |
2650 | const ZSTD_CCtx_params* cctxParams, | |
2651 | void* dst, size_t dstCapacity, | |
2652 | size_t srcSize, | |
c25283cf | 2653 | void* entropyWorkspace, size_t entropyWkspSize, |
255925c2 | 2654 | int bmi2) |
308047eb | 2655 | { |
5b566ebe | 2656 | size_t const cSize = ZSTD_entropyCompressSeqStore_internal( |
7b744051 YC |
2657 | seqStorePtr, prevEntropy, nextEntropy, cctxParams, |
2658 | dst, dstCapacity, | |
c25283cf | 2659 | entropyWorkspace, entropyWkspSize, bmi2); |
06b70179 | 2660 | if (cSize == 0) return 0; |
a146ee04 YC |
2661 | /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. |
2662 | * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. | |
308047eb | 2663 | */ |
255925c2 | 2664 | if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) |
2665 | return 0; /* block not compressed */ | |
2666 | FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); | |
2667 | ||
2668 | /* Check compressibility */ | |
2669 | { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); | |
2670 | if (cSize >= maxCSize) return 0; /* block not compressed */ | |
a146ee04 | 2671 | } |
5b566ebe | 2672 | DEBUGLOG(4, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); |
308047eb | 2673 | return cSize; |
14983e7a YC |
2674 | } |
2675 | ||
b44ab82f | 2676 | /* ZSTD_selectBlockCompressor() : |
360428c5 | 2677 | * Not static, but internal use only (used by long distance matcher) |
b44ab82f | 2678 | * assumption : strat is a valid strategy */ |
4694423c | 2679 | ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e useRowMatchFinder, ZSTD_dictMode_e dictMode) |
be2010ea | 2680 | { |
34b545ac | 2681 | static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { |
b44ab82f | 2682 | { ZSTD_compressBlock_fast /* default for 0 */, |
90cfc799 FH |
2683 | ZSTD_compressBlock_fast, |
2684 | ZSTD_compressBlock_doubleFast, | |
2685 | ZSTD_compressBlock_greedy, | |
2686 | ZSTD_compressBlock_lazy, | |
2687 | ZSTD_compressBlock_lazy2, | |
2688 | ZSTD_compressBlock_btlazy2, | |
2689 | ZSTD_compressBlock_btopt, | |
e9448cdf | 2690 | ZSTD_compressBlock_btultra, |
d613fd9a | 2691 | ZSTD_compressBlock_btultra2 }, |
b44ab82f | 2692 | { ZSTD_compressBlock_fast_extDict /* default for 0 */, |
90cfc799 FH |
2693 | ZSTD_compressBlock_fast_extDict, |
2694 | ZSTD_compressBlock_doubleFast_extDict, | |
2695 | ZSTD_compressBlock_greedy_extDict, | |
2696 | ZSTD_compressBlock_lazy_extDict, | |
2697 | ZSTD_compressBlock_lazy2_extDict, | |
2698 | ZSTD_compressBlock_btlazy2_extDict, | |
2699 | ZSTD_compressBlock_btopt_extDict, | |
e9448cdf | 2700 | ZSTD_compressBlock_btultra_extDict, |
90cfc799 | 2701 | ZSTD_compressBlock_btultra_extDict }, |
b67196f3 FH |
2702 | { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, |
2703 | ZSTD_compressBlock_fast_dictMatchState, | |
c10d1b40 | 2704 | ZSTD_compressBlock_doubleFast_dictMatchState, |
90cfc799 FH |
2705 | ZSTD_compressBlock_greedy_dictMatchState, |
2706 | ZSTD_compressBlock_lazy_dictMatchState, | |
2707 | ZSTD_compressBlock_lazy2_dictMatchState, | |
2708 | ZSTD_compressBlock_btlazy2_dictMatchState, | |
ccbf0679 | 2709 | ZSTD_compressBlock_btopt_dictMatchState, |
e9448cdf | 2710 | ZSTD_compressBlock_btultra_dictMatchState, |
34b545ac FH |
2711 | ZSTD_compressBlock_btultra_dictMatchState }, |
2712 | { NULL /* default for 0 */, | |
2713 | NULL, | |
2714 | NULL, | |
2715 | ZSTD_compressBlock_greedy_dedicatedDictSearch, | |
2716 | ZSTD_compressBlock_lazy_dedicatedDictSearch, | |
2717 | ZSTD_compressBlock_lazy2_dedicatedDictSearch, | |
2718 | NULL, | |
2719 | NULL, | |
2720 | NULL, | |
2721 | NULL } | |
7fe531e7 | 2722 | }; |
ae4fcf78 | 2723 | ZSTD_blockCompressor selectedCompressor; |
a5ffe3d3 | 2724 | ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); |
a9e57050 | 2725 | |
be9e561d | 2726 | assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); |
4694423c NT |
2727 | DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); |
2728 | if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { | |
2729 | static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = { | |
2730 | { ZSTD_compressBlock_greedy_row, | |
2731 | ZSTD_compressBlock_lazy_row, | |
2732 | ZSTD_compressBlock_lazy2_row }, | |
2733 | { ZSTD_compressBlock_greedy_extDict_row, | |
2734 | ZSTD_compressBlock_lazy_extDict_row, | |
2735 | ZSTD_compressBlock_lazy2_extDict_row }, | |
2736 | { ZSTD_compressBlock_greedy_dictMatchState_row, | |
2737 | ZSTD_compressBlock_lazy_dictMatchState_row, | |
2738 | ZSTD_compressBlock_lazy2_dictMatchState_row }, | |
2739 | { ZSTD_compressBlock_greedy_dedicatedDictSearch_row, | |
2740 | ZSTD_compressBlock_lazy_dedicatedDictSearch_row, | |
2741 | ZSTD_compressBlock_lazy2_dedicatedDictSearch_row } | |
2742 | }; | |
2743 | DEBUGLOG(4, "Selecting a row-based matchfinder"); | |
2744 | assert(useRowMatchFinder != ZSTD_urm_auto); | |
2745 | selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; | |
2746 | } else { | |
2747 | selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; | |
2748 | } | |
ae4fcf78 FH |
2749 | assert(selectedCompressor != NULL); |
2750 | return selectedCompressor; | |
be2010ea YC |
2751 | } |
2752 | ||
6a546efb SL |
2753 | static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, |
2754 | const BYTE* anchor, size_t lastLLSize) | |
2755 | { | |
c465f244 | 2756 | ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); |
6a546efb SL |
2757 | seqStorePtr->lit += lastLLSize; |
2758 | } | |
be2010ea | 2759 | |
a243020d | 2760 | void ZSTD_resetSeqStore(seqStore_t* ssPtr) |
9a11f70d YC |
2761 | { |
2762 | ssPtr->lit = ssPtr->litStart; | |
2763 | ssPtr->sequences = ssPtr->sequencesStart; | |
b1a43455 | 2764 | ssPtr->longLengthType = ZSTD_llt_none; |
9a11f70d YC |
2765 | } |
2766 | ||
f57ac7b0 EP |
2767 | typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; |
2768 | ||
2769 | static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) | |
59d7063f | 2770 | { |
af866b3a | 2771 | ZSTD_matchState_t* const ms = &zc->blockState.matchState; |
f57ac7b0 | 2772 | DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); |
0e2dbac1 | 2773 | assert(srcSize <= ZSTD_BLOCKSIZE_MAX); |
03103269 | 2774 | /* Assert that we have correctly flushed the ctx params into the ms's copy */ |
1f188ae6 | 2775 | ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); |
136b9e23 | 2776 | if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { |
4d01979b | 2777 | if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { |
2778 | ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); | |
2779 | } else { | |
2780 | ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); | |
2781 | } | |
f57ac7b0 | 2782 | return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ |
136b9e23 | 2783 | } |
19cab46f | 2784 | ZSTD_resetSeqStore(&(zc->seqStore)); |
3d7377b8 NT |
2785 | /* required for optimal parser to read stats from dictionary */ |
2786 | ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; | |
2787 | /* tell the optimal parser how we expect to compress literals */ | |
2788 | ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; | |
d9c7e671 | 2789 | /* a gap between an attached dict and the current window is not safe, |
8e0e495c YC |
2790 | * they must remain adjacent, |
2791 | * and when that stops being the case, the dict must be unset */ | |
d9c7e671 FH |
2792 | assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); |
2793 | ||
a9e57050 | 2794 | /* limited update after a very long match */ |
7e5e226c | 2795 | { const BYTE* const base = ms->window.base; |
a9e57050 | 2796 | const BYTE* const istart = (const BYTE*)src; |
f91ed5c7 | 2797 | const U32 curr = (U32)(istart-base); |
a1550613 | 2798 | if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ |
f91ed5c7 NT |
2799 | if (curr > ms->nextToUpdate + 384) |
2800 | ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); | |
a9e57050 | 2801 | } |
a1d4041e YC |
2802 | |
2803 | /* select and store sequences */ | |
b67196f3 | 2804 | { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); |
887cd4e3 | 2805 | size_t lastLLSize; |
a146ee04 YC |
2806 | { int i; |
2807 | for (i = 0; i < ZSTD_REP_NUM; ++i) | |
2808 | zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; | |
2809 | } | |
a9a6dcba NT |
2810 | if (zc->externSeqStore.pos < zc->externSeqStore.size) { |
2811 | assert(!zc->appliedParams.ldmParams.enableLdm); | |
2812 | /* Updates ldmSeqStore.pos */ | |
af866b3a | 2813 | lastLLSize = |
a9a6dcba | 2814 | ZSTD_ldm_blockCompress(&zc->externSeqStore, |
af866b3a NT |
2815 | ms, &zc->seqStore, |
2816 | zc->blockState.nextCBlock->rep, | |
4694423c | 2817 | zc->appliedParams.useRowMatchFinder, |
b67196f3 | 2818 | src, srcSize); |
a9a6dcba NT |
2819 | assert(zc->externSeqStore.pos <= zc->externSeqStore.size); |
2820 | } else if (zc->appliedParams.ldmParams.enableLdm) { | |
b9c8033c | 2821 | rawSeqStore_t ldmSeqStore = kNullRawSeqStore; |
a9a6dcba NT |
2822 | |
2823 | ldmSeqStore.seq = zc->ldmSequences; | |
2824 | ldmSeqStore.capacity = zc->maxNbLdmSequences; | |
2825 | /* Updates ldmSeqStore.size */ | |
501eb251 | 2826 | FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, |
a9a6dcba | 2827 | &zc->appliedParams.ldmParams, |
5e5f2626 | 2828 | src, srcSize), ""); |
a9a6dcba NT |
2829 | /* Updates ldmSeqStore.pos */ |
2830 | lastLLSize = | |
2831 | ZSTD_ldm_blockCompress(&ldmSeqStore, | |
2832 | ms, &zc->seqStore, | |
2833 | zc->blockState.nextCBlock->rep, | |
4694423c | 2834 | zc->appliedParams.useRowMatchFinder, |
b67196f3 | 2835 | src, srcSize); |
a9a6dcba | 2836 | assert(ldmSeqStore.pos == ldmSeqStore.size); |
a1d4041e | 2837 | } else { /* not long range mode */ |
4694423c NT |
2838 | ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, |
2839 | zc->appliedParams.useRowMatchFinder, | |
2840 | dictMode); | |
a6165c1b | 2841 | ms->ldmSeqStore = NULL; |
6cb24546 | 2842 | lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); |
887cd4e3 | 2843 | } |
a1d4041e YC |
2844 | { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; |
2845 | ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); | |
2846 | } } | |
f57ac7b0 EP |
2847 | return ZSTDbss_compress; |
2848 | } | |
2849 | ||
9e7bb55e BS |
2850 | static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) |
2851 | { | |
2852 | const seqStore_t* seqStore = ZSTD_getSeqStore(zc); | |
3a11c7eb | 2853 | const seqDef* seqStoreSeqs = seqStore->sequencesStart; |
2854 | size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; | |
1d221ecc | 2855 | size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); |
2856 | size_t literalsRead = 0; | |
2857 | size_t lastLLSize; | |
9e7bb55e BS |
2858 | |
2859 | ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; | |
3a11c7eb | 2860 | size_t i; |
be4ac6c5 | 2861 | repcodes_t updatedRepcodes; |
9e7bb55e BS |
2862 | |
2863 | assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); | |
1d221ecc | 2864 | /* Ensure we have enough space for last literals "sequence" */ |
2865 | assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); | |
9d936d61 | 2866 | ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); |
1d221ecc | 2867 | for (i = 0; i < seqStoreSeqSize; ++i) { |
4d4fd2c5 | 2868 | U32 rawOffset = seqStoreSeqs[i].offset - ZSTD_REP_NUM; |
3a11c7eb | 2869 | outSeqs[i].litLength = seqStoreSeqs[i].litLength; |
2870 | outSeqs[i].matchLength = seqStoreSeqs[i].matchLength + MINMATCH; | |
d4d0346b | 2871 | outSeqs[i].rep = 0; |
36528b96 BS |
2872 | |
2873 | if (i == seqStore->longLengthPos) { | |
b1a43455 | 2874 | if (seqStore->longLengthType == ZSTD_llt_literalLength) { |
36528b96 | 2875 | outSeqs[i].litLength += 0x10000; |
b1a43455 | 2876 | } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { |
36528b96 BS |
2877 | outSeqs[i].matchLength += 0x10000; |
2878 | } | |
2879 | } | |
2880 | ||
3a11c7eb | 2881 | if (seqStoreSeqs[i].offset <= ZSTD_REP_NUM) { |
be4ac6c5 | 2882 | /* Derive the correct offset corresponding to a repcode */ |
39627506 | 2883 | outSeqs[i].rep = seqStoreSeqs[i].offset; |
be4ac6c5 | 2884 | if (outSeqs[i].litLength != 0) { |
9d936d61 | 2885 | rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; |
4d4fd2c5 | 2886 | } else { |
be4ac6c5 | 2887 | if (outSeqs[i].rep == 3) { |
9d936d61 | 2888 | rawOffset = updatedRepcodes.rep[0] - 1; |
be4ac6c5 | 2889 | } else { |
9d936d61 | 2890 | rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; |
e3c58259 | 2891 | } |
9e7bb55e | 2892 | } |
4d4fd2c5 | 2893 | } |
2894 | outSeqs[i].offset = rawOffset; | |
9d936d61 | 2895 | /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode |
2896 | so we provide seqStoreSeqs[i].offset - 1 */ | |
2897 | updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, | |
2898 | seqStoreSeqs[i].offset - 1, | |
2899 | seqStoreSeqs[i].litLength == 0); | |
69bd5f06 | 2900 | literalsRead += outSeqs[i].litLength; |
9e7bb55e | 2901 | } |
32cac262 | 2902 | /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. |
2903 | * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker | |
2904 | * for the block boundary, according to the API. | |
2905 | */ | |
3ed5d053 | 2906 | assert(seqStoreLiteralsSize >= literalsRead); |
1d221ecc | 2907 | lastLLSize = seqStoreLiteralsSize - literalsRead; |
32cac262 | 2908 | outSeqs[i].litLength = (U32)lastLLSize; |
2909 | outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; | |
2910 | seqStoreSeqSize++; | |
3a11c7eb | 2911 | zc->seqCollector.seqIndex += seqStoreSeqSize; |
9e7bb55e BS |
2912 | } |
2913 | ||
51abd582 | 2914 | size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, |
779df995 | 2915 | size_t outSeqsSize, const void* src, size_t srcSize) |
9e7bb55e | 2916 | { |
f3c4fd17 | 2917 | const size_t dstCapacity = ZSTD_compressBound(srcSize); |
a686d306 | 2918 | void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); |
9e7bb55e | 2919 | SeqCollector seqCollector; |
be0bebd2 | 2920 | |
5e5f2626 | 2921 | RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); |
be0bebd2 | 2922 | |
9e7bb55e BS |
2923 | seqCollector.collectSequences = 1; |
2924 | seqCollector.seqStart = outSeqs; | |
2925 | seqCollector.seqIndex = 0; | |
2926 | seqCollector.maxSequences = outSeqsSize; | |
2927 | zc->seqCollector = seqCollector; | |
2928 | ||
f3c4fd17 | 2929 | ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); |
a686d306 | 2930 | ZSTD_customFree(dst, ZSTD_defaultCMem); |
9e7bb55e BS |
2931 | return zc->seqCollector.seqIndex; |
2932 | } | |
2933 | ||
7d1dea07 | 2934 | size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { |
779df995 | 2935 | size_t in = 0; |
2936 | size_t out = 0; | |
2937 | for (; in < seqsSize; ++in) { | |
2938 | if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { | |
2939 | if (in != seqsSize - 1) { | |
2940 | sequences[in+1].litLength += sequences[in].litLength; | |
a36fdada | 2941 | } |
779df995 | 2942 | } else { |
2943 | sequences[out] = sequences[in]; | |
2944 | ++out; | |
a36fdada | 2945 | } |
a36fdada | 2946 | } |
779df995 | 2947 | return out; |
9e7bb55e BS |
2948 | } |
2949 | ||
dcbbf7c0 | 2950 | /* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ |
2951 | static int ZSTD_isRLE(const BYTE* src, size_t length) { | |
2952 | const BYTE* ip = src; | |
2953 | const BYTE value = ip[0]; | |
e924a0fa | 2954 | const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); |
dcbbf7c0 | 2955 | const size_t unrollSize = sizeof(size_t) * 4; |
2956 | const size_t unrollMask = unrollSize - 1; | |
2957 | const size_t prefixLength = length & unrollMask; | |
cba5350f | 2958 | size_t i; |
dcbbf7c0 | 2959 | size_t u; |
2960 | if (length == 1) return 1; | |
2961 | /* Check if prefix is RLE first before using unrolled loop */ | |
2962 | if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { | |
2963 | return 0; | |
2964 | } | |
2965 | for (i = prefixLength; i != length; i += unrollSize) { | |
2966 | for (u = 0; u < unrollSize; u += sizeof(size_t)) { | |
2967 | if (MEM_readST(ip + i + u) != valueST) { | |
2968 | return 0; | |
2969 | } | |
2970 | } | |
cba5350f | 2971 | } |
2972 | return 1; | |
2973 | } | |
2974 | ||
e103d7b4 NT |
2975 | /* Returns true if the given block may be RLE. |
2976 | * This is just a heuristic based on the compressibility. | |
2977 | * It may return both false positives and false negatives. | |
2978 | */ | |
2979 | static int ZSTD_maybeRLE(seqStore_t const* seqStore) | |
2980 | { | |
2981 | size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); | |
2982 | size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); | |
2983 | ||
2984 | return nbSeqs < 4 && nbLits < 10; | |
2985 | } | |
2986 | ||
255925c2 | 2987 | static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) |
036b30b5 | 2988 | { |
255925c2 | 2989 | ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; |
2990 | bs->prevCBlock = bs->nextCBlock; | |
2991 | bs->nextCBlock = tmp; | |
036b30b5 NT |
2992 | } |
2993 | ||
98764493 | 2994 | /* Writes the block header */ |
2995 | static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) { | |
2996 | U32 const cBlockHeader = cSize == 1 ? | |
2997 | lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : | |
2998 | lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); | |
2999 | MEM_writeLE24(op, cBlockHeader); | |
f06f6626 | 3000 | DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); |
98764493 | 3001 | } |
3002 | ||
f06f6626 | 3003 | /** ZSTD_buildBlockEntropyStats_literals() : |
e2bb2151 | 3004 | * Builds entropy for the literals. |
c05c090c | 3005 | * Stores literals block type (raw, rle, compressed, repeat) and |
3006 | * huffman description table to hufMetadata. | |
e3433283 | 3007 | * Requires ENTROPY_WORKSPACE_SIZE workspace |
c05c090c | 3008 | * @return : size of huffman description table or error code */ |
f06f6626 | 3009 | static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, |
c05c090c | 3010 | const ZSTD_hufCTables_t* prevHuf, |
3011 | ZSTD_hufCTables_t* nextHuf, | |
3012 | ZSTD_hufCTablesMetadata_t* hufMetadata, | |
3013 | const int disableLiteralsCompression, | |
3014 | void* workspace, size_t wkspSize) | |
3015 | { | |
3016 | BYTE* const wkspStart = (BYTE*)workspace; | |
3017 | BYTE* const wkspEnd = wkspStart + wkspSize; | |
3018 | BYTE* const countWkspStart = wkspStart; | |
3019 | unsigned* const countWksp = (unsigned*)workspace; | |
3020 | const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); | |
3021 | BYTE* const nodeWksp = countWkspStart + countWkspSize; | |
3022 | const size_t nodeWkspSize = wkspEnd-nodeWksp; | |
e2bb2151 | 3023 | unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; |
c05c090c | 3024 | unsigned huffLog = HUF_TABLELOG_DEFAULT; |
3025 | HUF_repeat repeat = prevHuf->repeatMode; | |
f06f6626 | 3026 | DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); |
c05c090c | 3027 | |
3028 | /* Prepare nextEntropy assuming reusing the existing table */ | |
3029 | ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); | |
3030 | ||
3031 | if (disableLiteralsCompression) { | |
3032 | DEBUGLOG(5, "set_basic - disabled"); | |
3033 | hufMetadata->hType = set_basic; | |
3034 | return 0; | |
3035 | } | |
3036 | ||
3037 | /* small ? don't even attempt compression (speed opt) */ | |
e2bb2151 | 3038 | #ifndef COMPRESS_LITERALS_SIZE_MIN |
3039 | #define COMPRESS_LITERALS_SIZE_MIN 63 | |
3040 | #endif | |
c05c090c | 3041 | { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; |
3042 | if (srcSize <= minLitSize) { | |
3043 | DEBUGLOG(5, "set_basic - too small"); | |
3044 | hufMetadata->hType = set_basic; | |
3045 | return 0; | |
3046 | } | |
3047 | } | |
3048 | ||
3049 | /* Scan input and build symbol stats */ | |
3050 | { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize); | |
3051 | FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); | |
3052 | if (largest == srcSize) { | |
3053 | DEBUGLOG(5, "set_rle"); | |
3054 | hufMetadata->hType = set_rle; | |
3055 | return 0; | |
3056 | } | |
3057 | if (largest <= (srcSize >> 7)+4) { | |
3058 | DEBUGLOG(5, "set_basic - no gain"); | |
3059 | hufMetadata->hType = set_basic; | |
3060 | return 0; | |
3061 | } | |
3062 | } | |
3063 | ||
3064 | /* Validate the previous Huffman table */ | |
3065 | if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { | |
3066 | repeat = HUF_repeat_none; | |
3067 | } | |
3068 | ||
3069 | /* Build Huffman Tree */ | |
3070 | ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); | |
3071 | huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); | |
3072 | { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, | |
3073 | maxSymbolValue, huffLog, | |
3074 | nodeWksp, nodeWkspSize); | |
3075 | FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); | |
3076 | huffLog = (U32)maxBits; | |
3077 | { /* Build and write the CTable */ | |
3078 | size_t const newCSize = HUF_estimateCompressedSize( | |
3079 | (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); | |
41c3eae6 | 3080 | size_t const hSize = HUF_writeCTable_wksp( |
c05c090c | 3081 | hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), |
41c3eae6 SH |
3082 | (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, |
3083 | nodeWksp, nodeWkspSize); | |
c05c090c | 3084 | /* Check against repeating the previous CTable */ |
3085 | if (repeat != HUF_repeat_none) { | |
3086 | size_t const oldCSize = HUF_estimateCompressedSize( | |
3087 | (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); | |
3088 | if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { | |
3089 | DEBUGLOG(5, "set_repeat - smaller"); | |
3090 | ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); | |
3091 | hufMetadata->hType = set_repeat; | |
3092 | return 0; | |
3093 | } | |
3094 | } | |
3095 | if (newCSize + hSize >= srcSize) { | |
3096 | DEBUGLOG(5, "set_basic - no gains"); | |
3097 | ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); | |
3098 | hufMetadata->hType = set_basic; | |
3099 | return 0; | |
3100 | } | |
3101 | DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); | |
3102 | hufMetadata->hType = set_compressed; | |
3103 | nextHuf->repeatMode = HUF_repeat_check; | |
3104 | return hSize; | |
3105 | } | |
3106 | } | |
3107 | } | |
3108 | ||
f06f6626 | 3109 | /** ZSTD_buildBlockEntropyStats_sequences() : |
e2bb2151 | 3110 | * Builds entropy for the sequences. |
c05c090c | 3111 | * Stores symbol compression modes and fse table to fseMetadata. |
e3433283 | 3112 | * Requires ENTROPY_WORKSPACE_SIZE wksp. |
c05c090c | 3113 | * @return : size of fse tables or error code */ |
f06f6626 | 3114 | static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr, |
c05c090c | 3115 | const ZSTD_fseCTables_t* prevEntropy, |
3116 | ZSTD_fseCTables_t* nextEntropy, | |
3117 | const ZSTD_CCtx_params* cctxParams, | |
3118 | ZSTD_fseCTablesMetadata_t* fseMetadata, | |
3119 | void* workspace, size_t wkspSize) | |
3120 | { | |
c05c090c | 3121 | ZSTD_strategy const strategy = cctxParams->cParams.strategy; |
c05c090c | 3122 | size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; |
3123 | BYTE* const ostart = fseMetadata->fseTablesBuffer; | |
3124 | BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); | |
3125 | BYTE* op = ostart; | |
41c3eae6 SH |
3126 | unsigned* countWorkspace = (unsigned*)workspace; |
3127 | unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); | |
3128 | size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); | |
e3433283 | 3129 | ZSTD_symbolEncodingTypeStats_t stats; |
eb1ee868 | 3130 | |
f06f6626 | 3131 | DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); |
e3433283 | 3132 | stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, |
e3433283 SH |
3133 | prevEntropy, nextEntropy, op, oend, |
3134 | strategy, countWorkspace, | |
3135 | entropyWorkspace, entropyWorkspaceSize); | |
3136 | FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); | |
3137 | fseMetadata->llType = (symbolEncodingType_e) stats.LLtype; | |
3138 | fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype; | |
3139 | fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype; | |
2a907bf4 | 3140 | fseMetadata->lastCountSize = stats.lastCountSize; |
e3433283 | 3141 | return stats.size; |
c05c090c | 3142 | } |
3143 | ||
3144 | ||
f06f6626 | 3145 | /** ZSTD_buildBlockEntropyStats() : |
41c3eae6 | 3146 | * Builds entropy for the block. |
e3433283 | 3147 | * Requires workspace size ENTROPY_WORKSPACE_SIZE |
4694423c | 3148 | * |
e3433283 SH |
3149 | * @return : 0 on success or error code |
3150 | */ | |
f06f6626 | 3151 | size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, |
c05c090c | 3152 | const ZSTD_entropyCTables_t* prevEntropy, |
3153 | ZSTD_entropyCTables_t* nextEntropy, | |
3154 | const ZSTD_CCtx_params* cctxParams, | |
3155 | ZSTD_entropyCTablesMetadata_t* entropyMetadata, | |
3156 | void* workspace, size_t wkspSize) | |
3157 | { | |
3158 | size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; | |
c05c090c | 3159 | entropyMetadata->hufMetadata.hufDesSize = |
f06f6626 | 3160 | ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, |
c05c090c | 3161 | &prevEntropy->huf, &nextEntropy->huf, |
3162 | &entropyMetadata->hufMetadata, | |
3163 | ZSTD_disableLiteralsCompression(cctxParams), | |
3164 | workspace, wkspSize); | |
f06f6626 | 3165 | FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); |
c05c090c | 3166 | entropyMetadata->fseMetadata.fseTablesSize = |
f06f6626 | 3167 | ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, |
c05c090c | 3168 | &prevEntropy->fse, &nextEntropy->fse, |
3169 | cctxParams, | |
3170 | &entropyMetadata->fseMetadata, | |
3171 | workspace, wkspSize); | |
f06f6626 | 3172 | FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); |
c05c090c | 3173 | return 0; |
3174 | } | |
3175 | ||
e2bb2151 | 3176 | /* Returns the size estimate for the literals section (header + content) of a block */ |
41c3eae6 | 3177 | static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, |
c05c090c | 3178 | const ZSTD_hufCTables_t* huf, |
3179 | const ZSTD_hufCTablesMetadata_t* hufMetadata, | |
3180 | void* workspace, size_t wkspSize, | |
3181 | int writeEntropy) | |
3182 | { | |
3183 | unsigned* const countWksp = (unsigned*)workspace; | |
3184 | unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; | |
3185 | size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); | |
3186 | U32 singleStream = litSize < 256; | |
3187 | ||
3188 | if (hufMetadata->hType == set_basic) return litSize; | |
3189 | else if (hufMetadata->hType == set_rle) return 1; | |
3190 | else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { | |
3191 | size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); | |
3192 | if (ZSTD_isError(largest)) return litSize; | |
3193 | { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); | |
3194 | if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; | |
3195 | if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ | |
3196 | return cLitSizeEstimate + literalSectionHeaderSize; | |
3197 | } } | |
3198 | assert(0); /* impossible */ | |
3199 | return 0; | |
3200 | } | |
3201 | ||
e2bb2151 | 3202 | /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ |
41c3eae6 | 3203 | static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, |
e3433283 SH |
3204 | const BYTE* codeTable, size_t nbSeq, unsigned maxCode, |
3205 | const FSE_CTable* fseCTable, | |
c05c090c | 3206 | const U32* additionalBits, |
3207 | short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, | |
3208 | void* workspace, size_t wkspSize) | |
3209 | { | |
3210 | unsigned* const countWksp = (unsigned*)workspace; | |
3211 | const BYTE* ctp = codeTable; | |
3212 | const BYTE* const ctStart = ctp; | |
3213 | const BYTE* const ctEnd = ctStart + nbSeq; | |
3214 | size_t cSymbolTypeSizeEstimateInBits = 0; | |
3215 | unsigned max = maxCode; | |
3216 | ||
3217 | HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ | |
3218 | if (type == set_basic) { | |
3219 | /* We selected this encoding type, so it must be valid. */ | |
3220 | assert(max <= defaultMax); | |
e3433283 SH |
3221 | (void)defaultMax; |
3222 | cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); | |
c05c090c | 3223 | } else if (type == set_rle) { |
3224 | cSymbolTypeSizeEstimateInBits = 0; | |
3225 | } else if (type == set_compressed || type == set_repeat) { | |
3226 | cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); | |
3227 | } | |
2949a952 | 3228 | if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { |
2949a952 | 3229 | return nbSeq * 10; |
3230 | } | |
c05c090c | 3231 | while (ctp < ctEnd) { |
3232 | if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; | |
3233 | else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ | |
3234 | ctp++; | |
3235 | } | |
e2bb2151 | 3236 | return cSymbolTypeSizeEstimateInBits >> 3; |
c05c090c | 3237 | } |
3238 | ||
e2bb2151 | 3239 | /* Returns the size estimate for the sequences section (header + content) of a block */ |
41c3eae6 | 3240 | static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, |
c05c090c | 3241 | const BYTE* llCodeTable, |
3242 | const BYTE* mlCodeTable, | |
3243 | size_t nbSeq, | |
3244 | const ZSTD_fseCTables_t* fseTables, | |
3245 | const ZSTD_fseCTablesMetadata_t* fseMetadata, | |
3246 | void* workspace, size_t wkspSize, | |
3247 | int writeEntropy) | |
3248 | { | |
e2bb2151 | 3249 | size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); |
c05c090c | 3250 | size_t cSeqSizeEstimate = 0; |
e3433283 SH |
3251 | cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, |
3252 | fseTables->offcodeCTable, NULL, | |
c05c090c | 3253 | OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, |
3254 | workspace, wkspSize); | |
e3433283 SH |
3255 | cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, |
3256 | fseTables->litlengthCTable, LL_bits, | |
c05c090c | 3257 | LL_defaultNorm, LL_defaultNormLog, MaxLL, |
3258 | workspace, wkspSize); | |
e3433283 SH |
3259 | cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, |
3260 | fseTables->matchlengthCTable, ML_bits, | |
c05c090c | 3261 | ML_defaultNorm, ML_defaultNormLog, MaxML, |
3262 | workspace, wkspSize); | |
3263 | if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; | |
c05c090c | 3264 | return cSeqSizeEstimate + sequencesSectionHeaderSize; |
3265 | } | |
3266 | ||
e2bb2151 | 3267 | /* Returns the size estimate for a given stream of literals, of, ll, ml */ |
41c3eae6 SH |
3268 | static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, |
3269 | const BYTE* ofCodeTable, | |
3270 | const BYTE* llCodeTable, | |
3271 | const BYTE* mlCodeTable, | |
3272 | size_t nbSeq, | |
3273 | const ZSTD_entropyCTables_t* entropy, | |
3274 | const ZSTD_entropyCTablesMetadata_t* entropyMetadata, | |
3275 | void* workspace, size_t wkspSize, | |
3276 | int writeLitEntropy, int writeSeqEntropy) { | |
e3433283 | 3277 | size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, |
c05c090c | 3278 | &entropy->huf, &entropyMetadata->hufMetadata, |
3279 | workspace, wkspSize, writeLitEntropy); | |
e3433283 | 3280 | size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, |
c05c090c | 3281 | nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, |
3282 | workspace, wkspSize, writeSeqEntropy); | |
c05c090c | 3283 | return seqSize + literalsSize + ZSTD_blockHeaderSize; |
3284 | } | |
3285 | ||
de52de13 | 3286 | /* Builds entropy statistics and uses them for blocksize estimation. |
3287 | * | |
3288 | * Returns the estimated compressed size of the seqStore, or a zstd error. | |
3289 | */ | |
3290 | static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, const ZSTD_CCtx* zc) { | |
c05c090c | 3291 | ZSTD_entropyCTablesMetadata_t entropyMetadata; |
f06f6626 | 3292 | FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, |
c05c090c | 3293 | &zc->blockState.prevCBlock->entropy, |
3294 | &zc->blockState.nextCBlock->entropy, | |
3295 | &zc->appliedParams, | |
3296 | &entropyMetadata, | |
3297 | zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); | |
e3433283 SH |
3298 | return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), |
3299 | seqStore->ofCode, seqStore->llCode, seqStore->mlCode, | |
3300 | (size_t)(seqStore->sequences - seqStore->sequencesStart), | |
3301 | &zc->blockState.nextCBlock->entropy, &entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, | |
3302 | (int)(entropyMetadata.hufMetadata.hType == set_compressed), 1); | |
c05c090c | 3303 | } |
3304 | ||
f06f6626 | 3305 | /* Returns literals bytes represented in a seqStore */ |
41c3eae6 | 3306 | static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) { |
f06f6626 | 3307 | size_t literalsBytes = 0; |
e3433283 | 3308 | size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart; |
f06f6626 | 3309 | size_t i; |
3310 | for (i = 0; i < nbSeqs; ++i) { | |
386111ad | 3311 | seqDef seq = seqStore->sequencesStart[i]; |
3312 | literalsBytes += seq.litLength; | |
b1a43455 | 3313 | if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { |
386111ad | 3314 | literalsBytes += 0x10000; |
3315 | } | |
3316 | } | |
3317 | return literalsBytes; | |
3318 | } | |
3319 | ||
f06f6626 | 3320 | /* Returns match bytes represented in a seqStore */ |
41c3eae6 | 3321 | static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) { |
f06f6626 | 3322 | size_t matchBytes = 0; |
e3433283 | 3323 | size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart; |
f06f6626 | 3324 | size_t i; |
3325 | for (i = 0; i < nbSeqs; ++i) { | |
386111ad | 3326 | seqDef seq = seqStore->sequencesStart[i]; |
3327 | matchBytes += seq.matchLength + MINMATCH; | |
b1a43455 | 3328 | if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { |
386111ad | 3329 | matchBytes += 0x10000; |
3330 | } | |
3331 | } | |
3332 | return matchBytes; | |
3333 | } | |
3334 | ||
de52de13 | 3335 | /* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). |
e2bb2151 | 3336 | * Stores the result in resultSeqStore. |
f06f6626 | 3337 | */ |
41c3eae6 SH |
3338 | static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, |
3339 | const seqStore_t* originalSeqStore, | |
de52de13 | 3340 | size_t startIdx, size_t endIdx) { |
2949a952 | 3341 | BYTE* const litEnd = originalSeqStore->lit; |
0633bf17 | 3342 | size_t literalsBytes; |
3343 | size_t literalsBytesPreceding = 0; | |
386111ad | 3344 | |
de52de13 | 3345 | *resultSeqStore = *originalSeqStore; |
de52de13 | 3346 | if (startIdx > 0) { |
3347 | resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; | |
3348 | literalsBytesPreceding = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); | |
386111ad | 3349 | } |
3350 | ||
e2bb2151 | 3351 | /* Move longLengthPos into the correct position if necessary */ |
b1a43455 | 3352 | if (originalSeqStore->longLengthType != ZSTD_llt_none) { |
de52de13 | 3353 | if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { |
b1a43455 | 3354 | resultSeqStore->longLengthType = ZSTD_llt_none; |
de52de13 | 3355 | } else { |
0633bf17 | 3356 | resultSeqStore->longLengthPos -= (U32)startIdx; |
2949a952 | 3357 | } |
3358 | } | |
de52de13 | 3359 | resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; |
3360 | resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; | |
3361 | literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); | |
3362 | resultSeqStore->litStart += literalsBytesPreceding; | |
3363 | if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { | |
e2bb2151 | 3364 | /* This accounts for possible last literals if the derived chunk reaches the end of the block */ |
de52de13 | 3365 | resultSeqStore->lit = litEnd; |
2949a952 | 3366 | } else { |
de52de13 | 3367 | resultSeqStore->lit = resultSeqStore->litStart+literalsBytes; |
2949a952 | 3368 | } |
de52de13 | 3369 | resultSeqStore->llCode += startIdx; |
3370 | resultSeqStore->mlCode += startIdx; | |
3371 | resultSeqStore->ofCode += startIdx; | |
2949a952 | 3372 | } |
3373 | ||
550f76f1 SH |
3374 | /** |
3375 | * Returns the raw offset represented by the combination of offCode, ll0, and repcode history. | |
3376 | * offCode must be an offCode representing a repcode, therefore in the range of [0, 2]. | |
3377 | */ | |
3378 | static U32 ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offCode, const U32 ll0) { | |
3379 | U32 const adjustedOffCode = offCode + ll0; | |
3380 | assert(offCode < ZSTD_REP_NUM); | |
3381 | if (adjustedOffCode == ZSTD_REP_NUM) { | |
3382 | /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 */ | |
3383 | assert(rep[0] > 0); | |
3384 | return rep[0] - 1; | |
3385 | } | |
3386 | return rep[adjustedOffCode]; | |
3387 | } | |
3388 | ||
255925c2 | 3389 | /** |
f1e8b565 | 3390 | * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise |
3391 | * due to emission of RLE/raw blocks that disturb the offset history, and replaces any repcodes within | |
3392 | * the seqStore that may be invalid. | |
efa6dfa7 | 3393 | * |
f1e8b565 | 3394 | * dRepcodes are updated as would be on the decompression side. cRepcodes are updated exactly in |
3395 | * accordance with the seqStore. | |
255925c2 | 3396 | */ |
f1e8b565 | 3397 | static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, |
3398 | seqStore_t* const seqStore, U32 const nbSeq) { | |
255925c2 | 3399 | U32 idx = 0; |
255925c2 | 3400 | for (; idx < nbSeq; ++idx) { |
f1e8b565 | 3401 | seqDef* const seq = seqStore->sequencesStart + idx; |
3402 | U32 const ll0 = (seq->litLength == 0); | |
3403 | U32 offCode = seq->offset - 1; | |
3404 | assert(seq->offset > 0); | |
550f76f1 SH |
3405 | if (offCode <= ZSTD_REP_MOVE) { |
3406 | U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offCode, ll0); | |
3407 | U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offCode, ll0); | |
f1e8b565 | 3408 | /* Adjust simulated decompression repcode history if we come across a mismatch. Replace |
3409 | * the repcode with the offset it actually references, determined by the compression | |
3410 | * repcode history. | |
3411 | */ | |
550f76f1 SH |
3412 | if (dRawOffset != cRawOffset) { |
3413 | seq->offset = cRawOffset + ZSTD_REP_NUM; | |
f1e8b565 | 3414 | } |
f1e8b565 | 3415 | } |
550f76f1 SH |
3416 | /* Compression repcode history is always updated with values directly from the unmodified seqStore. |
3417 | * Decompression repcode history may use modified seq->offset value taken from compression repcode history. | |
3418 | */ | |
3419 | *dRepcodes = ZSTD_updateRep(dRepcodes->rep, seq->offset - 1, ll0); | |
f1e8b565 | 3420 | *cRepcodes = ZSTD_updateRep(cRepcodes->rep, offCode, ll0); |
255925c2 | 3421 | } |
255925c2 | 3422 | } |
3423 | ||
5b566ebe | 3424 | /* ZSTD_compressSeqStore_singleBlock(): |
f06f6626 | 3425 | * Compresses a seqStore into a block with a block header, into the buffer dst. |
4694423c | 3426 | * |
e2bb2151 | 3427 | * Returns the total size of that block (including header) or a ZSTD error code. |
f06f6626 | 3428 | */ |
f1e8b565 | 3429 | static size_t ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore, |
3430 | repcodes_t* const dRep, repcodes_t* const cRep, | |
3431 | void* dst, size_t dstCapacity, | |
3432 | const void* src, size_t srcSize, | |
3433 | U32 lastBlock, U32 isPartition) { | |
2949a952 | 3434 | const U32 rleMaxLength = 25; |
3435 | BYTE* op = (BYTE*)dst; | |
3436 | const BYTE* ip = (const BYTE*)src; | |
3437 | size_t cSize; | |
f1e8b565 | 3438 | size_t cSeqsSize; |
3439 | ||
3440 | /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ | |
3441 | repcodes_t const dRepOriginal = *dRep; | |
3442 | if (isPartition) | |
3443 | ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); | |
3444 | ||
3445 | cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, | |
2949a952 | 3446 | &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, |
3447 | &zc->appliedParams, | |
3448 | op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, | |
3449 | srcSize, | |
3450 | zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, | |
255925c2 | 3451 | zc->bmi2); |
5b566ebe | 3452 | FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); |
2949a952 | 3453 | |
3454 | if (!zc->isFirstBlock && | |
3455 | cSeqsSize < rleMaxLength && | |
255925c2 | 3456 | ZSTD_isRLE((BYTE const*)src, srcSize)) { |
2949a952 | 3457 | /* We don't want to emit our first block as a RLE even if it qualifies because |
3458 | * doing so will cause the decoder (cli only) to throw a "should consume all input error." | |
3459 | * This is only an issue for zstd <= v1.4.3 | |
3460 | */ | |
3461 | cSeqsSize = 1; | |
3462 | } | |
3463 | ||
3464 | if (zc->seqCollector.collectSequences) { | |
3465 | ZSTD_copyBlockSequences(zc); | |
255925c2 | 3466 | ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); |
2949a952 | 3467 | return 0; |
3468 | } | |
3469 | ||
3470 | if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) | |
3471 | zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; | |
3472 | ||
255925c2 | 3473 | if (cSeqsSize == 0) { |
2949a952 | 3474 | cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); |
3475 | FORWARD_IF_ERROR(cSize, "Nocompress block failed"); | |
e2bb2151 | 3476 | DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize); |
f1e8b565 | 3477 | *dRep = dRepOriginal; /* reset simulated decompression repcode history */ |
255925c2 | 3478 | } else if (cSeqsSize == 1) { |
2949a952 | 3479 | cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); |
3480 | FORWARD_IF_ERROR(cSize, "RLE compress block failed"); | |
e2bb2151 | 3481 | DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize); |
f1e8b565 | 3482 | *dRep = dRepOriginal; /* reset simulated decompression repcode history */ |
2949a952 | 3483 | } else { |
255925c2 | 3484 | ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); |
2949a952 | 3485 | writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); |
3486 | cSize = ZSTD_blockHeaderSize + cSeqsSize; | |
e2bb2151 | 3487 | DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize); |
2949a952 | 3488 | } |
3489 | return cSize; | |
386111ad | 3490 | } |
3491 | ||
de52de13 | 3492 | /* Struct to keep track of where we are in our recursive calls. */ |
3493 | typedef struct { | |
3494 | U32* splitLocations; /* Array of split indices */ | |
3495 | size_t idx; /* The current index within splitLocations being worked on */ | |
3496 | } seqStoreSplits; | |
3497 | ||
3498 | #define MIN_SEQUENCES_BLOCK_SPLITTING 300 | |
3499 | #define MAX_NB_SPLITS 196 | |
3500 | ||
3501 | /* Helper function to perform the recursive search for block splits. | |
e2bb2151 | 3502 | * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. |
3503 | * If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then | |
3504 | * we do not recurse. | |
4694423c | 3505 | * |
e2bb2151 | 3506 | * Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING. |
3507 | * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). | |
3508 | * In practice, recursion depth usually doesn't go beyond 4. | |
4694423c | 3509 | * |
e2bb2151 | 3510 | * Furthermore, the number of splits is capped by MAX_NB_SPLITS. At MAX_NB_SPLITS == 196 with the current existing blockSize |
3511 | * maximum of 128 KB, this value is actually impossible to reach. | |
de52de13 | 3512 | */ |
e2bb2151 | 3513 | static void ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, |
3514 | const ZSTD_CCtx* zc, const seqStore_t* origSeqStore) { | |
3515 | seqStore_t fullSeqStoreChunk; | |
de52de13 | 3516 | seqStore_t firstHalfSeqStore; |
3517 | seqStore_t secondHalfSeqStore; | |
3518 | size_t estimatedOriginalSize; | |
3519 | size_t estimatedFirstHalfSize; | |
3520 | size_t estimatedSecondHalfSize; | |
e2bb2151 | 3521 | size_t midIdx = (startIdx + endIdx)/2; |
de52de13 | 3522 | |
3523 | if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= MAX_NB_SPLITS) { | |
e2bb2151 | 3524 | return; |
de52de13 | 3525 | } |
e2bb2151 | 3526 | ZSTD_deriveSeqStoreChunk(&fullSeqStoreChunk, origSeqStore, startIdx, endIdx); |
3527 | ZSTD_deriveSeqStoreChunk(&firstHalfSeqStore, origSeqStore, startIdx, midIdx); | |
3528 | ZSTD_deriveSeqStoreChunk(&secondHalfSeqStore, origSeqStore, midIdx, endIdx); | |
3529 | estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&fullSeqStoreChunk, zc); | |
de52de13 | 3530 | estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&firstHalfSeqStore, zc); |
3531 | estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&secondHalfSeqStore, zc); | |
e3433283 SH |
3532 | DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", |
3533 | estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); | |
e2bb2151 | 3534 | if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { |
3535 | return; | |
3536 | } | |
de52de13 | 3537 | if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { |
e2bb2151 | 3538 | ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); |
0633bf17 | 3539 | splits->splitLocations[splits->idx] = (U32)midIdx; |
de52de13 | 3540 | splits->idx++; |
e2bb2151 | 3541 | ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); |
de52de13 | 3542 | } |
3543 | } | |
3544 | ||
e2bb2151 | 3545 | /* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio. |
de52de13 | 3546 | * |
3547 | * Returns the number of splits made (which equals the size of the partition table - 1). | |
3548 | */ | |
e2bb2151 | 3549 | static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) { |
3550 | seqStoreSplits splits = {partitions, 0}; | |
3551 | if (nbSeq <= 4) { | |
41c3eae6 | 3552 | DEBUGLOG(4, "ZSTD_deriveBlockSplits: Too few sequences to split"); |
e2bb2151 | 3553 | /* Refuse to try and split anything with less than 4 sequences */ |
3554 | return 0; | |
3555 | } | |
3556 | ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); | |
de52de13 | 3557 | splits.splitLocations[splits.idx] = nbSeq; |
550f76f1 | 3558 | DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); |
de52de13 | 3559 | return splits.idx; |
3560 | } | |
3561 | ||
f06f6626 | 3562 | /* ZSTD_compressBlock_splitBlock(): |
e2bb2151 | 3563 | * Attempts to split a given block into multiple blocks to improve compression ratio. |
4694423c | 3564 | * |
e2bb2151 | 3565 | * Returns combined size of all blocks (which includes headers), or a ZSTD error code. |
f06f6626 | 3566 | */ |
e2bb2151 | 3567 | static size_t ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, |
3568 | const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq) { | |
de52de13 | 3569 | size_t cSize = 0; |
386111ad | 3570 | const BYTE* ip = (const BYTE*)src; |
3571 | BYTE* op = (BYTE*)dst; | |
e2bb2151 | 3572 | U32 partitions[MAX_NB_SPLITS]; |
3573 | size_t i = 0; | |
e2bb2151 | 3574 | size_t srcBytesTotal = 0; |
3575 | size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); | |
41c3eae6 SH |
3576 | seqStore_t nextSeqStore; |
3577 | seqStore_t currSeqStore; | |
e2bb2151 | 3578 | |
f1e8b565 | 3579 | /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history |
3580 | * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two | |
3581 | * separate repcode histories that simulate repcode history on compression and decompression side, | |
3582 | * and use the histories to determine whether we must replace a particular repcode with its raw offset. | |
efa6dfa7 | 3583 | * |
f1e8b565 | 3584 | * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed |
3585 | * or RLE. This allows us to retrieve the offset value that an invalid repcode references within | |
3586 | * a nocompress/RLE block. | |
3587 | * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use | |
3588 | * the replacement offset value rather than the original repcode to update the repcode history. | |
3589 | * dRep also will be the final repcode history sent to the next block. | |
efa6dfa7 | 3590 | * |
f1e8b565 | 3591 | * See ZSTD_seqStore_resolveOffCodes() for more details. |
3592 | */ | |
3593 | repcodes_t dRep; | |
3594 | repcodes_t cRep; | |
3595 | ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); | |
3596 | ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); | |
3597 | ||
3598 | DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", | |
386111ad | 3599 | (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, |
3600 | (unsigned)zc->blockState.matchState.nextToUpdate); | |
e2bb2151 | 3601 | |
3602 | if (numSplits == 0) { | |
255925c2 | 3603 | size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, |
f1e8b565 | 3604 | &dRep, &cRep, |
255925c2 | 3605 | op, dstCapacity, |
3606 | ip, blockSize, | |
3607 | lastBlock, 0 /* isPartition */); | |
e2bb2151 | 3608 | FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); |
41c3eae6 | 3609 | DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); |
255925c2 | 3610 | assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize); |
e2bb2151 | 3611 | return cSizeSingleBlock; |
3612 | } | |
3613 | ||
41c3eae6 | 3614 | ZSTD_deriveSeqStoreChunk(&currSeqStore, &zc->seqStore, 0, partitions[0]); |
e2bb2151 | 3615 | for (i = 0; i <= numSplits; ++i) { |
e2bb2151 | 3616 | size_t srcBytes; |
3617 | size_t cSizeChunk; | |
255925c2 | 3618 | U32 const lastPartition = (i == numSplits); |
3619 | U32 lastBlockEntireSrc = 0; | |
e2bb2151 | 3620 | |
41c3eae6 | 3621 | srcBytes = ZSTD_countSeqStoreLiteralsBytes(&currSeqStore) + ZSTD_countSeqStoreMatchBytes(&currSeqStore); |
e2bb2151 | 3622 | srcBytesTotal += srcBytes; |
255925c2 | 3623 | if (lastPartition) { |
e2bb2151 | 3624 | /* This is the final partition, need to account for possible last literals */ |
3625 | srcBytes += blockSize - srcBytesTotal; | |
255925c2 | 3626 | lastBlockEntireSrc = lastBlock; |
41c3eae6 SH |
3627 | } else { |
3628 | ZSTD_deriveSeqStoreChunk(&nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); | |
de52de13 | 3629 | } |
e2bb2151 | 3630 | |
255925c2 | 3631 | cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, &currSeqStore, |
f1e8b565 | 3632 | &dRep, &cRep, |
255925c2 | 3633 | op, dstCapacity, |
3634 | ip, srcBytes, | |
3635 | lastBlockEntireSrc, 1 /* isPartition */); | |
e3433283 | 3636 | DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&currSeqStore, zc), cSizeChunk); |
e2bb2151 | 3637 | FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); |
e2bb2151 | 3638 | |
3639 | ip += srcBytes; | |
3640 | op += cSizeChunk; | |
3641 | dstCapacity -= cSizeChunk; | |
3642 | cSize += cSizeChunk; | |
41c3eae6 | 3643 | currSeqStore = nextSeqStore; |
255925c2 | 3644 | assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize); |
e2bb2151 | 3645 | } |
f1e8b565 | 3646 | /* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes |
3647 | * for the next block. | |
3648 | */ | |
3649 | ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t)); | |
e2bb2151 | 3650 | return cSize; |
3651 | } | |
3652 | ||
3653 | static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, | |
3654 | void* dst, size_t dstCapacity, | |
3655 | const void* src, size_t srcSize, U32 lastBlock) { | |
3656 | const BYTE* ip = (const BYTE*)src; | |
3657 | BYTE* op = (BYTE*)dst; | |
0633bf17 | 3658 | U32 nbSeq; |
e2bb2151 | 3659 | size_t cSize; |
3660 | DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); | |
3661 | ||
3662 | { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); | |
3663 | FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); | |
3664 | if (bss == ZSTDbss_noCompress) { | |
e2bb2151 | 3665 | if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) |
3666 | zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; | |
3667 | cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); | |
3668 | FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); | |
41c3eae6 | 3669 | DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block"); |
e2bb2151 | 3670 | return cSize; |
f06f6626 | 3671 | } |
0633bf17 | 3672 | nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); |
f06f6626 | 3673 | } |
e2bb2151 | 3674 | |
3675 | assert(zc->appliedParams.splitBlocks == 1); | |
3676 | cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); | |
3677 | FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); | |
386111ad | 3678 | return cSize; |
3679 | } | |
3680 | ||
f57ac7b0 EP |
3681 | static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, |
3682 | void* dst, size_t dstCapacity, | |
ce264ce5 | 3683 | const void* src, size_t srcSize, U32 frame) |
f57ac7b0 | 3684 | { |
96201d97 | 3685 | /* This the upper bound for the length of an rle block. |
3686 | * This isn't the actual upper bound. Finding the real threshold | |
3687 | * needs further investigation. | |
e5704bbf | 3688 | */ |
1f2bf77f | 3689 | const U32 rleMaxLength = 25; |
f57ac7b0 | 3690 | size_t cSize; |
4faf3a59 | 3691 | const BYTE* ip = (const BYTE*)src; |
3692 | BYTE* op = (BYTE*)dst; | |
f57ac7b0 | 3693 | DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", |
98692c28 YC |
3694 | (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, |
3695 | (unsigned)zc->blockState.matchState.nextToUpdate); | |
f57ac7b0 EP |
3696 | |
3697 | { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); | |
5e5f2626 | 3698 | FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); |
f57ac7b0 EP |
3699 | if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } |
3700 | } | |
a1d4041e | 3701 | |
bff6072e BS |
3702 | if (zc->seqCollector.collectSequences) { |
3703 | ZSTD_copyBlockSequences(zc); | |
255925c2 | 3704 | ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); |
bff6072e BS |
3705 | return 0; |
3706 | } | |
3707 | ||
a1d4041e | 3708 | /* encode sequences and literals */ |
5b566ebe | 3709 | cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, |
6391cd10 NT |
3710 | &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, |
3711 | &zc->appliedParams, | |
3712 | dst, dstCapacity, | |
7b744051 | 3713 | srcSize, |
a9077939 | 3714 | zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, |
255925c2 | 3715 | zc->bmi2); |
6391cd10 | 3716 | |
086513b5 | 3717 | if (zc->seqCollector.collectSequences) { |
3718 | ZSTD_copyBlockSequences(zc); | |
3719 | return 0; | |
3720 | } | |
3721 | ||
6391cd10 | 3722 | |
ce264ce5 | 3723 | if (frame && |
96201d97 | 3724 | /* We don't want to emit our first block as a RLE even if it qualifies because |
44e12205 | 3725 | * doing so will cause the decoder (cli only) to throw a "should consume all input error." |
a917cd59 | 3726 | * This is only an issue for zstd <= v1.4.3 |
e5704bbf | 3727 | */ |
96201d97 | 3728 | !zc->isFirstBlock && |
ce264ce5 | 3729 | cSize < rleMaxLength && |
3730 | ZSTD_isRLE(ip, srcSize)) | |
3731 | { | |
cba5350f | 3732 | cSize = 1; |
4faf3a59 | 3733 | op[0] = ip[0]; |
cba5350f | 3734 | } |
3735 | ||
6391cd10 | 3736 | out: |
cba5350f | 3737 | if (!ZSTD_isError(cSize) && cSize > 1) { |
255925c2 | 3738 | ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); |
887cd4e3 | 3739 | } |
6391cd10 NT |
3740 | /* We check that dictionaries have offset codes available for the first |
3741 | * block. After the first block, the offcode table might not have large | |
3742 | * enough codes to represent the offsets in the data. | |
3743 | */ | |
3744 | if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) | |
3745 | zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; | |
3746 | ||
3747 | return cSize; | |
59d7063f YC |
3748 | } |
3749 | ||
ffb04630 | 3750 | static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, |
e1913dc8 | 3751 | void* dst, size_t dstCapacity, |
ffb04630 | 3752 | const void* src, size_t srcSize, |
e1913dc8 | 3753 | const size_t bss, U32 lastBlock) |
ffb04630 | 3754 | { |
b1f53b1a | 3755 | DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); |
e1913dc8 | 3756 | if (bss == ZSTDbss_compress) { |
e103d7b4 NT |
3757 | if (/* We don't want to emit our first block as a RLE even if it qualifies because |
3758 | * doing so will cause the decoder (cli only) to throw a "should consume all input error." | |
3759 | * This is only an issue for zstd <= v1.4.3 | |
3760 | */ | |
3761 | !zc->isFirstBlock && | |
3762 | ZSTD_maybeRLE(&zc->seqStore) && | |
3763 | ZSTD_isRLE((BYTE const*)src, srcSize)) | |
3764 | { | |
3765 | return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); | |
3766 | } | |
036b30b5 NT |
3767 | /* Attempt superblock compression. |
3768 | * | |
3769 | * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the | |
3770 | * standard ZSTD_compressBound(). This is a problem, because even if we have | |
3771 | * space now, taking an extra byte now could cause us to run out of space later | |
3772 | * and violate ZSTD_compressBound(). | |
3773 | * | |
3774 | * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. | |
3775 | * | |
3776 | * In order to respect ZSTD_compressBound() we must attempt to emit a raw | |
3777 | * uncompressed block in these cases: | |
3778 | * * cSize == 0: Return code for an uncompressed block. | |
3779 | * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). | |
3780 | * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of | |
3781 | * output space. | |
3782 | * * cSize >= blockBound(srcSize): We have expanded the block too much so | |
3783 | * emit an uncompressed block. | |
3784 | */ | |
e103d7b4 NT |
3785 | { |
3786 | size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); | |
3787 | if (cSize != ERROR(dstSize_tooSmall)) { | |
3788 | size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); | |
5e5f2626 | 3789 | FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); |
e103d7b4 | 3790 | if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { |
255925c2 | 3791 | ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); |
e103d7b4 NT |
3792 | return cSize; |
3793 | } | |
036b30b5 | 3794 | } |
2ec556fe | 3795 | } |
7ce89187 | 3796 | } |
7ce89187 | 3797 | |
036b30b5 NT |
3798 | DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); |
3799 | /* Superblock compression failed, attempt to emit a single no compress block. | |
3800 | * The decoder will be able to stream this block since it is uncompressed. | |
3801 | */ | |
ffb04630 BS |
3802 | return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); |
3803 | } | |
3804 | ||
7ce89187 SH |
3805 | static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, |
3806 | void* dst, size_t dstCapacity, | |
3807 | const void* src, size_t srcSize, | |
ffb04630 BS |
3808 | U32 lastBlock) |
3809 | { | |
7ce89187 | 3810 | size_t cSize = 0; |
2ec556fe | 3811 | const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); |
7ce89187 SH |
3812 | DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", |
3813 | (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); | |
5e5f2626 | 3814 | FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); |
7ce89187 | 3815 | |
e1913dc8 | 3816 | cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); |
5e5f2626 | 3817 | FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); |
7ce89187 | 3818 | |
7ce89187 SH |
3819 | if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) |
3820 | zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; | |
3821 | ||
3822 | return cSize; | |
3823 | } | |
59d7063f | 3824 | |
bc020eec FH |
3825 | static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, |
3826 | ZSTD_cwksp* ws, | |
3827 | ZSTD_CCtx_params const* params, | |
3828 | void const* ip, | |
3829 | void const* iend) | |
674534a7 | 3830 | { |
34aff7ea NT |
3831 | U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); |
3832 | U32 const maxDist = (U32)1 << params->cParams.windowLog; | |
3833 | if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { | |
674534a7 NT |
3834 | U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); |
3835 | ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); | |
3836 | ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); | |
3837 | ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); | |
bc020eec | 3838 | ZSTD_cwksp_mark_tables_dirty(ws); |
674534a7 | 3839 | ZSTD_reduceIndex(ms, params, correction); |
bc020eec | 3840 | ZSTD_cwksp_mark_tables_clean(ws); |
674534a7 NT |
3841 | if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; |
3842 | else ms->nextToUpdate -= correction; | |
3843 | /* invalidate dictionaries on overflow correction */ | |
3844 | ms->loadedDictEnd = 0; | |
3845 | ms->dictMatchState = NULL; | |
3846 | } | |
3847 | } | |
3848 | ||
db8e21d5 | 3849 | /*! ZSTD_compress_frameChunk() : |
c991cc18 YC |
3850 | * Compress a chunk of data into one or multiple blocks. |
3851 | * All blocks will be terminated, all input will be consumed. | |
3852 | * Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. | |
3853 | * Frame is supposed already started (header already produced) | |
3854 | * @return : compressed size, or an error code | |
3855 | */ | |
e2bb2151 | 3856 | static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, |
f2a3b6e7 | 3857 | void* dst, size_t dstCapacity, |
c991cc18 YC |
3858 | const void* src, size_t srcSize, |
3859 | U32 lastFrameChunk) | |
f3eca253 | 3860 | { |
f2a3b6e7 | 3861 | size_t blockSize = cctx->blockSize; |
f3eca253 YC |
3862 | size_t remaining = srcSize; |
3863 | const BYTE* ip = (const BYTE*)src; | |
3864 | BYTE* const ostart = (BYTE*)dst; | |
3865 | BYTE* op = ostart; | |
c233bdba | 3866 | U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; |
5225dcfc | 3867 | |
4baecdf7 | 3868 | assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); |
9b11b46f | 3869 | |
293fad6b | 3870 | DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); |
1ad7c82e | 3871 | if (cctx->appliedParams.fParams.checksumFlag && srcSize) |
f2a3b6e7 YC |
3872 | XXH64_update(&cctx->xxhState, src, srcSize); |
3873 | ||
2ce49232 | 3874 | while (remaining) { |
aae267a2 | 3875 | ZSTD_matchState_t* const ms = &cctx->blockState.matchState; |
c991cc18 | 3876 | U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); |
f3eca253 | 3877 | |
cafc3b1b FH |
3878 | RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE, |
3879 | dstSize_tooSmall, | |
3880 | "not enough space to store compressed block"); | |
3e358271 | 3881 | if (remaining < blockSize) blockSize = remaining; |
89db5e00 | 3882 | |
bc020eec FH |
3883 | ZSTD_overflowCorrectIfNeeded( |
3884 | ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); | |
bc601bdc | 3885 | ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); |
a9680990 | 3886 | |
6453f815 | 3887 | /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ |
7e5e226c | 3888 | if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; |
89db5e00 | 3889 | |
d9646dcb | 3890 | { size_t cSize; |
b1f53b1a | 3891 | if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { |
7ce89187 | 3892 | cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); |
5e5f2626 | 3893 | FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); |
036b30b5 NT |
3894 | assert(cSize > 0); |
3895 | assert(cSize <= blockSize + ZSTD_blockHeaderSize); | |
c90e81a6 | 3896 | } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { |
e2bb2151 | 3897 | cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); |
3898 | FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); | |
c90e81a6 | 3899 | assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); |
a9e57050 | 3900 | } else { |
7ce89187 SH |
3901 | cSize = ZSTD_compressBlock_internal(cctx, |
3902 | op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, | |
d9646dcb | 3903 | ip, blockSize, 1 /* frame */); |
5e5f2626 | 3904 | FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); |
7ce89187 SH |
3905 | |
3906 | if (cSize == 0) { /* block is not compressible */ | |
3907 | cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); | |
5e5f2626 | 3908 | FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); |
7ce89187 | 3909 | } else { |
d9646dcb SH |
3910 | U32 const cBlockHeader = cSize == 1 ? |
3911 | lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : | |
3912 | lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); | |
3913 | MEM_writeLE24(op, cBlockHeader); | |
7ce89187 SH |
3914 | cSize += ZSTD_blockHeaderSize; |
3915 | } | |
a9e57050 | 3916 | } |
f3eca253 | 3917 | |
7ce89187 | 3918 | |
a9e57050 YC |
3919 | ip += blockSize; |
3920 | assert(remaining >= blockSize); | |
3921 | remaining -= blockSize; | |
3922 | op += cSize; | |
3923 | assert(dstCapacity >= cSize); | |
3924 | dstCapacity -= cSize; | |
96201d97 | 3925 | cctx->isFirstBlock = 0; |
2e97a6d4 | 3926 | DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", |
ededcfca | 3927 | (unsigned)cSize); |
a9e57050 | 3928 | } } |
f3eca253 | 3929 | |
62470b4b | 3930 | if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; |
4baecdf7 | 3931 | return (size_t)(op-ostart); |
f3eca253 YC |
3932 | } |
3933 | ||
3934 | ||
6236eba9 | 3935 | static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, |
77164547 | 3936 | const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) |
6236eba9 | 3937 | { BYTE* const op = (BYTE*)dst; |
31533bac | 3938 | U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ |
77164547 VN |
3939 | U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ |
3940 | U32 const checksumFlag = params->fParams.checksumFlag>0; | |
3941 | U32 const windowSize = (U32)1 << params->cParams.windowLog; | |
3942 | U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); | |
3943 | BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); | |
3944 | U32 const fcsCode = params->fParams.contentSizeFlag ? | |
55fc1f91 | 3945 | (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ |
a880ca23 | 3946 | BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); |
62568c9a | 3947 | size_t pos=0; |
c46fb924 | 3948 | |
77164547 | 3949 | assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); |
5e5f2626 FH |
3950 | RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, |
3951 | "dst buf is too small to fit worst-case frame header size."); | |
62568c9a | 3952 | DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", |
98165606 | 3953 | !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); |
77164547 | 3954 | if (params->format == ZSTD_f_zstd1) { |
62568c9a YC |
3955 | MEM_writeLE32(dst, ZSTD_MAGICNUMBER); |
3956 | pos = 4; | |
3957 | } | |
a880ca23 | 3958 | op[pos++] = frameHeaderDescriptionByte; |
e4d0265e | 3959 | if (!singleSegment) op[pos++] = windowLogByte; |
c46fb924 YC |
3960 | switch(dictIDSizeCode) |
3961 | { | |
cd2892fd | 3962 | default: assert(0); /* impossible */ |
c46fb924 YC |
3963 | case 0 : break; |
3964 | case 1 : op[pos] = (BYTE)(dictID); pos++; break; | |
d4180cad | 3965 | case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; |
c46fb924 YC |
3966 | case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; |
3967 | } | |
673f0d7c | 3968 | switch(fcsCode) |
6236eba9 | 3969 | { |
cd2892fd | 3970 | default: assert(0); /* impossible */ |
e4d0265e | 3971 | case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; |
673f0d7c YC |
3972 | case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; |
3973 | case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; | |
c46fb924 | 3974 | case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; |
6236eba9 | 3975 | } |
c46fb924 | 3976 | return pos; |
6236eba9 YC |
3977 | } |
3978 | ||
5c41490b | 3979 | /* ZSTD_writeSkippableFrame_advanced() : |
9d31c704 | 3980 | * Writes out a skippable frame with the specified magic number variant (16 are supported), |
7e11bd01 | 3981 | * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. |
9d31c704 | 3982 | * |
7e11bd01 | 3983 | * Returns the total number of bytes written, or a ZSTD error code. |
3984 | */ | |
5c41490b | 3985 | size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, |
3986 | const void* src, size_t srcSize, unsigned magicVariant) { | |
3987 | BYTE* op = (BYTE*)dst; | |
3988 | RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, | |
7e11bd01 | 3989 | dstSize_tooSmall, "Not enough room for skippable frame"); |
3990 | RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); | |
3991 | RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); | |
3992 | ||
3993 | MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); | |
3994 | MEM_writeLE32(op+4, (U32)srcSize); | |
3995 | ZSTD_memcpy(op+8, src, srcSize); | |
5c41490b | 3996 | return srcSize + ZSTD_SKIPPABLEHEADERSIZE; |
7e11bd01 | 3997 | } |
3998 | ||
a1d4041e YC |
3999 | /* ZSTD_writeLastEmptyBlock() : |
4000 | * output an empty Block with end-of-frame mark to complete a frame | |
4001 | * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) | |
a880ca23 | 4002 | * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize) |
a1d4041e YC |
4003 | */ |
4004 | size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) | |
4005 | { | |
5e5f2626 FH |
4006 | RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, |
4007 | "dst buf is too small to write frame trailer empty block."); | |
777d3c15 YC |
4008 | { U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1); /* 0 size */ |
4009 | MEM_writeLE24(dst, cBlockHeader24); | |
4010 | return ZSTD_blockHeaderSize; | |
4011 | } | |
a1d4041e YC |
4012 | } |
4013 | ||
a9a6dcba NT |
4014 | size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) |
4015 | { | |
5e5f2626 FH |
4016 | RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong, |
4017 | "wrong cctx stage"); | |
cafc3b1b | 4018 | RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm, |
5e5f2626 FH |
4019 | parameter_unsupported, |
4020 | "incompatible with ldm"); | |
a9a6dcba NT |
4021 | cctx->externSeqStore.seq = seq; |
4022 | cctx->externSeqStore.size = nbSeq; | |
4023 | cctx->externSeqStore.capacity = nbSeq; | |
4024 | cctx->externSeqStore.pos = 0; | |
ee84817f | 4025 | cctx->externSeqStore.posInSequence = 0; |
a9a6dcba NT |
4026 | return 0; |
4027 | } | |
4028 | ||
6236eba9 | 4029 | |
346efccc | 4030 | static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, |
7cbe79ab | 4031 | void* dst, size_t dstCapacity, |
bf42c8e5 | 4032 | const void* src, size_t srcSize, |
c991cc18 | 4033 | U32 frame, U32 lastFrameChunk) |
f3eca253 | 4034 | { |
0e2dbac1 | 4035 | ZSTD_matchState_t* const ms = &cctx->blockState.matchState; |
6236eba9 | 4036 | size_t fhSize = 0; |
ecd651bd | 4037 | |
2e233330 | 4038 | DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", |
ededcfca | 4039 | cctx->stage, (unsigned)srcSize); |
cafc3b1b FH |
4040 | RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, |
4041 | "missing init (ZSTD_compressBegin)"); | |
d4180cad | 4042 | |
346efccc | 4043 | if (frame && (cctx->stage==ZSTDcs_init)) { |
77164547 | 4044 | fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, |
6f1a21c7 | 4045 | cctx->pledgedSrcSizePlusOne-1, cctx->dictID); |
5e5f2626 | 4046 | FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); |
4baecdf7 | 4047 | assert(fhSize <= dstCapacity); |
6236eba9 YC |
4048 | dstCapacity -= fhSize; |
4049 | dst = (char*)dst + fhSize; | |
346efccc | 4050 | cctx->stage = ZSTDcs_ongoing; |
ecd651bd | 4051 | } |
f3eca253 | 4052 | |
23767e95 YC |
4053 | if (!srcSize) return fhSize; /* do not generate an empty block if no input */ |
4054 | ||
7e5e226c NT |
4055 | if (!ZSTD_window_update(&ms->window, src, srcSize)) { |
4056 | ms->nextToUpdate = ms->window.dictLimit; | |
4057 | } | |
404a7bfe | 4058 | if (cctx->appliedParams.ldmParams.enableLdm) { |
0a0e64c6 | 4059 | ZSTD_window_update(&cctx->ldmState.window, src, srcSize); |
404a7bfe YC |
4060 | } |
4061 | ||
4062 | if (!frame) { | |
4063 | /* overflow check and correction for block mode */ | |
bc020eec FH |
4064 | ZSTD_overflowCorrectIfNeeded( |
4065 | ms, &cctx->workspace, &cctx->appliedParams, | |
4066 | src, (BYTE const*)src + srcSize); | |
404a7bfe | 4067 | } |
89db5e00 | 4068 | |
ededcfca | 4069 | DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); |
23767e95 | 4070 | { size_t const cSize = frame ? |
db8e21d5 | 4071 | ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : |
ce264ce5 | 4072 | ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); |
6696933b | 4073 | FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); |
20d5e038 | 4074 | cctx->consumedSrcSize += srcSize; |
394eec69 | 4075 | cctx->producedCSize += (cSize + fhSize); |
3c3f59e6 NT |
4076 | assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); |
4077 | if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ | |
4078 | ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); | |
cafc3b1b FH |
4079 | RETURN_ERROR_IF( |
4080 | cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, | |
4081 | srcSize_wrong, | |
4082 | "error : pledgedSrcSize = %u, while realSrcSize >= %u", | |
4083 | (unsigned)cctx->pledgedSrcSizePlusOne-1, | |
4084 | (unsigned)cctx->consumedSrcSize); | |
48acaddf | 4085 | } |
6236eba9 | 4086 | return cSize + fhSize; |
23767e95 | 4087 | } |
f3eca253 YC |
4088 | } |
4089 | ||
5b56739b | 4090 | size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, |
7cbe79ab | 4091 | void* dst, size_t dstCapacity, |
bf42c8e5 YC |
4092 | const void* src, size_t srcSize) |
4093 | { | |
ededcfca | 4094 | DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); |
20d5e038 | 4095 | return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); |
5b56739b YC |
4096 | } |
4097 | ||
4098 | ||
fa3671ea | 4099 | size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) |
bf42c8e5 | 4100 | { |
878728dc YC |
4101 | ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; |
4102 | assert(!ZSTD_checkCParams(cParams)); | |
c233bdba | 4103 | return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); |
cf05b9d4 YC |
4104 | } |
4105 | ||
4106 | size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) | |
4107 | { | |
0b0b83e8 YC |
4108 | DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); |
4109 | { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); | |
5e5f2626 | 4110 | RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } |
0e2dbac1 | 4111 | |
20d5e038 | 4112 | return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); |
bf42c8e5 YC |
4113 | } |
4114 | ||
16a0b107 YC |
4115 | /*! ZSTD_loadDictionaryContent() : |
4116 | * @return : 0, or an error code | |
4117 | */ | |
295ab0db | 4118 | static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, |
5b0a452c | 4119 | ldmState_t* ls, |
bc020eec | 4120 | ZSTD_cwksp* ws, |
295ab0db NT |
4121 | ZSTD_CCtx_params const* params, |
4122 | const void* src, size_t srcSize, | |
4123 | ZSTD_dictTableLoadMethod_e dtlm) | |
417890ce | 4124 | { |
674534a7 | 4125 | const BYTE* ip = (const BYTE*) src; |
417890ce | 4126 | const BYTE* const iend = ip + srcSize; |
94db4398 NT |
4127 | int const loadLdmDict = params->ldmParams.enableLdm && ls != NULL; |
4128 | ||
4129 | /* Assert that we the ms params match the params we're being given */ | |
4130 | ZSTD_assertEqualCParams(params->cParams, ms->cParams); | |
0b88c258 | 4131 | |
94db4398 NT |
4132 | if (srcSize > ZSTD_CHUNKSIZE_MAX) { |
4133 | /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. | |
4134 | * Dictionaries right at the edge will immediately trigger overflow | |
4135 | * correction, but I don't want to insert extra constraints here. | |
4136 | */ | |
4137 | U32 const maxDictSize = ZSTD_CURRENT_MAX - 1; | |
4138 | /* We must have cleared our windows when our source is this large. */ | |
4139 | assert(ZSTD_window_isEmpty(ms->window)); | |
4140 | if (loadLdmDict) | |
4141 | assert(ZSTD_window_isEmpty(ls->window)); | |
4142 | /* If the dictionary is too large, only load the suffix of the dictionary. */ | |
4143 | if (srcSize > maxDictSize) { | |
4144 | ip = iend - maxDictSize; | |
4145 | src = ip; | |
4146 | srcSize = maxDictSize; | |
4147 | } | |
4148 | } | |
417890ce | 4149 | |
4694423c | 4150 | DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); |
7e5e226c | 4151 | ZSTD_window_update(&ms->window, src, srcSize); |
4af1fafe | 4152 | ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); |
887cd4e3 | 4153 | |
94db4398 | 4154 | if (loadLdmDict) { |
5b0a452c BS |
4155 | ZSTD_window_update(&ls->window, src, srcSize); |
4156 | ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); | |
4157 | } | |
4158 | ||
731ef16f | 4159 | if (srcSize <= HASH_READ_SIZE) return 0; |
417890ce | 4160 | |
94db4398 | 4161 | ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); |
45dc3562 | 4162 | |
94db4398 | 4163 | if (loadLdmDict) |
0b88c258 | 4164 | ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); |
417890ce | 4165 | |
94db4398 NT |
4166 | switch(params->cParams.strategy) |
4167 | { | |
4168 | case ZSTD_fast: | |
4169 | ZSTD_fillHashTable(ms, iend, dtlm); | |
4170 | break; | |
4171 | case ZSTD_dfast: | |
4172 | ZSTD_fillDoubleHashTable(ms, iend, dtlm); | |
4173 | break; | |
5b0a452c | 4174 | |
94db4398 NT |
4175 | case ZSTD_greedy: |
4176 | case ZSTD_lazy: | |
4177 | case ZSTD_lazy2: | |
0b88c258 NT |
4178 | assert(srcSize >= HASH_READ_SIZE); |
4179 | if (ms->dedicatedDictSearch) { | |
4180 | assert(ms->chainTable != NULL); | |
4181 | ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); | |
4182 | } else { | |
4183 | assert(params->useRowMatchFinder != ZSTD_urm_auto); | |
4184 | if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) { | |
4185 | size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16); | |
4186 | ZSTD_memset(ms->tagTable, 0, tagTableSize); | |
4187 | ZSTD_row_update(ms, iend-HASH_READ_SIZE); | |
4188 | DEBUGLOG(4, "Using row-based hash table for lazy dict"); | |
94db4398 | 4189 | } else { |
0b88c258 NT |
4190 | ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); |
4191 | DEBUGLOG(4, "Using chain-based hash table for lazy dict"); | |
13c5ec3e | 4192 | } |
674534a7 | 4193 | } |
94db4398 NT |
4194 | break; |
4195 | ||
4196 | case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ | |
4197 | case ZSTD_btopt: | |
4198 | case ZSTD_btultra: | |
4199 | case ZSTD_btultra2: | |
0b88c258 NT |
4200 | assert(srcSize >= HASH_READ_SIZE); |
4201 | ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); | |
94db4398 | 4202 | break; |
674534a7 | 4203 | |
94db4398 NT |
4204 | default: |
4205 | assert(0); /* not possible : not a valid strategy id */ | |
417890ce YC |
4206 | } |
4207 | ||
7e5e226c | 4208 | ms->nextToUpdate = (U32)(iend - ms->window.base); |
417890ce YC |
4209 | return 0; |
4210 | } | |
4211 | ||
f3eca253 | 4212 | |
f9c9af3c | 4213 | /* Dictionaries that assign zero probability to symbols that show up causes problems |
08981d26 NT |
4214 | * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check |
4215 | * and only dictionaries with 100% valid symbols can be assumed valid. | |
4216 | */ | |
4217 | static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) | |
4218 | { | |
f9c9af3c | 4219 | U32 s; |
08981d26 NT |
4220 | if (dictMaxSymbolValue < maxSymbolValue) { |
4221 | return FSE_repeat_check; | |
4222 | } | |
f9c9af3c | 4223 | for (s = 0; s <= maxSymbolValue; ++s) { |
08981d26 NT |
4224 | if (normalizedCounter[s] == 0) { |
4225 | return FSE_repeat_check; | |
4226 | } | |
f9c9af3c | 4227 | } |
08981d26 | 4228 | return FSE_repeat_valid; |
f9c9af3c NT |
4229 | } |
4230 | ||
0bcaf6db | 4231 | size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, |
a3a3c62b | 4232 | const void* const dict, size_t dictSize) |
b923f650 | 4233 | { |
08981d26 NT |
4234 | short offcodeNCount[MaxOff+1]; |
4235 | unsigned offcodeMaxValue = MaxOff; | |
d9c475f3 | 4236 | const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ |
52a0622b | 4237 | const BYTE* const dictEnd = dictPtr + dictSize; |
d9c475f3 | 4238 | dictPtr += 8; |
49c6d492 | 4239 | bs->entropy.huf.repeatMode = HUF_repeat_check; |
bea78e8f | 4240 | |
a86a7097 | 4241 | { unsigned maxSymbolValue = 255; |
49c6d492 | 4242 | unsigned hasZeroWeights = 1; |
a3a3c62b BS |
4243 | size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, |
4244 | dictEnd-dictPtr, &hasZeroWeights); | |
4245 | ||
4246 | /* We only set the loaded table as valid if it contains all non-zero | |
4247 | * weights. Otherwise, we set it to check */ | |
4248 | if (!hasZeroWeights) | |
4249 | bs->entropy.huf.repeatMode = HUF_repeat_valid; | |
a3a3c62b | 4250 | |
5e5f2626 FH |
4251 | RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); |
4252 | RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); | |
52a0622b | 4253 | dictPtr += hufHeaderSize; |
f2a3b6e7 YC |
4254 | } |
4255 | ||
f9c9af3c | 4256 | { unsigned offcodeLog; |
08981d26 | 4257 | size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); |
5e5f2626 FH |
4258 | RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); |
4259 | RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); | |
1a26ec6e | 4260 | /* fill all offset symbols to avoid garbage at end of table */ |
03e040a9 FH |
4261 | RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( |
4262 | bs->entropy.fse.offcodeCTable, | |
4263 | offcodeNCount, MaxOff, offcodeLog, | |
4264 | workspace, HUF_WORKSPACE_SIZE)), | |
5e5f2626 | 4265 | dictionary_corrupted, ""); |
08981d26 | 4266 | /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ |
52a0622b | 4267 | dictPtr += offcodeHeaderSize; |
f2a3b6e7 YC |
4268 | } |
4269 | ||
4270 | { short matchlengthNCount[MaxML+1]; | |
bfd943ac | 4271 | unsigned matchlengthMaxValue = MaxML, matchlengthLog; |
52a0622b | 4272 | size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); |
5e5f2626 FH |
4273 | RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); |
4274 | RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); | |
03e040a9 FH |
4275 | RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( |
4276 | bs->entropy.fse.matchlengthCTable, | |
4277 | matchlengthNCount, matchlengthMaxValue, matchlengthLog, | |
4278 | workspace, HUF_WORKSPACE_SIZE)), | |
5e5f2626 | 4279 | dictionary_corrupted, ""); |
08981d26 | 4280 | bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); |
52a0622b | 4281 | dictPtr += matchlengthHeaderSize; |
f2a3b6e7 YC |
4282 | } |
4283 | ||
4284 | { short litlengthNCount[MaxLL+1]; | |
bfd943ac | 4285 | unsigned litlengthMaxValue = MaxLL, litlengthLog; |
52a0622b | 4286 | size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); |
5e5f2626 FH |
4287 | RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); |
4288 | RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); | |
03e040a9 FH |
4289 | RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( |
4290 | bs->entropy.fse.litlengthCTable, | |
4291 | litlengthNCount, litlengthMaxValue, litlengthLog, | |
4292 | workspace, HUF_WORKSPACE_SIZE)), | |
5e5f2626 | 4293 | dictionary_corrupted, ""); |
08981d26 | 4294 | bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); |
52a0622b | 4295 | dictPtr += litlengthHeaderSize; |
f2a3b6e7 YC |
4296 | } |
4297 | ||
5e5f2626 | 4298 | RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); |
887cd4e3 NT |
4299 | bs->rep[0] = MEM_readLE32(dictPtr+0); |
4300 | bs->rep[1] = MEM_readLE32(dictPtr+4); | |
4301 | bs->rep[2] = MEM_readLE32(dictPtr+8); | |
52a0622b YC |
4302 | dictPtr += 12; |
4303 | ||
08981d26 NT |
4304 | { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); |
4305 | U32 offcodeMax = MaxOff; | |
4306 | if (dictContentSize <= ((U32)-1) - 128 KB) { | |
4307 | U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ | |
4308 | offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ | |
4309 | } | |
4310 | /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ | |
4311 | bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); | |
4312 | ||
4313 | /* All repCodes must be <= dictContentSize and != 0 */ | |
4314 | { U32 u; | |
4315 | for (u=0; u<3; u++) { | |
4316 | RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); | |
4317 | RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); | |
4318 | } } } | |
4319 | ||
0bcaf6db SH |
4320 | return dictPtr - (const BYTE*)dict; |
4321 | } | |
4322 | ||
4323 | /* Dictionary format : | |
4324 | * See : | |
0b39531d | 4325 | * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format |
0bcaf6db SH |
4326 | */ |
4327 | /*! ZSTD_loadZstdDictionary() : | |
4328 | * @return : dictID, or an error code | |
4329 | * assumptions : magic number supposed already checked | |
b39149e1 | 4330 | * dictSize supposed >= 8 |
0bcaf6db SH |
4331 | */ |
4332 | static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, | |
4333 | ZSTD_matchState_t* ms, | |
4334 | ZSTD_cwksp* ws, | |
4335 | ZSTD_CCtx_params const* params, | |
4336 | const void* dict, size_t dictSize, | |
4337 | ZSTD_dictTableLoadMethod_e dtlm, | |
4338 | void* workspace) | |
4339 | { | |
0bcaf6db SH |
4340 | const BYTE* dictPtr = (const BYTE*)dict; |
4341 | const BYTE* const dictEnd = dictPtr + dictSize; | |
04fb42b4 SH |
4342 | size_t dictID; |
4343 | size_t eSize; | |
d06b9069 SH |
4344 | ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); |
4345 | assert(dictSize >= 8); | |
4346 | assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); | |
4347 | ||
6ce33537 | 4348 | dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); |
08981d26 | 4349 | eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); |
5e5f2626 | 4350 | FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); |
6ce33537 | 4351 | dictPtr += eSize; |
52a0622b | 4352 | |
08981d26 NT |
4353 | { |
4354 | size_t const dictContentSize = (size_t)(dictEnd - dictPtr); | |
bc020eec | 4355 | FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( |
5e5f2626 | 4356 | ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); |
16a0b107 | 4357 | } |
08981d26 | 4358 | return dictID; |
b923f650 YC |
4359 | } |
4360 | ||
d1b26849 | 4361 | /** ZSTD_compress_insertDictionary() : |
16bd0fd4 | 4362 | * @return : dictID, or an error code */ |
a1550613 YC |
4363 | static size_t |
4364 | ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, | |
4365 | ZSTD_matchState_t* ms, | |
5b0a452c | 4366 | ldmState_t* ls, |
bc020eec | 4367 | ZSTD_cwksp* ws, |
a1550613 YC |
4368 | const ZSTD_CCtx_params* params, |
4369 | const void* dict, size_t dictSize, | |
4370 | ZSTD_dictContentType_e dictContentType, | |
4371 | ZSTD_dictTableLoadMethod_e dtlm, | |
4372 | void* workspace) | |
b923f650 | 4373 | { |
05dffe43 | 4374 | DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); |
60205fec | 4375 | if ((dict==NULL) || (dictSize<8)) { |
5e5f2626 | 4376 | RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); |
60205fec NT |
4377 | return 0; |
4378 | } | |
7b51a294 | 4379 | |
16bd0fd4 NT |
4380 | ZSTD_reset_compressedBlockState(bs); |
4381 | ||
7bd1a290 | 4382 | /* dict restricted modes */ |
6873fec6 | 4383 | if (dictContentType == ZSTD_dct_rawContent) |
5b0a452c | 4384 | return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm); |
d1b26849 | 4385 | |
7d381618 | 4386 | if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { |
6873fec6 | 4387 | if (dictContentType == ZSTD_dct_auto) { |
05dffe43 | 4388 | DEBUGLOG(4, "raw content dictionary detected"); |
bc020eec | 4389 | return ZSTD_loadDictionaryContent( |
5b0a452c | 4390 | ms, ls, ws, params, dict, dictSize, dtlm); |
204b6b7e | 4391 | } |
5e5f2626 | 4392 | RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); |
7bd1a290 YC |
4393 | assert(0); /* impossible */ |
4394 | } | |
4395 | ||
4396 | /* dict as full zstd dictionary */ | |
bc020eec FH |
4397 | return ZSTD_loadZstdDictionary( |
4398 | bs, ms, ws, params, dict, dictSize, dtlm, workspace); | |
ecd651bd YC |
4399 | } |
4400 | ||
676f8990 | 4401 | #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) |
86657931 | 4402 | #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) |
0c8df5c9 | 4403 | |
27caf2af | 4404 | /*! ZSTD_compressBegin_internal() : |
c3bce24e | 4405 | * @return : 0, or an error code */ |
f2d6db45 NT |
4406 | static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, |
4407 | const void* dict, size_t dictSize, | |
4408 | ZSTD_dictContentType_e dictContentType, | |
4409 | ZSTD_dictTableLoadMethod_e dtlm, | |
4410 | const ZSTD_CDict* cdict, | |
77164547 | 4411 | const ZSTD_CCtx_params* params, U64 pledgedSrcSize, |
f2d6db45 | 4412 | ZSTD_buffered_policy_e zbuff) |
f3eca253 | 4413 | { |
94db4398 | 4414 | size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; |
54a4998a | 4415 | #if ZSTD_TRACE |
6cee3c2c | 4416 | cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; |
54a4998a | 4417 | #endif |
77164547 | 4418 | DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); |
5ac72b41 | 4419 | /* params are supposed to be fully validated at this point */ |
98165606 | 4420 | assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); |
1880337c | 4421 | assert(!((dict) && (cdict))); /* either dict or cdict, not both */ |
0c8df5c9 SH |
4422 | if ( (cdict) |
4423 | && (cdict->dictContentSize > 0) | |
c2e1e54f SH |
4424 | && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF |
4425 | || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER | |
b9ede1c8 | 4426 | || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN |
676f8990 SH |
4427 | || cdict->compressionLevel == 0) |
4428 | && (params->attachDictPref != ZSTD_dictForceLoad) ) { | |
ca77822d | 4429 | return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); |
204b6b7e | 4430 | } |
1880337c | 4431 | |
c2183d7c | 4432 | FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, |
94db4398 | 4433 | dictContentSize, |
5e5f2626 | 4434 | ZSTDcrp_makeClean, zbuff) , ""); |
60205fec | 4435 | { size_t const dictID = cdict ? |
0c8df5c9 SH |
4436 | ZSTD_compress_insertDictionary( |
4437 | cctx->blockState.prevCBlock, &cctx->blockState.matchState, | |
5b0a452c | 4438 | &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, |
7083f790 | 4439 | cdict->dictContentSize, cdict->dictContentType, dtlm, |
80c26117 | 4440 | cctx->entropyWorkspace) |
0c8df5c9 SH |
4441 | : ZSTD_compress_insertDictionary( |
4442 | cctx->blockState.prevCBlock, &cctx->blockState.matchState, | |
5b0a452c | 4443 | &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, |
2ab484a5 | 4444 | dictContentType, dtlm, cctx->entropyWorkspace); |
5e5f2626 | 4445 | FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); |
80d6ccea | 4446 | assert(dictID <= UINT_MAX); |
16bd0fd4 | 4447 | cctx->dictID = (U32)dictID; |
94db4398 | 4448 | cctx->dictContentSize = dictContentSize; |
16bd0fd4 NT |
4449 | } |
4450 | return 0; | |
f3eca253 YC |
4451 | } |
4452 | ||
e28305fc | 4453 | size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, |
63b8c985 | 4454 | const void* dict, size_t dictSize, |
6873fec6 | 4455 | ZSTD_dictContentType_e dictContentType, |
295ab0db | 4456 | ZSTD_dictTableLoadMethod_e dtlm, |
e28305fc | 4457 | const ZSTD_CDict* cdict, |
77164547 | 4458 | const ZSTD_CCtx_params* params, |
63b8c985 SL |
4459 | unsigned long long pledgedSrcSize) |
4460 | { | |
77164547 | 4461 | DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); |
63b8c985 | 4462 | /* compression parameters verification and optimization */ |
5e5f2626 | 4463 | FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); |
e28305fc | 4464 | return ZSTD_compressBegin_internal(cctx, |
295ab0db | 4465 | dict, dictSize, dictContentType, dtlm, |
e28305fc | 4466 | cdict, |
63b8c985 SL |
4467 | params, pledgedSrcSize, |
4468 | ZSTDb_not_buffered); | |
4469 | } | |
083fcc82 | 4470 | |
27caf2af YC |
4471 | /*! ZSTD_compressBegin_advanced() : |
4472 | * @return : 0, or an error code */ | |
81e13ef7 | 4473 | size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, |
27caf2af | 4474 | const void* dict, size_t dictSize, |
52c04fe5 | 4475 | ZSTD_parameters params, unsigned long long pledgedSrcSize) |
27caf2af | 4476 | { |
7736549b NT |
4477 | ZSTD_CCtx_params cctxParams; |
4478 | ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); | |
e28305fc | 4479 | return ZSTD_compressBegin_advanced_internal(cctx, |
295ab0db | 4480 | dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, |
e28305fc | 4481 | NULL /*cdict*/, |
77164547 | 4482 | &cctxParams, pledgedSrcSize); |
27caf2af YC |
4483 | } |
4484 | ||
81e13ef7 | 4485 | size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) |
b923f650 | 4486 | { |
7736549b NT |
4487 | ZSTD_CCtx_params cctxParams; |
4488 | { | |
4489 | ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); | |
4490 | ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); | |
4491 | } | |
ededcfca | 4492 | DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); |
295ab0db | 4493 | return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, |
77164547 | 4494 | &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); |
1c8e1942 | 4495 | } |
88fcd291 | 4496 | |
b05c4828 | 4497 | size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) |
88fcd291 | 4498 | { |
b05c4828 | 4499 | return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); |
083fcc82 YC |
4500 | } |
4501 | ||
4502 | ||
62470b4b YC |
4503 | /*! ZSTD_writeEpilogue() : |
4504 | * Ends a frame. | |
88fcd291 | 4505 | * @return : nb of bytes written into dst (or an error code) */ |
62470b4b | 4506 | static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) |
2acb5d3d | 4507 | { |
c991cc18 YC |
4508 | BYTE* const ostart = (BYTE*)dst; |
4509 | BYTE* op = ostart; | |
6236eba9 | 4510 | size_t fhSize = 0; |
2acb5d3d | 4511 | |
6f7280fb | 4512 | DEBUGLOG(4, "ZSTD_writeEpilogue"); |
cafc3b1b | 4513 | RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); |
887e7da7 YC |
4514 | |
4515 | /* special case : empty frame */ | |
c991cc18 | 4516 | if (cctx->stage == ZSTDcs_init) { |
77164547 | 4517 | fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); |
5e5f2626 | 4518 | FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); |
6236eba9 YC |
4519 | dstCapacity -= fhSize; |
4520 | op += fhSize; | |
731ef16f | 4521 | cctx->stage = ZSTDcs_ongoing; |
ecd651bd | 4522 | } |
2acb5d3d | 4523 | |
c991cc18 YC |
4524 | if (cctx->stage != ZSTDcs_ending) { |
4525 | /* write one last empty block, make it the "last" block */ | |
4526 | U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; | |
5e5f2626 | 4527 | RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); |
c991cc18 YC |
4528 | MEM_writeLE32(op, cBlockHeader24); |
4529 | op += ZSTD_blockHeaderSize; | |
4530 | dstCapacity -= ZSTD_blockHeaderSize; | |
4531 | } | |
4532 | ||
1ad7c82e | 4533 | if (cctx->appliedParams.fParams.checksumFlag) { |
c991cc18 | 4534 | U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); |
5e5f2626 | 4535 | RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); |
ededcfca | 4536 | DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); |
c991cc18 YC |
4537 | MEM_writeLE32(op, checksum); |
4538 | op += 4; | |
f2a3b6e7 | 4539 | } |
2acb5d3d | 4540 | |
731ef16f | 4541 | cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ |
c991cc18 | 4542 | return op-ostart; |
2acb5d3d YC |
4543 | } |
4544 | ||
e59c9459 | 4545 | void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) |
54a4998a NT |
4546 | { |
4547 | #if ZSTD_TRACE | |
6cee3c2c | 4548 | if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { |
54a4998a | 4549 | int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; |
e59c9459 | 4550 | ZSTD_Trace trace; |
54a4998a NT |
4551 | ZSTD_memset(&trace, 0, sizeof(trace)); |
4552 | trace.version = ZSTD_VERSION_NUMBER; | |
4553 | trace.streaming = streaming; | |
4554 | trace.dictionaryID = cctx->dictID; | |
4555 | trace.dictionarySize = cctx->dictContentSize; | |
4556 | trace.uncompressedSize = cctx->consumedSrcSize; | |
4557 | trace.compressedSize = cctx->producedCSize + extraCSize; | |
4558 | trace.params = &cctx->appliedParams; | |
e59c9459 NT |
4559 | trace.cctx = cctx; |
4560 | ZSTD_trace_compress_end(cctx->traceCtx, &trace); | |
54a4998a | 4561 | } |
e59c9459 | 4562 | cctx->traceCtx = 0; |
54a4998a NT |
4563 | #else |
4564 | (void)cctx; | |
4565 | (void)extraCSize; | |
4566 | #endif | |
4567 | } | |
4568 | ||
62470b4b YC |
4569 | size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, |
4570 | void* dst, size_t dstCapacity, | |
4571 | const void* src, size_t srcSize) | |
4572 | { | |
4573 | size_t endResult; | |
009d604e YC |
4574 | size_t const cSize = ZSTD_compressContinue_internal(cctx, |
4575 | dst, dstCapacity, src, srcSize, | |
4576 | 1 /* frame mode */, 1 /* last chunk */); | |
5e5f2626 | 4577 | FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); |
62470b4b | 4578 | endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); |
5e5f2626 | 4579 | FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); |
3c3f59e6 NT |
4580 | assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); |
4581 | if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ | |
4582 | ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); | |
fbd5ab70 | 4583 | DEBUGLOG(4, "end of frame : controlling src size"); |
cafc3b1b FH |
4584 | RETURN_ERROR_IF( |
4585 | cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, | |
4586 | srcSize_wrong, | |
4587 | "error : pledgedSrcSize = %u, while realSrcSize = %u", | |
4588 | (unsigned)cctx->pledgedSrcSizePlusOne-1, | |
4589 | (unsigned)cctx->consumedSrcSize); | |
4590 | } | |
54a4998a | 4591 | ZSTD_CCtx_trace(cctx, endResult); |
62470b4b YC |
4592 | return cSize + endResult; |
4593 | } | |
4594 | ||
8537bfd8 | 4595 | size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, |
21588e37 YC |
4596 | void* dst, size_t dstCapacity, |
4597 | const void* src, size_t srcSize, | |
4598 | const void* dict,size_t dictSize, | |
4599 | ZSTD_parameters params) | |
4600 | { | |
05dffe43 | 4601 | DEBUGLOG(4, "ZSTD_compress_advanced"); |
5e5f2626 | 4602 | FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); |
c2183d7c | 4603 | ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); |
7736549b NT |
4604 | return ZSTD_compress_advanced_internal(cctx, |
4605 | dst, dstCapacity, | |
4606 | src, srcSize, | |
4607 | dict, dictSize, | |
c2183d7c | 4608 | &cctx->simpleApiParams); |
21588e37 YC |
4609 | } |
4610 | ||
63b8c985 | 4611 | /* Internal */ |
23fc0e41 | 4612 | size_t ZSTD_compress_advanced_internal( |
023b24e6 SL |
4613 | ZSTD_CCtx* cctx, |
4614 | void* dst, size_t dstCapacity, | |
4615 | const void* src, size_t srcSize, | |
4616 | const void* dict,size_t dictSize, | |
77164547 | 4617 | const ZSTD_CCtx_params* params) |
63b8c985 | 4618 | { |
ededcfca | 4619 | DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); |
501eb251 | 4620 | FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, |
8537bfd8 | 4621 | dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, |
5e5f2626 | 4622 | params, srcSize, ZSTDb_not_buffered) , ""); |
63b8c985 SL |
4623 | return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); |
4624 | } | |
4625 | ||
8537bfd8 YC |
4626 | size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, |
4627 | void* dst, size_t dstCapacity, | |
4628 | const void* src, size_t srcSize, | |
4629 | const void* dict, size_t dictSize, | |
4630 | int compressionLevel) | |
31683c0b | 4631 | { |
7736549b NT |
4632 | { |
4633 | ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); | |
4634 | assert(params.fParams.contentSizeFlag == 1); | |
c2183d7c | 4635 | ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); |
7736549b | 4636 | } |
4e051591 | 4637 | DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); |
c2183d7c | 4638 | return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); |
31683c0b YC |
4639 | } |
4640 | ||
8537bfd8 YC |
4641 | size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, |
4642 | void* dst, size_t dstCapacity, | |
4643 | const void* src, size_t srcSize, | |
4644 | int compressionLevel) | |
083fcc82 | 4645 | { |
ededcfca | 4646 | DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); |
8537bfd8 | 4647 | assert(cctx != NULL); |
a146ee04 | 4648 | return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); |
083fcc82 YC |
4649 | } |
4650 | ||
8537bfd8 YC |
4651 | size_t ZSTD_compress(void* dst, size_t dstCapacity, |
4652 | const void* src, size_t srcSize, | |
4653 | int compressionLevel) | |
f3eca253 | 4654 | { |
44fe9911 | 4655 | size_t result; |
6a1e526e NT |
4656 | #if ZSTD_COMPRESS_HEAPMODE |
4657 | ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
4658 | RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); | |
4659 | result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); | |
3cda5fae | 4660 | ZSTD_freeCCtx(cctx); |
6a1e526e | 4661 | #else |
5be2dd25 | 4662 | ZSTD_CCtx ctxBody; |
e3c42c73 | 4663 | ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); |
d1b26849 | 4664 | result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); |
e3c42c73 | 4665 | ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ |
6a1e526e | 4666 | #endif |
44fe9911 | 4667 | return result; |
f3eca253 | 4668 | } |
fdcad6d3 | 4669 | |
fd416f1e | 4670 | |
81e13ef7 YC |
4671 | /* ===== Dictionary API ===== */ |
4672 | ||
09ae03a5 | 4673 | /*! ZSTD_estimateCDictSize_advanced() : |
a1d6704d | 4674 | * Estimate amount of memory that will be needed to create a dictionary with following arguments */ |
c88fb926 SL |
4675 | size_t ZSTD_estimateCDictSize_advanced( |
4676 | size_t dictSize, ZSTD_compressionParameters cParams, | |
4677 | ZSTD_dictLoadMethod_e dictLoadMethod) | |
a1d6704d | 4678 | { |
ededcfca | 4679 | DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); |
19a0955e FH |
4680 | return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) |
4681 | + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) | |
4694423c NT |
4682 | /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small |
4683 | * in case we are using DDS with row-hash. */ | |
4684 | + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams), | |
4685 | /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) | |
19a0955e FH |
4686 | + (dictLoadMethod == ZSTD_dlm_byRef ? 0 |
4687 | : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); | |
a1d6704d YC |
4688 | } |
4689 | ||
09ae03a5 YC |
4690 | size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) |
4691 | { | |
d5c688e8 | 4692 | ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); |
ee657017 | 4693 | return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); |
09ae03a5 YC |
4694 | } |
4695 | ||
d7c6589d YC |
4696 | size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) |
4697 | { | |
4698 | if (cdict==NULL) return 0; /* support sizeof on NULL */ | |
ededcfca | 4699 | DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); |
eb6f69d9 FH |
4700 | /* cdict may be in the workspace */ |
4701 | return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) | |
4702 | + ZSTD_cwksp_sizeof(&cdict->workspace); | |
d7c6589d YC |
4703 | } |
4704 | ||
cdf7e822 YC |
4705 | static size_t ZSTD_initCDict_internal( |
4706 | ZSTD_CDict* cdict, | |
7bd1a290 | 4707 | const void* dictBuffer, size_t dictSize, |
c88fb926 | 4708 | ZSTD_dictLoadMethod_e dictLoadMethod, |
6873fec6 | 4709 | ZSTD_dictContentType_e dictContentType, |
9c628238 | 4710 | ZSTD_CCtx_params params) |
cdf7e822 | 4711 | { |
ededcfca | 4712 | DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); |
e8a44326 FH |
4713 | assert(!ZSTD_checkCParams(params.cParams)); |
4714 | cdict->matchState.cParams = params.cParams; | |
f1b428fd | 4715 | cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; |
c88fb926 | 4716 | if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { |
cdf7e822 YC |
4717 | cdict->dictContent = dictBuffer; |
4718 | } else { | |
077a2d7d | 4719 | void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); |
5e5f2626 | 4720 | RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); |
75d57436 | 4721 | cdict->dictContent = internalBuffer; |
c465f244 | 4722 | ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); |
cdf7e822 YC |
4723 | } |
4724 | cdict->dictContentSize = dictSize; | |
7083f790 | 4725 | cdict->dictContentType = dictContentType; |
cdf7e822 | 4726 | |
077a2d7d | 4727 | cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); |
786f2266 FH |
4728 | |
4729 | ||
9d967615 NT |
4730 | /* Reset the state to no dictionary */ |
4731 | ZSTD_reset_compressedBlockState(&cdict->cBlockState); | |
786f2266 FH |
4732 | FORWARD_IF_ERROR(ZSTD_reset_matchState( |
4733 | &cdict->matchState, | |
4734 | &cdict->workspace, | |
e8a44326 | 4735 | ¶ms.cParams, |
4694423c | 4736 | params.useRowMatchFinder, |
5b10bb5e | 4737 | ZSTDcrp_makeClean, |
0492b9a9 | 4738 | ZSTDirp_reset, |
5e5f2626 | 4739 | ZSTD_resetTarget_CDict), ""); |
9d967615 | 4740 | /* (Maybe) load the dictionary |
60205fec | 4741 | * Skips loading the dictionary if it is < 8 bytes. |
9d967615 | 4742 | */ |
9c628238 | 4743 | { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; |
16bd0fd4 | 4744 | params.fParams.contentSizeFlag = 1; |
6f4d0778 | 4745 | { size_t const dictID = ZSTD_compress_insertDictionary( |
5b0a452c | 4746 | &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, |
bc020eec | 4747 | ¶ms, cdict->dictContent, cdict->dictContentSize, |
786f2266 | 4748 | dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); |
5e5f2626 | 4749 | FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); |
16bd0fd4 NT |
4750 | assert(dictID <= (size_t)(U32)-1); |
4751 | cdict->dictID = (U32)dictID; | |
4752 | } | |
cdf7e822 YC |
4753 | } |
4754 | ||
4755 | return 0; | |
4756 | } | |
4757 | ||
9c628238 | 4758 | static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, |
c88fb926 | 4759 | ZSTD_dictLoadMethod_e dictLoadMethod, |
4694423c NT |
4760 | ZSTD_compressionParameters cParams, |
4761 | ZSTD_useRowMatchFinderMode_e useRowMatchFinder, | |
4762 | U32 enableDedicatedDictSearch, | |
4763 | ZSTD_customMem customMem) | |
81e13ef7 | 4764 | { |
9261476b | 4765 | if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; |
81e13ef7 | 4766 | |
7a2416a8 | 4767 | { size_t const workspaceSize = |
19a0955e FH |
4768 | ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + |
4769 | ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + | |
4694423c | 4770 | ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + |
7a2416a8 | 4771 | (dictLoadMethod == ZSTD_dlm_byRef ? 0 |
19a0955e | 4772 | : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); |
a686d306 | 4773 | void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); |
077a2d7d | 4774 | ZSTD_cwksp ws; |
7a2416a8 | 4775 | ZSTD_CDict* cdict; |
81e13ef7 | 4776 | |
7a2416a8 | 4777 | if (!workspace) { |
a686d306 | 4778 | ZSTD_customFree(workspace, customMem); |
81e13ef7 YC |
4779 | return NULL; |
4780 | } | |
7a2416a8 | 4781 | |
9dab03db | 4782 | ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); |
7a2416a8 | 4783 | |
077a2d7d | 4784 | cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); |
7a2416a8 | 4785 | assert(cdict != NULL); |
8549ae9f | 4786 | ZSTD_cwksp_move(&cdict->workspace, &ws); |
16bd0fd4 | 4787 | cdict->customMem = customMem; |
c62eb059 | 4788 | cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ |
4694423c | 4789 | cdict->useRowMatchFinder = useRowMatchFinder; |
9c628238 BS |
4790 | return cdict; |
4791 | } | |
4792 | } | |
4793 | ||
4794 | ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, | |
4795 | ZSTD_dictLoadMethod_e dictLoadMethod, | |
4796 | ZSTD_dictContentType_e dictContentType, | |
eee51a66 FH |
4797 | ZSTD_compressionParameters cParams, |
4798 | ZSTD_customMem customMem) | |
9c628238 | 4799 | { |
eee51a66 | 4800 | ZSTD_CCtx_params cctxParams; |
88fac5d5 | 4801 | ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); |
eee51a66 FH |
4802 | ZSTD_CCtxParams_init(&cctxParams, 0); |
4803 | cctxParams.cParams = cParams; | |
4804 | cctxParams.customMem = customMem; | |
4805 | return ZSTD_createCDict_advanced2( | |
4806 | dictBuffer, dictSize, | |
4807 | dictLoadMethod, dictContentType, | |
4808 | &cctxParams, customMem); | |
81e13ef7 YC |
4809 | } |
4810 | ||
bc6521a6 FH |
4811 | ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2( |
4812 | const void* dict, size_t dictSize, | |
4813 | ZSTD_dictLoadMethod_e dictLoadMethod, | |
4814 | ZSTD_dictContentType_e dictContentType, | |
eee51a66 | 4815 | const ZSTD_CCtx_params* originalCctxParams, |
bc6521a6 | 4816 | ZSTD_customMem customMem) |
75b63600 | 4817 | { |
eee51a66 FH |
4818 | ZSTD_CCtx_params cctxParams = *originalCctxParams; |
4819 | ZSTD_compressionParameters cParams; | |
4820 | ZSTD_CDict* cdict; | |
9c628238 | 4821 | |
eee51a66 FH |
4822 | DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); |
4823 | if (!customMem.customAlloc ^ !customMem.customFree) return NULL; | |
9c628238 | 4824 | |
eee51a66 FH |
4825 | if (cctxParams.enableDedicatedDictSearch) { |
4826 | cParams = ZSTD_dedicatedDictSearch_getCParams( | |
fadaab8c | 4827 | cctxParams.compressionLevel, dictSize); |
eee51a66 FH |
4828 | ZSTD_overrideCParams(&cParams, &cctxParams.cParams); |
4829 | } else { | |
4830 | cParams = ZSTD_getCParamsFromCCtxParams( | |
d5c688e8 | 4831 | &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); |
b30f71be | 4832 | } |
1f57c2ed | 4833 | |
eee51a66 FH |
4834 | if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { |
4835 | /* Fall back to non-DDSS params */ | |
4836 | cctxParams.enableDedicatedDictSearch = 0; | |
4837 | cParams = ZSTD_getCParamsFromCCtxParams( | |
d5c688e8 | 4838 | &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); |
81e13ef7 | 4839 | } |
9c628238 | 4840 | |
4694423c | 4841 | DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch); |
e8a44326 | 4842 | cctxParams.cParams = cParams; |
4694423c | 4843 | cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); |
81e13ef7 | 4844 | |
eee51a66 | 4845 | cdict = ZSTD_createCDict_advanced_internal(dictSize, |
e8a44326 | 4846 | dictLoadMethod, cctxParams.cParams, |
4694423c | 4847 | cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, |
eee51a66 | 4848 | customMem); |
9c628238 | 4849 | |
eee51a66 FH |
4850 | if (ZSTD_isError( ZSTD_initCDict_internal(cdict, |
4851 | dict, dictSize, | |
4852 | dictLoadMethod, dictContentType, | |
e8a44326 | 4853 | cctxParams) )) { |
eee51a66 FH |
4854 | ZSTD_freeCDict(cdict); |
4855 | return NULL; | |
75b63600 | 4856 | } |
eee51a66 FH |
4857 | |
4858 | return cdict; | |
75b63600 BS |
4859 | } |
4860 | ||
81e13ef7 YC |
4861 | ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) |
4862 | { | |
d5c688e8 | 4863 | ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); |
17b56f93 | 4864 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, |
23dac23a SH |
4865 | ZSTD_dlm_byCopy, ZSTD_dct_auto, |
4866 | cParams, ZSTD_defaultCMem); | |
8cb21744 | 4867 | if (cdict) |
17b56f93 | 4868 | cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; |
0c8df5c9 | 4869 | return cdict; |
1f57c2ed YC |
4870 | } |
4871 | ||
4872 | ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) | |
4873 | { | |
d5c688e8 | 4874 | ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); |
17b56f93 | 4875 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, |
6873fec6 | 4876 | ZSTD_dlm_byRef, ZSTD_dct_auto, |
7bd1a290 | 4877 | cParams, ZSTD_defaultCMem); |
64bd68e4 | 4878 | if (cdict) |
17b56f93 | 4879 | cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; |
64bd68e4 | 4880 | return cdict; |
81e13ef7 YC |
4881 | } |
4882 | ||
4883 | size_t ZSTD_freeCDict(ZSTD_CDict* cdict) | |
4884 | { | |
23b6e05d | 4885 | if (cdict==NULL) return 0; /* support free on NULL */ |
16bd0fd4 | 4886 | { ZSTD_customMem const cMem = cdict->customMem; |
ef0b5707 FH |
4887 | int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); |
4888 | ZSTD_cwksp_free(&cdict->workspace, cMem); | |
4889 | if (!cdictInWorkspace) { | |
a686d306 | 4890 | ZSTD_customFree(cdict, cMem); |
7a2416a8 | 4891 | } |
23b6e05d YC |
4892 | return 0; |
4893 | } | |
81e13ef7 YC |
4894 | } |
4895 | ||
cdf7e822 YC |
4896 | /*! ZSTD_initStaticCDict_advanced() : |
4897 | * Generate a digested dictionary in provided memory area. | |
4898 | * workspace: The memory area to emplace the dictionary into. | |
4899 | * Provided pointer must 8-bytes aligned. | |
4900 | * It must outlive dictionary usage. | |
4901 | * workspaceSize: Use ZSTD_estimateCDictSize() | |
4902 | * to determine how large workspace must be. | |
4903 | * cParams : use ZSTD_getCParams() to transform a compression level | |
4904 | * into its relevants cParams. | |
4905 | * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) | |
4906 | * Note : there is no corresponding "free" function. | |
4907 | * Since workspace was allocated externally, it must be freed externally. | |
4908 | */ | |
f3b8f90b YC |
4909 | const ZSTD_CDict* ZSTD_initStaticCDict( |
4910 | void* workspace, size_t workspaceSize, | |
7bd1a290 | 4911 | const void* dict, size_t dictSize, |
c88fb926 | 4912 | ZSTD_dictLoadMethod_e dictLoadMethod, |
6873fec6 | 4913 | ZSTD_dictContentType_e dictContentType, |
cdf7e822 YC |
4914 | ZSTD_compressionParameters cParams) |
4915 | { | |
4694423c NT |
4916 | ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams); |
4917 | /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ | |
4918 | size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); | |
19a0955e FH |
4919 | size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) |
4920 | + (dictLoadMethod == ZSTD_dlm_byRef ? 0 | |
4921 | : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) | |
4922 | + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) | |
4923 | + matchStateSize; | |
786f2266 | 4924 | ZSTD_CDict* cdict; |
def62e2d | 4925 | ZSTD_CCtx_params params; |
786f2266 | 4926 | |
cdf7e822 | 4927 | if ((size_t)workspace & 7) return NULL; /* 8-aligned */ |
786f2266 FH |
4928 | |
4929 | { | |
077a2d7d | 4930 | ZSTD_cwksp ws; |
9dab03db | 4931 | ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); |
077a2d7d | 4932 | cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); |
786f2266 | 4933 | if (cdict == NULL) return NULL; |
8549ae9f | 4934 | ZSTD_cwksp_move(&cdict->workspace, &ws); |
786f2266 FH |
4935 | } |
4936 | ||
3c1e3f8e | 4937 | DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", |
ededcfca | 4938 | (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); |
cdf7e822 YC |
4939 | if (workspaceSize < neededSize) return NULL; |
4940 | ||
8930c6e5 | 4941 | ZSTD_CCtxParams_init(¶ms, 0); |
e8a44326 | 4942 | params.cParams = cParams; |
4694423c NT |
4943 | params.useRowMatchFinder = useRowMatchFinder; |
4944 | cdict->useRowMatchFinder = useRowMatchFinder; | |
9c628238 | 4945 | |
cdf7e822 | 4946 | if (ZSTD_isError( ZSTD_initCDict_internal(cdict, |
7bd1a290 | 4947 | dict, dictSize, |
75d57436 | 4948 | dictLoadMethod, dictContentType, |
e8a44326 | 4949 | params) )) |
cdf7e822 YC |
4950 | return NULL; |
4951 | ||
4952 | return cdict; | |
4953 | } | |
4954 | ||
b8686532 YC |
4955 | ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) |
4956 | { | |
cb57c107 | 4957 | assert(cdict != NULL); |
4e3ecee9 | 4958 | return cdict->matchState.cParams; |
9516234e YC |
4959 | } |
4960 | ||
eac309c7 LP |
4961 | /*! ZSTD_getDictID_fromCDict() : |
4962 | * Provides the dictID of the dictionary loaded into `cdict`. | |
4963 | * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. | |
4964 | * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ | |
4965 | unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) | |
4966 | { | |
4967 | if (cdict==NULL) return 0; | |
4968 | return cdict->dictID; | |
4969 | } | |
4970 | ||
4971 | ||
715b9aa1 | 4972 | /* ZSTD_compressBegin_usingCDict_advanced() : |
4f818182 | 4973 | * cdict must be != NULL */ |
715b9aa1 | 4974 | size_t ZSTD_compressBegin_usingCDict_advanced( |
4f818182 YC |
4975 | ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, |
4976 | ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) | |
4cb21293 | 4977 | { |
7736549b | 4978 | ZSTD_CCtx_params cctxParams; |
05dffe43 | 4979 | DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); |
5e5f2626 | 4980 | RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); |
7736549b NT |
4981 | /* Initialize the cctxParams from the cdict */ |
4982 | { | |
4983 | ZSTD_parameters params; | |
4984 | params.fParams = fParams; | |
c2e1e54f SH |
4985 | params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF |
4986 | || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER | |
b9ede1c8 | 4987 | || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN |
7736549b | 4988 | || cdict->compressionLevel == 0 ) ? |
0c8df5c9 SH |
4989 | ZSTD_getCParamsFromCDict(cdict) |
4990 | : ZSTD_getCParams(cdict->compressionLevel, | |
23dac23a SH |
4991 | pledgedSrcSize, |
4992 | cdict->dictContentSize); | |
7736549b | 4993 | ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); |
2db72492 | 4994 | } |
7736549b NT |
4995 | /* Increase window log to fit the entire dictionary and source if the |
4996 | * source size is known. Limit the increase to 19, which is the | |
4997 | * window log for compression level 1 with the largest source size. | |
4998 | */ | |
4999 | if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { | |
5000 | U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); | |
5001 | U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; | |
5002 | cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog); | |
5003 | } | |
5004 | return ZSTD_compressBegin_internal(cctx, | |
5005 | NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, | |
5006 | cdict, | |
5007 | &cctxParams, pledgedSrcSize, | |
5008 | ZSTDb_not_buffered); | |
4cb21293 YC |
5009 | } |
5010 | ||
4f818182 YC |
5011 | /* ZSTD_compressBegin_usingCDict() : |
5012 | * pledgedSrcSize=0 means "unknown" | |
5013 | * if pledgedSrcSize>0, it will enable contentSizeFlag */ | |
768df129 | 5014 | size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) |
4f818182 | 5015 | { |
768df129 | 5016 | ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; |
3c1e3f8e | 5017 | DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); |
ad4524d6 | 5018 | return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); |
4f818182 YC |
5019 | } |
5020 | ||
f4bd857d YC |
5021 | size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, |
5022 | void* dst, size_t dstCapacity, | |
5023 | const void* src, size_t srcSize, | |
5024 | const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) | |
5025 | { | |
5e5f2626 | 5026 | FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ |
f4bd857d | 5027 | return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); |
4f818182 YC |
5028 | } |
5029 | ||
0763905f | 5030 | /*! ZSTD_compress_usingCDict() : |
4f818182 YC |
5031 | * Compression using a digested Dictionary. |
5032 | * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. | |
5033 | * Note that compression parameters are decided at CDict creation time | |
5034 | * while frame parameters are hardcoded */ | |
4cb21293 YC |
5035 | size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, |
5036 | void* dst, size_t dstCapacity, | |
5037 | const void* src, size_t srcSize, | |
5038 | const ZSTD_CDict* cdict) | |
81e13ef7 | 5039 | { |
4f818182 | 5040 | ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; |
f4bd857d | 5041 | return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); |
81e13ef7 YC |
5042 | } |
5043 | ||
5044 | ||
5045 | ||
104e5b07 YC |
5046 | /* ****************************************************************** |
5047 | * Streaming | |
5048 | ********************************************************************/ | |
5a0c8e24 | 5049 | |
5a0c8e24 YC |
5050 | ZSTD_CStream* ZSTD_createCStream(void) |
5051 | { | |
3aa2b27a | 5052 | DEBUGLOG(3, "ZSTD_createCStream"); |
ae728a43 | 5053 | return ZSTD_createCStream_advanced(ZSTD_defaultCMem); |
5a0c8e24 YC |
5054 | } |
5055 | ||
dde10b23 YC |
5056 | ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) |
5057 | { | |
5058 | return ZSTD_initStaticCCtx(workspace, workspaceSize); | |
5059 | } | |
5060 | ||
5a0c8e24 | 5061 | ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) |
ae728a43 | 5062 | { /* CStream and CCtx are now same object */ |
6fb2f241 | 5063 | return ZSTD_createCCtx_advanced(customMem); |
5a0c8e24 YC |
5064 | } |
5065 | ||
5066 | size_t ZSTD_freeCStream(ZSTD_CStream* zcs) | |
5067 | { | |
78553665 | 5068 | return ZSTD_freeCCtx(zcs); /* same object */ |
5a0c8e24 YC |
5069 | } |
5070 | ||
5a0c8e24 YC |
5071 | |
5072 | ||
104e5b07 YC |
5073 | /*====== Initialization ======*/ |
5074 | ||
fa3671ea | 5075 | size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } |
5a0c8e24 | 5076 | |
c17e020c YC |
5077 | size_t ZSTD_CStreamOutSize(void) |
5078 | { | |
fa3671ea | 5079 | return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; |
c17e020c | 5080 | } |
5a0c8e24 | 5081 | |
d5c688e8 NT |
5082 | static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) |
5083 | { | |
5084 | if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) | |
5085 | return ZSTD_cpm_attachDict; | |
5086 | else | |
5087 | return ZSTD_cpm_noAttachDict; | |
5088 | } | |
5089 | ||
213ef3b5 YC |
5090 | /* ZSTD_resetCStream(): |
5091 | * pledgedSrcSize == 0 means "unknown" */ | |
e55da9e9 | 5092 | size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) |
2db72492 | 5093 | { |
e55da9e9 NT |
5094 | /* temporary : 0 interpreted as "unknown" during transition period. |
5095 | * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. | |
5096 | * 0 will be interpreted as "empty" in the future. | |
5097 | */ | |
5098 | U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; | |
ededcfca | 5099 | DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); |
5e5f2626 FH |
5100 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5101 | FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); | |
e55da9e9 | 5102 | return 0; |
2db72492 SP |
5103 | } |
5104 | ||
204b6b7e | 5105 | /*! ZSTD_initCStream_internal() : |
e28305fc | 5106 | * Note : for lib/compress only. Used by zstdmt_compress.c. |
204b6b7e YC |
5107 | * Assumption 1 : params are valid |
5108 | * Assumption 2 : either dict, or cdict, is defined, not both */ | |
8c910d20 YC |
5109 | size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, |
5110 | const void* dict, size_t dictSize, const ZSTD_CDict* cdict, | |
c25283cf FH |
5111 | const ZSTD_CCtx_params* params, |
5112 | unsigned long long pledgedSrcSize) | |
5a0c8e24 | 5113 | { |
db1668a4 | 5114 | DEBUGLOG(4, "ZSTD_initCStream_internal"); |
5e5f2626 FH |
5115 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5116 | FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); | |
77164547 VN |
5117 | assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); |
5118 | zcs->requestedParams = *params; | |
8c910d20 | 5119 | assert(!((dict) && (cdict))); /* either dict or cdict, not both */ |
e55da9e9 | 5120 | if (dict) { |
5e5f2626 | 5121 | FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); |
8c910d20 | 5122 | } else { |
e55da9e9 | 5123 | /* Dictionary is cleared if !cdict */ |
5e5f2626 | 5124 | FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); |
5a0c8e24 | 5125 | } |
e55da9e9 | 5126 | return 0; |
e88034fe YC |
5127 | } |
5128 | ||
77bf59ef YC |
5129 | /* ZSTD_initCStream_usingCDict_advanced() : |
5130 | * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ | |
8c910d20 YC |
5131 | size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, |
5132 | const ZSTD_CDict* cdict, | |
5133 | ZSTD_frameParameters fParams, | |
5134 | unsigned long long pledgedSrcSize) | |
15768cab YC |
5135 | { |
5136 | DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); | |
5e5f2626 FH |
5137 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5138 | FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); | |
e55da9e9 | 5139 | zcs->requestedParams.fParams = fParams; |
5e5f2626 | 5140 | FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); |
e55da9e9 | 5141 | return 0; |
e88034fe YC |
5142 | } |
5143 | ||
77bf59ef YC |
5144 | /* note : cdict must outlive compression session */ |
5145 | size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) | |
5146 | { | |
15768cab | 5147 | DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); |
5e5f2626 FH |
5148 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5149 | FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); | |
e55da9e9 | 5150 | return 0; |
5a0c8e24 YC |
5151 | } |
5152 | ||
6f4d0778 | 5153 | |
213ef3b5 | 5154 | /* ZSTD_initCStream_advanced() : |
6f4d0778 | 5155 | * pledgedSrcSize must be exact. |
dfc697e9 | 5156 | * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. |
6323966e | 5157 | * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ |
4b987ad8 YC |
5158 | size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, |
5159 | const void* dict, size_t dictSize, | |
e55da9e9 | 5160 | ZSTD_parameters params, unsigned long long pss) |
9516234e | 5161 | { |
e55da9e9 NT |
5162 | /* for compatibility with older programs relying on this behavior. |
5163 | * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. | |
5164 | * This line will be removed in the future. | |
5165 | */ | |
5166 | U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; | |
5167 | DEBUGLOG(4, "ZSTD_initCStream_advanced"); | |
5e5f2626 FH |
5168 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5169 | FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); | |
5170 | FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); | |
7736549b | 5171 | ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); |
5e5f2626 | 5172 | FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); |
e55da9e9 | 5173 | return 0; |
9516234e YC |
5174 | } |
5175 | ||
5a0c8e24 YC |
5176 | size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) |
5177 | { | |
e55da9e9 | 5178 | DEBUGLOG(4, "ZSTD_initCStream_usingDict"); |
5e5f2626 FH |
5179 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5180 | FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); | |
5181 | FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); | |
e55da9e9 | 5182 | return 0; |
5a0c8e24 YC |
5183 | } |
5184 | ||
c029ee1f | 5185 | size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) |
e795c8a5 | 5186 | { |
e55da9e9 NT |
5187 | /* temporary : 0 interpreted as "unknown" during transition period. |
5188 | * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. | |
5189 | * 0 will be interpreted as "empty" in the future. | |
5190 | */ | |
5191 | U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; | |
5192 | DEBUGLOG(4, "ZSTD_initCStream_srcSize"); | |
5e5f2626 FH |
5193 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5194 | FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); | |
5195 | FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); | |
5196 | FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); | |
e55da9e9 | 5197 | return 0; |
e795c8a5 YC |
5198 | } |
5199 | ||
5a0c8e24 YC |
5200 | size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) |
5201 | { | |
3aa2b27a | 5202 | DEBUGLOG(4, "ZSTD_initCStream"); |
5e5f2626 FH |
5203 | FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); |
5204 | FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); | |
5205 | FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); | |
e55da9e9 | 5206 | return 0; |
cb327632 | 5207 | } |
5a0c8e24 | 5208 | |
104e5b07 | 5209 | /*====== Compression ======*/ |
5a0c8e24 | 5210 | |
6ced8f7c YC |
5211 | static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) |
5212 | { | |
5213 | size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; | |
5214 | if (hintInSize==0) hintInSize = cctx->blockSize; | |
5215 | return hintInSize; | |
5216 | } | |
5217 | ||
a3d9926c | 5218 | /** ZSTD_compressStream_generic(): |
6ced8f7c | 5219 | * internal function for all *compressStream*() variants |
5188749e | 5220 | * non-static, because can be called from zstdmt_compress.c |
a3d9926c | 5221 | * @return : hint size for next input */ |
e55da9e9 NT |
5222 | static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, |
5223 | ZSTD_outBuffer* output, | |
5224 | ZSTD_inBuffer* input, | |
5225 | ZSTD_EndDirective const flushMode) | |
5a0c8e24 | 5226 | { |
01b1549f | 5227 | const char* const istart = (const char*)input->src; |
659e9f05 NT |
5228 | const char* const iend = input->size != 0 ? istart + input->size : istart; |
5229 | const char* ip = input->pos != 0 ? istart + input->pos : istart; | |
01b1549f | 5230 | char* const ostart = (char*)output->dst; |
659e9f05 NT |
5231 | char* const oend = output->size != 0 ? ostart + output->size : ostart; |
5232 | char* op = output->pos != 0 ? ostart + output->pos : ostart; | |
5a0c8e24 | 5233 | U32 someMoreWork = 1; |
01b1549f | 5234 | |
58e8d793 | 5235 | /* check expectations */ |
ededcfca | 5236 | DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); |
d4e021fe NT |
5237 | if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { |
5238 | assert(zcs->inBuff != NULL); | |
5239 | assert(zcs->inBuffSize > 0); | |
5240 | } | |
fcf81cee NT |
5241 | if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { |
5242 | assert(zcs->outBuff != NULL); | |
5243 | assert(zcs->outBuffSize > 0); | |
5244 | } | |
58e8d793 YC |
5245 | assert(output->pos <= output->size); |
5246 | assert(input->pos <= input->size); | |
c91a0855 | 5247 | assert((U32)flushMode <= (U32)ZSTD_e_end); |
009d604e | 5248 | |
5a0c8e24 | 5249 | while (someMoreWork) { |
0be6fd34 | 5250 | switch(zcs->streamStage) |
5a0c8e24 | 5251 | { |
1ad7c82e | 5252 | case zcss_init: |
cafc3b1b | 5253 | RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); |
5a0c8e24 YC |
5254 | |
5255 | case zcss_load: | |
a3d9926c | 5256 | if ( (flushMode == ZSTD_e_end) |
987cb4ca NT |
5257 | && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ |
5258 | || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ | |
a3d9926c YC |
5259 | && (zcs->inBuffPos == 0) ) { |
5260 | /* shortcut to compression pass directly into output buffer */ | |
5261 | size_t const cSize = ZSTD_compressEnd(zcs, | |
5262 | op, oend-op, ip, iend-ip); | |
ededcfca | 5263 | DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); |
5e5f2626 | 5264 | FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); |
a3d9926c YC |
5265 | ip = iend; |
5266 | op += cSize; | |
5267 | zcs->frameEnded = 1; | |
5c686391 | 5268 | ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); |
a3d9926c | 5269 | someMoreWork = 0; break; |
2084b041 | 5270 | } |
24f72789 NT |
5271 | /* complete loading into inBuffer in buffered mode */ |
5272 | if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { | |
5273 | size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; | |
06589fe5 YC |
5274 | size_t const loaded = ZSTD_limitCopy( |
5275 | zcs->inBuff + zcs->inBuffPos, toLoad, | |
5276 | ip, iend-ip); | |
5a0c8e24 | 5277 | zcs->inBuffPos += loaded; |
659e9f05 NT |
5278 | if (loaded != 0) |
5279 | ip += loaded; | |
009d604e YC |
5280 | if ( (flushMode == ZSTD_e_continue) |
5281 | && (zcs->inBuffPos < zcs->inBuffTarget) ) { | |
5282 | /* not enough input to fill full block : stop here */ | |
5283 | someMoreWork = 0; break; | |
5284 | } | |
5285 | if ( (flushMode == ZSTD_e_flush) | |
5286 | && (zcs->inBuffPos == zcs->inToCompress) ) { | |
5287 | /* empty */ | |
5288 | someMoreWork = 0; break; | |
559ee82e YC |
5289 | } |
5290 | } | |
5a0c8e24 | 5291 | /* compress current block (note : this stage cannot be stopped in the middle) */ |
009d604e | 5292 | DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); |
24f72789 NT |
5293 | { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); |
5294 | void* cDst; | |
5a0c8e24 | 5295 | size_t cSize; |
5a0c8e24 | 5296 | size_t oSize = oend-op; |
24f72789 NT |
5297 | size_t const iSize = inputBuffered |
5298 | ? zcs->inBuffPos - zcs->inToCompress | |
5299 | : MIN((size_t)(iend - ip), zcs->blockSize); | |
6d5dc93d | 5300 | if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) |
559ee82e | 5301 | cDst = op; /* compress into output buffer, to skip flush stage */ |
5a0c8e24 YC |
5302 | else |
5303 | cDst = zcs->outBuff, oSize = zcs->outBuffSize; | |
24f72789 NT |
5304 | if (inputBuffered) { |
5305 | unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); | |
5306 | cSize = lastBlock ? | |
5307 | ZSTD_compressEnd(zcs, cDst, oSize, | |
5308 | zcs->inBuff + zcs->inToCompress, iSize) : | |
5309 | ZSTD_compressContinue(zcs, cDst, oSize, | |
5310 | zcs->inBuff + zcs->inToCompress, iSize); | |
5311 | FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); | |
5312 | zcs->frameEnded = lastBlock; | |
5313 | /* prepare next block */ | |
5314 | zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; | |
5315 | if (zcs->inBuffTarget > zcs->inBuffSize) | |
5316 | zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; | |
5317 | DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", | |
5318 | (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); | |
5319 | if (!lastBlock) | |
5320 | assert(zcs->inBuffTarget <= zcs->inBuffSize); | |
5321 | zcs->inToCompress = zcs->inBuffPos; | |
5322 | } else { | |
5323 | unsigned const lastBlock = (ip + iSize == iend); | |
5324 | assert(flushMode == ZSTD_e_end /* Already validated */); | |
5325 | cSize = lastBlock ? | |
5326 | ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : | |
5327 | ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); | |
5328 | /* Consume the input prior to error checking to mirror buffered mode. */ | |
5329 | if (iSize > 0) | |
5330 | ip += iSize; | |
5331 | FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); | |
5332 | zcs->frameEnded = lastBlock; | |
5333 | if (lastBlock) | |
5334 | assert(ip == iend); | |
5335 | } | |
009d604e YC |
5336 | if (cDst == op) { /* no need to flush */ |
5337 | op += cSize; | |
5338 | if (zcs->frameEnded) { | |
559ee82e | 5339 | DEBUGLOG(5, "Frame completed directly in outBuffer"); |
009d604e | 5340 | someMoreWork = 0; |
5c686391 | 5341 | ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); |
009d604e YC |
5342 | } |
5343 | break; | |
5344 | } | |
5a0c8e24 YC |
5345 | zcs->outBuffContentSize = cSize; |
5346 | zcs->outBuffFlushedSize = 0; | |
009d604e | 5347 | zcs->streamStage = zcss_flush; /* pass-through to flush stage */ |
5a0c8e24 | 5348 | } |
7cd7a756 | 5349 | /* fall-through */ |
5a0c8e24 | 5350 | case zcss_flush: |
009d604e | 5351 | DEBUGLOG(5, "flush stage"); |
6d5dc93d | 5352 | assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); |
5a0c8e24 | 5353 | { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; |
621adde3 | 5354 | size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), |
01b1549f | 5355 | zcs->outBuff + zcs->outBuffFlushedSize, toFlush); |
2cb9774f | 5356 | DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", |
ededcfca | 5357 | (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); |
659e9f05 NT |
5358 | if (flushed) |
5359 | op += flushed; | |
5a0c8e24 | 5360 | zcs->outBuffFlushedSize += flushed; |
01b1549f | 5361 | if (toFlush!=flushed) { |
2cb9774f YC |
5362 | /* flush not fully completed, presumably because dst is too small */ |
5363 | assert(op==oend); | |
01b1549f YC |
5364 | someMoreWork = 0; |
5365 | break; | |
5366 | } | |
5a0c8e24 | 5367 | zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; |
009d604e | 5368 | if (zcs->frameEnded) { |
559ee82e | 5369 | DEBUGLOG(5, "Frame completed on flush"); |
009d604e | 5370 | someMoreWork = 0; |
5c686391 | 5371 | ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); |
009d604e YC |
5372 | break; |
5373 | } | |
0be6fd34 | 5374 | zcs->streamStage = zcss_load; |
5a0c8e24 YC |
5375 | break; |
5376 | } | |
5377 | ||
cd2892fd YC |
5378 | default: /* impossible */ |
5379 | assert(0); | |
5a0c8e24 YC |
5380 | } |
5381 | } | |
5382 | ||
01b1549f YC |
5383 | input->pos = ip - istart; |
5384 | output->pos = op - ostart; | |
5a0c8e24 | 5385 | if (zcs->frameEnded) return 0; |
6ced8f7c YC |
5386 | return ZSTD_nextInputSizeHint(zcs); |
5387 | } | |
5388 | ||
5389 | static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) | |
5390 | { | |
5391 | #ifdef ZSTD_MULTITHREAD | |
5392 | if (cctx->appliedParams.nbWorkers >= 1) { | |
5393 | assert(cctx->mtctx != NULL); | |
5394 | return ZSTDMT_nextInputSizeHint(cctx->mtctx); | |
5a0c8e24 | 5395 | } |
6ced8f7c YC |
5396 | #endif |
5397 | return ZSTD_nextInputSizeHint(cctx); | |
5398 | ||
5a0c8e24 YC |
5399 | } |
5400 | ||
53e17fbd YC |
5401 | size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) |
5402 | { | |
5e5f2626 | 5403 | FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); |
6ced8f7c | 5404 | return ZSTD_nextInputSizeHint_MTorST(zcs); |
5a0c8e24 YC |
5405 | } |
5406 | ||
c74be3f6 NT |
5407 | /* After a compression call set the expected input/output buffer. |
5408 | * This is validated at the start of the next compression call. | |
5409 | */ | |
5410 | static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input) | |
5411 | { | |
5412 | if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { | |
5413 | cctx->expectedInBuffer = *input; | |
5414 | } | |
5415 | if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { | |
5416 | cctx->expectedOutBufferSize = output->size - output->pos; | |
5417 | } | |
5418 | } | |
5419 | ||
5420 | /* Validate that the input/output buffers match the expectations set by | |
5421 | * ZSTD_setBufferExpectations. | |
5422 | */ | |
5423 | static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, | |
5424 | ZSTD_outBuffer const* output, | |
5425 | ZSTD_inBuffer const* input, | |
5426 | ZSTD_EndDirective endOp) | |
5427 | { | |
5428 | if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { | |
5429 | ZSTD_inBuffer const expect = cctx->expectedInBuffer; | |
5430 | if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) | |
5431 | RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); | |
5432 | if (endOp != ZSTD_e_end) | |
5433 | RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); | |
5434 | } | |
5435 | if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { | |
5436 | size_t const outBufferSize = output->size - output->pos; | |
5437 | if (cctx->expectedOutBufferSize != outBufferSize) | |
5438 | RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!"); | |
5439 | } | |
5440 | return 0; | |
5441 | } | |
f35e2de6 | 5442 | |
48f67da8 | 5443 | static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, |
5444 | ZSTD_EndDirective endOp, | |
5445 | size_t inSize) { | |
5446 | ZSTD_CCtx_params params = cctx->requestedParams; | |
5447 | ZSTD_prefixDict const prefixDict = cctx->prefixDict; | |
5448 | FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ | |
5449 | ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ | |
5450 | assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ | |
f71aabb5 | 5451 | if (cctx->cdict && !cctx->localDict.cdict) { |
5452 | /* Let the cdict's compression level take priority over the requested params. | |
5453 | * But do not take the cdict's compression level if the "cdict" is actually a localDict | |
5454 | * generated from ZSTD_initLocalDict(). | |
5455 | */ | |
5456 | params.compressionLevel = cctx->cdict->compressionLevel; | |
5457 | } | |
48f67da8 | 5458 | DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); |
5459 | if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */ | |
5460 | { | |
5461 | size_t const dictSize = prefixDict.dict | |
5462 | ? prefixDict.dictSize | |
5463 | : (cctx->cdict ? cctx->cdict->dictContentSize : 0); | |
5464 | ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); | |
5465 | params.cParams = ZSTD_getCParamsFromCCtxParams( | |
5466 | ¶ms, cctx->pledgedSrcSizePlusOne-1, | |
5467 | dictSize, mode); | |
5468 | } | |
5469 | ||
5470 | if (ZSTD_CParams_shouldEnableLdm(¶ms.cParams)) { | |
5471 | /* Enable LDM by default for optimal parser and window size >= 128MB */ | |
5472 | DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)"); | |
5473 | params.ldmParams.enableLdm = 1; | |
5474 | } | |
5475 | ||
c90e81a6 SH |
5476 | if (ZSTD_CParams_useBlockSplitter(¶ms.cParams)) { |
5477 | DEBUGLOG(4, "Block splitter enabled by default (window size >= 128K, strategy >= btopt)"); | |
5478 | params.splitBlocks = 1; | |
5479 | } | |
5480 | ||
4694423c NT |
5481 | params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); |
5482 | ||
48f67da8 | 5483 | #ifdef ZSTD_MULTITHREAD |
5484 | if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { | |
5485 | params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ | |
5486 | } | |
5487 | if (params.nbWorkers > 0) { | |
54a4998a | 5488 | #if ZSTD_TRACE |
6cee3c2c | 5489 | cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; |
54a4998a | 5490 | #endif |
48f67da8 | 5491 | /* mt context creation */ |
5492 | if (cctx->mtctx == NULL) { | |
5493 | DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", | |
5494 | params.nbWorkers); | |
5495 | cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); | |
5496 | RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); | |
5497 | } | |
5498 | /* mt compression */ | |
5499 | DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); | |
5500 | FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( | |
5501 | cctx->mtctx, | |
5502 | prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, | |
5503 | cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); | |
54a4998a NT |
5504 | cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0; |
5505 | cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize; | |
5506 | cctx->consumedSrcSize = 0; | |
5507 | cctx->producedCSize = 0; | |
48f67da8 | 5508 | cctx->streamStage = zcss_load; |
5509 | cctx->appliedParams = params; | |
5510 | } else | |
5511 | #endif | |
5512 | { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; | |
5513 | assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); | |
5514 | FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, | |
5515 | prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, | |
5516 | cctx->cdict, | |
5517 | ¶ms, pledgedSrcSize, | |
5518 | ZSTDb_buffered) , ""); | |
5519 | assert(cctx->appliedParams.nbWorkers == 0); | |
5520 | cctx->inToCompress = 0; | |
5521 | cctx->inBuffPos = 0; | |
5522 | if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { | |
5523 | /* for small input: avoid automatic flush on reaching end of block, since | |
5524 | * it would require to add a 3-bytes null block to end frame | |
5525 | */ | |
5526 | cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); | |
5527 | } else { | |
5528 | cctx->inBuffTarget = 0; | |
5529 | } | |
5530 | cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; | |
5531 | cctx->streamStage = zcss_load; | |
5532 | cctx->frameEnded = 0; | |
5533 | } | |
5534 | return 0; | |
5535 | } | |
5536 | ||
d8e215cb YC |
5537 | size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, |
5538 | ZSTD_outBuffer* output, | |
5539 | ZSTD_inBuffer* input, | |
5540 | ZSTD_EndDirective endOp) | |
6d4fef36 | 5541 | { |
ededcfca | 5542 | DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); |
6d4fef36 | 5543 | /* check conditions */ |
c91a0855 YC |
5544 | RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); |
5545 | RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); | |
5546 | RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); | |
5547 | assert(cctx != NULL); | |
01b1549f | 5548 | |
a3d9926c | 5549 | /* transparent initialization stage */ |
6d4fef36 | 5550 | if (cctx->streamStage == zcss_init) { |
2bbdddf2 | 5551 | FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); |
48f67da8 | 5552 | ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ |
c74be3f6 | 5553 | } |
6ced8f7c | 5554 | /* end of transparent initialization stage */ |
f35e2de6 | 5555 | |
c74be3f6 | 5556 | FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); |
a3d9926c | 5557 | /* compression stage */ |
f129fd39 | 5558 | #ifdef ZSTD_MULTITHREAD |
209df52b | 5559 | if (cctx->appliedParams.nbWorkers > 0) { |
48a6427d | 5560 | size_t flushMin; |
4b525af5 | 5561 | if (cctx->cParamsChanged) { |
6f4d0778 | 5562 | ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); |
4b525af5 | 5563 | cctx->cParamsChanged = 0; |
9e73f2f3 | 5564 | } |
4c58cb83 NT |
5565 | for (;;) { |
5566 | size_t const ipos = input->pos; | |
5567 | size_t const opos = output->pos; | |
48a6427d | 5568 | flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); |
54a4998a NT |
5569 | cctx->consumedSrcSize += (U64)(input->pos - ipos); |
5570 | cctx->producedCSize += (U64)(output->pos - opos); | |
4b525af5 YC |
5571 | if ( ZSTD_isError(flushMin) |
5572 | || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ | |
54a4998a NT |
5573 | if (flushMin == 0) |
5574 | ZSTD_CCtx_trace(cctx, 0); | |
5c686391 | 5575 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); |
4b525af5 | 5576 | } |
5e5f2626 | 5577 | FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); |
4c58cb83 NT |
5578 | |
5579 | if (endOp == ZSTD_e_continue) { | |
5580 | /* We only require some progress with ZSTD_e_continue, not maximal progress. | |
5581 | * We're done if we've consumed or produced any bytes, or either buffer is | |
5582 | * full. | |
5583 | */ | |
5584 | if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size) | |
5585 | break; | |
5586 | } else { | |
5587 | assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end); | |
5588 | /* We require maximal progress. We're done when the flush is complete or the | |
5589 | * output buffer is full. | |
5590 | */ | |
5591 | if (flushMin == 0 || output->pos == output->size) | |
5592 | break; | |
5593 | } | |
5594 | } | |
48a6427d NT |
5595 | DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); |
5596 | /* Either we don't require maximum forward progress, we've finished the | |
5597 | * flush, or we are out of output space. | |
5598 | */ | |
4c58cb83 | 5599 | assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size); |
c74be3f6 | 5600 | ZSTD_setBufferExpectations(cctx, output, input); |
48a6427d NT |
5601 | return flushMin; |
5602 | } | |
f129fd39 | 5603 | #endif |
5e5f2626 | 5604 | FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); |
d8e215cb | 5605 | DEBUGLOG(5, "completed ZSTD_compressStream2"); |
c74be3f6 | 5606 | ZSTD_setBufferExpectations(cctx, output, input); |
deee6e52 | 5607 | return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ |
6d4fef36 YC |
5608 | } |
5609 | ||
d8e215cb | 5610 | size_t ZSTD_compressStream2_simpleArgs ( |
deee6e52 YC |
5611 | ZSTD_CCtx* cctx, |
5612 | void* dst, size_t dstCapacity, size_t* dstPos, | |
5613 | const void* src, size_t srcSize, size_t* srcPos, | |
5614 | ZSTD_EndDirective endOp) | |
6d4fef36 | 5615 | { |
deee6e52 YC |
5616 | ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; |
5617 | ZSTD_inBuffer input = { src, srcSize, *srcPos }; | |
d8e215cb YC |
5618 | /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ |
5619 | size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); | |
deee6e52 YC |
5620 | *dstPos = output.pos; |
5621 | *srcPos = input.pos; | |
58e8d793 | 5622 | return cErr; |
5a0c8e24 YC |
5623 | } |
5624 | ||
d8e215cb YC |
5625 | size_t ZSTD_compress2(ZSTD_CCtx* cctx, |
5626 | void* dst, size_t dstCapacity, | |
5627 | const void* src, size_t srcSize) | |
5628 | { | |
809b2f20 NT |
5629 | ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; |
5630 | ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; | |
4e051591 | 5631 | DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); |
c226a7b9 | 5632 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); |
809b2f20 NT |
5633 | /* Enable stable input/output buffers. */ |
5634 | cctx->requestedParams.inBufferMode = ZSTD_bm_stable; | |
5635 | cctx->requestedParams.outBufferMode = ZSTD_bm_stable; | |
c226a7b9 YC |
5636 | { size_t oPos = 0; |
5637 | size_t iPos = 0; | |
5638 | size_t const result = ZSTD_compressStream2_simpleArgs(cctx, | |
5639 | dst, dstCapacity, &oPos, | |
5640 | src, srcSize, &iPos, | |
5641 | ZSTD_e_end); | |
809b2f20 NT |
5642 | /* Reset to the original values. */ |
5643 | cctx->requestedParams.inBufferMode = originalInBufferMode; | |
5644 | cctx->requestedParams.outBufferMode = originalOutBufferMode; | |
5e5f2626 | 5645 | FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); |
c226a7b9 YC |
5646 | if (result != 0) { /* compression not completed, due to lack of output space */ |
5647 | assert(oPos == dstCapacity); | |
5e5f2626 | 5648 | RETURN_ERROR(dstSize_tooSmall, ""); |
c226a7b9 | 5649 | } |
3619c343 | 5650 | assert(iPos == srcSize); /* all input is expected consumed */ |
c226a7b9 | 5651 | return oPos; |
6ced8f7c | 5652 | } |
d8e215cb | 5653 | } |
5a0c8e24 | 5654 | |
74e95c05 | 5655 | typedef struct { |
347824ad | 5656 | U32 idx; /* Index in array of ZSTD_Sequence */ |
46824cb0 | 5657 | U32 posInSequence; /* Position within sequence at idx */ |
05c02296 | 5658 | size_t posInSrc; /* Number of bytes given by sequences provided so far */ |
46824cb0 | 5659 | } ZSTD_sequencePosition; |
74e95c05 | 5660 | |
347824ad | 5661 | /* Returns a ZSTD error code if sequence is not valid */ |
f6baad87 | 5662 | static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, |
4c5f3372 | 5663 | size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { |
05c02296 | 5664 | size_t offsetBound; |
347824ad | 5665 | U32 windowSize = 1 << windowLog; |
5666 | /* posInSrc represents the amount of data the the decoder would decode up to this point. | |
5667 | * As long as the amount of data decoded is less than or equal to window size, offsets may be | |
5668 | * larger than the total length of output decoded in order to reference the dict, even larger than | |
5669 | * window size. After output surpasses windowSize, we're limited to windowSize offsets again. | |
5670 | */ | |
05c02296 | 5671 | offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; |
f6baad87 | 5672 | RETURN_ERROR_IF(offCode > offsetBound + ZSTD_REP_MOVE, corruption_detected, "Offset too large!"); |
4c5f3372 | 5673 | RETURN_ERROR_IF(matchLength < minMatch, corruption_detected, "Matchlength too small"); |
347824ad | 5674 | return 0; |
5675 | } | |
347824ad | 5676 | |
55b90ef0 | 5677 | /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ |
0e32928b | 5678 | static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { |
347824ad | 5679 | U32 offCode = rawOffset + ZSTD_REP_MOVE; |
5680 | U32 repCode = 0; | |
5681 | ||
5682 | if (!ll0 && rawOffset == rep[0]) { | |
5683 | repCode = 1; | |
5684 | } else if (rawOffset == rep[1]) { | |
5685 | repCode = 2 - ll0; | |
5686 | } else if (rawOffset == rep[2]) { | |
5687 | repCode = 3 - ll0; | |
5688 | } else if (ll0 && rawOffset == rep[0] - 1) { | |
5689 | repCode = 3; | |
5690 | } | |
5691 | if (repCode) { | |
55b90ef0 | 5692 | /* ZSTD_storeSeq expects a number in the range [0, 2] to represent a repcode */ |
347824ad | 5693 | offCode = repCode - 1; |
5694 | } | |
5695 | return offCode; | |
5696 | } | |
5697 | ||
55b90ef0 | 5698 | /* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of |
5699 | * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. | |
5700 | */ | |
0e32928b | 5701 | static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, |
5702 | const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, | |
5703 | const void* src, size_t blockSize) { | |
05c02296 | 5704 | U32 idx = seqPos->idx; |
347824ad | 5705 | BYTE const* ip = (BYTE const*)(src); |
f6baad87 | 5706 | const BYTE* const iend = ip + blockSize; |
347824ad | 5707 | repcodes_t updatedRepcodes; |
5708 | U32 dictSize; | |
5709 | U32 litLength; | |
5710 | U32 matchLength; | |
5711 | U32 ll0; | |
5712 | U32 offCode; | |
5713 | ||
5714 | if (cctx->cdict) { | |
05c02296 | 5715 | dictSize = (U32)cctx->cdict->dictContentSize; |
55b90ef0 | 5716 | } else if (cctx->prefixDict.dict) { |
05c02296 | 5717 | dictSize = (U32)cctx->prefixDict.dictSize; |
347824ad | 5718 | } else { |
5719 | dictSize = 0; | |
5720 | } | |
347824ad | 5721 | ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); |
5722 | for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { | |
5723 | litLength = inSeqs[idx].litLength; | |
5724 | matchLength = inSeqs[idx].matchLength; | |
5725 | ll0 = litLength == 0; | |
5726 | offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); | |
5727 | updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); | |
5728 | ||
5729 | DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); | |
7742f076 | 5730 | if (cctx->appliedParams.validateSequences) { |
5731 | seqPos->posInSrc += litLength + matchLength; | |
5732 | FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, | |
4c5f3372 | 5733 | cctx->appliedParams.cParams.windowLog, dictSize, |
5734 | cctx->appliedParams.cParams.minMatch), | |
7742f076 | 5735 | "Sequence validation failed"); |
5736 | } | |
2cbd0385 | 5737 | RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, |
5738 | "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); | |
0e32928b | 5739 | ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); |
347824ad | 5740 | ip += matchLength + litLength; |
5741 | } | |
5742 | ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); | |
5743 | ||
5744 | if (inSeqs[idx].litLength) { | |
5745 | DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); | |
0e32928b | 5746 | ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); |
347824ad | 5747 | ip += inSeqs[idx].litLength; |
347824ad | 5748 | seqPos->posInSrc += inSeqs[idx].litLength; |
347824ad | 5749 | } |
5750 | RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); | |
5751 | seqPos->idx = idx+1; | |
5752 | return 0; | |
5753 | } | |
5754 | ||
5755 | /* Returns the number of bytes to move the current read position back by. Only non-zero | |
55b90ef0 | 5756 | * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something |
5757 | * went wrong. | |
db9e73cb | 5758 | * |
55b90ef0 | 5759 | * This function will attempt to scan through blockSize bytes represented by the sequences |
db9e73cb FH |
5760 | * in inSeqs, storing any (partial) sequences. |
5761 | * | |
55b90ef0 | 5762 | * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to |
5763 | * avoid splitting a match, or to avoid splitting a match such that it would produce a match | |
5764 | * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. | |
347824ad | 5765 | */ |
0e32928b | 5766 | static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, |
5767 | const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, | |
5768 | const void* src, size_t blockSize) { | |
05c02296 | 5769 | U32 idx = seqPos->idx; |
5770 | U32 startPosInSequence = seqPos->posInSequence; | |
5771 | U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; | |
5772 | size_t dictSize; | |
46824cb0 | 5773 | BYTE const* ip = (BYTE const*)(src); |
55b90ef0 | 5774 | BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ |
46824cb0 | 5775 | repcodes_t updatedRepcodes; |
5776 | U32 bytesAdjustment = 0; | |
347824ad | 5777 | U32 finalMatchSplit = 0; |
347824ad | 5778 | U32 litLength; |
5779 | U32 matchLength; | |
5780 | U32 rawOffset; | |
5781 | U32 offCode; | |
db9e73cb | 5782 | |
347824ad | 5783 | if (cctx->cdict) { |
55b90ef0 | 5784 | dictSize = cctx->cdict->dictContentSize; |
5785 | } else if (cctx->prefixDict.dict) { | |
347824ad | 5786 | dictSize = cctx->prefixDict.dictSize; |
55b90ef0 | 5787 | } else { |
5788 | dictSize = 0; | |
347824ad | 5789 | } |
05c02296 | 5790 | DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); |
5791 | DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); | |
46824cb0 | 5792 | ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); |
347824ad | 5793 | while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { |
5794 | const ZSTD_Sequence currSeq = inSeqs[idx]; | |
5795 | litLength = currSeq.litLength; | |
5796 | matchLength = currSeq.matchLength; | |
5797 | rawOffset = currSeq.offset; | |
46824cb0 | 5798 | |
5799 | /* Modify the sequence depending on where endPosInSequence lies */ | |
cfced934 | 5800 | if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { |
46824cb0 | 5801 | if (startPosInSequence >= litLength) { |
46824cb0 | 5802 | startPosInSequence -= litLength; |
5803 | litLength = 0; | |
5804 | matchLength -= startPosInSequence; | |
5805 | } else { | |
5806 | litLength -= startPosInSequence; | |
5807 | } | |
5808 | /* Move to the next sequence */ | |
cfced934 | 5809 | endPosInSequence -= currSeq.litLength + currSeq.matchLength; |
46824cb0 | 5810 | startPosInSequence = 0; |
cfced934 | 5811 | idx++; |
5812 | } else { | |
55b90ef0 | 5813 | /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence |
347824ad | 5814 | does not reach the end of the match. So, we have to split the sequence */ |
05c02296 | 5815 | DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", |
55b90ef0 | 5816 | currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); |
46824cb0 | 5817 | if (endPosInSequence > litLength) { |
f6baad87 | 5818 | U32 firstHalfMatchLength; |
347824ad | 5819 | litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; |
f6baad87 | 5820 | firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; |
3efe9c90 | 5821 | if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { |
347824ad | 5822 | /* Only ever split the match if it is larger than the block size */ |
5823 | U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; | |
3efe9c90 | 5824 | if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { |
5825 | /* Move the endPosInSequence backward so that it creates match of minMatch length */ | |
5826 | endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; | |
5827 | bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; | |
46824cb0 | 5828 | firstHalfMatchLength -= bytesAdjustment; |
5829 | } | |
5830 | matchLength = firstHalfMatchLength; | |
347824ad | 5831 | /* Flag that we split the last match - after storing the sequence, exit the loop, |
5832 | but keep the value of endPosInSequence */ | |
5833 | finalMatchSplit = 1; | |
46824cb0 | 5834 | } else { |
347824ad | 5835 | /* Move the position in sequence backwards so that we don't split match, and break to store |
5836 | * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence | |
55b90ef0 | 5837 | * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so |
5838 | * would cause the first half of the match to be too small | |
347824ad | 5839 | */ |
5840 | bytesAdjustment = endPosInSequence - currSeq.litLength; | |
5841 | endPosInSequence = currSeq.litLength; | |
46824cb0 | 5842 | break; |
c86151f5 | 5843 | } |
5844 | } else { | |
347824ad | 5845 | /* This sequence ends inside the literals, break to store the last literals */ |
c86151f5 | 5846 | break; |
337fac21 | 5847 | } |
337fac21 | 5848 | } |
347824ad | 5849 | /* Check if this offset can be represented with a repcode */ |
5850 | { U32 ll0 = (litLength == 0); | |
5851 | offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); | |
46824cb0 | 5852 | updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); |
89f38483 | 5853 | } |
347824ad | 5854 | |
7742f076 | 5855 | if (cctx->appliedParams.validateSequences) { |
5856 | seqPos->posInSrc += litLength + matchLength; | |
5857 | FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, | |
4c5f3372 | 5858 | cctx->appliedParams.cParams.windowLog, dictSize, |
5859 | cctx->appliedParams.cParams.minMatch), | |
7742f076 | 5860 | "Sequence validation failed"); |
5861 | } | |
347824ad | 5862 | DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); |
2cbd0385 | 5863 | RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, |
5864 | "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); | |
0e32928b | 5865 | ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); |
086513b5 | 5866 | ip += matchLength + litLength; |
5867 | } | |
05c02296 | 5868 | DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); |
8f3136a9 | 5869 | assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); |
46824cb0 | 5870 | seqPos->idx = idx; |
5871 | seqPos->posInSequence = endPosInSequence; | |
46824cb0 | 5872 | ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); |
5873 | ||
347824ad | 5874 | iend -= bytesAdjustment; |
5875 | if (ip != iend) { | |
5876 | /* Store any last literals */ | |
c86151f5 | 5877 | U32 lastLLSize = (U32)(iend - ip); |
dfef2983 | 5878 | assert(ip <= iend); |
347824ad | 5879 | DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); |
0e32928b | 5880 | ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); |
347824ad | 5881 | seqPos->posInSrc += lastLLSize; |
c86151f5 | 5882 | } |
347824ad | 5883 | |
46824cb0 | 5884 | return bytesAdjustment; |
086513b5 | 5885 | } |
5886 | ||
0e32928b | 5887 | typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, |
5888 | const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, | |
5889 | const void* src, size_t blockSize); | |
5890 | static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) { | |
7742f076 | 5891 | ZSTD_sequenceCopier sequenceCopier = NULL; |
0e32928b | 5892 | assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); |
7742f076 | 5893 | if (mode == ZSTD_sf_explicitBlockDelimiters) { |
5894 | return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; | |
5895 | } else if (mode == ZSTD_sf_noBlockDelimiters) { | |
5896 | return ZSTD_copySequencesToSeqStoreNoBlockDelim; | |
0e32928b | 5897 | } |
5898 | assert(sequenceCopier != NULL); | |
5899 | return sequenceCopier; | |
5900 | } | |
5901 | ||
5fd69f81 | 5902 | /* Compress, block-by-block, all of the sequences given. |
5903 | * | |
55b90ef0 | 5904 | * Returns the cumulative size of all compressed blocks (including their headers), otherwise a ZSTD error. |
5fd69f81 | 5905 | */ |
0e32928b | 5906 | static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, |
5907 | void* dst, size_t dstCapacity, | |
2bbdddf2 | 5908 | const ZSTD_Sequence* inSeqs, size_t inSeqsSize, |
46824cb0 | 5909 | const void* src, size_t srcSize) { |
05c02296 | 5910 | size_t cSize = 0; |
75b01f34 | 5911 | U32 lastBlock; |
05c02296 | 5912 | size_t blockSize; |
5913 | size_t compressedSeqsSize; | |
7eb6fa7b | 5914 | size_t remaining = srcSize; |
347824ad | 5915 | ZSTD_sequencePosition seqPos = {0, 0, 0}; |
db9e73cb | 5916 | |
75b01f34 | 5917 | BYTE const* ip = (BYTE const*)src; |
5918 | BYTE* op = (BYTE*)dst; | |
7742f076 | 5919 | ZSTD_sequenceCopier sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); |
75b01f34 | 5920 | |
dfef2983 | 5921 | DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); |
48405b46 | 5922 | /* Special case: empty frame */ |
5923 | if (remaining == 0) { | |
5924 | U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); | |
5925 | RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); | |
5926 | MEM_writeLE32(op, cBlockHeader24); | |
5927 | op += ZSTD_blockHeaderSize; | |
5928 | dstCapacity -= ZSTD_blockHeaderSize; | |
5929 | cSize += ZSTD_blockHeaderSize; | |
5930 | } | |
5931 | ||
7eb6fa7b | 5932 | while (remaining) { |
05c02296 | 5933 | size_t cBlockSize; |
5934 | size_t additionalByteAdjustment; | |
7eb6fa7b | 5935 | lastBlock = remaining <= cctx->blockSize; |
5936 | blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; | |
0e32928b | 5937 | ZSTD_resetSeqStore(&cctx->seqStore); |
05c02296 | 5938 | DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize); |
0e32928b | 5939 | |
5940 | additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize); | |
46824cb0 | 5941 | FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); |
5942 | blockSize -= additionalByteAdjustment; | |
0e32928b | 5943 | |
c44ce290 | 5944 | /* If blocks are too small, emit as a nocompress block */ |
7eb6fa7b | 5945 | if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { |
f51af9a6 | 5946 | cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); |
5947 | FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); | |
05c02296 | 5948 | DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); |
f51af9a6 | 5949 | cSize += cBlockSize; |
5950 | ip += blockSize; | |
5951 | op += cBlockSize; | |
7eb6fa7b | 5952 | remaining -= blockSize; |
f51af9a6 | 5953 | dstCapacity -= cBlockSize; |
7eb6fa7b | 5954 | continue; |
5955 | } | |
2cff8df1 | 5956 | |
5b566ebe | 5957 | compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, |
7eb6fa7b | 5958 | &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, |
5959 | &cctx->appliedParams, | |
c44ce290 | 5960 | op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, |
7eb6fa7b | 5961 | blockSize, |
5962 | cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, | |
255925c2 | 5963 | cctx->bmi2); |
c44ce290 | 5964 | FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); |
05c02296 | 5965 | DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize); |
293fad6b | 5966 | |
2db84412 | 5967 | if (!cctx->isFirstBlock && |
0e32928b | 5968 | ZSTD_maybeRLE(&cctx->seqStore) && |
2db84412 | 5969 | ZSTD_isRLE((BYTE const*)src, srcSize)) { |
5970 | /* We don't want to emit our first block as a RLE even if it qualifies because | |
5971 | * doing so will cause the decoder (cli only) to throw a "should consume all input error." | |
5972 | * This is only an issue for zstd <= v1.4.3 | |
5973 | */ | |
5974 | compressedSeqsSize = 1; | |
5975 | } | |
5976 | ||
1a449688 | 5977 | if (compressedSeqsSize == 0) { |
5978 | /* ZSTD_noCompressBlock writes the block header as well */ | |
e5fe485d | 5979 | cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); |
1a449688 | 5980 | FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); |
0e32928b | 5981 | DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize); |
2db84412 | 5982 | } else if (compressedSeqsSize == 1) { |
5983 | cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); | |
5984 | FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); | |
0e32928b | 5985 | DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize); |
7eb6fa7b | 5986 | } else { |
dfef2983 | 5987 | U32 cBlockHeader; |
1a449688 | 5988 | /* Error checking and repcodes update */ |
255925c2 | 5989 | ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); |
c44ce290 | 5990 | if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) |
5991 | cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; | |
5992 | ||
5993 | /* Write block header into beginning of block*/ | |
2db84412 | 5994 | cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); |
7eb6fa7b | 5995 | MEM_writeLE24(op, cBlockHeader); |
293fad6b | 5996 | cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; |
05c02296 | 5997 | DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize); |
7eb6fa7b | 5998 | } |
c44ce290 | 5999 | |
1a449688 | 6000 | cSize += cBlockSize; |
05c02296 | 6001 | DEBUGLOG(4, "cSize running total: %zu", cSize); |
db9e73cb | 6002 | |
7eb6fa7b | 6003 | if (lastBlock) { |
7eb6fa7b | 6004 | break; |
6005 | } else { | |
6006 | ip += blockSize; | |
293fad6b | 6007 | op += cBlockSize; |
7eb6fa7b | 6008 | remaining -= blockSize; |
293fad6b | 6009 | dstCapacity -= cBlockSize; |
2db84412 | 6010 | cctx->isFirstBlock = 0; |
7eb6fa7b | 6011 | } |
2cff8df1 | 6012 | } |
db9e73cb | 6013 | |
2cff8df1 | 6014 | return cSize; |
6015 | } | |
6016 | ||
b8e16a20 | 6017 | size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, |
46824cb0 | 6018 | const ZSTD_Sequence* inSeqs, size_t inSeqsSize, |
6019 | const void* src, size_t srcSize) { | |
086513b5 | 6020 | BYTE* op = (BYTE*)dst; |
2cff8df1 | 6021 | size_t cSize = 0; |
1a449688 | 6022 | size_t compressedBlocksSize = 0; |
3e930fd0 | 6023 | size_t frameHeaderSize = 0; |
3e930fd0 | 6024 | |
48f67da8 | 6025 | /* Transparent initialization stage, same as compressStream2() */ |
b8e16a20 | 6026 | DEBUGLOG(3, "ZSTD_compressSequences()"); |
013434e1 | 6027 | assert(cctx != NULL); |
6028 | FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); | |
c44ce290 | 6029 | /* Begin writing output, starting with frame header */ |
3e930fd0 | 6030 | frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); |
086513b5 | 6031 | op += frameHeaderSize; |
3e930fd0 | 6032 | dstCapacity -= frameHeaderSize; |
2cff8df1 | 6033 | cSize += frameHeaderSize; |
086513b5 | 6034 | if (cctx->appliedParams.fParams.checksumFlag && srcSize) { |
6035 | XXH64_update(&cctx->xxhState, src, srcSize); | |
6036 | } | |
2cff8df1 | 6037 | /* cSize includes block header size and compressed sequences size */ |
0e32928b | 6038 | compressedBlocksSize = ZSTD_compressSequences_internal(cctx, |
6039 | op, dstCapacity, | |
6040 | inSeqs, inSeqsSize, | |
55b90ef0 | 6041 | src, srcSize); |
5fd69f81 | 6042 | FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); |
1a449688 | 6043 | cSize += compressedBlocksSize; |
6044 | dstCapacity -= compressedBlocksSize; | |
086513b5 | 6045 | |
086513b5 | 6046 | if (cctx->appliedParams.fParams.checksumFlag) { |
6047 | U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); | |
6048 | RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); | |
3e930fd0 | 6049 | DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); |
dfef2983 | 6050 | MEM_writeLE32((char*)dst + cSize, checksum); |
086513b5 | 6051 | cSize += 4; |
6052 | } | |
6053 | ||
fd100071 | 6054 | DEBUGLOG(3, "Final compressed size: %zu", cSize); |
013434e1 | 6055 | return cSize; |
6056 | } | |
6057 | ||
104e5b07 | 6058 | /*====== Finalize ======*/ |
5a0c8e24 YC |
6059 | |
6060 | /*! ZSTD_flushStream() : | |
5188749e | 6061 | * @return : amount of data remaining to flush */ |
53e17fbd | 6062 | size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) |
5a0c8e24 | 6063 | { |
18ab5aff | 6064 | ZSTD_inBuffer input = { NULL, 0, 0 }; |
6ced8f7c | 6065 | return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); |
5a0c8e24 YC |
6066 | } |
6067 | ||
6068 | ||
53e17fbd | 6069 | size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) |
5a0c8e24 | 6070 | { |
18ab5aff | 6071 | ZSTD_inBuffer input = { NULL, 0, 0 }; |
6ced8f7c | 6072 | size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); |
5e5f2626 | 6073 | FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); |
6ced8f7c YC |
6074 | if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ |
6075 | /* single thread mode : attempt to calculate remaining to flush more precisely */ | |
2cb9774f | 6076 | { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; |
621adde3 | 6077 | size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); |
6ced8f7c | 6078 | size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; |
ededcfca | 6079 | DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); |
2cb9774f YC |
6080 | return toFlush; |
6081 | } | |
5a0c8e24 YC |
6082 | } |
6083 | ||
6084 | ||
70e8c389 | 6085 | /*-===== Pre-defined compression levels =====-*/ |
fd416f1e | 6086 | |
2c5eeea6 | 6087 | #define ZSTD_MAX_CLEVEL 22 |
41105348 | 6088 | int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } |
a54c86cf | 6089 | int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } |
e398744a | 6090 | int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } |
7d968c7b | 6091 | |
3b71925c | 6092 | static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { |
f9e4f892 | 6093 | { /* "default" - for any srcSize > 256 KB */ |
793c649e | 6094 | /* W, C, H, S, L, TL, strat */ |
a146ee04 | 6095 | { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ |
c2c47e24 | 6096 | { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ |
37e314a6 | 6097 | { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ |
5d5c895b YC |
6098 | { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ |
6099 | { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ | |
37e314a6 YC |
6100 | { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ |
6101 | { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ | |
6102 | { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ | |
b4250489 YC |
6103 | { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ |
6104 | { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ | |
37e314a6 YC |
6105 | { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ |
6106 | { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ | |
6107 | { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ | |
6108 | { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ | |
6109 | { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ | |
6110 | { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ | |
6111 | { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ | |
6112 | { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ | |
6113 | { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ | |
6114 | { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ | |
6115 | { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ | |
6116 | { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ | |
6117 | { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ | |
fd416f1e YC |
6118 | }, |
6119 | { /* for srcSize <= 256 KB */ | |
3b71925c | 6120 | /* W, C, H, S, L, T, strat */ |
a146ee04 | 6121 | { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ |
c2c47e24 | 6122 | { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ |
5d5c895b YC |
6123 | { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ |
6124 | { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ | |
b0b3fb51 YC |
6125 | { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ |
6126 | { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ | |
6127 | { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ | |
6128 | { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ | |
6129 | { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ | |
6130 | { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ | |
6131 | { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ | |
c9c4c7ec YC |
6132 | { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ |
6133 | { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ | |
6134 | { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ | |
6135 | { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ | |
6136 | { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ | |
6137 | { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ | |
6138 | { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ | |
6139 | { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ | |
6140 | { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ | |
6141 | { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ | |
6142 | { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ | |
6143 | { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ | |
fd416f1e YC |
6144 | }, |
6145 | { /* for srcSize <= 128 KB */ | |
3b71925c | 6146 | /* W, C, H, S, L, T, strat */ |
c9227ee1 | 6147 | { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ |
c2c47e24 YC |
6148 | { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ |
6149 | { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ | |
5d5c895b YC |
6150 | { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ |
6151 | { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ | |
63eeeaa1 YC |
6152 | { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ |
6153 | { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ | |
c9227ee1 | 6154 | { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ |
5894ea8d YC |
6155 | { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ |
6156 | { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ | |
6157 | { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ | |
8075d75f YC |
6158 | { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ |
6159 | { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ | |
6160 | { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ | |
6161 | { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ | |
6162 | { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ | |
6163 | { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ | |
6164 | { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ | |
6165 | { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ | |
6166 | { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ | |
6167 | { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ | |
6168 | { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ | |
6169 | { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ | |
fd416f1e YC |
6170 | }, |
6171 | { /* for srcSize <= 16 KB */ | |
3b71925c | 6172 | /* W, C, H, S, L, T, strat */ |
a146ee04 | 6173 | { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ |
c2c47e24 YC |
6174 | { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ |
6175 | { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ | |
5d5c895b | 6176 | { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ |
95b152ab | 6177 | { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ |
63eeeaa1 YC |
6178 | { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ |
6179 | { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ | |
6180 | { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ | |
6181 | { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ | |
6182 | { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ | |
6183 | { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ | |
6184 | { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ | |
95b152ab YC |
6185 | { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ |
6186 | { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ | |
6187 | { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ | |
6188 | { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ | |
6189 | { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ | |
6190 | { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ | |
6191 | { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ | |
6192 | { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ | |
6193 | { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ | |
6194 | { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ | |
6195 | { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ | |
fd416f1e YC |
6196 | }, |
6197 | }; | |
6198 | ||
fadaab8c | 6199 | static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) |
e36a373d | 6200 | { |
d5c688e8 | 6201 | ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); |
a3659fe1 FH |
6202 | switch (cParams.strategy) { |
6203 | case ZSTD_fast: | |
6204 | case ZSTD_dfast: | |
6205 | break; | |
6206 | case ZSTD_greedy: | |
6207 | case ZSTD_lazy: | |
6208 | case ZSTD_lazy2: | |
6209 | cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; | |
6210 | break; | |
6211 | case ZSTD_btlazy2: | |
6212 | case ZSTD_btopt: | |
6213 | case ZSTD_btultra: | |
6214 | case ZSTD_btultra2: | |
6215 | break; | |
6216 | } | |
6217 | return cParams; | |
e36a373d BS |
6218 | } |
6219 | ||
eee51a66 FH |
6220 | static int ZSTD_dedicatedDictSearch_isSupported( |
6221 | ZSTD_compressionParameters const* cParams) | |
e36a373d | 6222 | { |
77ae664b SH |
6223 | return (cParams->strategy >= ZSTD_greedy) |
6224 | && (cParams->strategy <= ZSTD_lazy2) | |
efa6dfa7 | 6225 | && (cParams->hashLog > cParams->chainLog) |
77ae664b | 6226 | && (cParams->chainLog <= 24); |
e36a373d BS |
6227 | } |
6228 | ||
7b5d2f72 FH |
6229 | /** |
6230 | * Reverses the adjustment applied to cparams when enabling dedicated dict | |
6231 | * search. This is used to recover the params set to be used in the working | |
6232 | * context. (Otherwise, those tables would also grow.) | |
6233 | */ | |
6234 | static void ZSTD_dedicatedDictSearch_revertCParams( | |
6235 | ZSTD_compressionParameters* cParams) { | |
6236 | switch (cParams->strategy) { | |
6237 | case ZSTD_fast: | |
6238 | case ZSTD_dfast: | |
6239 | break; | |
6240 | case ZSTD_greedy: | |
6241 | case ZSTD_lazy: | |
6242 | case ZSTD_lazy2: | |
6243 | cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; | |
efa6dfa7 | 6244 | if (cParams->hashLog < ZSTD_HASHLOG_MIN) { |
6245 | cParams->hashLog = ZSTD_HASHLOG_MIN; | |
6246 | } | |
7b5d2f72 FH |
6247 | break; |
6248 | case ZSTD_btlazy2: | |
6249 | case ZSTD_btopt: | |
6250 | case ZSTD_btultra: | |
6251 | case ZSTD_btultra2: | |
6252 | break; | |
6253 | } | |
6254 | } | |
6255 | ||
d5c688e8 NT |
6256 | static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) |
6257 | { | |
6258 | switch (mode) { | |
6259 | case ZSTD_cpm_unknown: | |
6260 | case ZSTD_cpm_noAttachDict: | |
6261 | case ZSTD_cpm_createCDict: | |
6262 | break; | |
6263 | case ZSTD_cpm_attachDict: | |
6264 | dictSize = 0; | |
6265 | break; | |
6266 | default: | |
6267 | assert(0); | |
6268 | break; | |
6269 | } | |
6270 | { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; | |
6271 | size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; | |
6272 | return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; | |
6273 | } | |
6274 | } | |
6275 | ||
8c474f98 | 6276 | /*! ZSTD_getCParams_internal() : |
f9e4f892 | 6277 | * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. |
8c474f98 | 6278 | * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. |
d5c688e8 NT |
6279 | * Use dictSize == 0 for unknown or unused. |
6280 | * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ | |
6281 | static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) | |
fd416f1e | 6282 | { |
d5c688e8 | 6283 | U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); |
f9e4f892 | 6284 | U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); |
52f8c07a | 6285 | int row; |
8c474f98 | 6286 | DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); |
52f8c07a | 6287 | |
6288 | /* row */ | |
6a9b41b7 | 6289 | if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ |
52f8c07a | 6290 | else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ |
6291 | else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; | |
6292 | else row = compressionLevel; | |
6293 | ||
6a9b41b7 | 6294 | { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; |
4694423c | 6295 | DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); |
52f8c07a | 6296 | /* acceleration factor */ |
6297 | if (compressionLevel < 0) { | |
6298 | int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); | |
6299 | cp.targetLength = (unsigned)(-clampedCompressionLevel); | |
6300 | } | |
8c474f98 | 6301 | /* refine parameters based on srcSize & dictSize */ |
d5c688e8 | 6302 | return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode); |
95b152ab | 6303 | } |
fd416f1e | 6304 | } |
3d2cd7f8 | 6305 | |
8c474f98 NT |
6306 | /*! ZSTD_getCParams() : |
6307 | * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. | |
6308 | * Size values are optional, provide 0 if not known or unused */ | |
6309 | ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) | |
6310 | { | |
6311 | if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; | |
d5c688e8 | 6312 | return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); |
8c474f98 NT |
6313 | } |
6314 | ||
3d2cd7f8 | 6315 | /*! ZSTD_getParams() : |
f9e4f892 YC |
6316 | * same idea as ZSTD_getCParams() |
6317 | * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). | |
6318 | * Fields of `ZSTD_frameParameters` are set to default values */ | |
d5c688e8 | 6319 | static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { |
3d2cd7f8 | 6320 | ZSTD_parameters params; |
d5c688e8 | 6321 | ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); |
50f763ec | 6322 | DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); |
c465f244 | 6323 | ZSTD_memset(¶ms, 0, sizeof(params)); |
3d2cd7f8 | 6324 | params.cParams = cParams; |
fb445166 | 6325 | params.fParams.contentSizeFlag = 1; |
3d2cd7f8 YC |
6326 | return params; |
6327 | } | |
8c474f98 NT |
6328 | |
6329 | /*! ZSTD_getParams() : | |
6330 | * same idea as ZSTD_getCParams() | |
6331 | * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). | |
6332 | * Fields of `ZSTD_frameParameters` are set to default values */ | |
6333 | ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { | |
6334 | if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; | |
d5c688e8 | 6335 | return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); |
8c474f98 | 6336 | } |