</b><p> These functions make it possible to estimate memory usage
of a future target object, before its allocation,
given a set of parameters, which vary depending on target object.
- The objective is to guide decision before allocation.
+ The objective is to guide decision before allocation.
+ Note : CCtx estimation is only correct for single-threaded compression
</p></pre><BR>
<pre><b>size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
ZSTD_p_rawContentDict, </b>/* load dictionary in "content-only" mode (no header analysis) (default:0) */<b>
</b>/* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */<b>
-#if 0
- </b>/* multi-threading parameters (not ready yet !) */<b>
+ </b>/* multi-threading parameters */<b>
ZSTD_p_nbThreads=400, </b>/* Select how many threads a compression job can spawn (default:1)<b>
- * More threads improve speed, but increases also memory usage */
- ZSTDMT_p_jobSize, </b>/* Size of a compression job. Each job is compressed in parallel.<b>
+ * More threads improve speed, but also increase memory usage.
+ * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+ * Special: value 0 means "do not change nbThreads" */
+ ZSTDMT_p_jobSize, </b>/* Size of a compression job. Each compression job is completed in parallel.<b>
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
* The minimum size is automatically and transparently enforced */
ZSTDMT_p_overlapSizeLog, </b>/* Size of previous input reloaded at the beginning of each job.<b>
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-#endif
</b>/* advanced parameters - may not remain available after API update */<b>
ZSTD_p_forceMaxWindow=1100, </b>/* Force back-references to remain < windowSize,<b>
#if defined(BIT_DEBUG) && (BIT_DEBUG>=1)
# include <assert.h>
#else
-# define assert(condition) ((void)0)
+# ifndef assert
+# define assert(condition) ((void)0)
+# endif
#endif
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
/* fall-through */
-
+
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
/* fall-through */
-
+
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
/* fall-through */
-
+
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
/* fall-through */
-
+
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
/* fall-through */
-
+
default: break;
}
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#ifndef XXH_STATIC_LINKING_ONLY
-# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+#endif
+#include "xxhash.h" /* XXH_reset, update, digest */
+
+
+/*-*************************************
+* Debug
+***************************************/
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
+# include <assert.h>
+#else
+# ifndef assert
+# define assert(condition) ((void)0)
+# endif
+#endif
+
+#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
+
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
+# include <stdio.h>
+ static unsigned g_debugLevel = ZSTD_DEBUG;
+# define DEBUGLOG(l, ...) { \
+ if (l<=g_debugLevel) { \
+ fprintf(stderr, __FILE__ ": "); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, " \n"); \
+ } }
+#else
+# define DEBUGLOG(l, ...) {} /* disabled */
#endif
-#include "xxhash.h" /* XXH_reset, update, digest */
/*-*************************************
#include "zstdmt_compress.h"
-/*-*************************************
-* Debug
-***************************************/
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
-# include <assert.h>
-#else
-# define assert(condition) ((void)0)
-#endif
-
-#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
-
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-# include <stdio.h>
- static unsigned g_debugLevel = ZSTD_DEBUG;
-# define DEBUGLOG(l, ...) { \
- if (l<=g_debugLevel) { \
- fprintf(stderr, __FILE__ ": "); \
- fprintf(stderr, __VA_ARGS__); \
- fprintf(stderr, " \n"); \
- } }
-#else
-# define DEBUGLOG(l, ...) {} /* disabled */
-#endif
-
-
/*-*************************************
* Constants
***************************************/
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode)
{
- U32 someMoreWork = 1;
const char* const istart = (const char*)input->src;
const char* const iend = istart + input->size;
const char* ip = istart + input->pos;
char* const ostart = (char*)output->dst;
char* const oend = ostart + output->size;
char* op = ostart + output->pos;
+ U32 someMoreWork = 1;
- /* expected to be already allocated */
+ /* check expectations */
assert(zcs->inBuff != NULL);
assert(zcs->outBuff!= NULL);
+ assert(output->pos <= output->size);
+ assert(input->pos <= input->size);
while (someMoreWork) {
switch(zcs->streamStage)
if (cctx->streamStage == zcss_init) {
/* transparent reset */
ZSTD_parameters params = cctx->requestedParams;
- DEBUGLOG(5, "ZSTD_compress_generic : transparent reset");
if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
params.cParams = ZSTD_getCParams(cctx->compressionLevel,
cctx->frameContentSize, 0 /* dictSize */);
DEBUGLOG(5, "starting ZSTD_compressStream_generic");
CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
- DEBUGLOG(5, "completing ZSTD_compress_generic_integral");
+ DEBUGLOG(5, "completing ZSTD_compress_generic");
return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
}
ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
ZSTD_inBuffer input = { src, srcSize, *srcPos };
/* ZSTD_compress_generic() will check validity of dstPos and srcPos */
- size_t const hint = ZSTD_compress_generic(cctx, &output, &input, endOp);
- if (ZSTD_isError(hint)) return hint;
-
+ size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp);
*dstPos = output.pos;
*srcPos = input.pos;
- return hint;
+ return cErr;
}
/* ====== Compiler specifics ====== */
#if defined(_MSC_VER)
-# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
/* ====== Dependencies ====== */
-#include <string.h> /* memcpy, memset */
-#include "pool.h" /* threadpool */
-#include "threading.h" /* mutex */
+#include <string.h> /* memcpy, memset */
+#include "pool.h" /* threadpool */
+#include "threading.h" /* mutex */
#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
#include "zstdmt_compress.h"
/* ====== Debug ====== */
-#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=1)
-# include <assert.h>
-#else
-# define assert(condition) ((void)0)
-#endif
-
#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=2)
# include <stdio.h>
static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
- const void* dict, size_t dictSize, unsigned updateDict,
- ZSTD_parameters params, unsigned long long pledgedSrcSize)
+ const void* dict, size_t dictSize, unsigned updateDict,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
- ZSTD_customMem const cmem = { NULL, NULL, NULL };
- DEBUGLOG(3, "Started new compression, with windowLog : %u",
- params.cParams.windowLog);
if (zcs->nbThreads==1)
return ZSTD_initCStream_advanced(zcs->cctxPool->cctx[0],
dict, dictSize,
params, pledgedSrcSize);
- if (zcs->allJobsCompleted == 0) { /* previous job not correctly finished */
+
+ if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */
ZSTDMT_waitForAllJobsCompleted(zcs);
ZSTDMT_releaseAllJobResources(zcs);
zcs->allJobsCompleted = 1;
if (updateDict) {
ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
if (dict && dictSize) {
- zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem);
+ zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0 /* byRef */,
+ params.cParams, zcs->cMem);
if (zcs->cdict == NULL) return ERROR(memory_allocation);
} }
zcs->frameContentSize = pledgedSrcSize;
#endif
-/* Note : All prototypes defined in this file shall be considered experimental.
- * There is no guarantee of API continuity (yet) on any of these prototypes */
+/* Note : All prototypes defined in this file must be considered experimental.
+ * There is no guarantee of API continuity on any of these prototypes */
/* === Dependencies === */
#include <stddef.h> /* size_t */
# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif
-ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
+ const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
+ ZSTD_parameters params,
+ unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
/* ZSDTMT_parameter :
#define FSE_isError ERR_isError
#define HUF_isError ERR_isError
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-# include <stdio.h>
- static unsigned g_debugLevel = ZSTD_DEBUG;
-# define DEBUGLOG(l, ...) { \
- if (l<=g_debugLevel) { \
- fprintf(stderr, __FILE__ ": "); \
- fprintf(stderr, __VA_ARGS__); \
- fprintf(stderr, " \n"); \
- } }
-#else
-# define DEBUGLOG(l, ...) {} /* disabled */
-#endif
-
/*_*******************************************************
* Memory operations
projects / thirdparty / zstd.git / commitdiff
