From: Yann Collet Date: Fri, 2 Jun 2017 00:56:14 +0000 (-0700) Subject: created ZSTDMT_sizeof_CCtx() and POOL_sizeof() X-Git-Tag: v1.3.0~1^2~17^2~66 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c4a5a21c5c4e75b24839267a4f0472948a0159cf;p=thirdparty%2Fzstd.git created ZSTDMT_sizeof_CCtx() and POOL_sizeof() required by ZSTD_sizeofCCtx() while adding a ZSTDMT_CCtx* --- diff --git a/lib/common/pool.c b/lib/common/pool.c index e439fe1b0..05adcf1ce 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -146,6 +146,13 @@ void POOL_free(POOL_ctx *ctx) { free(ctx); } +size_t POOL_sizeof(POOL_ctx *ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->numThreads * sizeof(pthread_t); +} + void POOL_add(void *ctxVoid, POOL_function function, void *opaque) { POOL_ctx *ctx = (POOL_ctx *)ctxVoid; if (!ctx) { return; } diff --git a/lib/common/pool.h b/lib/common/pool.h index 50cb25b12..386cd674b 100644 --- a/lib/common/pool.h +++ b/lib/common/pool.h @@ -32,6 +32,11 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize); */ void POOL_free(POOL_ctx *ctx); +/*! POOL_sizeof() : + return memory usage of pool returned by POOL_create(). +*/ +size_t POOL_sizeof(POOL_ctx *ctx); + /*! POOL_function : The function type that can be added to a thread pool. */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 175a7b2b0..7924968e4 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -25,6 +25,7 @@ #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_internal.h" /* includes zstd.h */ +#include "zstdmt_compress.h" /*-************************************* @@ -154,8 +155,12 @@ struct ZSTD_CCtx_s { size_t outBuffFlushedSize; ZSTD_cStreamStage streamStage; U32 frameEnded; + + /* Multi-threading */ + ZSTDMT_CCtx* mtctx; }; + ZSTD_CCtx* ZSTD_createCCtx(void) { return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); @@ -205,11 +210,13 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support free on NULL */ - assert(!cctx->staticSize); /* not compatible with static CCtx */ + if (cctx->staticSize) return ERROR(memory_allocation); /* not compatible with static CCtx */ ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL; ZSTD_freeCDict(cctx->cdictLocal); cctx->cdictLocal = NULL; + ZSTDMT_freeCCtx(cctx->mtctx); + cctx->mtctx = NULL; ZSTD_free(cctx, cctx->customMem); return 0; /* reserved as a potential error code in the future */ } @@ -219,7 +226,8 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) if (cctx==NULL) return 0; /* support sizeof on NULL */ return sizeof(*cctx) + cctx->workSpaceSize + ZSTD_sizeof_CDict(cctx->cdictLocal) - + cctx->outBuffSize + cctx->inBuffSize; + + cctx->outBuffSize + cctx->inBuffSize + + ZSTDMT_sizeof_CCtx(cctx->mtctx); } size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index b2c82e489..57c0360b0 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -123,6 +123,19 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) ZSTD_free(bufPool, bufPool->cMem); } +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool) + + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + for (u=0; utotalBuffers; u++) + totalBufferSize += bufPool->bTable[u].size; + + return poolSize + totalBufferSize; +} + /* assumption : invocation from main thread only ! */ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize) { @@ -181,7 +194,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) /* ZSTDMT_createCCtxPool() : * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads, - ZSTD_customMem cMem) + ZSTD_customMem cMem) { ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc( sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem); @@ -195,6 +208,20 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads, return cctxPool; } +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + unsigned const nbThreads = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool) + + (nbThreads-1)*sizeof(ZSTD_CCtx*); + unsigned u; + size_t totalCCtxSize = 0; + for (u=0; ucctx[u]); + + return poolSize + totalCCtxSize; +} + static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool) { if (pool->availCCtx) { @@ -393,6 +420,17 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) return 0; } +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->buffPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTD_sizeof_CDict(mtctx->cdict); +} + size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value) { switch(parameter) diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 94fef7c07..ff7f21687 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -24,14 +24,21 @@ #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ -/* === Simple one-pass functions === */ - +/* === Memory management === */ typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads); -ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem); -ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx); +ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, + ZSTD_customMem cMem); +ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); +ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams, + unsigned nbThreads); + + +/* === Simple buffer-to-butter one-pass function === */ -ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx, +ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);