created ZSTDMT_sizeof_CCtx() and POOL_sizeof()

required by ZSTD_sizeofCCtx() while adding a ZSTDMT_CCtx*

diff --git a/lib/common/pool.c b/lib/common/pool.c
index e439fe1b0dd6bc4344ec889121241a92f2d72b1f..05adcf1ce40b8a1e0c678bff507ce721cb17a883 100644 (file)
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -146,6 +146,13 @@ void POOL_free(POOL_ctx *ctx) {
     free(ctx);
 }
 
+size_t POOL_sizeof(POOL_ctx *ctx) {
+    if (ctx==NULL) return 0;  /* supports sizeof NULL */
+    return sizeof(*ctx)
+        + ctx->queueSize * sizeof(POOL_job)
+        + ctx->numThreads * sizeof(pthread_t);
+}
+
 void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
     POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
     if (!ctx) { return; }

diff --git a/lib/common/pool.h b/lib/common/pool.h
index 50cb25b12c9f165309e110ea9b19dc88b64ca182..386cd674b7c067c3b7c6e6b04b6525c906e09712 100644 (file)
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -32,6 +32,11 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
 */
 void POOL_free(POOL_ctx *ctx);
 
+/*! POOL_sizeof() :
+    return memory usage of pool returned by POOL_create().
+*/
+size_t POOL_sizeof(POOL_ctx *ctx);
+
 /*! POOL_function :
     The function type that can be added to a thread pool.
 */

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 175a7b2b04e4a9b1ad47ef33e562f39d5a1a269f..7924968e4af24c0ba86a9c2f9462d175f7e58abc 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -25,6 +25,7 @@
 #define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
 #include "zstd_internal.h"  /* includes zstd.h */
+#include "zstdmt_compress.h"
 
 
 /*-*************************************
@@ -154,8 +155,12 @@ struct ZSTD_CCtx_s {
     size_t outBuffFlushedSize;
     ZSTD_cStreamStage streamStage;
     U32    frameEnded;
+
+    /* Multi-threading */
+    ZSTDMT_CCtx* mtctx;
 };
 
+
 ZSTD_CCtx* ZSTD_createCCtx(void)
 {
     return ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
@@ -205,11 +210,13 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
 size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return 0;   /* support free on NULL */
-    assert(!cctx->staticSize);  /* not compatible with static CCtx */
+    if (cctx->staticSize) return ERROR(memory_allocation);   /* not compatible with static CCtx */
     ZSTD_free(cctx->workSpace, cctx->customMem);
     cctx->workSpace = NULL;
     ZSTD_freeCDict(cctx->cdictLocal);
     cctx->cdictLocal = NULL;
+    ZSTDMT_freeCCtx(cctx->mtctx);
+    cctx->mtctx = NULL;
     ZSTD_free(cctx, cctx->customMem);
     return 0;   /* reserved as a potential error code in the future */
 }
@@ -219,7 +226,8 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
     if (cctx==NULL) return 0;   /* support sizeof on NULL */
     return sizeof(*cctx) + cctx->workSpaceSize
            + ZSTD_sizeof_CDict(cctx->cdictLocal)
-           + cctx->outBuffSize + cctx->inBuffSize;
+           + cctx->outBuffSize + cctx->inBuffSize
+           + ZSTDMT_sizeof_CCtx(cctx->mtctx);
 }
 
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index b2c82e489b9b8919f679b73ab55c2e00bed85df7..57c0360b029789f0a63144ff2e3f0e4c72d41a5f 100644 (file)
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -123,6 +123,19 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
     ZSTD_free(bufPool, bufPool->cMem);
 }
 
+/* only works at initialization, not during compression */
+static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
+{
+    size_t const poolSize = sizeof(*bufPool)
+                            + (bufPool->totalBuffers - 1) * sizeof(buffer_t);
+    unsigned u;
+    size_t totalBufferSize = 0;
+    for (u=0; u<bufPool->totalBuffers; u++)
+        totalBufferSize += bufPool->bTable[u].size;
+
+    return poolSize + totalBufferSize;
+}
+
 /* assumption : invocation from main thread only ! */
 static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
 {
@@ -181,7 +194,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
 /* ZSTDMT_createCCtxPool() :
  * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */
 static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
-                                            ZSTD_customMem cMem)
+                                              ZSTD_customMem cMem)
 {
     ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
         sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
@@ -195,6 +208,20 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
     return cctxPool;
 }
 
+/* only works during initialization phase, not during compression */
+static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
+{
+    unsigned const nbThreads = cctxPool->totalCCtx;
+    size_t const poolSize = sizeof(*cctxPool)
+                            + (nbThreads-1)*sizeof(ZSTD_CCtx*);
+    unsigned u;
+    size_t totalCCtxSize = 0;
+    for (u=0; u<nbThreads; u++)
+        totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
+
+    return poolSize + totalCCtxSize;
+}
+
 static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
 {
     if (pool->availCCtx) {
@@ -393,6 +420,17 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     return 0;
 }
 
+size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
+{
+    if (mtctx == NULL) return 0;   /* supports sizeof NULL */
+    return sizeof(*mtctx)
+        + POOL_sizeof(mtctx->factory)
+        + ZSTDMT_sizeof_bufferPool(mtctx->buffPool)
+        + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription)
+        + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool)
+        + ZSTD_sizeof_CDict(mtctx->cdict);
+}
+
 size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
 {
     switch(parameter)

diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 94fef7c0712cf314e68939cc45192750f1e11878..ff7f21687ae3e7b18b82a84dcedcd3ef1fe5ca16 100644 (file)
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -24,14 +24,21 @@
 #include "zstd.h"     /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
 
 
-/* ===   Simple one-pass functions   === */
-
+/* ===   Memory management   === */
 typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
 ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
-ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem);
-ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
+ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
+                                                    ZSTD_customMem cMem);
+ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
+
+ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
+ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams,
+                                           unsigned nbThreads);
+
+
+/* ===   Simple buffer-to-butter one-pass function   === */
 
-ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
+ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
                            void* dst, size_t dstCapacity,
                      const void* src, size_t srcSize,
                            int compressionLevel);