From: Yann Collet Date: Tue, 11 Jul 2017 00:16:41 +0000 (-0700) Subject: improved ZSTDMT_compress() memory usage X-Git-Tag: v1.3.1^2~35^2~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4616fad18b2f8950144cdbcffc081b74cc115fb1;p=thirdparty%2Fzstd.git improved ZSTDMT_compress() memory usage does not need the input buffer for streaming operations also : reduced a few tests time length --- diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index a176c3eea..fbb86b008 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -134,9 +134,11 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize) /* large enough, but not too much */ return buf; /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(2, "existing buffer does not meet size conditions => freeing"); ZSTD_free(buf.start, pool->cMem); } /* create new buffer */ + DEBUGLOG(2, "create a new buffer"); { buffer_t buffer; void* const start = ZSTD_malloc(bSize, pool->cMem); if (start==NULL) bSize = 0; @@ -149,12 +151,14 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize) /* store buffer for later re-use, up to pool capacity */ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf) { + DEBUGLOG(2, "ZSTDMT_releaseBuffer"); if (buf.start == NULL) return; /* release on NULL */ if (pool->nbBuffers < pool->totalBuffers) { pool->bTable[pool->nbBuffers++] = buf; /* store for later re-use */ return; } /* Reached bufferPool capacity (should not happen) */ + DEBUGLOG(2, "buffer pool capacity reached => freeing "); ZSTD_free(buf.start, pool->cMem); } @@ -635,8 +639,8 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, if (zcs->nbThreads==1) { DEBUGLOG(4, "single thread mode"); return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0], - dict, dictSize, cdict, - params, pledgedSrcSize); + dict, dictSize, cdict, + params, pledgedSrcSize); } if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */ @@ -671,9 +675,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10)); zcs->marginSize = zcs->targetSectionSize >> 2; zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize; - zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize); - if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation); - zcs->inBuff.filled = 0; + zcs->inBuff.buffer = g_nullBuffer; zcs->dictSize = 0; zcs->doneJobID = 0; zcs->nextJobID = 0; @@ -899,13 +901,18 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, } /* fill input buffer */ - if ((input->src) && (mtctx->inBuff.buffer.start)) { /* support NULL input */ - size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled); - DEBUGLOG(5, "inBuff:%08X; inBuffSize=%u; ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad); - memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad); - input->pos += toLoad; - mtctx->inBuff.filled += toLoad; - } + if (input->src) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->buffPool, mtctx->inBuffSize); + if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation); + mtctx->inBuff.filled = 0; + } + { size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled); + DEBUGLOG(5, "inBuff:%08X; inBuffSize=%u; ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad); + memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad); + input->pos += toLoad; + mtctx->inBuff.filled += toLoad; + } } if ( (mtctx->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */ && (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) { /* avoid overwriting job round buffer */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 667de08ce..1e25383f5 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -129,7 +129,7 @@ static void* FUZ_mallocDebug(void* counter, size_t size) static void FUZ_freeDebug(void* counter, void* address) { mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; - DISPLAYLEVEL(4, "releasing %u KB \n", (U32)(malloc_size(address) >> 10)); + DISPLAYLEVEL(4, "freeing %u KB \n", (U32)(malloc_size(address) >> 10)); mcPtr->nbFree += 1; mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ free(address); diff --git a/tests/playTests.sh b/tests/playTests.sh index 2e1cc6826..88a1c2ab4 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -7,17 +7,17 @@ die() { roundTripTest() { if [ -n "$3" ]; then - local_c="$3" - local_p="$2" + cLevel="$3" + proba="$2" else - local_c="$2" - local_p="" + cLevel="$2" + proba="" fi rm -f tmp1 tmp2 - $ECHO "roundTripTest: ./datagen $1 $local_p | $ZSTD -v$local_c | $ZSTD -d" - ./datagen $1 $local_p | $MD5SUM > tmp1 - ./datagen $1 $local_p | $ZSTD --ultra -v$local_c | $ZSTD -d | $MD5SUM > tmp2 + $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d" + ./datagen $1 $proba | $MD5SUM > tmp1 + ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d | $MD5SUM > tmp2 $DIFF -q tmp1 tmp2 } @@ -625,16 +625,15 @@ roundTripTest -g35000000 -P75 10 roundTripTest -g35000000 -P75 11 roundTripTest -g35000000 -P75 12 -roundTripTest -g18000000 -P80 13 -roundTripTest -g18000000 -P80 14 -roundTripTest -g18000000 -P80 15 -roundTripTest -g18000000 -P80 16 -roundTripTest -g18000000 -P80 17 - -roundTripTest -g50000000 -P94 18 -roundTripTest -g50000000 -P94 19 +roundTripTest -g18000013 -P80 13 +roundTripTest -g18000014 -P80 14 +roundTripTest -g18000015 -P80 15 +roundTripTest -g18000016 -P80 16 +roundTripTest -g18000017 -P80 17 +roundTripTest -g18000018 -P94 18 +roundTripTest -g18000019 -P94 19 -roundTripTest -g99000000 -P99 20 +roundTripTest -g68000020 -P99 20 roundTripTest -g6000000000 -P99 1 fileRoundTripTest -g4193M -P99 1 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 9b2b8eaf8..8b84400db 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -95,19 +95,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr) return rand32 >> 5; } -static void* allocFunction(void* opaque, size_t size) -{ - void* address = malloc(size); - (void)opaque; - return address; -} - -static void freeFunction(void* opaque, void* address) -{ - (void)opaque; - free(address); -} - /*====================================================== * Basic Unit tests @@ -1543,7 +1530,6 @@ int main(int argc, const char** argv) int bigTests = (sizeof(size_t) == 8); e_api selected_api = simple_api; const char* const programName = argv[0]; - ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL }; ZSTD_customMem const customNULL = ZSTD_defaultCMem; /* Check command line */ @@ -1657,10 +1643,7 @@ int main(int argc, const char** argv) if (testNb==0) { result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */ - if (!result) { - DISPLAYLEVEL(3, "Unit tests using customMem :\n") - result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */ - } } + } if (!result) { switch(selected_api)