From: Yann Collet Date: Wed, 26 Oct 2016 00:47:02 +0000 (-0700) Subject: added ZSTD_initDStream_usingDDict() . X-Git-Tag: v1.1.1~17 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=335ad5d4d4310f224aaa8cd75b55c371de891641;p=thirdparty%2Fzstd.git added ZSTD_initDStream_usingDDict() . slightly optimized ZSTD_initDStream() when no dictionary . fixed ZSTD_sizeof_CStream() . --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 5409c233a..85a5ff22d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2949,7 +2949,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) { if (zcs==NULL) return 0; /* support sizeof on NULL */ - return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize; + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize; } /*====== Compression ======*/ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 2a1232eee..f3ff4ebff 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -170,20 +170,22 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) { ZSTD_decompressBegin(dstDCtx); /* init */ - dstDCtx->dictEnd = srcDCtx->dictEnd; - dstDCtx->vBase = srcDCtx->vBase; - dstDCtx->base = srcDCtx->base; - dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; - dstDCtx->dictID = srcDCtx->dictID; - dstDCtx->litEntropy = srcDCtx->litEntropy; - dstDCtx->fseEntropy = srcDCtx->fseEntropy; - dstDCtx->LLTptr = srcDCtx->LLTable; - dstDCtx->MLTptr = srcDCtx->MLTable; - dstDCtx->OFTptr = srcDCtx->OFTable; - dstDCtx->HUFptr = srcDCtx->hufTable; - dstDCtx->rep[0] = srcDCtx->rep[0]; - dstDCtx->rep[1] = srcDCtx->rep[1]; - dstDCtx->rep[2] = srcDCtx->rep[2]; + if (srcDCtx) { /* support refDCtx on NULL */ + dstDCtx->dictEnd = srcDCtx->dictEnd; + dstDCtx->vBase = srcDCtx->vBase; + dstDCtx->base = srcDCtx->base; + dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; + dstDCtx->dictID = srcDCtx->dictID; + dstDCtx->litEntropy = srcDCtx->litEntropy; + dstDCtx->fseEntropy = srcDCtx->fseEntropy; + dstDCtx->LLTptr = srcDCtx->LLTable; + dstDCtx->MLTptr = srcDCtx->MLTable; + dstDCtx->OFTptr = srcDCtx->OFTable; + dstDCtx->HUFptr = srcDCtx->hufTable; + dstDCtx->rep[0] = srcDCtx->rep[0]; + dstDCtx->rep[1] = srcDCtx->rep[1]; + dstDCtx->rep[2] = srcDCtx->rep[2]; + } } @@ -1483,7 +1485,8 @@ typedef enum { zdss_init, zdss_loadHeader, /* *** Resource management *** */ struct ZSTD_DStream_s { ZSTD_DCtx* dctx; - ZSTD_DDict* ddict; + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; ZSTD_frameParams fParams; ZSTD_dStreamStage stage; char* inBuff; @@ -1533,7 +1536,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) if (zds==NULL) return 0; /* support free on null */ { ZSTD_customMem const cMem = zds->customMem; ZSTD_freeDCtx(zds->dctx); - ZSTD_freeDDict(zds->ddict); + ZSTD_freeDDict(zds->ddictLocal); ZSTD_free(zds->inBuff, cMem); ZSTD_free(zds->outBuff, cMem); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) @@ -1555,9 +1558,12 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di { zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - ZSTD_freeDDict(zds->ddict); - zds->ddict = ZSTD_createDDict(dict, dictSize); - if (zds->ddict == NULL) return ERROR(memory_allocation); + ZSTD_freeDDict(zds->ddictLocal); + if (dict) { + zds->ddictLocal = ZSTD_createDDict(dict, dictSize); + if (zds->ddictLocal == NULL) return ERROR(memory_allocation); + } else zds->ddictLocal = NULL; + zds->ddict = zds->ddictLocal; zds->legacyVersion = 0; zds->hostageByte = 0; return ZSTD_frameHeaderSize_prefix; @@ -1568,9 +1574,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) /**< note : ddict will just be referenced, and must outlive decompression session */ +{ + size_t const initResult = ZSTD_initDStream(zds); + zds->ddict = ddict; + return initResult; +} + size_t ZSTD_resetDStream(ZSTD_DStream* zds) { - if (zds->ddict == NULL) return ERROR(stage_wrong); /* must be init at least once */ zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->legacyVersion = 0; @@ -1593,7 +1605,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { if (zds==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize; + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize; } @@ -1634,15 +1646,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { + const void* const dict = zds->ddict ? zds->ddict->dict : NULL; + size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, - zds->ddict->dict, zds->ddict->dictSize)); + dict, dictSize)); zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); } else { return hSize; /* error */ } } #else - return hSize; + return hSize; #endif if (hSize != 0) { /* need more input */ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ @@ -1657,7 +1671,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* Consume header */ - ZSTD_refDCtx(zds->dctx, zds->ddict->refContext); + { const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL; + ZSTD_refDCtx(zds->dctx, refContext); + } { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 963078d8a..38b6a76cc 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -309,6 +309,25 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); } + /* DDict scenario */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize); + { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, 128 KB); + size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict); + if (ZSTD_isError(initError)) goto _output_error; + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ + ZSTD_freeDDict(ddict); + DISPLAYLEVEL(4, "OK \n"); + } + /* test ZSTD_setDStreamParameter() resilience */ DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++); { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */