From: Yann Collet Date: Fri, 12 Aug 2016 11:04:27 +0000 (+0200) Subject: added : streaming decompression API X-Git-Tag: v0.8.1^2~18 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=104e5b072dadfe5dc0fc65d2a5b6671f0f28893c;p=thirdparty%2Fzstd.git added : streaming decompression API --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d14a1cde8..6e8f0fba7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2758,7 +2758,9 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, -/*-====== Streaming ======-*/ +/* ****************************************************************** +* Streaming +********************************************************************/ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; @@ -2815,7 +2817,10 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) } -/* Initialization */ +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } +size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, @@ -2863,7 +2868,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) } -/* Compression */ +/*====== Compression ======*/ typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; @@ -2969,7 +2974,7 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor } -/* Finalize */ +/*====== Finalize ======*/ /*! ZSTD_flushStream() : * @return : amount of data remaining to flush */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 052ccd601..57ca21112 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1301,3 +1301,243 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, dst, dstCapacity, src, srcSize); } + + +/*===================================== +* Streaming decompression +*====================================*/ + +typedef enum { zdss_init, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +/* *** Resource management *** */ +struct ZSTD_DStream_s { + ZSTD_DCtx* zd; + ZSTD_frameParams fParams; + ZSTD_dStreamStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t lhSize; + ZSTD_customMem customMem; +}; /* typedef'd to ZSTD_DStream within "zstd.h" */ + + +ZSTD_DStream* ZSTD_createDStream(void) +{ + return ZSTD_createDStream_advanced(defaultCustomMem); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_DStream* zds; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream)); + if (zds==NULL) return NULL; + memset(zds, 0, sizeof(ZSTD_DStream)); + memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); + zds->zd = ZSTD_createDCtx_advanced(customMem); + if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->stage = zdss_init; + return zds; +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + if (zds==NULL) return 0; /* support free on null */ + ZSTD_freeDCtx(zds->zd); + if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); + if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + zds->customMem.customFree(zds->customMem.opaque, zds); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } + +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize); +} + +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + return ZSTD_initDStream_usingDict(zds, NULL, 0); +} + + +/* *** Decompression *** */ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input) +{ + const char* const istart = (const char*)(input->ptr); + const char* const iend = istart + input->size; + const char* ip = istart; + char* const ostart = (char*)(output->ptr); + char* const oend = ostart + output->size; + char* op = ostart; + U32 someMoreWork = 1; + + while (someMoreWork) { + switch(zds->stage) + { + case zdss_init : + return ERROR(init_missing); + + case zdss_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize); + if (ZSTD_isError(hSize)) return hSize; + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); + zds->lhSize += iend-ip; + input->ptr = iend; + input->size = 0; + return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */ + size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size); + if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ + if (h1Size < zds->lhSize) { /* long header */ + size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd); + size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size); + if (ZSTD_isError(h2Result)) return h2Result; + } } + + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->fParams.windowSize + blockSize; + zds->blockSize = blockSize; + if (zds->inBuffSize < blockSize) { + zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); + zds->inBuffSize = blockSize; + zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + } + if (zds->outBuffSize < neededOutSize) { + zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + zds->outBuffSize = neededOutSize; + zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize); + if (zds->outBuff == NULL) return ERROR(memory_allocation); + } } + zds->stage = zdss_read; + /* pass-through */ + + case zdss_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + if (neededInSize==0) { /* end of frame */ + zds->stage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); + size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + break; + } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->stage = zdss_load; + /* pass-through */ + } + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); + size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, + zds->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zds->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + /* pass-through */ + } } + + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->stage = zdss_read; + if (zds->outStart + zds->blockSize > zds->outBuffSize) + zds->outStart = zds->outEnd = 0; + break; + } + /* cannot flush everything */ + someMoreWork = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + input->ptr = ip; + input->size = (size_t)(iend-ip); + output->ptr = op; + output->size = (size_t) (oend-op); + output->nbBytesWritten += (size_t)(op-ostart); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); + if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); + if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zds->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + + + + + diff --git a/lib/zstd.h b/lib/zstd.h index 3c563e38f..ff2f627e7 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -344,6 +344,9 @@ typedef struct ZSTD_CStream_s ZSTD_CStream; ZSTD_CStream* ZSTD_createCStream(void); size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +size_t ZSTD_CStreamInSize(void); /*!< recommended size for input buffer */ +size_t ZSTD_CStreamOutSize(void); /*!< recommended size for output buffer */ + size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input); size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); @@ -362,9 +365,16 @@ typedef struct ZSTD_DStream_s ZSTD_DStream; ZSTD_DStream* ZSTD_createDStream(void); size_t ZSTD_freeDStream(ZSTD_DStream* zds); +size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ + size_t ZSTD_initDStream(ZSTD_DStream* zds); size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input); +/* advanced */ +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + /* ******************************************************************