From: Yann Collet Date: Mon, 16 Feb 2015 17:06:26 +0000 (+0100) Subject: New streaming API behavior, to solve issue 19 (https://github.com/Cyan4973/zstd/issue... X-Git-Tag: v0.1.0~4^2^2~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c5d46b5c90197bd6305979224d62050340c2d287;p=thirdparty%2Fzstd.git New streaming API behavior, to solve issue 19 (https://github.com/Cyan4973/zstd/issues/19) --- diff --git a/lib/zstd.c b/lib/zstd.c index 38e7634a2..61beaf304 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -141,7 +141,7 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51C; /* Initial (limited) frame for #define KB *(1 <<10) #define MB *(1 <<20) -#define GB *(1U<<20) +#define GB *(1U<<30) #define BLOCKSIZE (128 KB) /* define, for static allocation */ static const U32 g_maxDistance = 512 KB; @@ -1708,24 +1708,24 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src } -/****************************** +/******************************* * Streaming Decompression API -******************************/ +*******************************/ typedef struct { U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)]; size_t expected; blockType_t bType; - U32 started; + U32 phase; } dctx_t; ZSTD_dctx_t ZSTD_createDCtx(void) { dctx_t* dctx = (dctx_t*)malloc(sizeof(dctx_t)); - dctx->expected = 4 + ZSTD_blockHeaderSize; // Frame Header + Block Header - dctx->started = 0; + dctx->expected = ZSTD_frameHeaderSize; + dctx->phase = 0; return (ZSTD_dctx_t)dctx; } @@ -1736,7 +1736,7 @@ size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx) } -size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx) +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx) { return ((dctx_t*)dctx)->expected; } @@ -1744,63 +1744,67 @@ size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx) size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { dctx_t* ctx = (dctx_t*)dctx; - size_t cSize = srcSize - ZSTD_blockHeaderSize; - size_t rSize; - // Sanity check + /* Sanity check */ if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_wrongSrcSize; - // Decompress - if (!ctx->started) + /* Decompress : frame header */ + if (ctx->phase == 0) { - // Just check correct magic header + /* Check frame magic header */ U32 magicNumber = ZSTD_readBE32(src); if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_wrongMagicNumber; - rSize = 0; + ctx->phase = 1; + ctx->expected = ZSTD_blockHeaderSize; + return 0; } - else + + /* Decompress : block header */ + if (ctx->phase == 1) { + blockProperties_t bp; + size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(blockSize)) return blockSize; + if (bp.blockType == bt_end) + { + ctx->expected = 0; + ctx->phase = 0; + } + else + { + ctx->expected = blockSize; + ctx->bType = bp.blockType; + ctx->phase = 2; + } + + return 0; + } + + /* Decompress : block content */ + { + size_t rSize; switch(ctx->bType) { case bt_compressed: - rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, cSize); + rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize); break; case bt_raw : - rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, cSize); + rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize); break; case bt_rle : return (size_t)-ZSTD_ERROR_GENERIC; /* not yet handled */ break; - case bt_end : + case bt_end : /* should never happen (filtered at phase 1) */ rSize = 0; break; default: return (size_t)-ZSTD_ERROR_GENERIC; } + ctx->phase = 1; + ctx->expected = ZSTD_blockHeaderSize; + return rSize; } - // Prepare next block - { - const BYTE* header = (const BYTE*)src; - blockProperties_t bp; - size_t blockSize; - header += cSize; - blockSize = ZSTD_getcBlockSize(header, ZSTD_blockHeaderSize, &bp); - if (ZSTD_isError(blockSize)) return blockSize; - if (bp.blockType == bt_end) - { - ctx->expected = 0; - ctx->started = 0; - } - else - { - ctx->expected = blockSize + ZSTD_blockHeaderSize; - ctx->bType = bp.blockType; - ctx->started = 1; - } - } - - return rSize; } diff --git a/lib/zstd.h b/lib/zstd.h index 47ce21fd5..cda16b88a 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -47,7 +47,7 @@ extern "C" { **************************************/ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */ -#define ZSTD_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) unsigned ZSTD_versionNumber (void); diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 59d3b02ad..9222a8c86 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -57,7 +57,7 @@ typedef void* ZSTD_dctx_t; ZSTD_dctx_t ZSTD_createDCtx(void); size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx); -size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx); +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx); size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); @@ -77,4 +77,4 @@ typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes; /* expo #if defined (__cplusplus) } -#endif \ No newline at end of file +#endif diff --git a/programs/fileio.c b/programs/fileio.c index fcf3d4f3a..7c5d12e65 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -327,10 +327,10 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* Init */ FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); dctx = ZSTD_createDCtx(); - toRead = ZSTD_getNextcBlockSize(dctx); - if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header"); /* check header */ + toRead = ZSTD_nextSrcSizeToDecompress(dctx); + if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header"); sizeCheck = fread(header, (size_t)1, toRead, finput); if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header @@ -348,7 +348,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); /* Main decompression Loop */ - toRead = ZSTD_getNextcBlockSize(dctx); + toRead = ZSTD_nextSrcSizeToDecompress(dctx); while (toRead) { size_t readSize, decodedSize; @@ -361,16 +361,19 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* Decode block */ decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); - /* Write block */ - sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file"); - filesize += decodedSize; + if (decodedSize) /* not a header */ + { + /* Write block */ + sizeCheck = fwrite(op, 1, decodedSize, foutput); + if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file"); + filesize += decodedSize; + op += decodedSize; + if (op==oend) op = outBuff; + DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) ); + } /* prepare for next Block */ - op += decodedSize; - if (op==oend) op = outBuff; - toRead = ZSTD_getNextcBlockSize(dctx); - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) ); + toRead = ZSTD_nextSrcSizeToDecompress(dctx); } DISPLAYLEVEL(2, "\r%79s\r", "");