From: Yann Collet Date: Sun, 5 Jun 2016 22:26:38 +0000 (+0200) Subject: new frame format, allowing custom window size X-Git-Tag: v0.7.0^2~49^2~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=673f0d7cdcd5d39d2c06d5c24ebaddaf05d57e22;p=thirdparty%2Fzstd.git new frame format, allowing custom window size --- diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index f8e8976f4..e449f6d3b 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -189,8 +189,7 @@ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); - -#endif +#endif /* ZBUFF_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 2f7ab1e87..b899656d1 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -180,6 +180,11 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, #define ZSTD_TARGETLENGTH_MIN 4 #define ZSTD_TARGETLENGTH_MAX 999 +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_min = 5; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ + /*--- Types ---*/ typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ @@ -305,15 +310,11 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci typedef struct { U64 frameContentSize; - U32 windowLog; + U32 windowSize; U32 dictID; U32 checksumFlag; } ZSTD_frameParams; -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ -static const size_t ZSTD_frameHeaderSize_min = 6; -static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; -static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 2e238faad..ebf3e880c 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -81,8 +81,8 @@ #define BIT1 2 #define BIT0 1 -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12 -static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1ae202772..39f073962 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -128,6 +128,7 @@ struct ZSTD_CCtx_s void* workSpace; size_t workSpaceSize; size_t blockSize; + U64 frameContentSize; XXH64_state_t xxhState; ZSTD_customMem customMem; @@ -257,7 +258,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface /*! ZSTD_resetCCtx_advanced() : note : 'params' is expected to be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, - ZSTD_parameters params, U32 reset) + ZSTD_parameters params, U64 frameContentSize, U32 reset) { /* note : params considered validated here */ const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); const U32 divider = (params.cParams.searchLength==3) ? 3 : 4; @@ -265,7 +266,10 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); const size_t hSize = ((size_t)1) << params.cParams.hashLog; - const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; + const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 : + ( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX : + ((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) ); + const size_t h3Size = 1 << hashLog3; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); /* Check if workSpace is large enough, alloc a new one if needed */ @@ -282,6 +286,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ XXH64_reset(&zc->xxhState, 0); + zc->hashLog3 = hashLog3; zc->hashTable3 = (U32*)(zc->workSpace); zc->hashTable = zc->hashTable3 + h3Size; zc->chainTable = zc->hashTable + hSize; @@ -298,6 +303,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->lowLimit = 0; zc->params = params; zc->blockSize = blockSize; + zc->frameContentSize = frameContentSize; if (params.cParams.strategy == ZSTD_btopt) { zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer); @@ -333,9 +339,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { if (srcCCtx->stage!=1) return ERROR(stage_wrong); - dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ /* copy tables */ @@ -403,8 +408,9 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) /* Frame format description Frame Header - [ Block Header - Block ] - Frame End 1) Frame Header - - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) - - 1 byte - Frame Descriptor + - 4 bytes : Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 1 byte : Frame Header Descriptor + - 1-13 bytes : Optional fields 2) Block Header - 3 bytes, starting with a 2-bits descriptor Uncompressed, Compressed, Frame End, unused @@ -417,13 +423,38 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) /* Frame descriptor - 1 byte, using : + // old + 1 byte - Alloc : bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) - bit 4 : minmatch 4(0) or 3(1) + bit 4 : reserved for windowLog (must be zero) bit 5 : reserved (must be zero) bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes - Optional : content size (0, 1, 2 or 8 bytes) + 1 byte - checker : + bit 0-1 : dictID (0, 1, 2 or 4 bytes) + bit 2-7 : reserved (must be zero) + + + // new + 1 byte - FrameHeaderDescription : + bit 0-1 : dictID (0, 1, 2 or 4 bytes) + bit 2-4 : reserved (must be zero) + bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present) + bit 6-7 : FrameContentFieldsize (0, 2, 4, or 8) + if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1; + + Optional : WindowLog (0 or 1 byte) + bit 0-2 : octal Fractional (1/8th) + bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB) + + Optional : dictID (0, 1, 2 or 4 bytes) + Automatic adaptation + 0 : no dictID + 1 : 1 - 255 + 2 : 256 - 65535 + 4 : all other values + + Optional : content size (0, 1, 2, 4 or 8 bytes) 0 : unknown 1 : 0-255 bytes 2 : 256 - 65535+256 @@ -2116,21 +2147,22 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; - U32 const fcsId = params.fParams.contentSizeFlag ? - (pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */ - 0; - BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */ - | (fcsId << 6) ); U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4)); + U32 const checksumFlag = params.fParams.checksumFlag; + U32 const windowSize = 1U << params.cParams.windowLog; + U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1)); + BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params.fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ + 0; + BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (directModeFlag<<5) + (fcsCode<<6) ); size_t pos; if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); MEM_writeLE32(dst, ZSTD_MAGICNUMBER); - op[4] = fAllocByte; - op[5] = fCheckByte; - pos = 6; + op[4] = frameHeaderDecriptionByte; pos=5; + if (!directModeFlag) op[pos++] = windowLogByte; switch(dictIDSizeCode) { default: /* impossible */ @@ -2139,12 +2171,12 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break; case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; } - switch(fcsId) + switch(fcsCode) { default: /* impossible */ - case 0 : break; - case 1 : op[pos] = (BYTE)(pledgedSrcSize); pos++; break; - case 2 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 0 : if (directModeFlag) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; } return pos; @@ -2161,7 +2193,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc, if (zc->stage==0) return ERROR(stage_wrong); if (frame && (zc->stage==1)) { /* copy saved header */ - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize, zc->dictID); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; dst = (char*)dst + fhSize; @@ -2344,11 +2376,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - { U32 const hashLog3 = (!pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); - zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; } - - { size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1); - if (ZSTD_isError(resetError)) return resetError; } + size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1); + if (ZSTD_isError(resetError)) return resetError; return ZSTD_compress_insertDictionary(zc, dict, dictSize); } diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index a6b99c976..b6e1806e3 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -191,10 +191,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (ZSTD_isError(h2Result)) return h2Result; } } - if (zbd->fParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) zbd->fParams.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for buffer allocation */ + zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); /* Frame header instruct buffer sizes */ - { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); + { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; if (zbd->inBuffSize < blockSize) { zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); @@ -202,7 +202,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } - { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; + { size_t const neededOutSize = zbd->fParams.windowSize + blockSize; if (zbd->outBuffSize < neededOutSize) { zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->outBuffSize = neededOutSize; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 63b674dfd..4630eb8a3 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -64,7 +64,7 @@ #include "huf.h" #include "zstd_internal.h" -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) # include "zstd_legacy.h" #endif @@ -218,22 +218,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) bit 0-1 : dictID (0, 1, 2 or 4 bytes) bit 2-7 : reserved (must be zero) - // new - 1 byte - Alloc : + 1 byte - FrameHeaderDescription : bit 0-1 : dictID (0, 1, 2 or 4 bytes) bit 2-4 : reserved (must be zero) - bit 5 : WindowLog skipped (note : if 1, then fcs > 0) - bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes - - OR : 0-3 : no windowLog, 1, 2, 4, 8 - 4-7 : windowLog, 0, 2, 4, 8 + bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present) + bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8) + if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1; - 1 byte - WindowLog (can be skipped if fcs>0) + Optional : WindowLog (0 or 1 byte) bit 0-2 : octal Fractional (1/8th) bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB) - Optional : dictID (0, 1, 2 or 4 bytes) Automatic adaptation 0 : no dictID @@ -241,11 +237,12 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) 2 : 256 - 65535 4 : all other values - Optional : content size (0, 1, 2 or 8 bytes) - 0 : unknown - 1 : 0-255 bytes - 2 : 256 - 65535+256 - 8 : up to 16 exa + Optional : content size (0, 1, 2, 4 or 8 bytes) + 0 : unknown (fcfs==0 and swl==0) + 1 : 0-255 bytes (fcfs==0 and swl==1) + 2 : 256 - 65535+256 (fcfs==1) + 4 : 0 - 4GB-1 (fcfs==2) + 8 : 0 - 16EB-1 (fcfs==3) */ @@ -321,15 +318,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); - { U32 const fcsId = (((const BYTE*)src)[4]) >> 6; - U32 const dictID =(((const BYTE*)src)[5]) & 3; - return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId] + ZSTD_did_fieldSize[dictID]; + { BYTE const fhd = ((const BYTE*)src)[4]; + U32 const dictID= fhd & 3; + U32 const directMode = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return ZSTD_frameHeaderSize_min + !directMode + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (directMode && !ZSTD_fcs_fieldSize[fcsId]); } } /** ZSTD_getFrameParams() : -* decode Frame Header, or provide expected `srcSize`. +* decode Frame Header, or require larger `srcSize`. * @return : 0, `fparamsPtr` is correctly filled, * >0, `srcSize` is too small, result is expected `srcSize`, * or an error code, which can be tested using ZSTD_isError() */ @@ -343,7 +343,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ memset(fparamsPtr, 0, sizeof(*fparamsPtr)); fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); - fparamsPtr->windowLog = 0; /* windowLog==0 means a frame is skippable */ + fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ return 0; } return ERROR(prefix_unknown); @@ -353,31 +353,48 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); if (srcSize < fhsize) return fhsize; } - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - { BYTE const allocByte = ip[4]; - BYTE const checkByte = ip[5]; - size_t pos = ZSTD_frameHeaderSize_min; - U32 const dictIDSizeCode = checkByte&3; - if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ - if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ - fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - fparamsPtr->checksumFlag = checkByte & 0x10; - switch(dictIDSizeCode) /* fcsId */ + { BYTE const fhdByte = ip[4]; + size_t pos = 5; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const directMode = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; + U32 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = 0; + if ((fhdByte & 0x18) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */ + if (!directMode) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + windowSize = (1U << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + + switch(dictIDSizeCode) { default: /* impossible */ - case 0 : fparamsPtr->dictID = 0; break; - case 1 : fparamsPtr->dictID = ip[pos]; pos++; break; - case 2 : fparamsPtr->dictID = MEM_readLE16(ip+pos); pos+=2; break; - case 3 : fparamsPtr->dictID = MEM_readLE32(ip+pos); pos+=4; break; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; } - switch(allocByte >> 6) /* fcsId */ + switch(fcsID) { default: /* impossible */ - case 0 : fparamsPtr->frameContentSize = 0; break; - case 1 : fparamsPtr->frameContentSize = ip[pos]; break; - case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+pos)+256; break; - case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+pos); break; - } } + case 0 : if (directMode) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (!windowSize) windowSize = (U32)frameContentSize; + if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + fparamsPtr->frameContentSize = frameContentSize; + fparamsPtr->windowSize = windowSize; + fparamsPtr->dictID = dictID; + fparamsPtr->checksumFlag = checksumFlag; + } return 0; } @@ -388,7 +405,6 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize) { size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); - if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); return result; @@ -1059,7 +1075,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c if (srcSize != dctx->expected) return ERROR(srcSize_wrong); if (dstCapacity) ZSTD_checkContinuity(dctx, dst); - /* Decompress : frame header; part 1 */ switch (dctx->stage) { case ZSTDds_getFrameHeaderSize : diff --git a/programs/fileio.c b/programs/fileio.c index 4d7da9130..37f6e7822 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -615,7 +615,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); /* Header loading (optional, saves one loop) */ - { size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */ + { size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */ size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); readSize = alreadyLoaded + loadedSize; } diff --git a/programs/playTests.sh b/programs/playTests.sh index 46e2fb6a4..f51d28ea2 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -23,7 +23,7 @@ roundTripTest() { isWindows=false ECHO="echo" case "$OS" in - Windows*) + Windows*) isWindows=true ECHO="echo -e" ;; @@ -42,8 +42,8 @@ file $ZSTD $ECHO "\n**** simple tests **** " ./datagen > tmp -$ZSTD -f tmp # trivial compression case, creates tmp.zst -$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) +$ZSTD -f tmp # trivial compression case, creates tmp.zst +$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) $ECHO "test : too large compression level (must fail)" $ZSTD -99 tmp && die "too large compression level undetected" $ECHO "test : compress to stdout" @@ -57,7 +57,7 @@ $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout $ZSTD --decompress tmpCompressed -c > tmpResult $ZSTD --decompress tmpCompressed --stdout > tmpResult if [ "$isWindows" = false ] ; then - $ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout + $ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout $ZSTD -d - < tmp.zst > /dev/null fi $ZSTD -dc < tmp.zst > /dev/null diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 78122e710..b890786cf 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -171,6 +171,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); cSize = skippableFrameSize + 8; + /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); @@ -186,14 +187,18 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo cSize += genSize; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); - /* Basic decompression test */ - DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + /* skippable frame test */ + DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSkipSize = cSize; genSize = CNBufferSize; { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); if (r != 0) goto _output_error; } if (genSize != 0) goto _output_error; /* skippable frame len is 0 */ + DISPLAYLEVEL(4, "OK \n"); + + /* Basic decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSize = cSize - readSkipSize; genSize = CNBufferSize; @@ -204,13 +209,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); { size_t i; - DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); for (i=0; i