From: Binh Vo Date: Fri, 11 Jun 2021 16:11:58 +0000 (-0400) Subject: Add API for fetching skippable frame content X-Git-Tag: v1.5.1~1^2~146^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F2708%2Fhead;p=thirdparty%2Fzstd.git Add API for fetching skippable frame content --- diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 910bc034c..ee707ce6c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -380,6 +380,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) return 0; } +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + /** ZSTD_frameHeaderSize_internal() : * srcSize must be large enough to reach header size fields. * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. @@ -503,7 +516,6 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); } - /** ZSTD_getFrameContentSize() : * compatible with legacy mode * @return : decompressed size of the single frame pointed to be `src` if known, otherwise @@ -544,6 +556,37 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) } } +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize) +{ + U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; +} + /** ZSTD_findDecompressedSize() : * compatible with legacy mode * `srcSize` must be the exact length of some number of ZSTD compressed and/or diff --git a/lib/zstd.h b/lib/zstd.h index 4651e6c4d..054fb9a5a 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1441,6 +1441,26 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned magicVariant); +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + /*************************************** * Memory management diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 553209f4d..960050f96 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1830,6 +1830,46 @@ static int basicUnitTests(U32 const seed, double compressibility) if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; DISPLAYLEVEL(3, "OK \n"); + /* Simple API skippable frame test */ + DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); + { + U32 i; + unsigned readMagic; + unsigned long long receivedSize; + size_t skippableSize; + const U32 skipLen = 129 KB; + char* const skipBuff = (char*)malloc(skipLen); + assert(skipBuff != NULL); + for (i = 0; i < skipLen; i++) + skipBuff[i] = (BYTE) ((seed + i) % 256); + skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, + skipBuff, skipLen, seed % 15); + CHECK_Z(skippableSize); + CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); + receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); + CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); + CHECK_EQ(seed % 15, readMagic); + if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error; + + free(skipBuff); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++); + { + unsigned readMagic; + unsigned long long receivedSize; + size_t skippableSize; + skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, + CNBuffer, 0, seed % 15); + CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); + CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); + receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); + CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); + CHECK_EQ(seed % 15, readMagic); + } + DISPLAYLEVEL(3, "OK \n"); + /* Dictionary and CCtx Duplication tests */ { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();