From: Yann Collet Date: Tue, 6 Dec 2016 00:21:06 +0000 (-0800) Subject: added : dictID retrieval functions. X-Git-Tag: v1.1.2~38 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e7a41a595558903db0a681f889150158f7c76ff4;p=thirdparty%2Fzstd.git added : dictID retrieval functions. added : unit tests for dictID retrieval functions --- diff --git a/NEWS b/NEWS index 580889542..127d1ac8f 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,8 @@ cli : fixed : status displays total amount decoded, even for file consisting of cli : fixed : zstdcat API : changed : zbuff prototypes now generate deprecation warnings API : changed : streaming decompression implicit reset on starting new frame +API : added experimental : dictID retrieval functions +zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski Changed : reduced stack memory use v1.1.1 diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 7c4f54bd3..1b235ed6d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1778,6 +1778,45 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; } +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0; + return MEM_readLE32((const char*)dict + 4); +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameParams zfp; + size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + /*! ZSTD_decompress_usingDDict() : * Decompression using a pre-digested Dictionary diff --git a/lib/zstd.h b/lib/zstd.h index b7a8bb989..a1b0e62bc 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -462,6 +462,30 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); * Gives the amount of memory used by a given ZSTD_DDict */ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + /******************************************************************** * Advanced streaming functions diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f8110498d..1c1ae470a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,7 +27,7 @@ #include /* clock_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "zstd.h" /* ZSTD_VERSION_STRING */ -#include "zstd_errors.h" /* ZSTD_getErrorCode */ +#include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" @@ -273,6 +273,18 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(4, "test%3i : retrieve dictID from dictionary : ", testNb++); + { U32 const dictID = ZSTD_getDictID_fromDict(dictBuffer, dictSize); + if (dictID != 0) goto _output_error; /* non-conformant (content-only) dictionary */ + } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++); + { U32 const dictID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (dictID != 0) goto _output_error; /* non-conformant (content-only) dictionary */ + } + DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize,