From: Yann Collet Date: Sat, 4 Jun 2016 18:17:38 +0000 (+0200) Subject: removed `ZSTD_highbit()` from `zstd_internal.h`, as it is only used by `zstd_compress.c` X-Git-Tag: v0.7.0^2~49^2~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=49bb0041afe1f696e08690884450a8c2fbc10807;p=thirdparty%2Fzstd.git removed `ZSTD_highbit()` from `zstd_internal.h`, as it is only used by `zstd_compress.c` --- diff --git a/lib/common/zstd.h b/lib/common/zstd.h index a84b59501..2f7ab1e87 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -162,7 +162,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, /*--- Constants ---*/ -#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) @@ -350,7 +350,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity) + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 712016369..2e238faad 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -85,7 +85,7 @@ static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; -#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */ +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; @@ -162,28 +162,6 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length) while (op < oend); } -MEM_STATIC unsigned ZSTD_highbit(U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - _BitScanReverse(&r, val); - return (unsigned)r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); -# else /* Software version */ - static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - int r; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; - return r; -# endif -} - /*-******************************************* * Private interfaces @@ -211,7 +189,7 @@ typedef struct { MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif // #if ZSTD_OPT_DEBUG == 3 +#endif /* #if ZSTD_OPT_DEBUG == 3 */ typedef struct { void* buffer; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 05b35de11..49c3d2663 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -71,6 +71,27 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat ***************************************/ size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } +static U32 ZSTD_highbit32(U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse(&r, val); + return (unsigned)r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} /*-************************************* * Sequence storage @@ -176,8 +197,6 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) } -static unsigned ZSTD_highbit(U32 val); - /** ZSTD_checkCParams_advanced() : temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize) @@ -205,7 +224,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1< srcLog) cPar.windowLog = srcLog; } } if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; @@ -649,7 +668,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) size_t u; for (u=0; u63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll]; + llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit32(ll) + LL_deltaCode : LL_Code[ll]; } if (seqStorePtr->longLengthID==1) llCodeTable[seqStorePtr->longLengthPos] = MaxLL; @@ -659,7 +678,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) { const U32* const offsetTable = seqStorePtr->offsetStart; BYTE* const ofCodeTable = seqStorePtr->offCodeStart; size_t u; - for (u=0; u127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml]; + mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit32(ml) + ML_deltaCode : ML_Code[ml]; } if (seqStorePtr->longLengthID==2) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; @@ -1425,7 +1444,7 @@ static size_t ZSTD_insertBtAndFindBestMatch ( if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ break; /* drop, to guarantee consistency (miss a little bit of compression) */ @@ -1711,14 +1730,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32; int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) matchLength = mlRep, offset = 0, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; /* search a better one */ @@ -1730,14 +1749,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32; int const gain2 = (int)(ml2 * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) matchLength = ml2, offset = 0, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; @@ -1883,7 +1902,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) matchLength = repLength, offset = 0, start = ip; } } @@ -1891,8 +1910,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* search match, depth 1 */ { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; /* search a better one */ @@ -1913,7 +1932,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; int gain2 = (int)(repLength * 4); - int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); + int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) matchLength = repLength, offset = 0, start = ip; } } @@ -1921,8 +1940,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* search match, depth 2 */ { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); - int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 86fa13067..46b2cb14a 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -41,10 +41,10 @@ ***************************************/ FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) { - ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1); - ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1); - ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1); - ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1); + ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1); + ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1); + ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1); + ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1); ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); } @@ -106,7 +106,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY U32 price, u; if (litLength == 0) - return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1); + return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1); /* literals */ if (ssPtr->cachedLiterals == literals) { @@ -114,13 +114,13 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; for (u=0; u < additional; u++) - price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1); + price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1); ssPtr->cachedPrice = price; ssPtr->cachedLitLength = litLength; } else { price = litLength * ssPtr->log2litSum; for (u=0; u < litLength; u++) - price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); + price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1); if (litLength >= 12) { ssPtr->cachedLiterals = literals; @@ -139,8 +139,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; - price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1); + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1); } return price; @@ -150,8 +150,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { /* offset */ - BYTE offCode = (BYTE)ZSTD_highbit(offset+1); - U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); + BYTE offCode = (BYTE)ZSTD_highbit32(offset+1); + U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); /* match Length */ { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, @@ -163,8 +163,8 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; - price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1); + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1); } return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; @@ -190,13 +190,13 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; seqStorePtr->litLengthFreq[llCode]++; seqStorePtr->litLengthSum++; } /* match offset */ - { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + { BYTE offCode = (BYTE)ZSTD_highbit32(offset+1); seqStorePtr->offCodeSum++; seqStorePtr->offCodeFreq[offCode]++; } @@ -211,7 +211,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; seqStorePtr->matchLengthFreq[mlCode]++; seqStorePtr->matchLengthSum++; } diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index ef2b1ae5b..ab79f7492 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -60,7 +60,7 @@ #include "error_private.h" #include "fse.h" #include "huf_static.h" -#include "zstd_internal.h" +#include "zstd_internal.h" /* includes zstd.h */ #include "xxhash.h" #include "divsufsort.h" #define ZDICT_STATIC_LINKING_ONLY