From: Yann Collet Date: Thu, 24 May 2018 23:21:02 +0000 (-0700) Subject: Merge branch 'dev' into staticDictCost X-Git-Tag: v1.3.5~3^2~41^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f6ad59ab5ccfe27ae40178a1669246681b8f5bab;p=thirdparty%2Fzstd.git Merge branch 'dev' into staticDictCost --- f6ad59ab5ccfe27ae40178a1669246681b8f5bab diff --cc lib/compress/zstd_compress.c index 1cd2977a0,538319bcb..df7cb8aa7 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@@ -1567,10 -1734,14 +1734,14 @@@ typedef enum ZSTD_defaultAllowed = 1 } ZSTD_defaultPolicy_e; -MEM_STATIC -symbolEncodingType_e ZSTD_selectEncodingType( +MEM_STATIC symbolEncodingType_e +ZSTD_selectEncodingType( - FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq, - U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed) + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) { #define MIN_SEQ_FOR_DYNAMIC_FSE 64 #define MAX_SEQ_FOR_STATIC_FSE 1000 @@@ -1863,13 -2064,17 +2064,17 @@@ MEM_STATIC size_t ZSTD_compressSequence { U32 max = MaxLL; size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace); DEBUGLOG(5, "Building LL table"); - nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; - LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed); + nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; + LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, count, max, mostFrequent, nbSeq, LLFSELog, prevEntropy->fse.litlengthCTable, LL_defaultNorm, LL_defaultNormLog, ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), - workspace, HUF_WORKSPACE_SIZE); + count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable), ++ prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), + workspace, HUF_WORKSPACE_SIZE); if (ZSTD_isError(countSize)) return countSize; + if (LLtype == set_compressed) + lastNCount = op; op += countSize; } } /* build CTable for Offsets */ @@@ -1878,26 -2083,32 +2083,32 @@@ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; DEBUGLOG(5, "Building OF table"); - nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; - Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy); + nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode; + Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, count, max, mostFrequent, nbSeq, OffFSELog, prevEntropy->fse.offcodeCTable, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); + assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), - workspace, HUF_WORKSPACE_SIZE); + count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable), ++ prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), + workspace, HUF_WORKSPACE_SIZE); if (ZSTD_isError(countSize)) return countSize; + if (Offtype == set_compressed) + lastNCount = op; op += countSize; } } /* build CTable for MatchLengths */ { U32 max = MaxML; size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace); DEBUGLOG(5, "Building ML table"); - nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; - MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed); + nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; + MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, count, max, mostFrequent, nbSeq, MLFSELog, prevEntropy->fse.matchlengthCTable, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); + assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), - workspace, HUF_WORKSPACE_SIZE); + count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable), ++ prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), + workspace, HUF_WORKSPACE_SIZE); if (ZSTD_isError(countSize)) return countSize; + if (MLtype == set_compressed) + lastNCount = op; op += countSize; } } diff --cc lib/compress/zstd_ldm.c index 423aedafd,03d1f54c4..819c62aa3 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@@ -594,7 -595,9 +593,7 @@@ size_t ZSTD_ldm_blockCompress(rawSeqSto { unsigned const minMatch = cParams->searchLength; ZSTD_blockCompressor const blockCompressor = - ZSTD_selectBlockCompressor(cParams->strategy, extDict); - ZSTD_selectBlockCompressor(cParams->strategy, - ZSTD_matchState_dictMode(ms)); - BYTE const* const base = ms->window.base; ++ ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms)); /* Input bounds */ BYTE const* const istart = (BYTE const*)src; BYTE const* const iend = istart + srcSize; diff --cc lib/compress/zstd_opt.c index 4295ec1bc,521fbbf39..2a699c2ac --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@@ -88,10 -39,15 +88,10 @@@ static void ZSTD_rescaleFreqs(optState_ optPtr->priceType = zop_predef; assert(optPtr->symbolCosts != NULL); - if (optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */ + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */ - if (srcSize <= 8192) /* heuristic */ - optPtr->priceType = zop_static; - else { - assert(optPtr->priceType == zop_dynamic); - } + optPtr->priceType = zop_dynamic; assert(optPtr->litFreq != NULL); - assert(optPtr->symbolCosts != NULL); optPtr->litSum = 0; { unsigned lit; for (lit=0; lit<=MaxLit; lit++) {