From: Yann Collet Date: Fri, 2 Dec 2016 00:24:04 +0000 (-0800) Subject: replaced usage of FSE_buildCTable by FSE_buildCTable_wksp, using less stack space... X-Git-Tag: v1.1.2~50 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=643d9a234b5022ec7d5717dcf6f582bd4e481e22;p=thirdparty%2Fzstd.git replaced usage of FSE_buildCTable by FSE_buildCTable_wksp, using less stack space in the process --- diff --git a/frame.txt b/frame.txt new file mode 100644 index 000000000..58902c328 --- /dev/null +++ b/frame.txt @@ -0,0 +1,128 @@ +common/fse_decompress.c:321:8: warning: stack frame size of 16984 bytes in function 'FSE_decompress' [-Wframe-larger-than=] +size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) + ^ +1 warning generated. +compress/fse_compress.c:185:8: warning: stack frame size of 4120 bytes in function 'FSE_buildCTable' [-Wframe-larger-than=] +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) + ^ +compress/fse_compress.c:423:8: warning: stack frame size of 4152 bytes in function 'FSE_countFast' [-Wframe-larger-than=] +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + ^ +compress/fse_compress.c:442:8: warning: stack frame size of 4120 bytes in function 'FSE_count' [-Wframe-larger-than=] +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, + ^ +compress/fse_compress.c:784:8: warning: stack frame size of 5752 bytes in function 'FSE_compress_wksp' [-Wframe-larger-than=] +size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) + ^ +compress/fse_compress.c:836:8: warning: stack frame size of 14392 bytes in function 'FSE_compress2' [-Wframe-larger-than=] +size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) + ^ +compress/fse_compress.c:844:8: warning: stack frame size of 14392 bytes in function 'FSE_compress' [-Wframe-larger-than=] +size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) + ^ +6 warnings generated. +compress/huf_compress.c:332:8: warning: stack frame size of 4472 bytes in function 'HUF_buildCTable' [-Wframe-larger-than=] +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) + ^ +compress/huf_compress.c:567:8: warning: stack frame size of 4136 bytes in function 'HUF_compress1X' [-Wframe-larger-than=] +size_t HUF_compress1X (void* dst, size_t dstSize, + ^ +compress/huf_compress.c:582:8: warning: stack frame size of 4136 bytes in function 'HUF_compress2' [-Wframe-larger-than=] +size_t HUF_compress2 (void* dst, size_t dstSize, + ^ +compress/huf_compress.c:590:8: warning: stack frame size of 4136 bytes in function 'HUF_compress' [-Wframe-larger-than=] +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) + ^ +4 warnings generated. +compress/zstd_compress.c:2711:8: warning: stack frame size of 8168 bytes in function 'ZSTD_compress' [-Wframe-larger-than=] +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) + ^ +1 warning generated. +decompress/huf_decompress.c:224:8: warning: stack frame size of 8264 bytes in function 'HUF_decompress1X2' [-Wframe-larger-than=] +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +decompress/huf_decompress.c:347:8: warning: stack frame size of 8280 bytes in function 'HUF_decompress4X2' [-Wframe-larger-than=] +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +decompress/huf_decompress.c:445:8: warning: stack frame size of 2408 bytes in function 'HUF_readDTableX4' [-Wframe-larger-than=] +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) + ^ +decompress/huf_decompress.c:636:8: warning: stack frame size of 16456 bytes in function 'HUF_decompress1X4' [-Wframe-larger-than=] +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +decompress/huf_decompress.c:758:8: warning: stack frame size of 16472 bytes in function 'HUF_decompress4X4' [-Wframe-larger-than=] +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +5 warnings generated. +dictBuilder/divsufsort.c:1441:1: warning: stack frame size of 2472 bytes in function 'sort_typeBstar' [-Wframe-larger-than=] +sort_typeBstar(const unsigned char *T, int *SA, +^ +1 warning generated. +dictBuilder/zdict.c:828:8: warning: stack frame size of 7176 bytes in function 'ZDICT_addEntropyTablesFromBuffer_advanced' [-Wframe-larger-than=] +size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + ^ +1 warning generated. +common/fse_decompress.c:321:8: warning: stack frame size of 16984 bytes in function 'FSE_decompress' [-Wframe-larger-than=] +size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) + ^ +1 warning generated. +compress/fse_compress.c:185:8: warning: stack frame size of 4120 bytes in function 'FSE_buildCTable' [-Wframe-larger-than=] +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) + ^ +compress/fse_compress.c:423:8: warning: stack frame size of 4152 bytes in function 'FSE_countFast' [-Wframe-larger-than=] +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + ^ +compress/fse_compress.c:442:8: warning: stack frame size of 4120 bytes in function 'FSE_count' [-Wframe-larger-than=] +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, + ^ +compress/fse_compress.c:784:8: warning: stack frame size of 5752 bytes in function 'FSE_compress_wksp' [-Wframe-larger-than=] +size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) + ^ +compress/fse_compress.c:836:8: warning: stack frame size of 14392 bytes in function 'FSE_compress2' [-Wframe-larger-than=] +size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) + ^ +compress/fse_compress.c:844:8: warning: stack frame size of 14392 bytes in function 'FSE_compress' [-Wframe-larger-than=] +size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) + ^ +6 warnings generated. +compress/huf_compress.c:332:8: warning: stack frame size of 4472 bytes in function 'HUF_buildCTable' [-Wframe-larger-than=] +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) + ^ +compress/huf_compress.c:567:8: warning: stack frame size of 4136 bytes in function 'HUF_compress1X' [-Wframe-larger-than=] +size_t HUF_compress1X (void* dst, size_t dstSize, + ^ +compress/huf_compress.c:582:8: warning: stack frame size of 4136 bytes in function 'HUF_compress2' [-Wframe-larger-than=] +size_t HUF_compress2 (void* dst, size_t dstSize, + ^ +compress/huf_compress.c:590:8: warning: stack frame size of 4136 bytes in function 'HUF_compress' [-Wframe-larger-than=] +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) + ^ +4 warnings generated. +compress/zstd_compress.c:2711:8: warning: stack frame size of 8168 bytes in function 'ZSTD_compress' [-Wframe-larger-than=] +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) + ^ +1 warning generated. +decompress/huf_decompress.c:224:8: warning: stack frame size of 8264 bytes in function 'HUF_decompress1X2' [-Wframe-larger-than=] +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +decompress/huf_decompress.c:347:8: warning: stack frame size of 8280 bytes in function 'HUF_decompress4X2' [-Wframe-larger-than=] +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +decompress/huf_decompress.c:445:8: warning: stack frame size of 2408 bytes in function 'HUF_readDTableX4' [-Wframe-larger-than=] +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) + ^ +decompress/huf_decompress.c:636:8: warning: stack frame size of 16456 bytes in function 'HUF_decompress1X4' [-Wframe-larger-than=] +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +decompress/huf_decompress.c:758:8: warning: stack frame size of 16472 bytes in function 'HUF_decompress4X4' [-Wframe-larger-than=] +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) + ^ +5 warnings generated. +dictBuilder/divsufsort.c:1441:1: warning: stack frame size of 2472 bytes in function 'sort_typeBstar' [-Wframe-larger-than=] +sort_typeBstar(const unsigned char *T, int *SA, +^ +1 warning generated. +dictBuilder/zdict.c:828:8: warning: stack frame size of 7176 bytes in function 'ZDICT_addEntropyTablesFromBuffer_advanced' [-Wframe-larger-than=] +size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + ^ +1 warning generated. diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 840a3fec5..6627facfe 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -774,8 +774,8 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize, size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } -#define CHECK_E_F(e, f) size_t const e = f; if (ERR_isError(e)) return f -#define CHECK_F(f) { CHECK_E_F(_var_err__, f); } +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } /* FSE_compress_wksp() : * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). @@ -801,7 +801,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; /* Scan input and build symbol stats */ - { CHECK_E_F(maxCount, FSE_count(count, &maxSymbolValue, src, srcSize) ); + { CHECK_V_F(maxCount, FSE_count(count, &maxSymbolValue, src, srcSize) ); if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ @@ -811,13 +811,13 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); /* Write table description header */ - { CHECK_E_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); op += nc_err; } /* Compress */ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); - { CHECK_E_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); if (cSize == 0) return 0; /* not enough space for compressed data */ op += cSize; } diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index ff2e82ae8..4ed093eee 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -56,8 +56,8 @@ * Error Management ****************************************************************/ #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -#define CHECK_E_F(e, f) size_t const e = f; if (ERR_isError(e)) return f -#define CHECK_F(f) { CHECK_E_F(_var_err__, f); } +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } /* ************************************************************** @@ -97,7 +97,7 @@ size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, if (wtSize <= 1) return 0; /* Not compressible */ /* Scan input and build symbol stats */ - { CHECK_E_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) ); + { CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) ); if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ } @@ -106,13 +106,13 @@ size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); /* Write table description header */ - { CHECK_E_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); op += hSize; } /* Compress */ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); - { CHECK_E_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); if (cSize == 0) return 0; /* not enough space for compressed data */ op += cSize; } @@ -148,7 +148,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, huffWeight[n] = bitsToWeight[CTable[n].nbBits]; /* attempt weights compression by FSE */ - { CHECK_E_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) ); + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) ); if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ op[0] = (BYTE)hSize; return hSize+1; @@ -173,7 +173,7 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si U32 nbSymbols = 0; /* get symbol weights */ - CHECK_E_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); /* check result */ if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); @@ -424,8 +424,8 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si /* init */ if (dstSize < 8) return 0; /* not enough space to compress */ - { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op); - if (HUF_isError(errorCode)) return 0; } + { size_t const initErr = BIT_initCStream(&bitC, op, oend-op); + if (HUF_isError(initErr)) return 0; } n = srcSize & ~3; /* join to mod 4 */ switch (srcSize & 3) @@ -468,28 +468,28 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si if (srcSize < 12) return 0; /* no saving possible : too small input */ op += 6; /* jumpTable */ - { CHECK_E_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); if (cSize==0) return 0; MEM_writeLE16(ostart, (U16)cSize); op += cSize; } ip += segmentSize; - { CHECK_E_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); if (cSize==0) return 0; MEM_writeLE16(ostart+2, (U16)cSize); op += cSize; } ip += segmentSize; - { CHECK_E_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); if (cSize==0) return 0; MEM_writeLE16(ostart+4, (U16)cSize); op += cSize; } ip += segmentSize; - { CHECK_E_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); if (cSize==0) return 0; op += cSize; } @@ -523,19 +523,19 @@ static size_t HUF_compress_internal ( if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; /* Scan input and build symbol stats */ - { CHECK_E_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace) ); + { CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace) ); if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ } /* Build Huffman Tree */ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { CHECK_E_F(maxBits, HUF_buildCTable (table.CTable, table.count, maxSymbolValue, huffLog) ); + { CHECK_V_F(maxBits, HUF_buildCTable (table.CTable, table.count, maxSymbolValue, huffLog) ); huffLog = (U32)maxBits; } /* Write table description header */ - { CHECK_E_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) ); + { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) ); if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ op += hSize; } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 853b0c3bf..30296989b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2509,6 +2509,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ const BYTE* const dictEnd = dictPtr + dictSize; short offcodeNCount[MaxOff+1]; unsigned offcodeMaxValue = MaxOff; + BYTE scratchBuffer[1<hufTable, 255, dict, dictSize); if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); @@ -2520,7 +2521,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ - CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + CHECK_E (FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -2531,7 +2532,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); /* Every match length code must have non-zero probability */ CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); - CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); + CHECK_E (FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } @@ -2542,7 +2543,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); /* Every literal length code must have non-zero probability */ CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); - CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); + CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); dictPtr += litlengthHeaderSize; }