From: Yann Collet Date: Sat, 14 Dec 2024 01:22:19 +0000 (-0800) Subject: add the compressSequences() benchmark scenario X-Git-Tag: v1.5.7^2~48^2~33 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8ab04097ed9736923405e4928f928e49654e2c9a;p=thirdparty%2Fzstd.git add the compressSequences() benchmark scenario --- diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bb92bd1ff..5aad13531 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -7058,7 +7058,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, size_t cSize = 0; /* Transparent initialization stage, same as compressStream2() */ - DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity); + DEBUGLOG(4, "ZSTD_compressSequences (nbSeqs=%zu,dstCapacity=%zu)", inSeqsSize, dstCapacity); assert(cctx != NULL); FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 90d462fd2..294835946 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -741,14 +741,6 @@ ZSTD_storeSeqOnly(SeqStore_t* seqStorePtr, U32 offBase, size_t matchLength) { -#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) - static const BYTE* g_start = NULL; - if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ - { U32 const pos = (U32)((const BYTE*)literals - g_start); - DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u", - pos, (U32)litLength, (U32)matchLength, (U32)offBase); - } -#endif assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); /* update seqStorePtr->lit, so that we know how many literals were or will be copied */ assert(seqStorePtr->maxNbLit <= 128 KB); diff --git a/lib/zstd.h b/lib/zstd.h index 3c0e836c9..435fe34d4 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1328,7 +1328,7 @@ typedef struct { * * Note: This field is optional. ZSTD_generateSequences() will calculate the value of * 'rep', but repeat offsets do not necessarily need to be calculated from an external - * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * sequence provider perspective. For example, ZSTD_compressSequences() does not * use this 'rep' field at all (as of now). */ } ZSTD_Sequence; @@ -1588,7 +1588,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); * @param zc The compression context to be used for ZSTD_compress2(). Set any * compression parameters you need on this context. * @param outSeqs The output sequences buffer of size @p outSeqsSize - * @param outSeqsSize The size of the output sequences buffer. + * @param outSeqsCapacity The size of the output sequences buffer. * ZSTD_sequenceBound(srcSize) is an upper bound on the number * of sequences that can be generated. * @param src The source buffer to generate sequences from of size @p srcSize. @@ -1606,7 +1606,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, - ZSTD_Sequence* outSeqs, size_t outSeqsSize, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, const void* src, size_t srcSize); /*! ZSTD_mergeBlockDelimiters() : diff --git a/tests/fullbench.c b/tests/fullbench.c index 7218bd6e6..ff512149a 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -12,6 +12,7 @@ /*_************************************ * Includes **************************************/ +#include #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still bench some deprecated functions */ #include "util.h" /* Compiler options, UTIL_GetFileSize */ #include /* malloc */ @@ -272,7 +273,7 @@ local_ZSTD_decodeLiteralsHeader(const void* src, size_t srcSize, void* dst, size return ZSTD_decodeLiteralsHeader(g_zdc, src, srcSize); } -static PrepResult prepSequences(const void* src, size_t srcSize, int cLevel) +static PrepResult prepSequences1stBlock(const void* src, size_t srcSize, int cLevel) { PrepResult r = PREPRESULT_INIT; size_t const dstCapacity = srcSize; @@ -456,7 +457,6 @@ local_ZSTD_decompressStream(const void* src, size_t srcSize, return buffOut.pos; } -#ifndef ZSTD_DLL_IMPORT static size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload) @@ -523,11 +523,54 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, return regeneratedSize; } -#endif + +static PrepResult prepSequences(const void* src, size_t srcSize, int cLevel) +{ + PrepResult r = PREPRESULT_INIT; + size_t const dstCapacity = ZSTD_compressBound(srcSize); + void* const dst = malloc(dstCapacity); + size_t const prepCapacity = dstCapacity * 4; + void* prepBuffer = malloc(prepCapacity); + void* sequencesStart = (char*)prepBuffer + 2*sizeof(unsigned); + ZSTD_Sequence* const seqs = sequencesStart; + size_t const seqsCapacity = prepCapacity / sizeof(ZSTD_Sequence); + size_t nbSeqs; + ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel); + nbSeqs = ZSTD_generateSequences(g_zcc, seqs, seqsCapacity, src, srcSize); + CONTROL(srcSize < UINT_MAX); + MEM_write32(prepBuffer, (U32)srcSize); + MEM_write32((char*)prepBuffer+4, (U32)nbSeqs); + memcpy(seqs + nbSeqs, src, srcSize); + r.prepBuffer = prepBuffer; + r.prepSize = 8 + sizeof(ZSTD_Sequence)*nbSeqs + srcSize; + r.dst = dst; + r.dstCapacity = dstCapacity; + return r; +} + +static size_t local_compressSequences(const void* input, size_t inputSize, + void* dst, size_t dstCapacity, + void* payload) +{ + const char* ip = input; + size_t srcSize = MEM_read32(ip); + size_t nbSeqs = MEM_read32(ip+=4); + const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4); + const void* src = (ip+=nbSeqs * sizeof(ZSTD_Sequence)); + ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); + assert(8 + nbSeqs * sizeof(ZSTD_Sequence) + srcSize == inputSize); (void)inputSize; + (void)payload; + + return ZSTD_compressSequences(g_zcc, dst, dstCapacity, seqs, nbSeqs, src, srcSize); +} + static PrepResult prepCopy(const void* src, size_t srcSize, int cLevel) { PrepResult r = PREPRESULT_INIT; + (void)cLevel; r.prepSize = srcSize; r.prepBuffer = malloc(srcSize); CONTROL(r.prepBuffer != NULL); @@ -577,10 +620,11 @@ static BenchScenario kScenarios[] = { { "compressStream2, continue", NULL, local_ZSTD_compressStream2_continue }, { "compressStream2, -T2, continue", NULL, local_ZSTD_compress_generic_T2_continue }, { "compressStream2, -T2, end", NULL, local_ZSTD_compress_generic_T2_end }, + { "compressSequences", prepSequences, local_compressSequences }, #ifndef ZSTD_DLL_IMPORT { "decodeLiteralsHeader (1st block)", prepLiterals, local_ZSTD_decodeLiteralsHeader }, { "decodeLiteralsBlock (1st block)", prepLiterals, local_ZSTD_decodeLiteralsBlock }, - { "decodeSeqHeaders (1st block)", prepSequences, local_ZSTD_decodeSeqHeaders }, + { "decodeSeqHeaders (1st block)", prepSequences1stBlock, local_ZSTD_decodeSeqHeaders }, #endif }; #define NB_SCENARIOS (sizeof(kScenarios) / sizeof(kScenarios[0]))