From: Yann Collet Date: Fri, 1 Apr 2016 13:48:48 +0000 (+0200) Subject: minor variable isolation X-Git-Tag: v0.6.0^2~17^2~28^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=27caf2afca1ade42e7fd42d49ecdaa768758ec60;p=thirdparty%2Fzstd.git minor variable isolation --- diff --git a/lib/zbuff.c b/lib/zbuff.c index e26072d81..1027c6973 100644 --- a/lib/zbuff.c +++ b/lib/zbuff.c @@ -128,18 +128,14 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - size_t neededInBuffSize; - - ZSTD_adjustCParams(¶ms.cParams, pledgedSrcSize, dictSize); - neededInBuffSize = (size_t)1 << params.cParams.windowLog; - /* allocate buffers */ - if (zbc->inBuffSize < neededInBuffSize) { - zbc->inBuffSize = neededInBuffSize; - free(zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)malloc(neededInBuffSize); - if (zbc->inBuff == NULL) return ERROR(memory_allocation); - } + { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + if (zbc->inBuffSize < neededInBuffSize) { + zbc->inBuffSize = neededInBuffSize; + free(zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)malloc(neededInBuffSize); + if (zbc->inBuff == NULL) return ERROR(memory_allocation); + } } zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, zbc->inBuffSize); if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; @@ -148,23 +144,24 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, if (zbc->outBuff == NULL) return ERROR(memory_allocation); } - zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize); - if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize; + { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize); + if (ZSTD_isError(errorCode)) return errorCode; } zbc->inToCompress = 0; zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */ + zbc->stage = ZBUFFcs_load; return 0; /* ready to go */ } -ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) +size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); params.fParams.contentSizeFlag = 0; + ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); } @@ -190,11 +187,11 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, { U32 notDone = 1; const char* const istart = (const char*)src; - const char* ip = istart; const char* const iend = istart + *srcSizePtr; + const char* ip = istart; char* const ostart = (char*)dst; - char* op = ostart; char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; while (notDone) { switch(zbc->stage) @@ -203,19 +200,17 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, case ZBUFFcs_load: /* complete inBuffer */ - { - size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos; - size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); + { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos; + size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); zbc->inBuffPos += loaded; ip += loaded; if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) { notDone = 0; break; /* not enough input to get a full block : stop there, wait for more */ } } /* compress current block (note : this stage cannot be stopped in the middle) */ - { - void* cDst; + { void* cDst; size_t cSize; - size_t iSize = zbc->inBuffPos - zbc->inToCompress; + size_t const iSize = zbc->inBuffPos - zbc->inToCompress; size_t oSize = oend-op; if (oSize >= ZSTD_compressBound(iSize)) cDst = op; /* compress directly into output buffer (avoid flush stage) */ @@ -231,15 +226,13 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, if (cDst == op) { op += cSize; break; } /* no need to flush */ zbc->outBuffContentSize = cSize; zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_flush; - // break; /* flush stage follows */ + zbc->stage = ZBUFFcs_flush; /* continue to flush stage */ } case ZBUFFcs_flush: /* flush into dst */ - { - size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; - size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); + { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; + size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); op += flushed; zbc->outBuffFlushedSize += flushed; if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */ @@ -255,8 +248,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, *srcSizePtr = ip - istart; *dstCapacityPtr = op - ostart; - { - size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; + { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; if (hintInSize==0) hintInSize = zbc->blockSize; return hintInSize; } @@ -276,7 +268,7 @@ size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) { size_t srcSize = 0; - ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */ + ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL */ return zbc->outBuffContentSize - zbc->outBuffFlushedSize; } @@ -284,8 +276,8 @@ size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) { BYTE* const ostart = (BYTE*)dst; - BYTE* op = ostart; BYTE* const oend = ostart + *dstCapacityPtr; + BYTE* op = ostart; size_t outSize = *dstCapacityPtr; size_t epilogueSize, remaining; ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */ @@ -411,7 +403,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, return headerSize; } } - /* Frame header provides buffer sizes */ + /* Frame header instruct buffer sizes */ { size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); @@ -419,8 +411,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, zbc->inBuff = (char*)malloc(neededInSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } } - { - size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog; + { size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog; if (zbc->outBuffSize < neededOutSize) { free(zbc->outBuff); zbc->outBuffSize = neededOutSize; @@ -430,8 +421,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, zbc->stage = ZBUFFds_read; case ZBUFFds_read: - { - size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); if (neededInSize==0) { /* end of frame */ zbc->stage = ZBUFFds_init; notDone = 0; @@ -454,8 +444,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, } case ZBUFFds_load: - { - size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */ size_t loadedSize; if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */ @@ -463,8 +452,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, ip += loadedSize; zbc->inPos += loadedSize; if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ - { - size_t const decodedSize = ZSTD_decompressContinue(zbc->zc, + /* decode loaded input */ + { size_t const decodedSize = ZSTD_decompressContinue(zbc->zc, zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, zbc->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -475,8 +464,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, // break; /* ZBUFFds_flush follows */ } } case ZBUFFds_flush: - { - size_t const toFlushSize = zbc->outEnd - zbc->outStart; + { size_t const toFlushSize = zbc->outEnd - zbc->outStart; size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize); op += flushedSize; zbc->outStart += flushedSize; @@ -496,8 +484,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, /* result */ *srcSizePtr = ip-istart; *dstCapacityPtr = op-ostart; - { - size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ nextSrcSizeHint -= zbc->inPos; /* already loaded*/ return nextSrcSizeHint; diff --git a/lib/zbuff.h b/lib/zbuff.h index 7a914a4a7..e2482054c 100644 --- a/lib/zbuff.h +++ b/lib/zbuff.h @@ -31,11 +31,6 @@ #ifndef ZSTD_BUFFERED_H #define ZSTD_BUFFERED_H -/* The objects defined into this file should be considered experimental. - * They are not considered stable, as their prototype may change in the future. - * You can use them for tests, provide feedback, or if you can endure risk of future changes. - */ - #if defined (__cplusplus) extern "C" { #endif @@ -89,12 +84,12 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap * *srcSizePtr and *dstCapacityPtr can be any size. * The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst . +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst . * @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) * or an error code, which can be tested using ZBUFF_isError(). * * At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush(). -* The nb of bytes written into @dst will be reported into *dstCapacityPtr. +* The nb of bytes written into `dst` will be reported into *dstCapacityPtr. * Note that the function cannot output more than *dstCapacityPtr, * therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small. * @return : nb of bytes still present into internal buffer (0 if it's empty) @@ -139,13 +134,13 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, * *srcSizePtr and *dstCapacityPtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), * or 0 when a frame is completely decoded, * or an error code, which can be tested using ZBUFF_isError(). * * Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. * input : ZBUFF_recommendedDInSize == 128KB + 3; * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * *******************************************************************************/ diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 3cec2778f..b25291228 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -208,7 +208,8 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface return ccsize; } } - +/*! ZSTD_resetCCtx_advanced() : + note : 'params' is expected to be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params) { /* note : params considered validated here */ @@ -1200,8 +1201,8 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); /* check immediate repcode */ while (ip <= ilimit) { - U32 current2 = (U32)(ip-base); - const U32 repIndex2 = current2 - offset_2; + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; if ( ((repIndex2 <= dictLimit-4) || (repIndex2 >= dictLimit)) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { @@ -2233,18 +2234,12 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si } -/*! ZSTD_compressBegin_advanced() : +/*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - /* compression parameters verification and optimization */ - { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); - if (ZSTD_isError(errorCode)) return errorCode; } - - ZSTD_adjustCParams(¶ms.cParams, pledgedSrcSize, dictSize); - { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); if (ZSTD_isError(errorCode)) return errorCode; } @@ -2272,13 +2267,28 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, } +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, + const void* dict, size_t dictSize, + ZSTD_parameters params, U64 pledgedSrcSize) +{ + /* compression parameters verification and optimization */ + { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); + if (ZSTD_isError(errorCode)) return errorCode; } + + return ZSTD_compressBegin_internal(zc, dict, dictSize, params, pledgedSrcSize); +} + + size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); params.fParams.contentSizeFlag = 0; + ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); - return ZSTD_compressBegin_advanced(zc, dict, dictSize, params, 0); + return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); } size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) @@ -2342,7 +2352,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* ctx, BYTE* op = ostart; /* Init */ - { size_t const errorCode = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params, srcSize); + { size_t const errorCode = ZSTD_compressBegin_internal(ctx, dict, dictSize, params, srcSize); if(ZSTD_isError(errorCode)) return errorCode; } /* body (compression) */ diff --git a/programs/result b/programs/result new file mode 100644 index 000000000..0d18ea646 --- /dev/null +++ b/programs/result @@ -0,0 +1,519 @@ +/* + bench.c - Demo module to benchmark open-source compression algorithms + Copyright (C) Yann Collet 2012-2015 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd + - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* ************************************** +* Compiler Options +****************************************/ +/* Disable some Visual warning messages */ +#ifdef _MSC_VER +# define _CRT_SECURE_NO_WARNINGS /* fopen */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +/* Unix Large Files support (>4GB) */ +#define _FILE_OFFSET_BITS 64 +#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE +#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ +# define _LARGEFILE64_SOURCE +#endif + + +/* ************************************* +* Includes +***************************************/ +#include /* malloc, free */ +#include /* memset */ +#include /* fprintf, fopen, ftello64 */ +#include /* stat64 */ +#include /* stat64 */ +#include /* clock_t, clock, CLOCKS_PER_SEC */ + +/* sleep : posix - windows - others */ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) +# include /* sleep */ +# include /* setpriority */ +# define BMK_sleep(s) sleep(s) +# define HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +#elif defined(_WIN32) +# include +# define BMK_sleep(s) Sleep(1000*s) +# define HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS); +#else +# define BMK_sleep(s) /* disabled */ +# define HIGH_PRIORITY +#endif + +#include "mem.h" +#include "zstd_static.h" +#include "xxhash.h" +#include "datagen.h" /* RDG_genBuffer */ + + +/* ************************************* +* Compiler specifics +***************************************/ +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + +#ifdef _MSC_VER +#define snprintf sprintf_s +#endif + + +/* ************************************* +* Constants +***************************************/ +#define NBLOOPS 3 +#define TIMELOOP_S 1 +#define ACTIVEPERIOD_S 70 +#define COOLPERIOD_S 10 + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); +#define DEFAULT_CHUNKSIZE (4 MB) + +static U32 g_compressibilityDefault = 50; + + +/* ************************************* +* console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ + + +/* ************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +/* ************************************* +* Benchmark Parameters +***************************************/ +static U32 g_nbIterations = NBLOOPS; +static size_t g_blockSize = 0; + +void BMK_SetNbIterations(unsigned nbLoops) +{ + g_nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", g_nbIterations); +} + +void BMK_SetBlockSize(size_t blockSize) +{ + g_blockSize = blockSize; + DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10)); +} + + +/* ******************************************************** +* Private functions +**********************************************************/ +static clock_t BMK_clockSpan( clock_t clockStart ) +{ + return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */ +} + + +static U64 BMK_getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + return (U64)statbuf.st_size; +} + + +/* ******************************************************** +* Bench functions +**********************************************************/ +typedef struct +{ + const char* srcPtr; + size_t srcSize; + char* cPtr; + size_t cRoom; + size_t cSize; + char* resPtr; + size_t resSize; +} blockParam_t; + +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +static int BMK_benchMem(const void* srcBuffer, size_t srcSize, + const char* displayName, int cLevel, + const size_t* fileSizes, U32 nbFiles, + const void* dictBuffer, size_t dictBufferSize) +{ + size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); + size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + void* const compressedBuffer = malloc(maxCompressedSize); + void* const resultBuffer = malloc(srcSize); + ZSTD_CCtx* refCtx = ZSTD_createCCtx(); + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* refDCtx = ZSTD_createDCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + U32 nbBlocks; + + /* checks */ + if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx) + EXM_THROW(31, "not enough memory"); + + /* init */ + if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + HIGH_PRIORITY; + + /* Init blockTable data */ + { const char* srcPtr = (const char*)srcBuffer; + char* cPtr = (char*)compressedBuffer; + char* resPtr = (char*)resultBuffer; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_S * CLOCKS_PER_SEC) { + DISPLAY("\rcooling down ... \r"); + BMK_sleep(COOLPERIOD_S); + coolTime = clock(); + } + + /* Compression */ + DISPLAY("%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); + memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + + clockStart = clock(); + while (clock() == clockStart); + clockStart = clock(); + { U32 nbLoops; + for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { + U32 blockNb; + ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); + for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", + testNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) ); + + (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ +#if 1 + /* Decompression */ + memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + + clockStart = clock(); + while (clock() == clockStart); + clockStart = clock(); + + { U32 nbLoops; + for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { + U32 blockNb; + ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize); + for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + testNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC), + (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) ); + + /* CRC Checking */ + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if (crcOrig!=crcCheck) { + size_t u; + DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u u) break; + bacc += blockTable[segNb].srcSize; + } + pos = (U32)(u - bacc); + bNb = pos / (128 KB); + DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); + break; + } + if (u==srcSize-1) { /* should never happen */ + DISPLAY("no difference detected\n"); + } } + break; + } } /* CRC Checking */ +#endif + } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ + DISPLAY("%2i#\n", cLevel); + } /* Bench */ + + /* clean up */ + free(compressedBuffer); + free(resultBuffer); + ZSTD_freeCCtx(refCtx); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(refDCtx); + ZSTD_freeDCtx(dctx); + return 0; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t const step = 64 MB; + BYTE* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += step; + if (requiredMem > maxMemory) requiredMem = maxMemory; + + do { + testmem = (BYTE*)malloc((size_t)requiredMem); + requiredMem -= step; + } while (!testmem); + + free(testmem); + return (size_t)(requiredMem); +} + +static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, + const char* displayName, int cLevel, + const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize) +{ + if (cLevel < 0) { /* range mode : test all levels from 1 to l */ + int l; + for (l=1; l <= -cLevel; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); + } + return; + } + BMK_benchMem(srcBuffer, benchedSize, + displayName, cLevel, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); +} + +static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) +{ + U64 total = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + fclose(f); + } +} + +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel) +{ + void* srcBuffer; + size_t benchedSize; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; + size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles); + char mfName[20] = {0}; + const char* displayName = NULL; + + if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + + /* Load dictionary */ + if (dictFileName != NULL) { + U64 dictFileSize = BMK_getFileSize(dictFileName); + if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName); + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); + BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1); + } + + /* Memory allocation & restrictions */ + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; + if (benchedSize < totalSizeToLoad) + DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); + srcBuffer = malloc(benchedSize); + if (!srcBuffer) EXM_THROW(12, "not enough memory"); + + /* Load input buffer */ + BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); + + /* Bench */ + snprintf (mfName, sizeof(mfName), " %u files", nbFiles); + if (nbFiles > 1) displayName = mfName; + else displayName = fileNamesTable[0]; + + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); + + /* clean up */ + free(srcBuffer); + free(dictBuffer); + free(fileSizes); +} + + +static void BMK_syntheticTest(int cLevel, double compressibility) +{ + char name[20] = {0}; + size_t benchedSize = 10000000; + void* srcBuffer = malloc(benchedSize); + + /* Memory allocation */ + if (!srcBuffer) EXM_THROW(21, "not enough memory"); + + /* Fill input buffer */ + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); + + /* Bench */ + snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0); + + /* clean up */ + free(srcBuffer); +} + + +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel) +{ + double const compressibility = (double)g_compressibilityDefault / 100; + + if (nbFiles == 0) + BMK_syntheticTest(cLevel, compressibility); + else + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel); + return 0; +} + diff --git a/programs/tmp.mgz b/programs/tmp.mgz new file mode 100644 index 000000000..165a506f9 Binary files /dev/null and b/programs/tmp.mgz differ diff --git a/programs/tmp.zstd b/programs/tmp.zstd new file mode 100644 index 000000000..6f0296f66 Binary files /dev/null and b/programs/tmp.zstd differ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index aa57b576a..fd18c9e59 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -1,6 +1,6 @@ /* Fuzzer test tool for zstd_buffered - Copyright (C) Yann Collet 2105 + Copyright (C) Yann Collet 2015-2016 GPL v2 License @@ -19,11 +19,10 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - ZSTD source repository : https://github.com/Cyan4973/zstd - - ZSTD public forum : https://groups.google.com/forum/#!forum/lz4c + - ZSTD homepage : https://www.zstd.net/ */ -/************************************** +/*-************************************ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ @@ -33,7 +32,7 @@ #endif -/************************************** +/*-************************************ * Includes **************************************/ #include /* free */ @@ -47,8 +46,8 @@ #include "xxhash.h" /* XXH64 */ -/************************************** - Constants +/*-************************************ +* Constants **************************************/ #ifndef ZSTD_VERSION # define ZSTD_VERSION "" @@ -66,7 +65,7 @@ static const U32 prime2 = 2246822519U; -/************************************** +/*-************************************ * Display Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) @@ -83,7 +82,7 @@ static U32 g_displayTime = 0; static U32 g_testTime = 0; -/********************************************************* +/*-******************************************************* * Fuzzer functions *********************************************************/ #define MAX(a,b) ((a)>(b)?(a):(b)) @@ -149,8 +148,7 @@ static int basicUnitTests(U32 seed, double compressibility) CNBuffer = malloc(CNBufferSize); compressedBuffer = malloc(compressedBufferSize); decodedBuffer = malloc(decodedBufferSize); - if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) - { + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { DISPLAY("Not enough memory, aborting\n"); goto _output_error; } @@ -183,11 +181,9 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ - { - size_t i; + { size_t i; DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); - for (i=0; i>= 3; - if (buffNb & 7) - { + if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; - } - else - { + } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; - } - } + } } srcBuffer = cNoiseBuffer[buffNb]; /* Multi - segments compression test */ @@ -327,8 +316,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit totalTestSize = 0; cSize = 0; - for (n=0; n='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; nbTests += *argument - '0'; argument++; @@ -532,8 +511,7 @@ int main(int argc, char** argv) case 'T': argument++; nbTests=0; g_testTime=0; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { g_testTime *= 10; g_testTime += *argument - '0'; argument++; @@ -547,8 +525,7 @@ int main(int argc, char** argv) argument++; seed=0; seedset=1; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { seed *= 10; seed += *argument - '0'; argument++; @@ -558,8 +535,7 @@ int main(int argc, char** argv) case 't': argument++; testNb=0; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { testNb *= 10; testNb += *argument - '0'; argument++; @@ -569,8 +545,7 @@ int main(int argc, char** argv) case 'P': /* compressibility % */ argument++; proba=0; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { proba *= 10; proba += *argument - '0'; argument++; @@ -582,9 +557,7 @@ int main(int argc, char** argv) default: return FUZ_usage(programName); } - } - } - } + } } } /* Get Seed */ DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION); @@ -598,8 +571,8 @@ int main(int argc, char** argv) if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ if (!result) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); - if (mainPause) - { + + if (mainPause) { int unused; DISPLAY("Press Enter \n"); unused = getchar();