#define BIT0 1
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
+#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
case ZSTD_p_enableLongDistanceMatching :
if (value != 0) {
ZSTD_cLevelToCCtxParams(params);
- params->cParams.windowLog = ZSTD_LDM_WINDOW_LOG;
+ params->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
}
return ZSTD_ldm_initializeParameters(¶ms->ldmParams, value);
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
- U32 const maxDist = 1 << cctx->appliedParams.cParams.windowLog;
+ U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
if (cctx->appliedParams.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize);
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
*/
if (cctx->lowLimit > (3U<<29)) {
- U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
+ U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
U32 const current = (U32)(ip - cctx->base);
- U32 const newCurrent = (current & cycleMask) + (1 << cctx->appliedParams.cParams.windowLog);
+ U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog);
U32 const correction = current - newCurrent;
ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
U32 const checksumFlag = params.fParams.checksumFlag>0;
- U32 const windowSize = 1U << params.cParams.windowLog;
+ U32 const windowSize = (U32)1 << params.cParams.windowLog;
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
U32 const fcsCode = params.fParams.contentSizeFlag ?
{
ZSTD_compressionParameters const cParams =
ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0);
- return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
+ return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
}
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
case ZSTD_btopt:
case ZSTD_btultra:
if (srcSize >= HASH_READ_SIZE)
- ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
+ ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
break;
default:
#define LDM_BUCKET_SIZE_LOG 3
#define LDM_MIN_MATCH_LENGTH 64
-#define LDM_HASH_LOG 20
+#define LDM_HASH_RLOG 7
#define LDM_HASH_CHAR_OFFSET 10
size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
{
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
params->enableLdm = enableLdm>0;
- params->hashLog = LDM_HASH_LOG;
+ params->hashLog = 0;
params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
params->minMatchLength = LDM_MIN_MATCH_LENGTH;
params->hashEveryLog = ZSTD_LDM_HASHEVERYLOG_NOTSET;
void ZSTD_ldm_adjustParameters(ldmParams_t* params, U32 windowLog)
{
+ if (params->hashLog == 0) {
+ params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG);
+ assert(params->hashLog <= ZSTD_HASHLOG_MAX);
+ }
if (params->hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET) {
params->hashEveryLog =
windowLog < params->hashLog ? 0 : windowLog - params->hashLog;
* Long distance matching
***************************************/
-#define ZSTD_LDM_WINDOW_LOG 27
+#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
#define ZSTD_LDM_HASHEVERYLOG_NOTSET 9999
/** ZSTD_compressBlock_ldm_generic() :
* Frames requiring more memory will be rejected.
*/
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U64)1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
+# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
#endif
offset = 0;
else {
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
- ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
+ ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
assert(ofBits <= MaxOff);
if (MEM_32bits() && longOffsets) {
U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
offset = 0;
else {
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
- ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
+ ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
assert(ofBits <= MaxOff);
if (MEM_32bits() && longOffsets) {
U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
-#define ZSTD_WINDOWLOG_MAX_32 27
-#define ZSTD_WINDOWLOG_MAX_64 27
+#define ZSTD_WINDOWLOG_MAX_32 30
+#define ZSTD_WINDOWLOG_MAX_64 31
#define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
#define ZSTD_WINDOWLOG_MIN 10
#define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
* Special: value 0 means "do not change cLevel". */
ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
- * Special: value 0 means "do not change windowLog". */
+ * Special: value 0 means "do not change windowLog".
+ * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
+ * requires setting the maximum window size at least as large during decompression. */
ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
* Resulting table size is (1 << (hashLog+2)).
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
* Larger values increase memory usage and compression ratio, but decrease
* compression speed.
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
- * (default: 20). */
+ * (default: windowlog - 7). */
ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher.
* Larger/too small values usually decrease compression ratio.
* Must be clamped between ZSTD_LDM_MINMATCH_MIN
# include <io.h>
#endif
+#include "bitstream.h"
#include "mem.h"
#include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
return 0;
}
+static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName)
+{
+ ZSTD_frameHeader header;
+ /* No special help for these errors */
+ if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge)
+ return;
+ /* Try to decode the frame header */
+ ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
+ if (ret == 0) {
+ U32 const windowSize = (U32)header.windowSize;
+ U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
+ U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1));
+ assert(header.windowSize <= (U64)((U32)-1));
+ assert(g_memLimit > 0);
+ DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
+ srcFileName, header.windowSize, g_memLimit);
+ if (windowLog <= ZSTD_WINDOWLOG_MAX) {
+ DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
+ srcFileName, windowLog, windowMB);
+ return;
+ }
+ } else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) {
+ DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n",
+ srcFileName, ZSTD_getErrorName(ret));
+ return;
+ }
+ DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n",
+ srcFileName, ZSTD_WINDOWLOG_MAX);
+}
/** FIO_decompressFrame() :
* @return : size of decoded zstd frame, or an error code
ZSTD_resetDStream(ress->dctx);
if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
- /* Header loading (optional, saves one loop) */
- { size_t const toRead = 9;
+ /* Header loading : ensures ZSTD_getFrameHeader() will succeed */
+ { size_t const toRead = ZSTD_FRAMEHEADERSIZE_MAX;
if (ress->srcBufferLoaded < toRead)
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
}
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint));
+ FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
return FIO_ERROR_FRAME_DECODING;
}
unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
.
.TP
-\fB\-\-long\fR
-enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
+\fB\-\-long[=#]\fR
+enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
+.
+.IP
+Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
.TP
\fB\-T#\fR, \fB\-\-threads=#\fR
Limit dictionary to specified size (default: 112640)\.
.
.TP
+\fB\-B#\fR
+Split input files in blocks of size # (default: no split)
+.
+.TP
\fB\-\-dictID=#\fR
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
.
Specify the maximum number of bits for a match distance\.
.
.IP
-The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
+The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
+.
+.IP
+Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
.TP
\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
.
.TP
-\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR
+\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
Specify the minimum searched length of a match for long distance matching\.
.
.IP
* `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Note that decompression will also require more memory when using these levels.
-* `--long`:
- enables long distance matching.
+* `--long[=#]`:
+ enables long distance matching with `#` `windowLog`, if not `#` is not
+ present it defaults to `27`.
This increases the window size (`windowLog`) and memory usage for both the
- compressor and decompressor. This setting is designed to improve the
- compression ratio for files with long matches at a large distance
- (up to the maximum window size, 128 MiB).
+ compressor and decompressor.
+ This setting is designed to improve the compression ratio for files with
+ long matches at a large distance.
+
+ Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
+ `--memory=windowSize` needs to be passed to the decompressor.
* `-T#`, `--threads=#`:
Compress using `#` threads (default: 1).
If `#` is 0, attempt to detect and use the number of physical CPU cores.
The higher number of increases the chance to find a match which usually
improves compression ratio.
It also increases memory requirements for the compressor and decompressor.
- The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
+ The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
+ platforms and 31 (2 GiB) on 64-bit platforms.
+
+ Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
+ `--memory=windowSize` needs to be passed to the decompressor.
- `hashLog`=_hlog_, `hlog`=_hlog_:
Specify the maximum number of bits for a hash table.
static const unsigned g_defaultMaxDictSize = 110 KB;
static const int g_defaultDictCLevel = 3;
static const unsigned g_defaultSelectivityLevel = 9;
+static const unsigned g_defaultMaxWindowLog = 27;
#define OVERLAP_LOG_DEFAULT 9999
#define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
DISPLAY( " -l : print information about zstd compressed files \n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
- DISPLAY( "--long : enable long distance matching (requires more memory)\n");
+ DISPLAY( "--long[=#] : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog);
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : use # threads for compression (default:1) \n");
DISPLAY( " -B# : select size of each job (default:0==automatic) \n");
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
- if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
+ if (longCommandWArg(&argument, "--long")) {
+ unsigned ldmWindowLog = 0;
+ ldmFlag = 1;
+ /* Parse optional window log */
+ if (*argument == '=') {
+ ++argument;
+ ldmWindowLog = readU32FromChar(&argument);
+ } else if (*argument != 0) {
+ /* Invalid character following --long */
+ CLEAN_RETURN(badusage(programName));
+ }
+ /* Only set windowLog if not already set by --zstd */
+ if (compressionParams.windowLog == 0)
+ compressionParams.windowLog = ldmWindowLog;
+ continue;
+ }
/* fall-through, will trigger bad_usage() later on */
}
#endif
} else { /* decompression or test */
#ifndef ZSTD_NODECOMPRESS
+ if (memLimit == 0) {
+ if (compressionParams.windowLog == 0)
+ memLimit = (U32)1 << g_defaultMaxWindowLog;
+ else {
+ memLimit = (U32)1 << (compressionParams.windowLog & 31);
+ }
+ }
FIO_setMemLimit(memLimit);
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
cLevel="$2"
proba=""
fi
+ if [ -n "$4" ]; then
+ dLevel="$4"
+ else
+ dLevel="$cLevel"
+ fi
rm -f tmp1 tmp2
- $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
+ $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
./datagen $1 $proba | $MD5SUM > tmp1
- ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d | $MD5SUM > tmp2
+ ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2
}
local_c="$2"
local_p=""
fi
+ if [ -n "$4" ]; then
+ local_d="$4"
+ else
+ local_d="$local_c"
+ fi
rm -f tmp.zstd tmp.md5.1 tmp.md5.2
- $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
+ $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
./datagen $1 $local_p > tmp
cat tmp | $MD5SUM > tmp.md5.1
- $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
+ $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
$DIFF -q tmp.md5.1 tmp.md5.2
}
roundTripTest -g70000000 -P70 "8 --long"
roundTripTest -g18000001 -P80 "18 --long"
fileRoundTripTest -g4100M -P99 "1 --long"
+# Test large window logs
+roundTripTest -g4100M -P50 "1 --long=30"
+roundTripTest -g4100M -P50 "1 --long --zstd=wlog=30,clog=30"
+# Test parameter parsing
+roundTripTest -g1M -P50 "1 --long=30" " --memory=1024MB"
+roundTripTest -g1M -P50 "1 --long=30 --zstd=wlog=29" " --memory=512MB"
+roundTripTest -g1M -P50 "1 --long=30" " --long=29 --memory=1024MB"
+roundTripTest -g1M -P50 "1 --long=30" " --zstd=wlog=29 --memory=1024MB"
if [ -n "$hasMT" ]
then
$ECHO "\n**** zstdmt long round-trip tests **** "
- roundTripTest -g99000000 -P99 "20 -T2"
- roundTripTest -g6000000000 -P99 "1 -T2"
- roundTripTest -g1500000000 -P97 "1 -T999"
- fileRoundTripTest -g4195M -P98 " -T0"
- roundTripTest -g1500000000 -P97 "1 --long -T999"
+ roundTripTest -g99000000 -P99 "20 -T2" " "
+ roundTripTest -g6000000000 -P99 "1 -T2" " "
+ roundTripTest -g1500000000 -P97 "1 -T999" " "
+ fileRoundTripTest -g4195M -P98 " -T0" " "
+ roundTripTest -g1500000000 -P97 "1 --long=23 -T2" " "
else
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
fi
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime32 = 2654435761U;
+static const U32 windowLogMax = 27;
/*-************************************
/* mess with compression parameters */
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
+ cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
projects / thirdparty / zstd.git / commitdiff
