U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */
+ U32 loadedDictEnd;
U32 stage;
ZSTD_parameters params;
void* workSpace;
zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
zc->hbSize = 0;
zc->stage = 0;
+ zc->loadedDictEnd = 0;
return 0;
}
}
-/* *************************************
+/*-*************************************
* Fast Scan
***************************************/
#define FILLHASHSTEP 3
const U32 hBits = zc->params.hashLog;
const BYTE* const base = zc->base;
const BYTE* ip = base + zc->nextToUpdate;
- const BYTE* const iend = (const BYTE*) end;
+ const BYTE* const iend = ((const BYTE*)end) - 8;
while(ip <= iend) {
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
- if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */
+ if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */
blockCompressor(zc, src, srcSize);
return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize);
}
-static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
+static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
- size_t blockSize = ctxPtr->blockSize;
+ size_t blockSize = zc->blockSize;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
- const U32 maxDist = 1 << ctxPtr->params.windowLog;
+ const U32 maxDist = 1 << zc->params.windowLog;
- while (remaining)
- {
+ while (remaining) {
size_t cSize;
- if (maxDstSize < 3 + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
+ if (maxDstSize < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
- if ((U32)(ip+blockSize - (ctxPtr->base + ctxPtr->lowLimit)) > maxDist)
- {
- /* respect windowLog contract */
- ctxPtr->lowLimit = (U32)(ip+blockSize - ctxPtr->base) - maxDist;
- if (ctxPtr->dictLimit < ctxPtr->lowLimit) ctxPtr->dictLimit = ctxPtr->lowLimit;
+ if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { /* enforce maxDist */
+ zc->lowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
+ if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
}
- cSize = ZSTD_compressBlock_internal(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
+ cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
- if (cSize == 0)
- {
- cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
+ if (cSize == 0) { /* block is not compressible */
+ cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
- }
- else
- {
+ } else {
op[0] = (BYTE)(cSize>>16);
op[1] = (BYTE)(cSize>>8);
op[2] = (BYTE)cSize;
const BYTE* const ip = (const BYTE*) src;
size_t hbSize = 0;
- if (frame && (zc->stage==0))
- {
+ if (frame && (zc->stage==0)) {
hbSize = zc->hbSize;
if (dstSize <= hbSize) return ERROR(dstSize_tooSmall);
zc->stage = 1;
}
/* Check if blocks follow each other */
- if (src != zc->nextSrc)
- {
+ if (src != zc->nextSrc) {
/* not contiguous */
size_t delta = zc->nextSrc - ip;
zc->lowLimit = zc->dictLimit;
}
/* preemptive overflow correction */
- if (zc->lowLimit > (1<<30))
- {
+ if (zc->lowLimit > (1<<30)) {
U32 btplus = (zc->params.strategy == ZSTD_btlazy2);
U32 contentMask = (1 << (zc->params.contentLog - btplus)) - 1;
U32 newLowLimit = zc->lowLimit & contentMask; /* preserve position % contentSize */
}
/* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
- if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit))
- {
+ if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) {
zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
if (zc->lowLimit > zc->dictLimit) zc->lowLimit = zc->dictLimit;
}
zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit;
+ zc->loadedDictEnd = (U32)(iend - zc->base);
zc->nextSrc = iend;
if (srcSize <= 8) return 0;
switch(zc->params.strategy)
{
case ZSTD_fast:
- ZSTD_fillHashTable (zc, iend-8, zc->params.searchLength);
+ ZSTD_fillHashTable (zc, iend, zc->params.searchLength);
break;
case ZSTD_greedy:
case ZSTD_btlazy2:
ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength);
- zc->nextToUpdate = (U32)(iend - zc->base);
break;
default:
return ERROR(GENERIC); /* strategy doesn't exist; impossible */
}
+ zc->nextToUpdate = zc->loadedDictEnd;
return 0;
}
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
- if (dict && dictSize)
- {
+ if (dict && (dictSize>4)) {
U32 magic = MEM_readLE32(dict);
size_t eSize;
if (magic != ZSTD_DICT_MAGIC)
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
{
- return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, 0));
+ return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, 128 KB));
}
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{
- return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize+dictSize));
+ return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
}
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
} blockParam_t;
#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliSpan(milliTime) < TIMELOOP) {
- ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, dictBufferSize+largestBlockSize));
+ ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, MAX(dictBufferSize, largestBlockSize)));
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
-/**************************************
+/*-************************************
* Includes
**************************************/
#include <stdlib.h> /* malloc */
#include <string.h> /* memcpy */
-/**************************************
+/*-************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
#endif
-/**************************************
+/*-************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
#endif
-/**************************************
+/*-************************************
* Constants
**************************************/
#define KB *(1 <<10)
#define PRIME2 2246822519U
-/**************************************
+/*-************************************
* Local types
**************************************/
#define LTLOG 13
-/*********************************************************
+/*-*******************************************************
* Local Functions
*********************************************************/
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
BYTE firstChar = '(';
BYTE lastChar = '}';
- if (ld==0.0)
- {
+ if (ld==0.0) {
character = 0;
firstChar = 0;
lastChar =255;
}
- while (i<LTSIZE)
- {
+ while (i<LTSIZE) {
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
U32 end;
if (weight + i > LTSIZE) weight = LTSIZE-i;
U32 prevOffset = 1;
/* special case : sparse content */
- while (matchProba >= 1.0)
- {
+ while (matchProba >= 1.0) {
size_t size0 = RDG_rand(seed) & 3;
size0 = (size_t)1 << (16 + size0 * 2);
size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
- if (buffSize < pos + size0)
- {
+ if (buffSize < pos + size0) {
memset(buffPtr+pos, 0, buffSize-pos);
return;
}
if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1;
/* Generate compressible data */
- while (pos < buffSize)
- {
+ while (pos < buffSize) {
/* Select : Literal (char) or Match (within 32K) */
- if (RDG_RAND15BITS < matchProba32)
- {
+ if (RDG_RAND15BITS < matchProba32) {
/* Copy (within 32K) */
size_t match;
size_t d;
d = pos + length;
if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
- }
- else
- {
+ } else {
/* Literal (noise) */
size_t d;
size_t length = RDG_RANDLENGTH;
d = pos + length;
if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt);
- }
- }
+ } }
}
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed);
/* Generate compressible data */
- while (total < size)
- {
+ while (total < size) {
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed);
if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
total += genBlockSize;
memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
}
- // cleanup
+ /* cleanup */
free(buff);
}
static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
const char* dstFileName, const char* srcFileName)
{
- if (!strcmp (srcFileName, stdinmark))
- {
+ if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
*fileInPtr = stdin;
SET_BINARY_MODE(stdin);
- }
- else
- {
+ } else {
*fileInPtr = fopen(srcFileName, "rb");
}
- if ( *fileInPtr==0 )
- {
+ if ( *fileInPtr==0 ) {
DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
return 1;
}
- if (!strcmp (dstFileName, stdoutmark))
- {
+ if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
*fileOutPtr = stdout;
SET_BINARY_MODE(stdout);
- }
- else
- {
- /* Check if destination file already exists */
- if (!g_overwrite)
- {
+ } else {
+ if (!g_overwrite) { /* Check if destination file already exists */
*fileOutPtr = fopen( dstFileName, "rb" );
- if (*fileOutPtr != 0)
- {
- /* prompt for overwrite authorization */
+ if (*fileOutPtr != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(*fileOutPtr);
DISPLAY("Warning : %s already exists \n", dstFileName);
- if ((g_displayLevel <= 1) || (*fileInPtr == stdin))
- {
+ if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) {
/* No interaction possible */
DISPLAY("Operation aborted : %s already exists \n", dstFileName);
return 1;
DISPLAY("Overwrite ? (y/N) : ");
{
int ch = getchar();
- if ((ch!='Y') && (ch!='y'))
- {
+ if ((ch!='Y') && (ch!='y')) {
DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
return 1;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
- }
- }
- }
+ } } }
*fileOutPtr = fopen( dstFileName, "wb" );
}
U64 fileSize;
*bufferPtr = NULL;
- if (fileName == NULL)
- return 0;
+ if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "Error opening file %s", fileName);
fileSize = FIO_getFileSize(fileName);
- if (fileSize > MAX_DICT_SIZE)
- {
+ if (fileSize > MAX_DICT_SIZE) {
int seekResult;
if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */
DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, MAX_DICT_SIZE);
ress = FIO_createCResources(dictFileName);
/* loop on each file */
- for (u=0; u<nbFiles; u++)
- {
+ for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]);
/* Main decompression Loop */
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
- while (1)
- {
+ while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
/* for each frame */
- for ( ; ; )
- {
+ for ( ; ; ) {
size_t sizeCheck;
/* check magic number -> version */
size_t toRead = 4;
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
- if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer)))
- {
+ if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer))) {
filesize += FIO_decompressLegacyFrame(dstFile, srcFile, MEM_readLE32(ress.srcBuffer));
continue;
}
if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
ress = FIO_createDResources(dictFileName);
- for (u=0; u<nbFiles; u++)
- {
+ for (u=0; u<nbFiles; u++) {
const char* srcFileName = srcNamesTable[u];
size_t sfnSize = strlen(srcFileName);
const char* suffixPtr = srcFileName + sfnSize - suffixSize;
if (dfnSize <= sfnSize-suffixSize+1) { free(dstFileName); dfnSize = sfnSize + 20; dstFileName = (char*)malloc(dfnSize); if (dstFileName==NULL) EXM_THROW(71, "not enough memory for dstFileName"); }
- if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0)
- {
+ if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
DISPLAYLEVEL(1, "File extension doesn't match expected extension (%4s); will not process file: %s\n", suffix, srcFileName);
skippedFiles++;
continue;
projects / thirdparty / zstd.git / commitdiff
