env:
- ZSTD_TRAVIS_CI_ENV=travis-install
- ZSTD_TRAVIS_CI_ENV=test-all
+ - ZSTD_TRAVIS_CI_ENV=clangtest
+ - ZSTD_TRAVIS_CI_ENV=gpptest
matrix:
fast_finish: true
# ################################################################
# Version number
-export VERSION=0.0.1
+export VERSION=0.0.2
export RELEASE=r$(VERSION)
DESTDIR?=
endif
+.PHONY: clean
+
default: zstdprograms
all:
prg-travis:
@cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV)
+clangtest: clean
+ $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
+
+gpptest: clean
+ $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+
+
endif
return maxSymbolValue ? maxHeaderSize : FSE_MAX_HEADERSIZE;
}
+#ifndef __clang_analyzer__ /* clang static analyzer has difficulties with this function : seems to believe normalizedCounter is uninitialized */
+
static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned safeWrite)
while (charnum >= start+24)
{
start+=24;
- bitStream += 0xFFFF<<bitCount;
+ bitStream += 0xFFFFU << bitCount;
if ((!safeWrite) && (out > oend-2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
- out[0] = (BYTE)bitStream;
+ out[0] = (BYTE) bitStream;
out[1] = (BYTE)(bitStream>>8);
out+=2;
bitStream>>=16;
short count = normalizedCounter[charnum++];
const short max = (short)((2*threshold-1)-remaining);
remaining -= FSE_abs(count);
- if (remaining<0) return (size_t)-FSE_ERROR_GENERIC;
+ if (remaining<1) return (size_t)-FSE_ERROR_GENERIC;
count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount;
return (out-ostart);
}
+#endif // __clang_analyzer__
size_t FSE_writeHeader (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
U32 weight = sEnd - sStart;
if (weight < 1)
return (size_t)-FSE_ERROR_GENERIC;
- norm[s] = weight;
+ norm[s] = (short)weight;
tmpTotal = end;
}
}
void FSE_initCState(FSE_CState_t* statePtr, const void* CTable)
{
- const U32 tableLog = ( (U16*) CTable) [0];
+ const U32 tableLog = ( (const U16*) CTable) [0];
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = ((const U16*) CTable) + 2;
statePtr->symbolTT = (const U32*)CTable + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
const void* cSrc, size_t cSrcSize)
{
if (cSrcSize != 1) return (size_t)-FSE_ERROR_srcSize_wrong;
- memset(dst, *(BYTE*)cSrc, originalSize);
+ memset(dst, *(const BYTE*)cSrc, originalSize);
return originalSize;
}
if (srcSize >= sizeof(bitD_t))
{
U32 contain32;
- bitD->start = (char*)srcBuffer;
- bitD->ptr = (char*)srcBuffer + srcSize - sizeof(bitD_t);
+ bitD->start = (const char*)srcBuffer;
+ bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD_t);
bitD->bitContainer = FSE_readLEST(bitD->ptr);
- contain32 = ((BYTE*)srcBuffer)[srcSize-1];
+ contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
}
else
{
U32 contain32;
- bitD->start = (char*)srcBuffer;
+ bitD->start = (const char*)srcBuffer;
bitD->ptr = bitD->start;
- bitD->bitContainer = *(BYTE*)(bitD->start);
+ bitD->bitContainer = *(const BYTE*)(bitD->start);
switch(srcSize)
{
- case 7: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16);
- case 6: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24);
- case 5: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32);
- case 4: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[3]) << 24;
- case 3: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[2]) << 16;
- case 2: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[1]) << 8;
+ case 7: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16);
+ case 6: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24);
+ case 5: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32);
+ case 4: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[3]) << 24;
+ case 3: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[2]) << 16;
+ case 2: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[1]) << 8;
default:;
}
- contain32 = ((BYTE*)srcBuffer)[srcSize-1];
+ contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
bitD->bitsConsumed += (U32)(sizeof(bitD_t) - srcSize)*8;
/* FSE_readBits
* Read next n bits from the bitContainer.
- * Use the fast variant *only* if n > 0.
- * Note : for this function to work properly on 32-bits, don't read more than maxNbBits==25
+ * On 32-bits, don't read more than maxNbBits==25
+ * On 64-bits, don't read more than maxNbBits==57
+ * Use the fast variant *only* if n >= 1.
* return : value extracted.
*/
bitD_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits)
{
- bitD_t value = ((bitD->bitContainer << bitD->bitsConsumed) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits);
+ bitD_t value = ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD_t)*8)-1))) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits);
bitD->bitsConsumed += nbBits;
return value;
}
-bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 */
+bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */
{
bitD_t value = (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD_t)*8)-nbBits);
bitD->bitsConsumed += nbBits;
unsigned FSE_endOfDStream(const FSE_DStream_t* bitD)
{
- return FSE_reloadDStream((FSE_DStream_t*)bitD)==2;
+ return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD_t)*8));
}
unsigned FSE_endOfDState(const FSE_DState_t* statePtr)
}
/* tail */
+ /* note : FSE_reloadDStream(&bitD) >= 1; Ends at exactly 2 */
while (1)
{
- if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state1) && FSE_endOfDStream(&bitD)) )
+ if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) )
break;
*op++ = fast ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
- if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state2) && FSE_endOfDStream(&bitD)) )
+ if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) )
break;
*op++ = fast ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
}
+#ifndef __clang_analyzer__ /* clang static analyzer doesn't understand that tableSymbol is necessarily entirely initialized */
+
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
return 0;
}
+#endif // __clang_analyzer__
+
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION)
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
U32 position = 0;
U32 highThreshold = tableSize-1;
- const S16 largeLimit= 1 << (tableLog-1);
+ const S16 largeLimit= (S16)(1 << (tableLog-1));
U32 noLarge = 1;
U32 s;
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
-#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
unsigned ZSTD_versionNumber (void);
or an error code if it fails (which can be tested using ZSTD_isError())
ZSTD_decompress() :
- compressedSize : is obviously the source size
+ compressedSize : is the exact source size
maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.
It must be equal or larger than originalSize, otherwise decompression will fail.
return : the number of bytes decompressed into destination buffer (originalSize)
/**************************************
* Tool functions
**************************************/
-size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size */
+size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size (worst case scenario) */
/* Error Management */
unsigned ZSTD_isError(size_t code); /* tells if a return value is an error code */
/**************************************
-* Basic Types
+* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
}
-int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
+int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
{
double compressibility = (double)g_compressibilityDefault / 100;
*/
#pragma once
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
/* Main function */
-int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
+int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
/* Set Parameters */
void BMK_SetNbIterations(int nbLoops);
-
-#if defined (__cplusplus)
-}
-#endif
* Local Functions
*********************************************************/
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-static unsigned int RDG_rand(U32* src)
+static U32 RDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
#define LTSIZE 8192
#define LTMASK (LTSIZE-1)
-static void* RDG_createLiteralDistrib(double ld)
+static void* RDG_createLiteralDistrib(const double ld)
{
- BYTE* lt = malloc(LTSIZE);
+ BYTE* lt = (BYTE*)malloc(LTSIZE);
U32 i = 0;
BYTE character = '0';
BYTE firstChar = '(';
BYTE lastChar = '}';
- if (ld==0.0)
+ if (ld<=0.02)
{
- character = 0;
firstChar = 0;
- lastChar =255;
+ lastChar = 254;
+ character = firstChar;
}
+ if (ld==0.0) lastChar = 255;
while (i<LTSIZE)
{
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
static char RDG_genChar(U32* seed, const void* ltctx)
{
- const BYTE* lt = ltctx;
+ const BYTE* lt = (const BYTE*)ltctx;
U32 id = RDG_rand(seed) & LTMASK;
return lt[id];
}
ctx = ZSTD_createCCtx();
/* Allocate Memory */
- inBuff = malloc(inBuffSize);
- outBuff = malloc(outBuffSize);
+ inBuff = (BYTE*)malloc(inBuffSize);
+ outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff) EXM_THROW(21, "Allocation error : not enough memory");
inSlot = inBuff;
inEnd = inBuff + inBuffSize;
/* Fill input Buffer */
if (inSlot + blockSize > inEnd) inSlot = inBuff;
inSize = fread(inSlot, (size_t)1, blockSize, finput);
+ DISPLAY("Read block of size %u at pos %u \n", (U32)inSize, (U32)(inSlot-inBuff));
if (inSize==0) break;
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Allocate Memory */
inBuffSize = blockSize + FIO_blockHeaderSize;
- inBuff = malloc(inBuffSize);
+ inBuff = (BYTE*)malloc(inBuffSize);
outBuffSize = wNbBlocks * blockSize;
- outBuff = malloc(outBuffSize);
+ outBuff = (BYTE*)malloc(outBuffSize);
op = outBuff;
oend = outBuff + outBuffSize;
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
while (toRead)
{
size_t readSize, decodedSize;
+ static U32 nbReads = 0;
/* Fill input buffer */
readSize = fread(inBuff, 1, toRead, finput);
EXM_THROW(34, "Read error");
/* Decode block */
+ if (nbReads==55)
+ DISPLAY("!");
decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
+ DISPLAY("nbReads : %u \n", nbReads++);
if (decodedSize) /* not a header */
{
/* Write block */
+ DISPLAY("writing %u bytes from pos %u \n", (U32)decodedSize, (U32)(op-outBuff));
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
filesize += decodedSize;
{
U32 i;
size_t total = 0;
- BYTE* data = buff2;
+ BYTE* data = (BYTE*)buff2;
(void)dst; (void)dstSize; (void)src;
for (i=0; i < srcSize; i++)
/* Allocation */
dstBuffSize = ZSTD_compressBound(srcSize);
- dstBuff = malloc(dstBuffSize);
- buff2 = malloc(dstBuffSize);
+ dstBuff = (BYTE*)malloc(dstBuffSize);
+ buff2 = (BYTE*)malloc(dstBuffSize);
if ((!dstBuff) || (!buff2))
{
DISPLAY("\nError: not enough memory!\n");
if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
+ /* long rle test */
+ {
+ size_t sampleSize = 0;
+ DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
+ FUZ_generateSynthetic(CNBuffer, sampleSize, compressibility, &randState);
+ memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
+ sampleSize += 256 KB - 1;
+ FUZ_generateSynthetic((char*)CNBuffer+sampleSize, 96 KB, compressibility, &randState);
+ sampleSize += 96 KB;
+ cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize);
+ if (ZSTD_isError(cSize)) goto _output_error;
+ result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
+ if (ZSTD_isError(result)) goto _output_error;
+ if (result!=sampleSize) goto _output_error;
+ DISPLAYLEVEL(4, "OK \n");
+ }
+
+
+
_end:
free(CNBuffer);
free(compressedBuffer);
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{
- const BYTE* b1 = buf1;
- const BYTE* b2 = buf2;
+ const BYTE* b1 = (const BYTE*)buf1;
+ const BYTE* b2 = (const BYTE*)buf2;
size_t i;
for (i=0; i<max; i++)
{
(void)startTest; (void)compressibility;
/* allocation */
- srcBuffer = malloc (srcBufferSize);
- dstBuffer = malloc (dstBufferSize);
- cBuffer = malloc (cBufferSize);
+ srcBuffer = (BYTE*)malloc (srcBufferSize);
+ dstBuffer = (BYTE*)malloc (dstBufferSize);
+ cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled");
/* Create initial sample */
if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
/* Check if benchmark is selected */
- if (bench) { BMK_bench(argv+fileNameStart, nbFiles, 0); goto _end; }
+ if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, 0); goto _end; }
/* No output filename ==> try to select one automatically (when possible) */
while (!outFileName)