default: all
-all: main-circular-buffer main-integrated main-hf
+all: main-circular-buffer main-integrated main-64
#main-basic : basic_table.c ldm.c main-ldm.c
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
main-circular-buffer: circular_buffer_table.c ldm.c main-ldm.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-main-hf: ldm_hf_test.c main-ldm.c
+main-64: ldm_64_hash.c main-ldm.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
main-integrated: ldm_with_table.c main-ldm.c
clean:
@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
- main-basic main-circular-buffer main-integrated main-hf
+ main-basic main-circular-buffer main-integrated main-64
@echo Cleaning completed
const BYTE *pIn,
const BYTE *pEnd,
const BYTE *pAnchor,
- U32 *pForwardMatchLength,
- U32 *pBackwardMatchLength) {
+ U64 *pForwardMatchLength,
+ U64 *pBackwardMatchLength) {
LDM_hashEntry *bucket = getBucket(table, hash);
LDM_hashEntry *cur = bucket;
LDM_hashEntry *bestEntry = NULL;
- U32 bestMatchLength = 0;
+ U64 bestMatchLength = 0;
for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
const BYTE *pMatch = cur->offset + table->offsetBase;
// Check checksum for faster check.
if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
- U32 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
- U32 backwardMatchLength, totalMatchLength;
+ U64 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
+ U64 backwardMatchLength, totalMatchLength;
if (forwardMatchLength < table->minMatchLength) {
continue;
* matchLength contains the forward length of the match.
*/
static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
- U32 *matchLength, U32 *backwardMatchLength) {
+ U64 *matchLength, U64 *backwardMatchLength) {
LDM_hashEntry *entry = NULL;
cctx->nextIp = cctx->ip + cctx->step;
}
void LDM_encodeLiteralLengthAndLiterals(
- LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+ LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
/* Encode the literal length. */
if (literalLength >= RUN_MASK) {
int len = (int)literalLength - RUN_MASK;
}
void LDM_outputBlock(LDM_CCtx *cctx,
- const U32 literalLength,
+ const U64 literalLength,
const U32 offset,
- const U32 matchLength) {
+ const U64 matchLength) {
BYTE *pToken = cctx->op++;
/* Encode the literal length and literals. */
/* Encode the match length. */
if (matchLength >= ML_MASK) {
- unsigned matchLengthRemaining = matchLength;
+ U64 matchLengthRemaining = matchLength;
*pToken += ML_MASK;
matchLengthRemaining -= ML_MASK;
MEM_write32(cctx->op, 0xFFFFFFFF);
// Note that this is not the number of buckets.
// Currently this should be less than WINDOW_SIZE_LOG + 4?
#define LDM_MEMORY_USAGE 22
-#define HASH_BUCKET_SIZE_LOG 3 // MAX is 4 for now
+#define HASH_BUCKET_SIZE_LOG 1 // MAX is 4 for now
// Defines the lag in inserting elements into the hash table.
#define LDM_LAG 0
* This is followed by literalLength bytes corresponding to the literals.
*/
void LDM_encodeLiteralLengthAndLiterals(
- LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength);
+ LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength);
/**
* Write current block (literals, literal length, match offset,
* match length).
*/
void LDM_outputBlock(LDM_CCtx *cctx,
- const U32 literalLength,
+ const U64 literalLength,
const U32 offset,
- const U32 matchLength);
+ const U64 matchLength);
/**
* Decompresses src into dst.
*
* We count only bytes where pMatch > pBaes and pIn > pAnchor.
*/
-U32 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
+U64 countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
const BYTE *pMatch, const BYTE *pBase) {
- U32 matchLength = 0;
+ U64 matchLength = 0;
while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
pIn--;
pMatch--;
LDM_hashEntry *HASH_getBestEntry(const LDM_CCtx *cctx,
const hash_t hash,
const U32 checksum,
- U32 *pForwardMatchLength,
- U32 *pBackwardMatchLength) {
+ U64 *pForwardMatchLength,
+ U64 *pBackwardMatchLength) {
LDM_hashTable *table = cctx->hashTable;
LDM_hashEntry *bucket = getBucket(table, hash);
LDM_hashEntry *cur = bucket;
BYTE toRemove, BYTE toAdd) {
// TODO: deduplicate.
static const U64 prime8bytes = 11400714785074694791ULL;
-// static const U64 prime8bytes = 5;
+
sum -= ((toRemove + CHECKSUM_CHAR_OFFSET) *
ipow(prime8bytes, len - 1));
sum *= prime8bytes;
void LDM_outputConfiguration(void) {
printf("=====================\n");
printf("Configuration\n");
- printf("Window size log: %d\n", LDM_WINDOW_SIZE_LOG);
- printf("Min match, hash length: %d, %d\n",
+ printf("LDM_WINDOW_SIZE_LOG: %d\n", LDM_WINDOW_SIZE_LOG);
+ printf("LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH: %d, %d\n",
LDM_MIN_MATCH_LENGTH, LDM_HASH_LENGTH);
printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
+ printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
printf("LDM_LAG %d\n", LDM_LAG);
printf("=====================\n");
}
* forwardMatchLength contains the forward length of the match.
*/
static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
- U32 *forwardMatchLength, U32 *backwardMatchLength) {
+ U64 *forwardMatchLength, U64 *backwardMatchLength) {
LDM_hashEntry *entry = NULL;
cctx->nextIp = cctx->ip + cctx->step;
}
void LDM_encodeLiteralLengthAndLiterals(
- LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+ LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
/* Encode the literal length. */
if (literalLength >= RUN_MASK) {
- int len = (int)literalLength - RUN_MASK;
+ U64 len = (U64)literalLength - RUN_MASK;
*pToken = (RUN_MASK << ML_BITS);
for (; len >= 255; len -= 255) {
*(cctx->op)++ = 255;
}
void LDM_outputBlock(LDM_CCtx *cctx,
- const U32 literalLength,
+ const U64 literalLength,
const U32 offset,
- const U32 matchLength) {
+ const U64 matchLength) {
BYTE *pToken = cctx->op++;
/* Encode the literal length and literals. */
/* Encode the match length. */
if (matchLength >= ML_MASK) {
- unsigned matchLengthRemaining = matchLength;
+ U64 matchLengthRemaining = matchLength;
*pToken += ML_MASK;
matchLengthRemaining -= ML_MASK;
MEM_write32(cctx->op, 0xFFFFFFFF);
void *dst, size_t maxDstSize) {
LDM_CCtx cctx;
const BYTE *match = NULL;
- U32 forwardMatchLength = 0;
- U32 backwardsMatchLength = 0;
+ U64 forwardMatchLength = 0;
+ U64 backwardsMatchLength = 0;
LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
LDM_outputConfiguration();
* length) and update pointers and hashes.
*/
{
- const U32 literalLength = cctx.ip - cctx.anchor;
+ const U64 literalLength = cctx.ip - cctx.anchor;
const U32 offset = cctx.ip - match;
- const U32 matchLength = forwardMatchLength +
+ const U64 matchLength = forwardMatchLength +
backwardsMatchLength -
LDM_MIN_MATCH_LENGTH;
/* Encode the last literals (no more matches). */
{
- const U32 lastRun = cctx.iend - cctx.anchor;
+ const U64 lastRun = cctx.iend - cctx.anchor;
BYTE *pToken = cctx.op++;
LDM_encodeLiteralLengthAndLiterals(&cctx, pToken, lastRun);
}
size_t LDM_decompress(const void *src, size_t compressedSize,
void *dst, size_t maxDecompressedSize) {
+
LDM_DCtx dctx;
LDM_initializeDCtx(&dctx, src, compressedSize, dst, maxDecompressedSize);
const BYTE *pIn,
const BYTE *pEnd,
const BYTE *pAnchor,
- U32 *matchLength,
- U32 *backwardsMatchLength);
+ U64 *matchLength,
+ U64 *backwardsMatchLength);
hash_t HASH_hashU32(U32 value);
}
void LDM_encodeLiteralLengthAndLiterals(
- LDM_CCtx *cctx, BYTE *pToken, const U32 literalLength) {
+ LDM_CCtx *cctx, BYTE *pToken, const U64 literalLength) {
/* Encode the literal length. */
if (literalLength >= RUN_MASK) {
- int len = (int)literalLength - RUN_MASK;
+ U64 len = (U64)literalLength - RUN_MASK;
*pToken = (RUN_MASK << ML_BITS);
for (; len >= 255; len -= 255) {
*(cctx->op)++ = 255;
}
void LDM_outputBlock(LDM_CCtx *cctx,
- const U32 literalLength,
+ const U64 literalLength,
const U32 offset,
- const U32 matchLength) {
+ const U64 matchLength) {
BYTE *pToken = cctx->op++;
/* Encode the literal length and literals. */
* length) and update pointers and hashes.
*/
{
- const U32 literalLength = cctx.ip - cctx.anchor;
+ const U64 literalLength = cctx.ip - cctx.anchor;
const U32 offset = cctx.ip - match;
- const U32 matchLength = forwardMatchLength +
+ const U64 matchLength = forwardMatchLength +
backwardsMatchLength -
LDM_MIN_MATCH_LENGTH;
// Truncate file to compressedSize.
ftruncate(fdout, compressedSize);
- printf("%25s : %6u -> %7u - %s (%.1f%%)\n", fname,
- (unsigned)statbuf.st_size, (unsigned)compressedSize, oname,
+ printf("%25s : %10lu -> %10lu - %s (%.1f%%)\n", fname,
+ (size_t)statbuf.st_size, (size_t)compressedSize, oname,
(double)compressedSize / (statbuf.st_size) * 100);
timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
dst, decompressedSize);
printf("Ret size out: %zu\n", outSize);
- ftruncate(fdout, outSize);
+// ftruncate(fdout, decompressedSize);
close(fdin);
close(fdout);
printf("ldm = [%s]\n", ldmFilename);
printf("dec = [%s]\n", decFilename);
-
/* Compress */
{
if (compress(inpFilename, ldmFilename)) {
projects / thirdparty / zstd.git / commitdiff
