+++ /dev/null
-#include <stdlib.h>
-#include <stdio.h>
-
-#include "ldm.h"
-#include "ldm_hashtable.h"
-#include "mem.h"
-
-// THe number of elements per hash bucket.
-// HASH_BUCKET_SIZE_LOG is defined in ldm.h.
-#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
-
-// The number of hash buckets.
-#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
-
-// If ZSTD_SKIP is defined, then the first entry is returned in HASH_getBestEntry
-// (without looking at other entries in the bucket).
-//#define ZSTD_SKIP
-
-struct LDM_hashTable {
- U32 numBuckets; // The number of buckets.
- U32 numEntries; // numBuckets * HASH_BUCKET_SIZE.
- LDM_hashEntry *entries;
- BYTE *bucketOffsets; // A pointer (per bucket) to the next insert position.
-
- const BYTE *offsetBase; // Corresponds to offset=0 in LDM_hashEntry.
- U32 minMatchLength;
- U32 maxWindowSize;
-};
-
-LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
- U32 minMatchLength, U32 maxWindowSize) {
- LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
- table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
- table->numEntries = size;
- table->entries = calloc(size, sizeof(LDM_hashEntry));
- table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
- table->offsetBase = offsetBase;
- table->minMatchLength = minMatchLength;
- table->maxWindowSize = maxWindowSize;
- return table;
-}
-
-static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
- return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
-}
-
-// From lib/compress/zstd_compress.c
-static unsigned ZSTD_NbCommonBytes (register size_t val)
-{
- if (MEM_isLittleEndian()) {
- if (MEM_64bits()) {
-# if defined(_MSC_VER) && defined(_WIN64)
- unsigned long r = 0;
- _BitScanForward64( &r, (U64)val );
- return (unsigned)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3)
- return (__builtin_ctzll((U64)val) >> 3);
-# else
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
- 0, 3, 1, 3, 1, 4, 2, 7,
- 0, 2, 3, 6, 1, 5, 3, 5,
- 1, 3, 4, 4, 2, 5, 6, 7,
- 7, 0, 1, 2, 3, 3, 4, 6,
- 2, 6, 5, 5, 3, 4, 5, 6,
- 7, 1, 2, 4, 6, 4, 4, 5,
- 7, 2, 6, 5, 7, 6, 7, 7 };
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-# endif
- } else { /* 32 bits */
-# if defined(_MSC_VER)
- unsigned long r=0;
- _BitScanForward( &r, (U32)val );
- return (unsigned)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3)
- return (__builtin_ctz((U32)val) >> 3);
-# else
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
- 3, 2, 2, 1, 3, 2, 0, 1,
- 3, 3, 1, 2, 2, 2, 2, 0,
- 3, 1, 2, 0, 1, 0, 1, 1 };
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-# endif
- }
- } else { /* Big Endian CPU */
- if (MEM_64bits()) {
-# if defined(_MSC_VER) && defined(_WIN64)
- unsigned long r = 0;
- _BitScanReverse64( &r, val );
- return (unsigned)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3)
- return (__builtin_clzll(val) >> 3);
-# else
- unsigned r;
- const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
- if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
- r += (!val);
- return r;
-# endif
- } else { /* 32 bits */
-# if defined(_MSC_VER)
- unsigned long r = 0;
- _BitScanReverse( &r, (unsigned long)val );
- return (unsigned)(r>>3);
-# elif defined(__GNUC__) && (__GNUC__ >= 3)
- return (__builtin_clz((U32)val) >> 3);
-# else
- unsigned r;
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
- r += (!val);
- return r;
-# endif
- } }
-}
-
-/**
- * From lib/compress/zstd_compress.c
- * Returns the number of bytes (consecutively) in common between pIn and pMatch
- * up to pInLimit.
- */
-static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
- const BYTE *const pInLimit) {
- const BYTE * const pStart = pIn;
- const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
-
- while (pIn < pInLoopLimit) {
- size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
- if (!diff) {
- pIn += sizeof(size_t);
- pMatch += sizeof(size_t);
- continue;
- }
- pIn += ZSTD_NbCommonBytes(diff);
- return (size_t)(pIn - pStart);
- }
-
- if (MEM_64bits()) {
- if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
- pIn += 4;
- pMatch += 4;
- }
- }
- if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
- pIn += 2;
- pMatch += 2;
- }
- if ((pIn < pInLimit) && (*pMatch == *pIn)) {
- pIn++;
- }
- return (size_t)(pIn - pStart);
-}
-
-/**
- * Returns the number of bytes in common between pIn and pMatch,
- * counting backwards, with pIn having a lower limit of pAnchor and
- * pMatch having a lower limit of pBase.
- */
-static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
- const BYTE *pMatch, const BYTE *pBase) {
- size_t matchLength = 0;
- while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
- pIn--;
- pMatch--;
- matchLength++;
- }
- return matchLength;
-}
-
-LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
- const hash_t hash,
- const U32 checksum,
- const BYTE *pIn,
- const BYTE *pEnd,
- const BYTE *pAnchor,
- U64 *pForwardMatchLength,
- U64 *pBackwardMatchLength) {
- LDM_hashEntry *bucket = getBucket(table, hash);
- LDM_hashEntry *cur = bucket;
- LDM_hashEntry *bestEntry = NULL;
- U64 bestMatchLength = 0;
- for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
- const BYTE *pMatch = cur->offset + table->offsetBase;
-
- // Check checksum for faster check.
- if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
- U64 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
- U64 backwardMatchLength, totalMatchLength;
-
- // Only take matches where the forwardMatchLength is large enough
- // for speed.
- if (forwardMatchLength < table->minMatchLength) {
- continue;
- }
- backwardMatchLength =
- countBackwardsMatch(pIn, pAnchor, cur->offset + table->offsetBase,
- table->offsetBase);
-
- totalMatchLength = forwardMatchLength + backwardMatchLength;
-
- if (totalMatchLength >= bestMatchLength) {
- bestMatchLength = totalMatchLength;
- *pForwardMatchLength = forwardMatchLength;
- *pBackwardMatchLength = backwardMatchLength;
-
- bestEntry = cur;
-
-#ifdef ZSTD_SKIP
- return cur;
-#endif
- }
- }
- }
- if (bestEntry != NULL) {
- return bestEntry;
- }
- return NULL;
-}
-
-hash_t HASH_hashU32(U32 value) {
- return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
-}
-
-void HASH_insert(LDM_hashTable *table,
- const hash_t hash, const LDM_hashEntry entry) {
- // Circular buffer.
- *(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
- table->bucketOffsets[hash]++;
- table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
-}
-
-U32 HASH_getSize(const LDM_hashTable *table) {
- return table->numBuckets;
-}
-
-void HASH_destroyTable(LDM_hashTable *table) {
- free(table->entries);
- free(table->bucketOffsets);
- free(table);
-}
-
-void HASH_outputTableOccupancy(const LDM_hashTable *table) {
- U32 ctr = 0;
- LDM_hashEntry *cur = table->entries;
- LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
- for (; cur < end; ++cur) {
- if (cur->offset == 0) {
- ctr++;
- }
- }
-
- printf("Num buckets, bucket size: %d, %d\n",
- table->numBuckets, HASH_BUCKET_SIZE);
- printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
- table->numEntries, ctr,
- 100.0 * (double)(ctr) / table->numEntries);
-}
(double) stats->numMatches);
}
printf("\n");
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
/*
printf("Lower bit distribution\n");
for (i = 0; i < (1 << HASH_ONLY_EVERY_LOG); i++) {
return (hash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
}
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
static U32 lowerBitsFromHfHash(U64 hash) {
// The number of bits used so far is LDM_HASHLOG + 32.
// So there are 32 - LDM_HASHLOG bits left.
cctx->lastPosHashed[LDM_HASH_LENGTH]);
cctx->nextPosHashed = cctx->nextIp;
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
{
U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextHash);
cctx->stats.TMP_totalHashCount++;
// Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
// Note: this works only when cctx->step is 1.
#if LDM_LAG
- if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
- // TODO: Off by one, but not important.
- if (cctx->lagIp - cctx->ibase > 0) {
+ if (cctx -> lagIp - cctx->ibase > 0) {
+#ifdef INSERT_BY_TAG
+ U32 hashEveryMask = lowerBitsFromHfHash(cctx->lagHash);
+ if (hashEveryMask == HASH_ONLY_EVERY) {
+#else
+ if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+#endif
U32 smallHash = getSmallHash(cctx->lagHash);
# if USE_CHECKSUM
# else
HASH_insert(cctx->hashTable, smallHash, entry);
# endif
- } else {
-# if USE_CHECKSUM
+ }
+ } else {
+#ifdef INSERT_BY_TAG
+ U32 hashEveryMask = lowerBitsFromHfHash(hash);
+ if (hashEveryMask == HASH_ONLY_EVERY) {
+#else
+ if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
+#endif
+ U32 smallHash = getSmallHash(hash);
+
+#if USE_CHECKSUM
U32 checksum = getChecksum(hash);
- const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, checksum };
-# else
- const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase };
-# endif
+ const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
+#else
+ const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
+#endif
-# ifdef TMP_EVICTION
+#ifdef TMP_EVICTION
HASH_insert(cctx->hashTable, smallHash, entry, cctx);
-# else
+#else
HASH_insert(cctx->hashTable, smallHash, entry);
-# endif
+#endif
}
}
#else
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
U32 hashEveryMask = lowerBitsFromHfHash(hash);
if (hashEveryMask == HASH_ONLY_EVERY) {
#else
U64 hash;
hash_t smallHash;
U32 checksum;
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
U32 hashEveryMask;
#endif
setNextHash(cctx);
hash = cctx->nextHash;
smallHash = getSmallHash(hash);
checksum = getChecksum(hash);
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
hashEveryMask = lowerBitsFromHfHash(hash);
#endif
if (cctx->ip > cctx->imatchLimit) {
return 1;
}
-#ifdef TMP_TAG_INSERT
+#ifdef INSERT_BY_TAG
if (hashEveryMask == HASH_ONLY_EVERY) {
entry = HASH_getBestEntry(cctx, smallHash, checksum,
printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
- printf("LDM_LAG %d\n", LDM_LAG);
- printf("USE_CHECKSUM %d\n", USE_CHECKSUM);
+ printf("LDM_LAG: %d\n", LDM_LAG);
+ printf("USE_CHECKSUM: %d\n", USE_CHECKSUM);
+#ifdef INSERT_BY_TAG
+ printf("INSERT_BY_TAG: %d\n", 1);
+#else
+ printf("INSERT_BY_TAG: %d\n", 0);
+#endif
+ printf("HASH_CHAR_OFFSET: %d\n", HASH_CHAR_OFFSET);
printf("=====================\n");
}
-
-
// TODO: implement and test hash function
void LDM_test(const BYTE *src) {
const U32 diff = 100;
+++ /dev/null
-/**
- * A "hash" table used in LDM compression.
- *
- * This is not exactly a hash table in the sense that inserted entries
- * are not guaranteed to remain in the hash table.
- */
-
-#ifndef LDM_HASHTABLE_H
-#define LDM_HASHTABLE_H
-
-#include "mem.h"
-
-// The log size of LDM_hashEntry in bytes.
-#define LDM_HASH_ENTRY_SIZE_LOG 3
-
-typedef U32 hash_t;
-
-typedef struct LDM_hashEntry {
- U32 offset; // Represents the offset of the entry from offsetBase.
- U32 checksum; // A checksum to select entries with the same hash value.
-} LDM_hashEntry;
-
-typedef struct LDM_hashTable LDM_hashTable;
-
-/**
- * Create a table that can contain size elements. This does not necessarily
- * correspond to the number of hash buckets. The number of hash buckets
- * is size / (1 << HASH_BUCKET_SIZE_LOG)
- *
- * minMatchLength is the minimum match length required in HASH_getBestEntry.
- *
- * maxWindowSize is the maximum distance from pIn in HASH_getBestEntry.
- * The window is defined to be (pIn - offsetBase - offset).
- */
-LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
- U32 minMatchLength, U32 maxWindowSize);
-
-/**
- * Return the "best" entry from the table with the same hash and checksum.
- *
- * pIn: a pointer to the current input position.
- * pEnd: a pointer to the maximum input position.
- * pAnchor: a pointer to the minimum input position.
- *
- * This function computes the forward and backward match length from pIn
- * and writes it to forwardMatchLength and backwardsMatchLength.
- *
- * E.g. for the two strings "aaabbbb" "aaabbbb" with pIn and the
- * entry pointing at the first "b", the forward match length would be
- * four (representing the "b" matches) and the backward match length would
- * three (representing the "a" matches before the pointer).
- */
-LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
- const hash_t hash,
- const U32 checksum,
- const BYTE *pIn,
- const BYTE *pEnd,
- const BYTE *pAnchor,
- U64 *forwardMatchLength,
- U64 *backwardsMatchLength);
-
-/**
- * Return a hash of the value.
- */
-hash_t HASH_hashU32(U32 value);
-
-/**
- * Insert an LDM_hashEntry into the bucket corresponding to hash.
- *
- * An entry may be evicted in the process.
- */
-void HASH_insert(LDM_hashTable *table, const hash_t hash,
- const LDM_hashEntry entry);
-
-/**
- * Return the number of distinct hash buckets.
- */
-U32 HASH_getSize(const LDM_hashTable *table);
-
-/**
- * Destroy the table.
- */
-void HASH_destroyTable(LDM_hashTable *table);
-
-/**
- * Prints the percentage of the hash table occupied (where occupied is defined
- * as the entry being non-zero).
- */
-void HASH_outputTableOccupancy(const LDM_hashTable *hashTable);
-
-#endif /* LDM_HASHTABLE_H */
projects / thirdparty / zstd.git / commitdiff
