# This Makefile presumes libzstd is installed, using `sudo make install`
+CFLAGS ?= -O3
+DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
+ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+ -Wredundant-decls
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS = $(CPPFLAGS) $(CFLAGS)
LDFLAGS += -lzstd
#main : ldm.c main.c
# $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
-main-ldm : ldm.c main-ldm.c
+main-ldm : util.c ldm.c main-ldm.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
clean:
#include <stdio.h>
#include "ldm.h"
+#include "util.h"
#define HASH_EVERY 7
// typedef uint64_t tag;
+/*
static unsigned LDM_isLittleEndian(void) {
const union { U32 u; BYTE c[4]; } one = { 1 };
return one.c[0];
static void LDM_copy8(void *dst, const void *src) {
memcpy(dst, src, 8);
}
+
+*/
typedef struct LDM_hashEntry {
offset_t offset;
} LDM_hashEntry;
return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
}
+/*
static hash_t LDM_hash5(U64 sequence) {
static const U64 prime5bytes = 889523592379ULL;
static const U64 prime8bytes = 11400714785074694791ULL;
else
return (((sequence >> 24) * prime8bytes) >> (64 - hashLog));
}
+*/
static hash_t LDM_hash_position(const void * const p) {
return LDM_hash(LDM_read32(p));
}
-static void LDM_put_position_on_hash(const BYTE *p, hash_t h,
- void *tableBase, const BYTE *srcBase) {
+static void LDM_putHashOfPosition(const BYTE *p, hash_t h,
+ void *tableBase, const BYTE *srcBase) {
+ LDM_hashEntry *hashTable;
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
return;
}
- LDM_hashEntry *hashTable = (LDM_hashEntry *) tableBase;
- hashTable[h] = (LDM_hashEntry) { (hash_t )(p - srcBase) };
+ hashTable = (LDM_hashEntry *) tableBase;
+ hashTable[h] = (LDM_hashEntry) { (hash_t)(p - srcBase) };
}
static void LDM_putPosition(const BYTE *p, void *tableBase,
- const BYTE *srcBase) {
+ const BYTE *srcBase) {
+ hash_t hash;
if (((p - srcBase) & HASH_EVERY) != HASH_EVERY) {
return;
}
- hash_t const h = LDM_hash_position(p);
- LDM_put_position_on_hash(p, h, tableBase, srcBase);
+ hash = LDM_hash_position(p);
+ LDM_putHashOfPosition(p, hash, tableBase, srcBase);
}
static void LDM_putHashOfCurrentPosition(LDM_CCtx *const cctx) {
- LDM_putPosition(cctx->ip, cctx->hashTable, cctx->ibase);
+ hash_t hash = LDM_hash_position(cctx->ip);
+ LDM_putHashOfPosition(cctx->ip, hash, cctx->hashTable, cctx->ibase);
+ cctx->lastPosHashed = cctx->ip;
+ cctx->lastHash = hash;
}
-
static const BYTE *LDM_get_position_on_hash(
hash_t h, void *tableBase, const BYTE *srcBase) {
const LDM_hashEntry * const hashTable = (LDM_hashEntry *)tableBase;
return (unsigned)(pIn - pStart);
}
-void LDM_read_header(const void *src, size_t *compressSize,
- size_t *decompressSize) {
+void LDM_readHeader(const void *src, size_t *compressSize,
+ size_t *decompressSize) {
const U32 *ip = (const U32 *)src;
*compressSize = *ip++;
*decompressSize = *ip;
cctx->imatchLimit = cctx->iend - MINMATCH;
cctx->obase = (BYTE *)dst;
- cctx->op = (BYTE *)cctx->obase;
+ cctx->op = (BYTE *)dst;
cctx->anchor = cctx->ibase;
size_t LDM_compress(const void *src, size_t srcSize,
void *dst, size_t maxDstSize) {
LDM_CCtx cctx;
+ U32 forwardH;
LDM_initializeCCtx(&cctx, src, srcSize, dst, maxDstSize);
- U32 forwardH;
/* Hash the first position and put it into the hash table. */
LDM_putHashOfCurrentPosition(&cctx);
- const BYTE *lastHash = cctx.ip;
cctx.ip++;
forwardH = LDM_hash_position(cctx.ip);
match = LDM_get_position_on_hash(h, cctx.hashTable, cctx.ibase);
forwardH = LDM_hash_position(forwardIp);
- LDM_put_position_on_hash(cctx.ip, h, cctx.hashTable, cctx.ibase);
- lastHash = cctx.ip;
+ LDM_putHashOfPosition(cctx.ip, h, cctx.hashTable, cctx.ibase);
} while (cctx.ip - match > WINDOW_SIZE ||
LDM_read64(match) != LDM_read64(cctx.ip));
}
memcpy(cctx.op, cctx.anchor, litLength);
cctx.op += litLength;
}
-_next_match:
+
/* Encode offset */
{
/*
/* Encode Match Length */
{
unsigned matchCode;
+ unsigned ctr = 1;
matchCode = LDM_count(cctx.ip + MINMATCH, match + MINMATCH,
cctx.ihashLimit);
#ifdef LDM_DEBUG
printf("\n");
#endif
cctx.stats.total_match_length += matchCode + MINMATCH;
- unsigned ctr = 1;
cctx.ip++;
for (; ctr < MINMATCH + matchCode; cctx.ip++, ctr++) {
LDM_putHashOfCurrentPosition(&cctx);
LDM_putPosition(cctx.ip, cctx.hashTable, cctx.ibase);
forwardH = LDM_hash_position(++cctx.ip);
- lastHash = cctx.ip;
}
_last_literals:
/* Encode last literals */
cctx.op += lastRun;
}
LDM_printCompressStats(&cctx.stats);
- return (cctx.op - (BYTE *)cctx.obase);
+ return (cctx.op - (const BYTE *)cctx.obase);
}
typedef struct LDM_DCtx {
LDM_DCtx dctx;
LDM_initializeDCtx(&dctx, src, compressSize, dst, maxDecompressSize);
- BYTE *cpy;
- size_t length;
- const BYTE *match;
- size_t offset;
-
while (dctx.ip < dctx.iend) {
+ BYTE *cpy;
+ size_t length;
+ const BYTE *match;
+ size_t offset;
+
/* get literal length */
unsigned const token = *(dctx.ip)++;
if ((length = (token >> ML_BITS)) == RUN_MASK) {
size_t LDM_decompress(const void *src, size_t srcSize,
void *dst, size_t maxDstSize);
-void LDM_read_header(const void *src, size_t *compressSize,
- size_t *decompressSize);
+void LDM_readHeader(const void *src, size_t *compressSize,
+ size_t *decompressSize);
#endif /* LDM_H */
int fdin, fdout;
struct stat statbuf;
char *src, *dst;
+ size_t maxCompressSize, compressSize;
/* Open the input file. */
if ((fdin = open(fname, O_RDONLY)) < 0) {
return 1;
}
- size_t maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
+ maxCompressSize = statbuf.st_size + LDM_HEADER_SIZE;
/* Go to the location corresponding to the last byte. */
- /* TODO: fallocate? */
+ /* TODO: fallocate? */
if (lseek(fdout, maxCompressSize - 1, SEEK_SET) == -1) {
perror("lseek error");
return 1;
}
#ifdef ZSTD
- size_t compressSize = ZSTD_compress(dst, statbuf.st_size,
+ compressSize = ZSTD_compress(dst, statbuf.st_size,
src, statbuf.st_size, 1);
#else
- size_t compressSize = LDM_HEADER_SIZE +
+ compressSize = LDM_HEADER_SIZE +
LDM_compress(src, statbuf.st_size,
dst + LDM_HEADER_SIZE, statbuf.st_size);
- // Write compress and decompress size to header
+ // Write compress and decompress size to header
// TODO: should depend on LDM_DECOMPRESS_SIZE write32
memcpy(dst, &compressSize, 4);
memcpy(dst + 4, &(statbuf.st_size), 4);
/* Decompress file compressed using LDM_compress.
* The input file should have the LDM_HEADER followed by payload.
- * Returns 0 if succesful, and an error code otherwise.
+ * Returns 0 if succesful, and an error code otherwise.
*/
static int decompress(const char *fname, const char *oname) {
int fdin, fdout;
struct stat statbuf;
char *src, *dst;
+ size_t compressSize, decompressSize, outSize;
/* Open the input file. */
if ((fdin = open(fname, O_RDONLY)) < 0) {
}
/* Read the header. */
- size_t compressSize, decompressSize;
- LDM_read_header(src, &compressSize, &decompressSize);
+ LDM_readHeader(src, &compressSize, &decompressSize);
#ifdef DEBUG
printf("Size, compressSize, decompressSize: %zu %zu %zu\n",
}
#ifdef ZSTD
- size_t outSize = ZSTD_decompress(dst, decomrpessed_size,
+ outSize = ZSTD_decompress(dst, decomrpessed_size,
src + LDM_HEADER_SIZE,
statbuf.st_size - LDM_HEADER_SIZE);
#else
- size_t outSize = LDM_decompress(
+ outSize = LDM_decompress(
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
dst, decompressSize);
FILE *decFp = fopen(decFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
- const int cmp = compare(inpFp, decFp);
- if(0 == cmp) {
- printf("verify : OK\n");
- } else {
- printf("verify : NG\n");
- }
+ {
+ const int cmp = compare(inpFp, decFp);
+ if(0 == cmp) {
+ printf("verify : OK\n");
+ } else {
+ printf("verify : NG\n");
+ }
+ }
fclose(decFp);
fclose(inpFp);
printf("ldm = [%s]\n", ldmFilename);
printf("dec = [%s]\n", decFilename);
- struct timeval tv1, tv2;
/* Compress */
-
- gettimeofday(&tv1, NULL);
- if (compress(inpFilename, ldmFilename)) {
- printf("Compress error");
- return 1;
+ {
+ struct timeval tv1, tv2;
+ gettimeofday(&tv1, NULL);
+ if (compress(inpFilename, ldmFilename)) {
+ printf("Compress error");
+ return 1;
+ }
+ gettimeofday(&tv2, NULL);
+ printf("Total time = %f seconds\n",
+ (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+ (double) (tv2.tv_sec - tv1.tv_sec));
}
- gettimeofday(&tv2, NULL);
- printf("Total time = %f seconds\n",
- (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
- (double) (tv2.tv_sec - tv1.tv_sec));
/* Decompress */
-
- gettimeofday(&tv1, NULL);
- if (decompress(ldmFilename, decFilename)) {
- printf("Decompress error");
- return 1;
+ {
+ struct timeval tv1, tv2;
+ gettimeofday(&tv1, NULL);
+ if (decompress(ldmFilename, decFilename)) {
+ printf("Decompress error");
+ return 1;
+ }
+ gettimeofday(&tv2, NULL);
+ printf("Total time = %f seconds\n",
+ (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
+ (double) (tv2.tv_sec - tv1.tv_sec));
}
- gettimeofday(&tv2, NULL);
- printf("Total time = %f seconds\n",
- (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
- (double) (tv2.tv_sec - tv1.tv_sec));
-
/* verify */
verify(inpFilename, decFilename);
return 0;
--- /dev/null
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "util.h"
+
+typedef uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef int32_t S32;
+typedef uint64_t U64;
+
+unsigned LDM_isLittleEndian(void) {
+ const union { U32 u; BYTE c[4]; } one = { 1 };
+ return one.c[0];
+}
+
+U16 LDM_read16(const void *memPtr) {
+ U16 val;
+ memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+U16 LDM_readLE16(const void *memPtr) {
+ if (LDM_isLittleEndian()) {
+ return LDM_read16(memPtr);
+ } else {
+ const BYTE *p = (const BYTE *)memPtr;
+ return (U16)((U16)p[0] + (p[1] << 8));
+ }
+}
+
+void LDM_write16(void *memPtr, U16 value){
+ memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_write32(void *memPtr, U32 value) {
+ memcpy(memPtr, &value, sizeof(value));
+}
+
+void LDM_writeLE16(void *memPtr, U16 value) {
+ if (LDM_isLittleEndian()) {
+ LDM_write16(memPtr, value);
+ } else {
+ BYTE* p = (BYTE *)memPtr;
+ p[0] = (BYTE) value;
+ p[1] = (BYTE)(value>>8);
+ }
+}
+
+U32 LDM_read32(const void *ptr) {
+ return *(const U32 *)ptr;
+}
+
+U64 LDM_read64(const void *ptr) {
+ return *(const U64 *)ptr;
+}
+
+void LDM_copy8(void *dst, const void *src) {
+ memcpy(dst, src, 8);
+}
+
+
--- /dev/null
+#ifndef LDM_UTIL_H
+#define LDM_UTIL_H
+
+unsigned LDM_isLittleEndian(void);
+
+uint16_t LDM_read16(const void *memPtr);
+
+uint16_t LDM_readLE16(const void *memPtr);
+
+void LDM_write16(void *memPtr, uint16_t value);
+
+void LDM_write32(void *memPtr, uint32_t value);
+
+void LDM_writeLE16(void *memPtr, uint16_t value);
+
+uint32_t LDM_read32(const void *ptr);
+
+uint64_t LDM_read64(const void *ptr);
+
+void LDM_copy8(void *dst, const void *src);
+
+
+#endif /* LDM_UTIL_H */
projects / thirdparty / zstd.git / commitdiff
