minor split optimization

let's fill the initial stats directly into target fingerprint

diff --git a/lib/compress/zstd_preSplit.c b/lib/compress/zstd_preSplit.c
index ceac5c91f6ac03933ab31aa583ec66e865bb1049..f831dabdec3125c0c3756a128e635ee0edb96601 100644 (file)
--- a/lib/compress/zstd_preSplit.c
+++ b/lib/compress/zstd_preSplit.c
@@ -18,7 +18,7 @@
 #define BLOCKSIZE_MIN 3500
 #define THRESHOLD_PENALTY_RATE 16
 #define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2)
-#define THRESHOLD_PENALTY 4
+#define THRESHOLD_PENALTY 3
 
 #define HASHLENGTH 2
 #define HASHLOG 10
@@ -84,8 +84,8 @@ static int compareFingerprints(const FingerPrint* ref,
                             const FingerPrint* newfp,
                             int penalty)
 {
-    if (ref->nbEvents <= BLOCKSIZE_MIN)
-        return 0;
+    assert(ref->nbEvents > 0);
+    assert(newfp->nbEvents > 0);
     {   S64 p50 = ref->nbEvents * newfp->nbEvents;
         S64 deviation = fpDistance(ref, newfp);
         S64 threshold = p50 * (THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
@@ -140,7 +140,8 @@ size_t ZSTD_splitBlock_4k(const void* src, size_t srcSize,
     assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
 
     initStats(fpstats);
-    for (pos = 0; pos < blockSizeMax;) {
+    recordFingerprint(&fpstats->pastEvents, p, CHUNKSIZE);
+    for (pos = CHUNKSIZE; pos < blockSizeMax; pos += CHUNKSIZE) {
         assert(pos <= blockSizeMax - CHUNKSIZE);
         recordFingerprint(&fpstats->newEvents, p + pos, CHUNKSIZE);
         if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty)) {
@@ -150,7 +151,6 @@ size_t ZSTD_splitBlock_4k(const void* src, size_t srcSize,
             ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents));
             penalty = penalty - 1 + (penalty == 0);
         }
-        pos += CHUNKSIZE;
     }
     return blockSizeMax;
     (void)flushEvents; (void)removeEvents;

diff --git a/lib/compress/zstd_preSplit.h b/lib/compress/zstd_preSplit.h
index b0b6bb76282fa980eda3dd42a51202e4f4ffba99..7b6aadd0b19565f740ba4542c37ef7699ed32ad7 100644 (file)
--- a/lib/compress/zstd_preSplit.h
+++ b/lib/compress/zstd_preSplit.h
@@ -22,6 +22,10 @@ extern "C" {
 /* note:
  * @workspace must be aligned on 8-bytes boundaries
  * @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE
+ * note2:
+ * for the time being, this function only accepts full 128 KB blocks,
+ * therefore @blockSizeMax must be == 128 KB.
+ * This could be extended to smaller sizes in the future.
  */
 size_t ZSTD_splitBlock_4k(const void* src, size_t srcSize, size_t blockSizeMax, void* workspace, size_t wkspSize);