Incorporate fast-zlib algorithm changes into longest_match.

diff --git a/match_tpl.h b/match_tpl.h
index 04eac1aa8f2d419cedae32d154eab6d3c0c56061..3d8ac4e3c5e77fe4901b7fd29e4b73373cf62ed3 100644 (file)
--- a/match_tpl.h
+++ b/match_tpl.h
@@ -1,3 +1,12 @@
+/* match_tpl.h -- find longest match template for compare256 variants
+ *
+ * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ *
+ * Portions copyright (C) 2014-2021 Konstantin Nosov
+ *  Fast-zlib optimized longest_match
+ *  https://github.com/gildor2/fast_zlib
+ */
 
 #include "zbuild.h"
 #include "deflate.h"
@@ -36,10 +45,12 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) {
     Z_REGISTER unsigned char *mbase_start = window;
     Z_REGISTER unsigned char *mbase_end;
     const Pos *prev = s->prev;
-    Pos limit;
+    Pos limit, limit_base;
     int32_t early_exit;
     uint32_t chain_length, nice_match, best_len, offset;
     uint32_t lookahead = s->lookahead;
+    Pos match_offset = 0;
+    int32_t rolling_hash;
     bestcmp_t scan_end;
 #ifndef UNALIGNED_OK
     bestcmp_t scan_end0;
@@ -55,7 +66,7 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) {
     /* The code is optimized for STD_MAX_MATCH-2 multiple of 16. */
     Assert(STD_MAX_MATCH == 258, "Code too clever");
 
-    best_len = s->prev_length ? s->prev_length : 1;
+    best_len = s->prev_length ? s->prev_length : STD_MIN_MATCH-1;
 
     /* Calculate read offset which should only extend an extra byte
      * to find the next best match length.
@@ -81,6 +92,7 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) {
 
     /* Do not waste too much time if we already have a good match */
     chain_length = s->max_chain_length;
+    rolling_hash = chain_length > 1024;
     early_exit = s->level < EARLY_EXIT_TRIGGER_LEVEL;
     if (best_len >= s->good_match)
         chain_length >>= 2;
@@ -89,7 +101,38 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) {
     /* Stop when cur_match becomes <= limit. To simplify the code,
      * we prevent matches with the string of window index 0
      */
-    limit = strstart > MAX_DIST(s) ? (Pos)(strstart - MAX_DIST(s)) : 0;
+    limit = limit_base = strstart > MAX_DIST(s) ? (Pos)(strstart - MAX_DIST(s)) : 0;
+
+    if (best_len >= STD_MIN_MATCH && rolling_hash) {
+        /* We're continuing search (lazy evaluation). */
+        uint32_t i, hash;
+        Pos pos;
+
+        /* Find a most distant chain starting from scan with index=1 (index=0 corresponds
+         * to cur_match). We cannot use s->prev[strstart+1,...] immediately, because
+         * these strings are not yet inserted into the hash table.
+         */
+        hash = s->update_hash(s, 0, scan[1]);
+        hash = s->update_hash(s, hash, scan[2]);
+
+        for (i = 3; i <= best_len; i++) {
+            hash = s->update_hash(s, hash, scan[i]);
+
+            /* If we're starting with best_len >= 3, we can use offset search. */
+            pos = s->head[hash];
+            if (pos < cur_match) {
+                match_offset = (Pos)(i - 2);
+                cur_match = pos;
+            }
+        }
+
+        /* Update offset-dependent variables */
+        limit = limit_base+match_offset;
+        if (cur_match <= limit)
+            goto break_matching;
+        mbase_start -= match_offset;
+        mbase_end -= match_offset;
+    }
 
     Assert((unsigned long)strstart <= s->window_size - MIN_LOOKAHEAD, "need lookahead");
     for (;;) {
@@ -140,7 +183,9 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) {
         Assert(scan+len <= window+(unsigned)(s->window_size-1), "wild scan");
 
         if (len > best_len) {
-            s->match_start = cur_match;
+            uint32_t match_start = cur_match - match_offset;
+            s->match_start = match_start;
+
             /* Do not look for matches beyond the end of the input. */
             if (len > lookahead)
                 return lookahead;
@@ -162,7 +207,56 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) {
 #ifndef UNALIGNED_OK
             scan_end0 = *(bestcmp_t *)(scan+offset+1);
 #endif
-            mbase_end = (mbase_start+offset);
+            /* Look for a better string offset */
+            if (len > STD_MIN_MATCH && match_start + len < strstart && rolling_hash) {
+                Pos pos, next_pos;
+                uint32_t i, hash;
+                unsigned char *scan_endstr;
+
+                /* Go back to offset 0 */
+                cur_match -= match_offset;
+                match_offset = 0;
+                next_pos = cur_match;
+                for (i = 0; i <= len - STD_MIN_MATCH; i++) {
+                    pos = prev[(cur_match + i) & wmask];
+                    if (pos < next_pos) {
+                        /* Hash chain is more distant, use it */
+                        if (pos <= limit_base + i)
+                            goto break_matching;
+                        next_pos = pos;
+                        match_offset = (Pos)i;
+                    }
+                }
+                /* Switch cur_match to next_pos chain */
+                cur_match = next_pos;
+
+                /* Try hash head at len-(STD_MIN_MATCH-1) position to see if we could get
+                 * a better cur_match at the end of string. Using (STD_MIN_MATCH-1) lets
+                 * us include one more byte into hash - the byte which will be checked
+                 * in main loop now, and which allows to grow match by 1.
+                 */
+                scan_endstr = scan + len - (STD_MIN_MATCH+1);
+
+                hash = s->update_hash(s, 0, scan_endstr[0]);
+                hash = s->update_hash(s, hash, scan_endstr[1]);
+                hash = s->update_hash(s, hash, scan_endstr[2]);
+
+                pos = s->head[hash];
+                if (pos < cur_match) {
+                    match_offset = (Pos)(len - (STD_MIN_MATCH+1));
+                    if (pos <= limit_base + match_offset)
+                        goto break_matching;
+                    cur_match = pos;
+                }
+
+                /* Update offset-dependent variables */
+                limit = limit_base+match_offset;
+                mbase_start = window-match_offset;
+                mbase_end = (mbase_start+offset);
+                continue;
+            } else {
+                mbase_end = (mbase_start+offset);
+            }
         } else if (UNLIKELY(early_exit)) {
             /* The probability of finding a match later if we here is pretty low, so for
              * performance it's best to outright stop here for the lower compression levels
@@ -171,8 +265,14 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, Pos cur_match) {
         }
         GOTO_NEXT_CHAIN;
     }
-
     return best_len;
+
+break_matching:
+
+    if (best_len < s->lookahead)
+        return best_len;
+
+    return s->lookahead;
 }
 
 #undef LONGEST_MATCH