Short-circuit longest_match on short candidates via XOR and ctz

When best_len is under 8, the whole candidate match fits in the
first 8 bytes, so a single load XORed against scan_start and run
through ctz gives the exact match length with no COMPARE256 call
and no separate tail sentinel. The best_len-to-offset mapping
collapses to a single subtract since the graduated sizeof(uint32_t)
and sizeof(uint64_t) brackets no longer cover anything distinct.

Benefits every deflate strategy that calls longest_match: level 3
gains 10-12% at 128KB+ inputs and levels 1, 6, and 7 each trim 1-3%.
Compressed output is byte-identical across corpora.

Prior art: zlib-rs first applied the same XOR+ctz technique in
trifectatechfoundation/zlib-rs#284.

Co-authored-by: Brian Pane <brianp@brianp.net>

diff --git a/arch/power/compare256_power9.c b/arch/power/compare256_power9.c
index 4ab542949eb90af2a653443134dd7a7bdd27d5c7..99c096effcfcd5e0c6aead35792c4f55c7a34777 100644 (file)
--- a/arch/power/compare256_power9.c
+++ b/arch/power/compare256_power9.c
@@ -6,9 +6,10 @@
 #ifdef POWER9
 
 #include "zbuild.h"
+#include "zendian.h"
 #include "zmemory.h"
 #include "deflate.h"
-#include "zendian.h"
+#include "fallback_builtins.h"
 
 #include <altivec.h>
 

diff --git a/arch/riscv/compare256_rvv.c b/arch/riscv/compare256_rvv.c
index f4d577863686df44d5e9f5ec8a61b5cdd8d1a65a..e7fe7d9dc2027eda2754ba70987943d23ca42777 100644 (file)
--- a/arch/riscv/compare256_rvv.c
+++ b/arch/riscv/compare256_rvv.c
@@ -7,8 +7,10 @@
 #ifdef RISCV_RVV
 
 #include "zbuild.h"
+#include "zendian.h"
 #include "zmemory.h"
 #include "deflate.h"
+#include "fallback_builtins.h"
 
 #include <riscv_vector.h>
 

diff --git a/match_tpl.h b/match_tpl.h
index e1f8ebd37c33cdf3dddc7f1443d24cc7b5919ae3..34f7911a42f1138e3fdd877df1270056a504dadb 100644 (file)
--- a/match_tpl.h
+++ b/match_tpl.h
@@ -57,15 +57,11 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, uint32_t cur_match) {
     if (UNLIKELY(best_len >= lookahead))
         return lookahead;
 
-    /* Calculate read offset which should only extend an extra byte
-     * to find the next best match length.
+    /* Calculate read offset which should only extend an extra byte to find the
+     * next best match length. When best_len is shorter than the read width, we
+     * diff the mismatched bytes instead.
      */
-    offset = best_len-1;
-    if (UNLIKELY(best_len >= sizeof(uint32_t))) {
-        offset -= 2;
-        if (best_len >= sizeof(uint64_t))
-            offset -= 4;
-    }
+    offset = best_len >= sizeof(uint64_t) ? best_len - 7 : 0;
 
     scan = window + strstart;
     scan_start = zng_memread_8(scan);
@@ -128,32 +124,36 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, uint32_t cur_match) {
          * that depend on those values. However the length of the match is limited to the
          * lookahead, so the output of deflate is not affected by the uninitialized values.
          */
-        if (best_len < sizeof(uint32_t)) {
-            for (;;) {
-                if (zng_memcmp_2(mbase_end+cur_match, &scan_end) == 0 &&
-                    zng_memcmp_2(mbase_start+cur_match, &scan_start) == 0)
-                    break;
-                GOTO_NEXT_CHAIN;
+        uint32_t len;
+        if (best_len < sizeof(uint64_t)) {
+            uint64_t cand_start = zng_memread_8(mbase_start + cur_match);
+            if (scan_start != cand_start) {
+                uint64_t diff = scan_start ^ cand_start;
+                len = zng_first_diff_byte64(diff);
+                if (len <= best_len) {
+                    GOTO_NEXT_CHAIN;
+                }
+                goto short_match_accept;
             }
-        } else if (best_len >= sizeof(uint64_t)) {
+            /* All 8 bytes match, fallthrough to compare256 for the tail. */
+        } else {
+            /* Pre-filter the candidate on the start and end sentinels before compare256 using
+             * simple 8-byte comparison since best_len >= 8. */
             for (;;) {
+                /* First check the end of the candidate at best_len+1 due to the higher
+                 * likelihood of a mismatch. */
                 if (zng_memcmp_8(mbase_end+cur_match, &scan_end) == 0 &&
                     zng_memcmp_8(mbase_start+cur_match, &scan_start) == 0)
                     break;
                 GOTO_NEXT_CHAIN;
             }
-        } else {
-            for (;;) {
-                if (zng_memcmp_4(mbase_end+cur_match, &scan_end) == 0 &&
-                    zng_memcmp_4(mbase_start+cur_match, &scan_start) == 0)
-                    break;
-                GOTO_NEXT_CHAIN;
-            }
         }
-        uint32_t len = COMPARE256(scan+2, mbase_start+cur_match+2) + 2;
+        len = COMPARE256(scan+2, mbase_start+cur_match+2) + 2;
         Assert(scan+len <= window+(unsigned)(s->window_size-1), "wild scan");
 
-        if (len > best_len) {
+        if (len > best_len)
+short_match_accept:
+        {
             uint32_t match_start = cur_match - match_offset;
             s->match_start = match_start;
 
@@ -167,12 +167,7 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, uint32_t cur_match) {
 
             best_len = len;
 
-            offset = best_len-1;
-            if (LIKELY(best_len >= sizeof(uint32_t))) {
-                offset -= 2;
-                if (best_len >= sizeof(uint64_t))
-                    offset -= 4;
-            }
+            offset = best_len >= sizeof(uint64_t) ? best_len - 7 : 0;
 
             scan_end = zng_memread_8(scan+offset);