Reject short-match candidates with a masked test instead of ctz

A candidate can only improve best_len when its first best_len+1 bytes
all match, so the rejection path does not need the exact mismatch
position. Compute a loop-invariant mask of those bytes and reject with a
single AND and zero-test, running ctz only on the rare accepting
candidate. This keeps the multi-cycle ctz off the hot rejection path,
which helps dense-chain inputs (low-alphabet data, slow cores) while
leaving compressed output unchanged.

diff --git a/fallback_builtins.h b/fallback_builtins.h
index 16ed1e70d58a59394f0a0bc7c22029da94342077..667924774c22042a8dbfa2a81b954d5da1d9d03c 100644 (file)
--- a/fallback_builtins.h
+++ b/fallback_builtins.h
@@ -130,6 +130,14 @@ Z_FORCEINLINE static uint32_t zng_clz64(uint64_t value) {
 #  define zng_first_diff_byte64(diff) (zng_ctz64(diff) / 8)
 #endif
 
+/* Builds a mask of the first n bytes (1..8) so an XOR diff can be tested for
+ * a leading n-byte match with a single AND. */
+#if BYTE_ORDER == BIG_ENDIAN
+#  define zng_first_bytes_mask64(n) (UINT64_MAX << ((8 - (n)) * 8))
+#else
+#  define zng_first_bytes_mask64(n) (UINT64_MAX >> ((8 - (n)) * 8))
+#endif
+
 Z_FORCEINLINE static uint16_t zng_bitreverse16(uint16_t value) {
 #if __has_builtin(__builtin_bitreverse16)
     return (uint16_t)__builtin_bitreverse16(value);

diff --git a/match_tpl.h b/match_tpl.h
index 2c453b70393115374bb26b121362c6c20d84740d..c82675579a06a55adb6cd846f2932be421ba688a 100644 (file)
--- a/match_tpl.h
+++ b/match_tpl.h
@@ -131,10 +131,14 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, uint32_t cur_match) {
                 /* Peel the first candidate out of the loop. A full 8-byte match falls straight
                  * through to compare256, and single-candidate chains (barely-compressible data)
                  * run with no loop overhead. */
+                uint64_t first_mask = zng_first_bytes_mask64(best_len + 1);
                 uint64_t diff = scan_start ^ cand_start;
-                len = zng_first_diff_byte64(diff);
-                if (len > best_len)
+                /* A candidate beats best_len only when its first best_len+1 bytes match, i.e.
+                 * those bytes of the XOR are zero. The masked test rejects without running ctz. */
+                if (UNLIKELY((diff & first_mask) == 0)) {
+                    len = zng_first_diff_byte64(diff);
                     goto short_match_accept;
+                }
                 if (--chain_length == 0 || (cur_match = prev[cur_match & wmask]) <= limit)
                     return best_len;
                 cand_start = zng_memread_8(mbase_start + cur_match);
@@ -142,9 +146,10 @@ Z_INTERNAL uint32_t LONGEST_MATCH(deflate_state *const s, uint32_t cur_match) {
                     /* Walk the remaining candidates with the chain advance kept inline. */
                     for (;;) {
                         diff = scan_start ^ cand_start;
-                        len = zng_first_diff_byte64(diff);
-                        if (len > best_len)
+                        if (UNLIKELY((diff & first_mask) == 0)) {
+                            len = zng_first_diff_byte64(diff);
                             goto short_match_accept;
+                        }
                         if (--chain_length == 0 || (cur_match = prev[cur_match & wmask]) <= limit)
                             return best_len;
                         cand_start = zng_memread_8(mbase_start + cur_match);