Skip redundant hash slides for non-chain-walking strategies

deflate_quick reads only the head of each hash chain and never walks
prev, while Z_HUFFMAN_ONLY and Z_RLE read neither. Sliding the whole
hash on every window move is wasted work for them. Add a slide_hash_head
functable entry that slides only the head table, so deflate_quick slides
just the head and the huffman and rle strategies skip the slide entirely.

About 8% faster on the level-1 no-CRC deflate benchmark, output unchanged.

diff --git a/arch/arm/arm_functions.h b/arch/arm/arm_functions.h
index 6eb72a3dac5441524a348367505550928a146622..afb1ae0b7e36643168fd699d83276550ab65bcb3 100644 (file)
--- a/arch/arm/arm_functions.h
+++ b/arch/arm/arm_functions.h
@@ -16,6 +16,7 @@ void inflate_fast_neon(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 uint32_t longest_match_neon(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_neon(deflate_state *const s, uint32_t cur_match);
 void slide_hash_neon(deflate_state *s);
+void slide_hash_head_neon(deflate_state *s);
 #endif
 
 #ifdef ARM_NEON_DOTPROD
@@ -48,6 +49,7 @@ uint32_t crc32_copy_armv8_pmull_eor3(uint32_t crc, uint8_t *dst, const uint8_t *
 
 #ifdef ARM_SIMD
 void slide_hash_armv6(deflate_state *s);
+void slide_hash_head_armv6(deflate_state *s);
 #endif
 
 #ifdef DISABLE_RUNTIME_CPU_DETECTION
@@ -55,6 +57,8 @@ void slide_hash_armv6(deflate_state *s);
 #  ifdef ARM_SIMD_NATIVE
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_armv6
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_armv6
 #  endif
 // ARM - NEON
 #  ifdef ARM_NEON_NATIVE
@@ -74,6 +78,8 @@ void slide_hash_armv6(deflate_state *s);
 #    define native_longest_match_roll longest_match_roll_neon
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_neon
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_neon
 #  endif
 // ARM - NEON DotProd
 #  ifdef ARM_NEON_DOTPROD_NATIVE

diff --git a/arch/arm/slide_hash_armv6.c b/arch/arm/slide_hash_armv6.c
index b241e6c5e64a5eecae9e6aacf29c07ca093772e0..735687d274e158af003e34a17121fddac4adca2a 100644 (file)
--- a/arch/arm/slide_hash_armv6.c
+++ b/arch/arm/slide_hash_armv6.c
@@ -46,4 +46,11 @@ Z_INTERNAL void slide_hash_armv6(deflate_state *s) {
     slide_hash_chain(s->head, HASH_SIZE, wsize);
     slide_hash_chain(s->prev, wsize, wsize);
 }
+
+Z_INTERNAL void slide_hash_head_armv6(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+
+    slide_hash_chain(s->head, HASH_SIZE, wsize);
+}
 #endif

diff --git a/arch/arm/slide_hash_neon.c b/arch/arm/slide_hash_neon.c
index 2f9e94a33d9596c3a8e89dc1e71ce43a661d1db8..9efb6275513ee285bb5bbef6091cb7e1f9582a95 100644 (file)
--- a/arch/arm/slide_hash_neon.c
+++ b/arch/arm/slide_hash_neon.c
@@ -45,4 +45,11 @@ Z_INTERNAL void slide_hash_neon(deflate_state *s) {
     slide_hash_chain(s->head, HASH_SIZE, wsize);
     slide_hash_chain(s->prev, wsize, wsize);
 }
+
+Z_INTERNAL void slide_hash_head_neon(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+
+    slide_hash_chain(s->head, HASH_SIZE, wsize);
+}
 #endif

diff --git a/arch/generic/generic_functions.h b/arch/generic/generic_functions.h
index 47b565201e2eb3cd288a91bc1c280515efb7330c..663054383682146bed68126865e308a28bf4cf04 100644 (file)
--- a/arch/generic/generic_functions.h
+++ b/arch/generic/generic_functions.h
@@ -50,6 +50,7 @@ uint32_t longest_match_roll_c(deflate_state *const s, uint32_t cur_match);
 #endif
 #ifdef SLIDE_HASH_FALLBACK
 void     slide_hash_c(deflate_state *s);
+void     slide_hash_head_c(deflate_state *s);
 #endif
 
 #ifdef DISABLE_RUNTIME_CPU_DETECTION
@@ -100,6 +101,9 @@ void     slide_hash_c(deflate_state *s);
 #    ifndef native_slide_hash
 #      define native_slide_hash slide_hash_c
 #    endif
+#    ifndef native_slide_hash_head
+#      define native_slide_hash_head slide_hash_head_c
+#    endif
 #  endif
 #endif
 

diff --git a/arch/generic/slide_hash_c.c b/arch/generic/slide_hash_c.c
index 8fdc478cbbfaa1105e2b12f6764d21557f4775f7..0078b1098aef347e8b6ed1a2aafebbd9c1836260 100644 (file)
--- a/arch/generic/slide_hash_c.c
+++ b/arch/generic/slide_hash_c.c
@@ -55,4 +55,10 @@ Z_INTERNAL void slide_hash_c(deflate_state *s) {
     slide_hash_c_chain(s->prev, wsize, wsize);
 }
 
+Z_INTERNAL void slide_hash_head_c(deflate_state *s) {
+    uint16_t wsize = (uint16_t)s->w_size;
+
+    slide_hash_c_chain(s->head, HASH_SIZE, wsize);
+}
+
 #endif /* SLIDE_HASH_FALLBACK */

diff --git a/arch/loongarch/loongarch_functions.h b/arch/loongarch/loongarch_functions.h
index 6516991fed1bc575421306cb216fac140eaf65d2..ed00ec7699abe00fde26029a171e6d8fd60b681a 100644 (file)
--- a/arch/loongarch/loongarch_functions.h
+++ b/arch/loongarch/loongarch_functions.h
@@ -28,6 +28,7 @@ void inflate_fast_lsx(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 uint32_t longest_match_lsx(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_lsx(deflate_state *const s, uint32_t cur_match);
 void slide_hash_lsx(deflate_state *s);
+void slide_hash_head_lsx(deflate_state *s);
 #endif
 
 #ifndef LOONGARCH_LSX_NATIVE
@@ -46,6 +47,7 @@ void inflate_fast_lasx(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 uint32_t longest_match_lasx(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_lasx(deflate_state *const s, uint32_t cur_match);
 void slide_hash_lasx(deflate_state *s);
+void slide_hash_head_lasx(deflate_state *s);
 #endif
 
 #ifdef DISABLE_RUNTIME_CPU_DETECTION
@@ -73,6 +75,8 @@ void slide_hash_lasx(deflate_state *s);
 #    define native_longest_match_roll longest_match_roll_lsx
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_lsx
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_lsx
 #  endif
 #  ifdef LOONGARCH_LASX_NATIVE
 #    undef native_adler32
@@ -91,6 +95,8 @@ void slide_hash_lasx(deflate_state *s);
 #    define native_longest_match_roll longest_match_roll_lasx
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_lasx
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_lasx
 #  endif
 #endif
 

diff --git a/arch/loongarch/slide_hash_lasx.c b/arch/loongarch/slide_hash_lasx.c
index f46477909072728784b5c8a43422a28a46721221..d0d6fba9375d58ef50ea00079f0c40ee24c20318 100644 (file)
--- a/arch/loongarch/slide_hash_lasx.c
+++ b/arch/loongarch/slide_hash_lasx.c
@@ -46,4 +46,12 @@ Z_INTERNAL void slide_hash_lasx(deflate_state *s) {
     slide_hash_chain(s->prev, wsize, ymm_wsize);
 }
 
+Z_INTERNAL void slide_hash_head_lasx(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+    const __m256i ymm_wsize = __lasx_xvreplgr2vr_h((short)wsize);
+
+    slide_hash_chain(s->head, HASH_SIZE, ymm_wsize);
+}
+
 #endif

diff --git a/arch/loongarch/slide_hash_lsx.c b/arch/loongarch/slide_hash_lsx.c
index f4c94ea70d6889ba7a3fc68d8b71d455baf6c7eb..3b845612390507ea61afe3922bb4dfb30c1a32fc 100644 (file)
--- a/arch/loongarch/slide_hash_lsx.c
+++ b/arch/loongarch/slide_hash_lsx.c
@@ -51,4 +51,13 @@ Z_INTERNAL void slide_hash_lsx(deflate_state *s) {
     slide_hash_chain(s->prev, wsize, xmm_wsize);
 }
 
+Z_INTERNAL void slide_hash_head_lsx(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+    const __m128i xmm_wsize = __lsx_vreplgr2vr_h((short)wsize);
+
+    assert(((uintptr_t)s->head & 15) == 0);
+    slide_hash_chain(s->head, HASH_SIZE, xmm_wsize);
+}
+
 #endif

diff --git a/arch/power/power_functions.h b/arch/power/power_functions.h
index 031e9d09ccb9e649a933706dae4ca481ab380a2d..60a611feb70b5d7d68de7d48737945502bbfc383 100644 (file)
--- a/arch/power/power_functions.h
+++ b/arch/power/power_functions.h
@@ -13,6 +13,7 @@
 uint32_t adler32_vmx(uint32_t adler, const uint8_t *buf, size_t len);
 uint32_t adler32_copy_vmx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
 void slide_hash_vmx(deflate_state *s);
+void slide_hash_head_vmx(deflate_state *s);
 #endif
 
 #ifdef POWER8_VSX
@@ -22,6 +23,7 @@ uint8_t* chunkmemset_safe_power8(uint8_t *out, uint8_t *from, size_t len, size_t
 uint32_t crc32_power8(uint32_t crc, const uint8_t *buf, size_t len);
 uint32_t crc32_copy_power8(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
 void slide_hash_power8(deflate_state *s);
+void slide_hash_head_power8(deflate_state *s);
 void inflate_fast_power8(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 #endif
 
@@ -56,6 +58,8 @@ uint32_t longest_match_roll_power9(deflate_state *const s, uint32_t cur_match);
 #    define native_adler32_copy adler32_copy_vmx
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_vmx
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_vmx
 #  endif
 // Power8 - VSX
 #  ifdef POWER8_VSX_NATIVE
@@ -69,6 +73,8 @@ uint32_t longest_match_roll_power9(deflate_state *const s, uint32_t cur_match);
 #    define native_inflate_fast inflate_fast_power8
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_power8
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_power8
 #  endif
 #  ifdef POWER8_VSX_CRC32_NATIVE
 #    undef native_crc32

diff --git a/arch/power/slide_hash_power8.c b/arch/power/slide_hash_power8.c
index d01e0acd5661574db3537822d3c15d1b6e5ca3a9..1cc9374c0d97da552916c537a2689b7dfb1ffdbf 100644 (file)
--- a/arch/power/slide_hash_power8.c
+++ b/arch/power/slide_hash_power8.c
@@ -7,6 +7,7 @@
 #ifdef POWER8_VSX
 
 #define SLIDE_PPC slide_hash_power8
+#define SLIDE_PPC_HEAD slide_hash_head_power8
 #include "slide_ppc_tpl.h"
 
 #endif /* POWER8_VSX */

diff --git a/arch/power/slide_hash_vmx.c b/arch/power/slide_hash_vmx.c
index 5a87ef7d9aa0b1e54554f7ab6049238681fc0a6b..e50ace1c0a2bf688ca0aca5a06d8fae85bd37f3f 100644 (file)
--- a/arch/power/slide_hash_vmx.c
+++ b/arch/power/slide_hash_vmx.c
@@ -5,6 +5,7 @@
 #ifdef PPC_VMX
 
 #define SLIDE_PPC slide_hash_vmx
+#define SLIDE_PPC_HEAD slide_hash_head_vmx
 #include "slide_ppc_tpl.h"
 
 #endif /* PPC_VMX */

diff --git a/arch/power/slide_ppc_tpl.h b/arch/power/slide_ppc_tpl.h
index 24629b4039318b5aaa10ee0dddc5ee37c189f8fa..ac611450b9533482bb5fba3d515751e97c1460f2 100644 (file)
--- a/arch/power/slide_ppc_tpl.h
+++ b/arch/power/slide_ppc_tpl.h
@@ -42,3 +42,9 @@ void Z_INTERNAL SLIDE_PPC(deflate_state *s) {
     slide_hash_chain(s->head, HASH_SIZE, wsize);
     slide_hash_chain(s->prev, wsize, wsize);
 }
+
+void Z_INTERNAL SLIDE_PPC_HEAD(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+    slide_hash_chain(s->head, HASH_SIZE, wsize);
+}

diff --git a/arch/riscv/riscv_functions.h b/arch/riscv/riscv_functions.h
index 821b014d3cd4071078480ad5b2bd5a25c48ff1cb..b82c7aef7c920c818cf3b9332ee9a587d9fb38c0 100644 (file)
--- a/arch/riscv/riscv_functions.h
+++ b/arch/riscv/riscv_functions.h
@@ -22,6 +22,7 @@ uint32_t compare256_rvv(const uint8_t *src0, const uint8_t *src1);
 uint32_t longest_match_rvv(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_rvv(deflate_state *const s, uint32_t cur_match);
 void slide_hash_rvv(deflate_state *s);
+void slide_hash_head_rvv(deflate_state *s);
 void inflate_fast_rvv(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 #endif
 
@@ -56,6 +57,8 @@ uint32_t crc32_copy_riscv64_zbc(uint32_t crc, uint8_t *dst, const uint8_t *src,
 #    define native_longest_match_roll longest_match_roll_rvv
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_rvv
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_rvv
 #  endif
 // RISCV - CRC32
 #  ifdef RISCV_ZBC_NATIVE

diff --git a/arch/riscv/slide_hash_rvv.c b/arch/riscv/slide_hash_rvv.c
index e794c38204e86c322d4a7d37b768ad15690bc28d..55ec273876de9e61c1038a00a896dab5750660b6 100644 (file)
--- a/arch/riscv/slide_hash_rvv.c
+++ b/arch/riscv/slide_hash_rvv.c
@@ -30,4 +30,11 @@ Z_INTERNAL void slide_hash_rvv(deflate_state *s) {
     slide_hash_chain(s->prev, wsize, wsize);
 }
 
+Z_INTERNAL void slide_hash_head_rvv(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+
+    slide_hash_chain(s->head, HASH_SIZE, wsize);
+}
+
 #endif // RISCV_RVV

diff --git a/arch/s390/s390_functions.h b/arch/s390/s390_functions.h
index c0043a6e85879b7d16851c3aa5496bc66cc497a2..3b06d0cf119710f49d539ef7bf511029fa3c5a79 100644 (file)
--- a/arch/s390/s390_functions.h
+++ b/arch/s390/s390_functions.h
@@ -20,6 +20,7 @@
 uint32_t crc32_s390_vx(uint32_t crc, const uint8_t *buf, size_t len);
 uint32_t crc32_copy_s390_vx(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
 void slide_hash_vx(deflate_state *s);
+void slide_hash_head_vx(deflate_state *s);
 
 #ifdef __clang__
 #  if ((__clang_major__ == 18) || (__clang_major__ == 19 && (__clang_minor__ < 1 || (__clang_minor__ == 1 && __clang_patchlevel__ < 2))))
@@ -38,6 +39,8 @@ void slide_hash_vx(deflate_state *s);
 #    define native_crc32_copy crc32_copy_s390_vx
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_vx
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_vx
 #  endif
 #endif
 

diff --git a/arch/s390/slide_hash_vx.c b/arch/s390/slide_hash_vx.c
index 0939bc984f622a37a203407abfbf2aea145bb142..1b50e6d912fdcd384eba52dfe0d578d392c73a18 100644 (file)
--- a/arch/s390/slide_hash_vx.c
+++ b/arch/s390/slide_hash_vx.c
@@ -32,4 +32,11 @@ Z_INTERNAL void slide_hash_vx(deflate_state *s) {
     slide_hash_chain(s->head, HASH_SIZE, wsize);
     slide_hash_chain(s->prev, wsize, wsize);
 }
+
+Z_INTERNAL void slide_hash_head_vx(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+
+    slide_hash_chain(s->head, HASH_SIZE, wsize);
+}
 #endif

diff --git a/arch/x86/slide_hash_avx2.c b/arch/x86/slide_hash_avx2.c
index 241ea305e3ddb73838841f77d8bee211a51178dd..257605ed9d0e4bfa7a58306c44bb453eda133a36 100644 (file)
--- a/arch/x86/slide_hash_avx2.c
+++ b/arch/x86/slide_hash_avx2.c
@@ -45,4 +45,12 @@ Z_INTERNAL void slide_hash_avx2(deflate_state *s) {
     slide_hash_chain(s->prev, wsize, ymm_wsize);
 }
 
+Z_INTERNAL void slide_hash_head_avx2(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+    const __m256i ymm_wsize = _mm256_set1_epi16((short)wsize);
+
+    slide_hash_chain(s->head, HASH_SIZE, ymm_wsize);
+}
+
 #endif

diff --git a/arch/x86/slide_hash_sse2.c b/arch/x86/slide_hash_sse2.c
index 0d2da26d9f50e2c56fb50a7f590aa012773f5d48..0f283208c84abf0e53b1c75096b4a739699700b1 100644 (file)
--- a/arch/x86/slide_hash_sse2.c
+++ b/arch/x86/slide_hash_sse2.c
@@ -45,4 +45,12 @@ Z_INTERNAL void slide_hash_sse2(deflate_state *s) {
     slide_hash_chain(s->prev, wsize, xmm_wsize);
 }
 
+Z_INTERNAL void slide_hash_head_sse2(deflate_state *s) {
+    Assert(s->w_size <= UINT16_MAX, "w_size should fit in uint16_t");
+    uint16_t wsize = (uint16_t)s->w_size;
+    const __m128i xmm_wsize = _mm_set1_epi16((short)wsize);
+
+    slide_hash_chain(s->head, HASH_SIZE, xmm_wsize);
+}
+
 #endif

diff --git a/arch/x86/x86_functions.h b/arch/x86/x86_functions.h
index 6f8dce6b5d9fa585da37ee3faeb5c505effcaea4..2c30090d6bc31021dba1fe385d76561dc6079a83 100644 (file)
--- a/arch/x86/x86_functions.h
+++ b/arch/x86/x86_functions.h
@@ -22,6 +22,7 @@ void inflate_fast_sse2(PREFIX3(stream)* strm, uint32_t start, int safe_mode);
 uint32_t longest_match_sse2(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_sse2(deflate_state *const s, uint32_t cur_match);
 void slide_hash_sse2(deflate_state *s);
+void slide_hash_head_sse2(deflate_state *s);
 
 #  ifdef CRC32_CHORBA_SSE_FALLBACK
     uint32_t crc32_chorba_sse2(uint32_t crc, const uint8_t *buf, size_t len);
@@ -67,6 +68,7 @@ void inflate_fast_avx2(PREFIX3(stream)* strm, uint32_t start, int safe_mode);
 uint32_t longest_match_avx2(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_avx2(deflate_state *const s, uint32_t cur_match);
 void slide_hash_avx2(deflate_state *s);
+void slide_hash_head_avx2(deflate_state *s);
 #endif
 #ifdef X86_AVX512
 uint32_t adler32_avx512(uint32_t adler, const uint8_t *buf, size_t len);
@@ -121,6 +123,8 @@ uint32_t crc32_copy_vpclmulqdq_avx512(uint32_t crc, uint8_t *dst, const uint8_t
 #    endif
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_sse2
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_sse2
 #  endif
 // X86 - SSSE3
 #  ifdef X86_SSSE3_NATIVE
@@ -172,6 +176,8 @@ uint32_t crc32_copy_vpclmulqdq_avx512(uint32_t crc, uint8_t *dst, const uint8_t
 #    define native_longest_match_roll longest_match_roll_avx2
 #    undef native_slide_hash
 #    define native_slide_hash slide_hash_avx2
+#    undef native_slide_hash_head
+#    define native_slide_hash_head slide_hash_head_avx2
 #  endif
 // X86 - AVX2 (VNNI)
 #    if defined(X86_AVX2VNNI) && defined(__AVXVNNI__)

diff --git a/deflate.c b/deflate.c
index 03fe962509b6f6269e2db4d34a9635bf447e140c..13ad2404452ed0d573ee720279eb79f42876953a 100644 (file)
--- a/deflate.c
+++ b/deflate.c
@@ -1233,7 +1233,14 @@ void Z_INTERNAL PREFIX(fill_window)(deflate_state *s) {
             s->block_start -= (int)wsize;
             if (s->insert > s->strstart)
                 s->insert = s->strstart;
-            FUNCTABLE_CALL(slide_hash)(s);
+            if (s->strategy != Z_HUFFMAN_ONLY && s->strategy != Z_RLE) {
+                /* Z_HUFFMAN_ONLY and Z_RLE never read the hash chain. deflate_quick
+                 * reads the chain head but never walks prev, so it slides head only. */
+                if (HAVE_QUICK_STRATEGY && level == 1)
+                    FUNCTABLE_CALL(slide_hash_head)(s);
+                else
+                    FUNCTABLE_CALL(slide_hash)(s);
+            }
             more += wsize;
         }
         if (strm->avail_in == 0)

diff --git a/deflate.h b/deflate.h
index f8790d744ef12f24519bf594d72f81b0c71b9fbc..b2768ceb70b0d450b0f9a41b8d20e7b6bcec3450 100644 (file)
--- a/deflate.h
+++ b/deflate.h
@@ -424,6 +424,7 @@ static inline void put_uint64(deflate_state *s, uint64_t lld) {
 
 void Z_INTERNAL PREFIX(fill_window)(deflate_state *s);
 void Z_INTERNAL slide_hash_c(deflate_state *s);
+void Z_INTERNAL slide_hash_head_c(deflate_state *s);
 
         /* in trees.c */
 void Z_INTERNAL zng_tr_init(deflate_state *s);

diff --git a/functable.c b/functable.c
index f7ee9398d2d5a7cd3e830714e74cda0500df4522..ebfdde85e2d78a44e6d1001dbca5205adf7ac233 100644 (file)
--- a/functable.c
+++ b/functable.c
@@ -95,6 +95,7 @@ static int init_functable(void) {
 #endif
 #ifdef SLIDE_HASH_FALLBACK
     ft.slide_hash = &slide_hash_c;
+    ft.slide_hash_head = &slide_hash_head_c;
 #endif
 
     // Select arch-optimized functions
@@ -119,6 +120,7 @@ static int init_functable(void) {
         ft.longest_match = &longest_match_sse2;
         ft.longest_match_roll = &longest_match_roll_sse2;
         ft.slide_hash = &slide_hash_sse2;
+        ft.slide_hash_head = &slide_hash_head_sse2;
 #  endif
 #  if defined(CRC32_CHORBA_SSE_FALLBACK) && !defined(X86_SSE41_NATIVE) && !defined(X86_PCLMULQDQ_NATIVE)
         ft.crc32 = &crc32_chorba_sse2;
@@ -193,6 +195,7 @@ static int init_functable(void) {
         ft.longest_match_roll = &longest_match_roll_avx2;
 #  endif
         ft.slide_hash = &slide_hash_avx2;
+        ft.slide_hash_head = &slide_hash_head_avx2;
     }
 #endif
 #ifdef X86_AVX2VNNI
@@ -256,6 +259,7 @@ static int init_functable(void) {
 #  endif
     {
         ft.slide_hash = &slide_hash_armv6;
+        ft.slide_hash_head = &slide_hash_head_armv6;
     }
 #endif
     // ARM - NEON
@@ -272,6 +276,7 @@ static int init_functable(void) {
         ft.longest_match = &longest_match_neon;
         ft.longest_match_roll = &longest_match_roll_neon;
         ft.slide_hash = &slide_hash_neon;
+        ft.slide_hash_head = &slide_hash_head_neon;
     }
 #endif
     // ARM - NEON DotProd
@@ -314,6 +319,7 @@ static int init_functable(void) {
         ft.adler32 = &adler32_vmx;
         ft.adler32_copy = &adler32_copy_vmx;
         ft.slide_hash = &slide_hash_vmx;
+        ft.slide_hash_head = &slide_hash_head_vmx;
     }
 #endif
     // Power8 - VSX
@@ -327,6 +333,7 @@ static int init_functable(void) {
         ft.chunkmemset_safe = &chunkmemset_safe_power8;
         ft.inflate_fast = &inflate_fast_power8;
         ft.slide_hash = &slide_hash_power8;
+        ft.slide_hash_head = &slide_hash_head_power8;
     }
 #endif
 #ifdef POWER8_VSX_CRC32
@@ -365,6 +372,7 @@ static int init_functable(void) {
         ft.longest_match = &longest_match_rvv;
         ft.longest_match_roll = &longest_match_roll_rvv;
         ft.slide_hash = &slide_hash_rvv;
+        ft.slide_hash_head = &slide_hash_head_rvv;
     }
 #endif
 
@@ -388,6 +396,7 @@ static int init_functable(void) {
         ft.crc32 = &crc32_s390_vx;
         ft.crc32_copy = &crc32_copy_s390_vx;
         ft.slide_hash = &slide_hash_vx;
+        ft.slide_hash_head = &slide_hash_head_vx;
     }
 #endif
 
@@ -414,6 +423,7 @@ static int init_functable(void) {
         ft.longest_match = &longest_match_lsx;
         ft.longest_match_roll = &longest_match_roll_lsx;
         ft.slide_hash = &slide_hash_lsx;
+        ft.slide_hash_head = &slide_hash_head_lsx;
     }
 #endif
 #ifdef LOONGARCH_LASX
@@ -429,6 +439,7 @@ static int init_functable(void) {
         ft.longest_match = &longest_match_lasx;
         ft.longest_match_roll = &longest_match_roll_lasx;
         ft.slide_hash = &slide_hash_lasx;
+        ft.slide_hash_head = &slide_hash_head_lasx;
     }
 #endif
 
@@ -446,6 +457,7 @@ static int init_functable(void) {
     FUNCTABLE_VERIFY_ASSIGN(ft, longest_match);
     FUNCTABLE_VERIFY_ASSIGN(ft, longest_match_roll);
     FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash);
+    FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash_head);
 
     // Memory barrier for weak memory order CPUs
     FUNCTABLE_BARRIER();
@@ -514,6 +526,11 @@ static void slide_hash_stub(deflate_state* s) {
     functable.slide_hash(s);
 }
 
+static void slide_hash_head_stub(deflate_state *s) {
+    FUNCTABLE_INIT_ABORT;
+    functable.slide_hash_head(s);
+}
+
 /* functable init */
 Z_INTERNAL struct functable_s functable = {
     force_init_stub,
@@ -527,6 +544,7 @@ Z_INTERNAL struct functable_s functable = {
     longest_match_stub,
     longest_match_roll_stub,
     slide_hash_stub,
+    slide_hash_head_stub,
 };
 
 #endif

diff --git a/functable.h b/functable.h
index 76af499aa4029c6e88cb9c18ac996db0facc1012..9e654a31bd538809f9a785c374d8e5f27a403a77 100644 (file)
--- a/functable.h
+++ b/functable.h
@@ -34,6 +34,7 @@ struct functable_s {
     uint32_t (* longest_match)      (deflate_state *const s, uint32_t cur_match);
     uint32_t (* longest_match_roll) (deflate_state *const s, uint32_t cur_match);
     void     (* slide_hash)         (deflate_state *s);
+    void     (* slide_hash_head)    (deflate_state *s);
 };
 
 Z_INTERNAL extern struct functable_s functable;