Improve inflate_fast performance for small output buffers

Lowers the inflate_fast entry threshold from 260 to 3 bytes of
available output by adding a safe_mode parameter that uses
bounds-checked copies and bails to the MATCH state when output
space is insufficient. This eliminates the performance cliff
where libpng-style row-by-row decompression falls back to the
slow inflate path for the last 260 bytes of each row.

diff --git a/arch/arm/arm_functions.h b/arch/arm/arm_functions.h
index 2b33443bbd1c3081b1b98a6bc7cda916573561f2..f2975fadfaeb1276d3777f7a13bcf521970ee43c 100644 (file)
--- a/arch/arm/arm_functions.h
+++ b/arch/arm/arm_functions.h
@@ -12,7 +12,7 @@ uint32_t adler32_neon(uint32_t adler, const uint8_t *buf, size_t len);
 uint32_t adler32_copy_neon(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
 uint8_t* chunkmemset_safe_neon(uint8_t *out, uint8_t *from, size_t len, size_t left);
 uint32_t compare256_neon(const uint8_t *src0, const uint8_t *src1);
-void inflate_fast_neon(PREFIX3(stream) *strm, uint32_t start);
+void inflate_fast_neon(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 uint32_t longest_match_neon(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_neon(deflate_state *const s, uint32_t cur_match);
 void slide_hash_neon(deflate_state *s);

diff --git a/arch/generic/generic_functions.h b/arch/generic/generic_functions.h
index b128a0208198a90da94413c93b1d5b2f8649230a..a01778b71837da173b65dbaa0cf56edbef006055 100644 (file)
--- a/arch/generic/generic_functions.h
+++ b/arch/generic/generic_functions.h
@@ -40,7 +40,7 @@ uint32_t crc32_copy_braid(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t
   uint32_t crc32_copy_chorba(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
 #endif
 #ifdef CHUNKSET_FALLBACK
-void     inflate_fast_c(PREFIX3(stream) *strm, uint32_t start);
+void     inflate_fast_c(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 #endif
 #ifdef COMPARE256_FALLBACK
 uint32_t longest_match_c(deflate_state *const s, uint32_t cur_match);

diff --git a/arch/loongarch/loongarch_functions.h b/arch/loongarch/loongarch_functions.h
index 8a6a387c431491c777dd91de7cf54f8ff7ba1ae0..6516991fed1bc575421306cb216fac140eaf65d2 100644 (file)
--- a/arch/loongarch/loongarch_functions.h
+++ b/arch/loongarch/loongarch_functions.h
@@ -24,7 +24,7 @@ uint32_t adler32_lsx(uint32_t adler, const uint8_t *src, size_t len);
 uint32_t adler32_copy_lsx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
 uint8_t* chunkmemset_safe_lsx(uint8_t *out, uint8_t *from, size_t len, size_t left);
 uint32_t compare256_lsx(const uint8_t *src0, const uint8_t *src1);
-void inflate_fast_lsx(PREFIX3(stream) *strm, uint32_t start);
+void inflate_fast_lsx(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 uint32_t longest_match_lsx(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_lsx(deflate_state *const s, uint32_t cur_match);
 void slide_hash_lsx(deflate_state *s);
@@ -42,7 +42,7 @@ uint32_t adler32_lasx(uint32_t adler, const uint8_t *src, size_t len);
 uint32_t adler32_copy_lasx(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
 uint8_t* chunkmemset_safe_lasx(uint8_t *out, uint8_t *from, size_t len, size_t left);
 uint32_t compare256_lasx(const uint8_t *src0, const uint8_t *src1);
-void inflate_fast_lasx(PREFIX3(stream) *strm, uint32_t start);
+void inflate_fast_lasx(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 uint32_t longest_match_lasx(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_lasx(deflate_state *const s, uint32_t cur_match);
 void slide_hash_lasx(deflate_state *s);

diff --git a/arch/power/power_functions.h b/arch/power/power_functions.h
index 3f89f709a696151c1f793f5835c416263add7946..031e9d09ccb9e649a933706dae4ca481ab380a2d 100644 (file)
--- a/arch/power/power_functions.h
+++ b/arch/power/power_functions.h
@@ -22,7 +22,7 @@ uint8_t* chunkmemset_safe_power8(uint8_t *out, uint8_t *from, size_t len, size_t
 uint32_t crc32_power8(uint32_t crc, const uint8_t *buf, size_t len);
 uint32_t crc32_copy_power8(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
 void slide_hash_power8(deflate_state *s);
-void inflate_fast_power8(PREFIX3(stream) *strm, uint32_t start);
+void inflate_fast_power8(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 #endif
 
 #if !defined(PPC_VMX_NATIVE) && !defined(POWER8_VSX_NATIVE)

diff --git a/arch/riscv/riscv_functions.h b/arch/riscv/riscv_functions.h
index bc17c1b81926cd19b69952bf60af323611ddc862..821b014d3cd4071078480ad5b2bd5a25c48ff1cb 100644 (file)
--- a/arch/riscv/riscv_functions.h
+++ b/arch/riscv/riscv_functions.h
@@ -22,7 +22,7 @@ uint32_t compare256_rvv(const uint8_t *src0, const uint8_t *src1);
 uint32_t longest_match_rvv(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_rvv(deflate_state *const s, uint32_t cur_match);
 void slide_hash_rvv(deflate_state *s);
-void inflate_fast_rvv(PREFIX3(stream) *strm, uint32_t start);
+void inflate_fast_rvv(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 #endif
 
 #ifndef RISCV_RVV_NATIVE

diff --git a/arch/x86/x86_functions.h b/arch/x86/x86_functions.h
index cf28c6ac0883c7ce5d3e138e1bc33b1fba024984..9c9357d128b948a828aa860fa16ca90d222df750 100644 (file)
--- a/arch/x86/x86_functions.h
+++ b/arch/x86/x86_functions.h
@@ -18,7 +18,7 @@
 #ifdef X86_SSE2
 uint8_t* chunkmemset_safe_sse2(uint8_t *out, uint8_t *from, size_t len, size_t left);
 uint32_t compare256_sse2(const uint8_t *src0, const uint8_t *src1);
-void inflate_fast_sse2(PREFIX3(stream)* strm, uint32_t start);
+void inflate_fast_sse2(PREFIX3(stream)* strm, uint32_t start, int safe_mode);
 uint32_t longest_match_sse2(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_sse2(deflate_state *const s, uint32_t cur_match);
 void slide_hash_sse2(deflate_state *s);
@@ -40,7 +40,7 @@ void slide_hash_sse2(deflate_state *s);
 uint32_t adler32_ssse3(uint32_t adler, const uint8_t *buf, size_t len);
 uint32_t adler32_copy_ssse3(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
 uint8_t* chunkmemset_safe_ssse3(uint8_t *out, uint8_t *from, size_t len, size_t left);
-void inflate_fast_ssse3(PREFIX3(stream) *strm, uint32_t start);
+void inflate_fast_ssse3(PREFIX3(stream) *strm, uint32_t start, int safe_mode);
 #endif
 
 #ifndef X86_SSSE3_NATIVE
@@ -63,7 +63,7 @@ uint32_t adler32_avx2(uint32_t adler, const uint8_t *buf, size_t len);
 uint32_t adler32_copy_avx2(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
 uint8_t* chunkmemset_safe_avx2(uint8_t *out, uint8_t *from, size_t len, size_t left);
 uint32_t compare256_avx2(const uint8_t *src0, const uint8_t *src1);
-void inflate_fast_avx2(PREFIX3(stream)* strm, uint32_t start);
+void inflate_fast_avx2(PREFIX3(stream)* strm, uint32_t start, int safe_mode);
 uint32_t longest_match_avx2(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_avx2(deflate_state *const s, uint32_t cur_match);
 void slide_hash_avx2(deflate_state *s);
@@ -73,7 +73,7 @@ uint32_t adler32_avx512(uint32_t adler, const uint8_t *buf, size_t len);
 uint32_t adler32_copy_avx512(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len);
 uint8_t* chunkmemset_safe_avx512(uint8_t *out, uint8_t *from, size_t len, size_t left);
 uint32_t compare256_avx512(const uint8_t *src0, const uint8_t *src1);
-void inflate_fast_avx512(PREFIX3(stream)* strm, uint32_t start);
+void inflate_fast_avx512(PREFIX3(stream)* strm, uint32_t start, int safe_mode);
 uint32_t longest_match_avx512(deflate_state *const s, uint32_t cur_match);
 uint32_t longest_match_roll_avx512(deflate_state *const s, uint32_t cur_match);
 #endif

diff --git a/functable.c b/functable.c
index 449ab446253d74069c34e0ac14e8b27727237e85..6ad05f32e11e8b71131759662077a2797e94d8fb 100644 (file)
--- a/functable.c
+++ b/functable.c
@@ -478,9 +478,9 @@ static uint32_t crc32_copy_stub(uint32_t crc, uint8_t *dst, const uint8_t *src,
     return functable.crc32_copy(crc, dst, src, len);
 }
 
-static void inflate_fast_stub(PREFIX3(stream) *strm, uint32_t start) {
+static void inflate_fast_stub(PREFIX3(stream) *strm, uint32_t start, int safe_mode) {
     FUNCTABLE_INIT_ABORT;
-    functable.inflate_fast(strm, start);
+    functable.inflate_fast(strm, start, safe_mode);
 }
 
 static uint32_t longest_match_stub(deflate_state* const s, uint32_t cur_match) {

diff --git a/functable.h b/functable.h
index 2a0bb88e0e7aacf01abdc64120e6681a8f9728bf..76af499aa4029c6e88cb9c18ac996db0facc1012 100644 (file)
--- a/functable.h
+++ b/functable.h
@@ -30,7 +30,7 @@ struct functable_s {
     uint32_t (* compare256)         (const uint8_t *src0, const uint8_t *src1);
     uint32_t (* crc32)              (uint32_t crc, const uint8_t *buf, size_t len);
     uint32_t (* crc32_copy)         (uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len);
-    void     (* inflate_fast)       (PREFIX3(stream) *strm, uint32_t start);
+    void     (* inflate_fast)       (PREFIX3(stream) *strm, uint32_t start, int safe_mode);
     uint32_t (* longest_match)      (deflate_state *const s, uint32_t cur_match);
     uint32_t (* longest_match_roll) (deflate_state *const s, uint32_t cur_match);
     void     (* slide_hash)         (deflate_state *s);

diff --git a/infback.c b/infback.c
index 2dc15458d57240ab3d4b0d11381504426d399b54..c4d6214faf1179342deb843d85c5bbb98c187319 100644 (file)
--- a/infback.c
+++ b/infback.c
@@ -361,12 +361,14 @@ int32_t Z_EXPORT PREFIX(inflateBack)(PREFIX3(stream) *strm, in_func in, void *in
 
         case LEN:
             /* use inflate_fast() if we have enough input and output */
-            if (have >= INFLATE_FAST_MIN_HAVE &&
-                left >= INFLATE_FAST_MIN_LEFT) {
+            if (have >= INFLATE_FAST_MIN_HAVE && left >= INFLATE_FAST_MIN_SAFE) {
                 RESTORE();
                 if (state->whave < state->wsize)
                     state->whave = state->wsize - left;
-                FUNCTABLE_CALL(inflate_fast)(strm, state->wsize);
+                /* inflateBack() writes directly into the window, so out and window
+                   always overlap. Pass safe_mode=1 to use safe chunk copy functions
+                   that prevent overwriting window data needed by future back-references. */
+                FUNCTABLE_CALL(inflate_fast)(strm, state->wsize, 1);
                 LOAD();
                 break;
             }
@@ -486,6 +488,28 @@ int32_t Z_EXPORT PREFIX(inflateBack)(PREFIX3(stream) *strm, in_func in, void *in
             } while (state->length != 0);
             break;
 
+        case MATCH:
+            /* Copy back-reference that inflate_fast() could not complete due to
+               insufficient output space. state->length and state->offset were set
+               by the safe_mode MATCH bailout in inflate_fast(). */
+            do {
+                ROOM();
+                copy = state->wsize - state->offset;
+                if (copy < left) {
+                    from = put + copy;
+                    copy = left - copy;
+                    copy = MIN(copy, state->length);
+                    put = chunkcopy_safe(put, from, copy, put + left);
+                } else {
+                    copy = MIN(state->length, left);
+                    put = FUNCTABLE_CALL(chunkmemset_safe)(put, put - state->offset, copy, left);
+                }
+                state->length -= copy;
+                left -= copy;
+            } while (state->length != 0);
+            state->mode = LEN;
+            break;
+
         case DONE:
             /* inflate stream terminated properly */
             ret = Z_STREAM_END;

diff --git a/inffast_tpl.h b/inffast_tpl.h
index 368dde58e7cdaf6c3d5062b6f2fc8f55118a6292..0557231c0b663c57bec2e3848f62244a5667493b 100644 (file)
--- a/inffast_tpl.h
+++ b/inffast_tpl.h
@@ -49,7 +49,7 @@
       requires strm->avail_out >= 258 for each loop to avoid checking for
       output space.
  */
-void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
+void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start, int safe_mode) {
     /* start: inflate()'s starting value for strm->avail_out */
     struct inflate_state *state;
     z_const unsigned char *in;  /* local strm->next_in */
@@ -112,7 +112,6 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
     unsigned len;               /* match length, unused bytes */
     unsigned char *from;        /* where to copy match from */
     unsigned dist;              /* match distance */
-    unsigned extra_safe;        /* copy chunks safely in all cases */
     uint64_t old;               /* look-behind buffer for extra bits */
 
     /* copy state to local variables */
@@ -121,8 +120,8 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
     last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
     out = strm->next_out;
     beg = out - (start - strm->avail_out);
-    end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
     safe = out + strm->avail_out;
+    end = safe - (safe_mode ? INFLATE_FAST_MIN_SAFE : INFLATE_FAST_MIN_LEFT) + 1;
     wsize = state->wsize;
     whave = state->whave;
     wnext = state->wnext;
@@ -134,11 +133,6 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
     lmask = (1U << state->lenbits) - 1;
     dmask = (1U << state->distbits) - 1;
 
-    /* Detect if out and window point to the same memory allocation. In this instance it is
-       necessary to use safe chunk copy functions to prevent overwriting the window. If the
-       window is overwritten then future matches with far distances will fail to copy correctly. */
-    extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + state->wbufsize);
-
     /* decode literals and length/distances until end-of-block or not enough
        input data or output space */
     do {
@@ -191,6 +185,16 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
                 }
 #endif
                 TRACE_DISTANCE(dist);
+
+                /* In safe mode, if there isn't enough output space for the full copy,
+                   bail to the slow path's MATCH state which handles partial copies. */
+                if (UNLIKELY(safe_mode && len > (unsigned)(safe - out))) {
+                    state->mode = MATCH;
+                    state->length = len;
+                    state->offset = dist;
+                    break;
+                }
+
                 op = (unsigned)(out - beg);     /* max distance in output */
                 if (UNLIKELY(dist > op)) {      /* see if copy from window */
                     op = dist - op;             /* distance back in window */
@@ -237,7 +241,7 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
                     }
                     if (UNLIKELY(op < len)) {           /* still need some from output */
                         len -= op;
-                        if (LIKELY(!extra_safe)) {
+                        if (LIKELY(!safe_mode)) {
                             out = CHUNKCOPY_SAFE(out, from, op, safe);
                             out = CHUNKUNROLL(out, &dist, &len);
                             out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
@@ -247,14 +251,14 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
                         }
                     } else {
 #ifndef HAVE_MASKED_READWRITE
-                        if (UNLIKELY(extra_safe))
+                        if (UNLIKELY(safe_mode))
                             out = chunkcopy_safe(out, from, len, safe);
                         else
 #endif
                             out = CHUNKCOPY_SAFE(out, from, len, safe);
                     }
 #ifndef HAVE_MASKED_READWRITE
-                } else if (UNLIKELY(extra_safe)) {
+                } else if (UNLIKELY(safe_mode)) {
                     /* Whole reference is in range of current output. */
                         out = chunkcopy_safe(out, out - dist, len, safe);
 #endif
@@ -302,8 +306,7 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
     strm->next_out = out;
     strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
                                           : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
-    strm->avail_out = (unsigned)(out < end ? (INFLATE_FAST_MIN_LEFT - 1) + (end - out)
-                                           : (INFLATE_FAST_MIN_LEFT - 1) - (out - end));
+    strm->avail_out = (unsigned)(safe - out);
 
     Assert(bits <= 32, "Remaining bits greater than 32");
     state->hold = (uint32_t)hold;

diff --git a/inflate.c b/inflate.c
index 3aab8722588b17f8a372684a7caa3dda81825ef3..5707082d6e682027472fb4b7fa15ddad5efd1929 100644 (file)
--- a/inflate.c
+++ b/inflate.c
@@ -925,9 +925,9 @@ int32_t Z_EXPORT PREFIX(inflate)(PREFIX3(stream) *strm, int32_t flush) {
 
         case LEN:
             /* use inflate_fast() if we have enough input and output */
-            if (have >= INFLATE_FAST_MIN_HAVE && left >= INFLATE_FAST_MIN_LEFT) {
+            if (have >= INFLATE_FAST_MIN_HAVE && left >= INFLATE_FAST_MIN_SAFE) {
                 RESTORE();
-                FUNCTABLE_CALL(inflate_fast)(strm, out);
+                FUNCTABLE_CALL(inflate_fast)(strm, out, left < INFLATE_FAST_MIN_LEFT);
                 LOAD();
                 if (state->mode == TYPE)
                     state->back = -1;

diff --git a/inflate_p.h b/inflate_p.h
index 0d04f72bb8444a3b332181193473c7178532340f..cd9e1f214cbe330b681a3fcfbffdabb0302161c7 100644 (file)
--- a/inflate_p.h
+++ b/inflate_p.h
@@ -191,8 +191,9 @@ typedef unsigned bits_t;
 #define TRACE_END_OF_BLOCK() \
     Tracevv((stderr, "inflate:         end of block\n"))
 
-#define INFLATE_FAST_MIN_HAVE 15
-#define INFLATE_FAST_MIN_LEFT 260
+#define INFLATE_FAST_MIN_HAVE 15   /* max input bits per length/distance pair */
+#define INFLATE_FAST_MIN_LEFT 260  /* max output per token (258) + 2 */
+#define INFLATE_FAST_MIN_SAFE 3    /* max unchecked literal writes per iteration */
 
 /* Load 64 bits from IN and place the bytes at offset BITS in the result. */
 static inline uint64_t load_64_bits(const unsigned char *in, unsigned bits) {