Inflate: refill unconditionally

diff --git a/inffast_tpl.h b/inffast_tpl.h
index e17e59241ec39f30b9d31ee66885d8a0261dea71..c209021b92e4e81c1e0df52f01c5c37aa8340a97 100644 (file)
--- a/inffast_tpl.h
+++ b/inffast_tpl.h
@@ -145,14 +145,22 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
        window is overwritten then future matches with far distances will fail to copy correctly. */
     extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + wsize);
 
+    /* This is extremely latency sensitive, so empty inline assembly blocks are
+       used to prevent the compiler from reassociating. */
+#define REFILL() do { \
+        hold |= load_64_bits(in, bits); \
+        in += 7; \
+        __asm__ volatile ("" : "+r"(in)); \
+        uint64_t tmp = ((bits >> 3) & 7); \
+        __asm__ volatile ("" : "+r"(tmp)); \
+        in -= tmp; \
+        bits |= 56; \
+    } while (0)
+
     /* decode literals and length/distances until end-of-block or not enough
        input data or output space */
     do {
-        if (bits < MAX_BITS) {
-            hold |= load_64_bits(in, bits);
-            in += 6;
-            bits += 48;
-        }
+        REFILL();
         here = lcode + (hold & lmask);
       dolen:
         DROPBITS(here->bits);
@@ -164,20 +172,10 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
             *out++ = (unsigned char)(here->val);
         } else if (op & 16) {                     /* length base */
             len = here->val;
-            op &= 15;                           /* number of extra bits */
-            if (bits < op) {
-                hold |= load_64_bits(in, bits);
-                in += 6;
-                bits += 48;
-            }
+            op &= MAX_BITS;                       /* number of extra bits */
             len += BITS(op);
             DROPBITS(op);
             Tracevv((stderr, "inflate:         length %u\n", len));
-            if (bits < MAX_BITS) {
-                hold |= load_64_bits(in, bits);
-                in += 6;
-                bits += 48;
-            }
             here = dcode + (hold & dmask);
           dodist:
             DROPBITS(here->bits);
@@ -185,11 +183,6 @@ void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
             if (op & 16) {                      /* distance base */
                 dist = here->val;
                 op &= MAX_BITS;                 /* number of extra bits */
-                if (bits < op) {
-                    hold |= load_64_bits(in, bits);
-                    in += 6;
-                    bits += 48;
-                }
                 dist += BITS(op);
 #ifdef INFLATE_STRICT
                 if (dist > dmax) {