Split chunkcopy_safe to allow the first part to be inlined more often.

diff --git a/inflate.c b/inflate.c
index cfcbf5235944d4269112c6d34b7bf6b4e3026683..de91b93a2c34b647ab92c85b6b9c3f738ddf852f 100644 (file)
--- a/inflate.c
+++ b/inflate.c
@@ -1451,3 +1451,67 @@ unsigned long Z_EXPORT PREFIX(inflateCodesUsed)(PREFIX3(stream) *strm) {
     state = (struct inflate_state *)strm->state;
     return (unsigned long)(state->next - state->codes);
 }
+
+/* Second half of chunkcopy_safe, this is split out to allow the first half to be inlined */
+Z_INTERNAL uint8_t* chunkcopy_safe_unrolled(uint8_t *out, uint8_t *from, uint64_t len) {
+    /* We are emulating a self-modifying copy loop here. To do this in a way that doesn't produce undefined behavior,
+     * we have to get a bit clever. First if the overlap is such that src falls between dst and dst+len, we can do the
+     * initial bulk memcpy of the nonoverlapping region. Then, we can leverage the size of this to determine the safest
+     * atomic memcpy size we can pick such that we have non-overlapping regions. This effectively becomes a safe look
+     * behind or lookahead distance. */
+    uint64_t non_olap_size = llabs(from - out); // llabs vs labs for compatibility with windows
+    uint64_t tocopy;
+
+    memcpy(out, from, (size_t)non_olap_size);
+    out += non_olap_size;
+    from += non_olap_size;
+    len -= non_olap_size;
+
+    /* So this doesn't give use a worst case scenario of function calls in a loop,
+     * we want to instead break this down into copy blocks of fixed lengths */
+    while (len) {
+        tocopy = MIN(non_olap_size, len);
+        len -= tocopy;
+
+        while (tocopy >= 32) {
+            memcpy(out, from, 32);
+            out += 32;
+            from += 32;
+            tocopy -= 32;
+        }
+
+        if (tocopy >= 16) {
+            memcpy(out, from, 16);
+            out += 16;
+            from += 16;
+            tocopy -= 16;
+        }
+
+        if (tocopy >= 8) {
+            memcpy(out, from, 8);
+            out += 8;
+            from += 8;
+            tocopy -= 8;
+        }
+
+        if (tocopy >= 4) {
+            memcpy(out, from, 4);
+            out += 4;
+            from += 4;
+            tocopy -= 4;
+        }
+
+        if (tocopy >= 2) {
+            memcpy(out, from, 2);
+            out += 2;
+            from += 2;
+            tocopy -= 2;
+        }
+
+        if (tocopy) {
+            *out++ = *from++;
+        }
+    }
+
+    return out;
+}

diff --git a/inflate.h b/inflate.h
index 30ff7db3b756e9dc146394d576188d5b0cb09cc4..dd23afe5257b4cd9ca9a32ad5d0e1a610d5b96cf 100644 (file)
--- a/inflate.h
+++ b/inflate.h
@@ -161,5 +161,6 @@ struct ALIGNED_(64) inflate_state {
 void Z_INTERNAL PREFIX(fixedtables)(struct inflate_state *state);
 Z_INTERNAL inflate_allocs* alloc_inflate(PREFIX3(stream) *strm);
 Z_INTERNAL void free_inflate(PREFIX3(stream) *strm);
+Z_INTERNAL uint8_t* chunkcopy_safe_unrolled(uint8_t *out, uint8_t *from, uint64_t len);
 
 #endif /* INFLATE_H_ */

diff --git a/inflate_p.h b/inflate_p.h
index c324b0486a1404068fb2220ce8564eaaed05941c..558a8da956a73aa7540e59a0364546600a9d2c28 100644 (file)
--- a/inflate_p.h
+++ b/inflate_p.h
@@ -154,7 +154,6 @@ static inline uint8_t* chunkcopy_safe(uint8_t *out, uint8_t *from, uint64_t len,
     len = MIN(len, safelen);
     int32_t olap_src = from >= out && from < out + len;
     int32_t olap_dst = out >= from && out < from + len;
-    uint64_t tocopy;
 
     /* For all cases without overlap, memcpy is ideal */
     if (!(olap_src || olap_dst)) {
@@ -167,65 +166,7 @@ static inline uint8_t* chunkcopy_safe(uint8_t *out, uint8_t *from, uint64_t len,
         return out + len;
     }
 
-    /* We are emulating a self-modifying copy loop here. To do this in a way that doesn't produce undefined behavior,
-     * we have to get a bit clever. First if the overlap is such that src falls between dst and dst+len, we can do the
-     * initial bulk memcpy of the nonoverlapping region. Then, we can leverage the size of this to determine the safest
-     * atomic memcpy size we can pick such that we have non-overlapping regions. This effectively becomes a safe look
-     * behind or lookahead distance. */
-    uint64_t non_olap_size = llabs(from - out); // llabs vs labs for compatibility with windows
-
-    memcpy(out, from, (size_t)non_olap_size);
-    out += non_olap_size;
-    from += non_olap_size;
-    len -= non_olap_size;
-
-    /* So this doesn't give use a worst case scenario of function calls in a loop,
-     * we want to instead break this down into copy blocks of fixed lengths */
-    while (len) {
-        tocopy = MIN(non_olap_size, len);
-        len -= tocopy;
-
-        while (tocopy >= 32) {
-            memcpy(out, from, 32);
-            out += 32;
-            from += 32;
-            tocopy -= 32;
-        }
-
-        if (tocopy >= 16) {
-            memcpy(out, from, 16);
-            out += 16;
-            from += 16;
-            tocopy -= 16;
-        }
-
-        if (tocopy >= 8) {
-            memcpy(out, from, 8);
-            out += 8;
-            from += 8;
-            tocopy -= 8;
-        }
-
-        if (tocopy >= 4) {
-            memcpy(out, from, 4);
-            out += 4;
-            from += 4;
-            tocopy -= 4;
-        }
-
-        if (tocopy >= 2) {
-            memcpy(out, from, 2);
-            out += 2;
-            from += 2;
-            tocopy -= 2;
-        }
-
-        if (tocopy) {
-            *out++ = *from++;
-        }
-    }
-
-    return out;
+    return chunkcopy_safe_unrolled(out, from, len);
 }
 
 #endif