add expand128() implementation for NEON

diff --git a/src/util/arch/arm/bitutils.h b/src/util/arch/arm/bitutils.h
index 1d1e01673c0319bf7abca051ca89d4d3399e5229..ddca35c9efcc72a1bab38699c863c94a99081cee 100644 (file)
--- a/src/util/arch/arm/bitutils.h
+++ b/src/util/arch/arm/bitutils.h
@@ -106,7 +106,6 @@ u64a compress64_impl(u64a x, u64a m) {
 
 static really_inline
 m128 compress128_impl(m128 x, m128 m) {
-
     m128 one = set1_2x64(1);
     m128 bitset = one;
     m128 vres = zeroes128();
@@ -118,7 +117,7 @@ m128 compress128_impl(m128 x, m128 m) {
        m128 mask = not128(eq64_m128(tv, zeroes128()));
        mask = vandq_s64(bitset, mask);
         vres = or128(vres, mask);
-       m = and128(m, sub_2x64(m, set1_2x64(1)));
+       m = and128(m, sub_2x64(m, one));
         bitset = lshift64_m128(bitset, 1);
     }
     return vres;
@@ -134,6 +133,25 @@ u64a expand64_impl(u64a x, u64a m) {
     return expand64_impl_c(x, m);
 }
 
+static really_inline
+m128 expand128_impl(m128 x, m128 m) {
+    m128 one = set1_2x64(1);
+    m128 bitset = one;
+    m128 vres = zeroes128();
+    while (isnonzero128(m)) {
+       m128 tv = and128(x, m);
+
+       m128 mm = sub_2x64(zeroes128(), m);
+       m128 mask = not128(eq64_m128(tv, zeroes128()));
+       mask = vandq_s64(bitset, mask);
+       mask = and128(mask, mm);
+        vres = or128(vres, mask);
+       m = and128(m, sub_2x64(m, one));
+        bitset = lshift64_m128(bitset, 1);
+    }
+    return vres;
+}
+
 /* returns the first set bit after begin (if not ~0U). If no bit is set after
  * begin returns ~0U
  */

diff --git a/src/util/arch/common/bitutils.h b/src/util/arch/common/bitutils.h
index 88e71bbaa82ec8a1ff8b721c93cf2541d10955aa..723e4a18239299e5e6c314cd4eb9494dae88da46 100644 (file)
--- a/src/util/arch/common/bitutils.h
+++ b/src/util/arch/common/bitutils.h
@@ -301,6 +301,18 @@ u64a expand64_impl_c(u64a x, u64a m) {
     return x & m0; // clear out extraneous bits*/
 }
 
+static really_inline
+m128 expand128_impl_c(m128 xvec, m128 mvec) {
+    u64a ALIGN_ATTR(16) x[2];
+    u64a ALIGN_ATTR(16) m[2];
+    store128(x, xvec);
+    store128(m, mvec);
+
+    expand64_impl_c(x[0], m[0]);
+    expand64_impl_c(x[1], m[1]);
+
+    return xvec;
+}
 
 /* returns the first set bit after begin (if not ~0U). If no bit is set after
  * begin returns ~0U

diff --git a/src/util/arch/x86/bitutils.h b/src/util/arch/x86/bitutils.h
index 33fff7c25a2c9f825ba5b2a2a5e0e771f9c5631a..1a9c3f7ca4287fb68a80ac286ca2966bad6aeb28 100644 (file)
--- a/src/util/arch/x86/bitutils.h
+++ b/src/util/arch/x86/bitutils.h
@@ -239,6 +239,11 @@ u64a expand64_impl(u64a x, u64a m) {
 #endif
 }
 
+static really_inline
+m128 expand128_impl(m128 x, m128 m) {
+    return expand128_impl_c(x, m);
+}
+
 /* returns the first set bit after begin (if not ~0U). If no bit is set after
  * begin returns ~0U
  */

diff --git a/src/util/bitutils.h b/src/util/bitutils.h
index 21d35388527fdc9e90bcc3af37981d336418d986..684945073db5098b71e09f88c214573db3c34209 100644 (file)
--- a/src/util/bitutils.h
+++ b/src/util/bitutils.h
@@ -135,6 +135,10 @@ u64a expand64(u64a x, u64a m) {
     return expand64_impl(x, m);
 }
 
+static really_inline
+m128 expand128(m128 x, m128 m) {
+    return expand128_impl(x, m);
+}
 
 /* returns the first set bit after begin (if not ~0U). If no bit is set after
  * begin returns ~0U

diff --git a/src/util/state_compress.c b/src/util/state_compress.c
index 5c26f04332108ae5474e57ec6fbf9d069274747e..66cd4daff52bbffa2cea624882cb433f9cbc3191 100644 (file)
--- a/src/util/state_compress.c
+++ b/src/util/state_compress.c
@@ -162,14 +162,16 @@ m128 loadcompressed128_64bit(const void *ptr, m128 mvec) {
     u64a ALIGN_ATTR(16) m[2];
     store128(m, mvec);
 
-    u32 bits[2] = { popcount64(m[0]), popcount64(m[1]) };
-    u64a ALIGN_ATTR(16) v[2];
+    // Count the number of bits of compressed state we're writing out per
+    // chunk.
+    u32 ALIGN_ATTR(16) bits[2] = { popcount64(m[0]), popcount64(m[1]) };
 
+    u64a ALIGN_ATTR(16) v[2];
     unpack_bits_64(v, (const u8 *)ptr, bits, 2);
+    m128 xvec = load128(v);
 
-    u64a x[2] = { expand64(v[0], m[0]), expand64(v[1], m[1]) };
-
-    return set2x64(x[1], x[0]);
+    // Expand vector
+    return expand128(xvec, mvec);
 }
 #endif