simd: convert rshift64 macros to functions and fix simd_utils bugs (#376)

Convert rshift64_m128/m256/m512 macros to inline functions that
support runtime (non-constant) shift amounts on x86, matching the
existing lshift64 function implementations.

Also fix:
- lshift64_m256/rshift64_m256 parameter type from int to unsigned in
  the non-256-bit fallback path (common/simd_utils.h)
- isnonzero512: remove redundant self-OR operations
- load512: fix alignment assertion to check m512 instead of m256

Fixes: 3f0f9e60526d ("move x86 implementations of simd_utils.h to util/arch/x86/")
Fixes: 6ff47528ba22 ("add scalar versions of the vectorized functions for architectures that don't support 256-bit/512-bit SIMD vectors such as ARM")
Fixes: 75aadb76f82e ("split arch-agnostic simd_utils.h functions into the common file")
Signed-off-by: Byeonguk Jeong <jungbu2855@gmail.com>

diff --git a/src/util/arch/common/simd_utils.h b/src/util/arch/common/simd_utils.h
index 109b11584a781927669d76912d5287dc53a4988f..ad5f9b2ab93461da6546a783a827725dd23db441 100644 (file)
--- a/src/util/arch/common/simd_utils.h
+++ b/src/util/arch/common/simd_utils.h
@@ -115,7 +115,7 @@ ALIGN_CL_DIRECTIVE static const u8 simd_onebit_masks[] = {
 #if !defined(HAVE_SIMD_256_BITS)
 
 static really_really_inline
-m256 lshift64_m256(m256 a, int b) {
+m256 lshift64_m256(m256 a, unsigned b) {
     m256 rv = a;
     rv.lo = lshift64_m128(rv.lo, b);
     rv.hi = lshift64_m128(rv.hi, b);
@@ -123,7 +123,7 @@ m256 lshift64_m256(m256 a, int b) {
 }
 
 static really_inline
-m256 rshift64_m256(m256 a, int b) {
+m256 rshift64_m256(m256 a, unsigned b) {
     m256 rv = a;
     rv.lo = rshift64_m128(rv.lo, b);
     rv.hi = rshift64_m128(rv.hi, b);
@@ -687,9 +687,7 @@ int diff512(m512 a, m512 b) {
 
 static really_inline
 int isnonzero512(m512 a) {
-    m256 x = or256(a.lo, a.lo);
-    m256 y = or256(a.hi, a.hi);
-    return isnonzero256(or256(x, y));
+    return isnonzero256(or256(a.lo, a.hi));
 }
 
 /**
@@ -715,7 +713,7 @@ u32 diffrich64_512(m512 a, m512 b) {
 // aligned load
 static really_inline
 m512 load512(const void *ptr) {
-    assert(ISALIGNED_N(ptr, alignof(m256)));
+    assert(ISALIGNED_N(ptr, alignof(m512)));
     // cppcheck-suppress cstyleCast
     m512 rv = { load256(ptr), load256((const char *)ptr + 32) };
     return rv;

diff --git a/src/util/arch/x86/simd_utils.h b/src/util/arch/x86/simd_utils.h
index 4df740086fcf454050be068da2449dd73a6f00df..17ca6ea6605faea129f14f6c22ba72d7d15b18d2 100644 (file)
--- a/src/util/arch/x86/simd_utils.h
+++ b/src/util/arch/x86/simd_utils.h
@@ -146,7 +146,17 @@ m128 lshift64_m128(m128 a, unsigned b) {
     return _mm_sll_epi64(a, x);
 }
 
-#define rshift64_m128(a, b) _mm_srli_epi64((a), (b))
+static really_really_inline
+m128 rshift64_m128(m128 a, unsigned b) {
+#if defined(HAVE__BUILTIN_CONSTANT_P)
+    if (__builtin_constant_p(b)) {
+        return _mm_srli_epi64(a, b);
+    }
+#endif
+    m128 x = _mm_cvtsi32_si128(b);
+    return _mm_srl_epi64(a, x);
+}
+
 #define eq128(a, b)         _mm_cmpeq_epi8((a), (b))
 #define eq64_m128(a, b)     _mm_cmpeq_epi64((a), (b))
 #define movemask128(a)      ((u32)_mm_movemask_epi8((a)))
@@ -519,7 +529,16 @@ m256 lshift64_m256(m256 a, unsigned b) {
     return _mm256_sll_epi64(a, x);
 }
 
-#define rshift64_m256(a, b) _mm256_srli_epi64((a), (b))
+static really_really_inline
+m256 rshift64_m256(m256 a, unsigned b) {
+#if defined(HAVE__BUILTIN_CONSTANT_P)
+    if (__builtin_constant_p(b)) {
+        return _mm256_srli_epi64(a, b);
+    }
+#endif
+    m128 x = _mm_cvtsi32_si128(b);
+    return _mm256_srl_epi64(a, x);
+}
 
 static really_inline m256 set1_4x64(u64a c) {
     return _mm256_set1_epi64x(c);
@@ -944,7 +963,17 @@ m512 lshift64_m512(m512 a, unsigned b) {
     return _mm512_sll_epi64(a, x);
 }
 
-#define rshift64_m512(a, b) _mm512_srli_epi64((a), (b))
+static really_really_inline
+m512 rshift64_m512(m512 a, unsigned b) {
+#if defined(HAVE__BUILTIN_CONSTANT_P)
+    if (__builtin_constant_p(b)) {
+        return _mm512_srli_epi64(a, b);
+    }
+#endif
+    m128 x = _mm_cvtsi32_si128(b);
+    return _mm512_srl_epi64(a, x);
+}
+
 #define rshift128_m512(a, count_immed) _mm512_bsrli_epi128(a, count_immed)
 #define lshift128_m512(a, count_immed) _mm512_bslli_epi128(a, count_immed)