fix failing corner case, add pshufb_maskz()

diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp
index cbfd23badf7069fbc2b5baf8d98d29763d9f6bd8..86b20deb35717f193b7a48ca5972d030a1907806 100644 (file)
--- a/src/nfa/shufti_simd.hpp
+++ b/src/nfa/shufti_simd.hpp
@@ -235,6 +235,44 @@ const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, Super
     return firstMatch<S>(buf, z);
 }
 
+template <uint16_t S>
+static really_inline const u8 *shuftiDoubleMini(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, SuperVector<S> mask2_lo, SuperVector<S> mask2_hi,
+                       const u8 *buf, const u8 *buf_end){
+    uintptr_t len = buf_end - buf;
+    assert(len < S);
+
+    const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
+
+    DEBUG_PRINTF("buf %p buf_end %p \n", buf, buf_end);
+    SuperVector<S> chars = SuperVector<S>::loadu_maskz(buf, len);
+    chars.print8("chars");
+
+    SuperVector<S> chars_lo = chars & low4bits;
+    chars_lo.print8("chars_lo");
+    SuperVector<S> chars_hi = chars.rshift64(4) & low4bits;
+    chars_hi.print8("chars_hi");
+    SuperVector<S> c1_lo = mask1_lo.pshufb_maskz(chars_lo, len);
+    c1_lo.print8("c1_lo");
+    SuperVector<S> c1_hi = mask1_hi.pshufb_maskz(chars_hi, len);
+    c1_hi.print8("c1_hi");
+    SuperVector<S> t1 = c1_lo | c1_hi;
+    t1.print8("t1");
+
+    SuperVector<S> c2_lo = mask2_lo.pshufb_maskz(chars_lo, len);
+    c2_lo.print8("c2_lo");
+    SuperVector<S> c2_hi = mask2_hi.pshufb_maskz(chars_hi, len);
+    c2_hi.print8("c2_hi");
+    SuperVector<S> t2 = c2_lo | c2_hi;
+    t2.print8("t2");
+    t2.rshift128(1).print8("t2.rshift128(1)");
+    SuperVector<S> t = t1 | (t2.rshift128(1));
+    t.print8("t");
+
+    typename SuperVector<S>::movemask_type z = t.eqmask(SuperVector<S>::Ones());
+    DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
+    return firstMatch<S>(buf, z);
+}
+
 template <uint16_t S>
 const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi,
                            const u8 *buf, const u8 *buf_end) {
@@ -284,8 +322,7 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128
     // finish off tail
 
     if (d != buf_end) {
-        SuperVector<S> chars = SuperVector<S>::loadu(buf_end - S);
-        rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, buf_end - S);
+        rv = shuftiDoubleMini(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, d, buf_end);
         DEBUG_PRINTF("rv %p \n", rv);
         if (rv >= buf && rv < buf_end) return rv;
     }

diff --git a/src/util/supervector/arch/arm/impl.cpp b/src/util/supervector/arch/arm/impl.cpp
index e40b6a38c6a50f3eacf8473ec966e1c48b7615b4..65d0faa576a894610145e3b3e670431fadfc64c1 100644 (file)
--- a/src/util/supervector/arch/arm/impl.cpp
+++ b/src/util/supervector/arch/arm/impl.cpp
@@ -348,6 +348,13 @@ really_inline SuperVector<16> SuperVector<16>::pshufb(SuperVector<16> b)
     return {vqtbl1q_s8((int8x16_t)u.v128[0], (uint8x16_t)btranslated)};
 }
 
+template<>
+really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len)
+{
+    SuperVector<16> mask = Ones().rshift128_var(16 -len);
+    return mask & pshufb(b);
+}
+
 #ifdef HS_OPTIMIZE
 template<>
 really_inline SuperVector<16> SuperVector<16>::lshift64(uint8_t const N)

diff --git a/src/util/supervector/arch/x86/impl.cpp b/src/util/supervector/arch/x86/impl.cpp
index e64583e1f7297fd3d08d1032e55ec195e34b5275..3c305d4b8c7fec266e71981db9affc1ce01836be 100644 (file)
--- a/src/util/supervector/arch/x86/impl.cpp
+++ b/src/util/supervector/arch/x86/impl.cpp
@@ -312,6 +312,13 @@ really_inline SuperVector<16> SuperVector<16>::pshufb(SuperVector<16> b)
     return {_mm_shuffle_epi8(u.v128[0], b.u.v128[0])};
 }
 
+template<>
+really_inline SuperVector<16> SuperVector<16>::pshufb_maskz(SuperVector<16> b, uint8_t const len)
+{
+    SuperVector<16> mask = Ones().rshift128_var(16 -len);
+    return mask & pshufb(b);
+}
+
 #ifdef HS_OPTIMIZE
 template<>
 really_inline SuperVector<16> SuperVector<16>::lshift64(uint8_t const N)
@@ -733,6 +740,13 @@ really_inline SuperVector<32> SuperVector<32>::pshufb(SuperVector<32> b)
     return {_mm256_shuffle_epi8(u.v256[0], b.u.v256[0])};
 }
 
+template<>
+really_inline SuperVector<32> SuperVector<32>::pshufb_maskz(SuperVector<32> b, uint8_t const len)
+{
+    SuperVector<32> mask = Ones().rshift128_var(32 -len);
+    return mask & pshufb(b);
+}
+
 #ifdef HS_OPTIMIZE
 template<>
 really_inline SuperVector<32> SuperVector<32>::lshift64(uint8_t const N)
@@ -1176,6 +1190,13 @@ really_inline SuperVector<64> SuperVector<64>::pshufb(SuperVector<64> b)
     return {_mm512_shuffle_epi8(u.v512[0], b.u.v512[0])};
 }
 
+template<>
+really_inline SuperVector<64> SuperVector<64>::pshufb_maskz(SuperVector<64> b, uint8_t const len)
+{
+    u64a mask = (~0ULL) >> (64 - len);
+    DEBUG_PRINTF("mask = %016llx\n", mask);
+    return {_mm512_maskz_shuffle_epi8(mask, u.v512[0], b.u.v512[0])};
+}
 
 #ifdef HS_OPTIMIZE
 template<>

diff --git a/src/util/supervector/supervector.hpp b/src/util/supervector/supervector.hpp
index bd7fd18a95031e305848e9f1e600f0367bfd7a08..e834fef0b8aa3b1360ed94242cf032f1ad8e802e 100644 (file)
--- a/src/util/supervector/supervector.hpp
+++ b/src/util/supervector/supervector.hpp
@@ -216,6 +216,7 @@ public:
   SuperVector alignr(SuperVector &other, int8_t offset);
 
   SuperVector pshufb(SuperVector b);
+  SuperVector pshufb_maskz(SuperVector b, uint8_t const len);
   SuperVector lshift64(uint8_t const N);
   SuperVector rshift64(uint8_t const N);
   SuperVector lshift128(uint8_t const N);