remove low4bits from the arguments, fix cases that mostly affect loading large (64...

diff --git a/src/nfa/shufti_simd.hpp b/src/nfa/shufti_simd.hpp
index 2f18e8d8558d5b71e8223cea2e4ea881295a03e9..cbfd23badf7069fbc2b5baf8d98d29763d9f6bd8 100644 (file)
--- a/src/nfa/shufti_simd.hpp
+++ b/src/nfa/shufti_simd.hpp
@@ -45,40 +45,29 @@
 template <uint16_t S>
 static really_inline
 typename SuperVector<S>::movemask_type block(SuperVector<S> mask_lo, SuperVector<S> mask_hi,
-            SuperVector<S> chars, const SuperVector<S> low4bits) {
+            SuperVector<S> chars) {
+    const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
+
     SuperVector<S> c_lo = chars & low4bits;
-    //printv_u8("c_lo", c_lo);
     c_lo = mask_lo.pshufb(c_lo);
-    //printv_u8("c_lo", c_lo);
     SuperVector<S> c_hi = mask_hi.pshufb(chars.rshift64(4) & low4bits);
     SuperVector<S> t = c_lo & c_hi;
 
-    /*printv_u8("low4bits", low4bits);
-    printv_u8("mask_lo", mask_lo);
-    printv_u8("mask_hi", mask_hi);
-    printv_u8("chars", chars);
-    printv_u8("c_lo", c_lo);
-    printv_u8("c_hi", c_hi);
-    printv_u8("t", t);*/
-
     return t.eqmask(SuperVector<S>::Zeroes());
 }
 
-
 template <uint16_t S>
 static really_inline
-const u8 *fwdBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, 
-                      const SuperVector<S> low4bits, const u8 *buf) {
-    typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars, low4bits);
+const u8 *fwdBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
+    typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars);
     DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
 
     return firstMatch<S>(buf, z);
 }
-
+/*
 template <uint16_t S>
 static really_inline
-const u8 *shortShufti(SuperVector<S> mask_lo, SuperVector<S> mask_hi, const u8 *buf,
-                         const u8 *buf_end, const SuperVector<S> low4bits) {
+const u8 *shortShufti(SuperVector<S> mask_lo, SuperVector<S> mask_hi, const u8 *buf, const u8 *buf_end) {
     DEBUG_PRINTF("short shufti %p len %zu\n", buf, buf_end - buf);
     uintptr_t len = buf_end - buf;
     assert(len <= S);
@@ -88,20 +77,19 @@ const u8 *shortShufti(SuperVector<S> mask_lo, SuperVector<S> mask_hi, const u8 *
     uint8_t alignment = (uintptr_t)(buf) & 15;
     typename SuperVector<S>::movemask_type maskb = 1 << alignment;
     typename SuperVector<S>::movemask_type maske = SINGLE_LOAD_MASK(len - alignment);
-    typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars, low4bits);
+    typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars);
     // reuse the load mask to indicate valid bytes
     DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
     z &= maskb | maske;
     DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
     
     return firstMatch<S>(buf, z);
-}
+}*/
 
 template <uint16_t S>
 static really_inline
-const u8 *revBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, 
-                   const SuperVector<S> low4bits, const u8 *buf) {
-    typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars, low4bits);
+const u8 *revBlock(SuperVector<S> mask_lo, SuperVector<S> mask_hi, SuperVector<S> chars, const u8 *buf) {
+    typename SuperVector<S>::movemask_type z = block(mask_lo, mask_hi, chars);
     DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
     return lastMatch<S>(buf, z);
 }
@@ -113,7 +101,6 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
     DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf);
     DEBUG_PRINTF("b %s\n", buf);
 
-    const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
     const SuperVector<S> wide_mask_lo(mask_lo);
     const SuperVector<S> wide_mask_hi(mask_hi);
 
@@ -128,7 +115,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
         DEBUG_PRINTF("until aligned %p \n", d1);
         if (d1 != d) {
             rv = shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d, d1);
-            // rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1, low4bits);
+            // rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1);
             if (rv != d1) {
                 return rv;
             }
@@ -145,7 +132,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
             __builtin_prefetch(base + 256);
 
             SuperVector<S> chars = SuperVector<S>::load(d);
-            rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, low4bits, d);
+            rv = fwdBlock(wide_mask_lo, wide_mask_hi, chars, d);
             if (rv) return rv;
         }
     }
@@ -156,7 +143,7 @@ const u8 *shuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *bu
     rv = buf_end;
     if (d != buf_end) {
         rv = shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d, buf_end);
-        // rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end, low4bits);
+        // rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end);
         DEBUG_PRINTF("rv %p \n", rv);
     }
     
@@ -170,7 +157,6 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b
     DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf);
     DEBUG_PRINTF("b %s\n", buf);
 
-    const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
     const SuperVector<S> wide_mask_lo(mask_lo);
     const SuperVector<S> wide_mask_hi(mask_hi);
 
@@ -186,30 +172,30 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b
         if (d1 != d) {
             rv = shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, d1, d);
             DEBUG_PRINTF("rv %p \n", rv);
-            // rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1, low4bits);
+            // rv = shortShufti(wide_mask_lo, wide_mask_hi, d, d1);
             if (rv != d1 - 1) return rv;
             d = d1;
         }
 
         while (d - S >= buf) {
+            DEBUG_PRINTF("aligned %p \n", d);
             d -= S;
-            DEBUG_PRINTF("d %p \n", d);
             const u8 *base = ROUNDDOWN_PTR(buf, S);
             // On large packet buffers, this prefetch appears to get us about 2%.
             __builtin_prefetch(base + 256);
 
             SuperVector<S> chars = SuperVector<S>::load(d);
-            rv = revBlock(wide_mask_lo, wide_mask_hi, chars, low4bits, d);
+            rv = revBlock(wide_mask_lo, wide_mask_hi, chars, d);
             if (rv) return rv;
         }
     }
 
-    DEBUG_PRINTF("d %p e %p \n", buf, d);
+    DEBUG_PRINTF("tail d %p e %p \n", buf, d);
     // finish off tail
 
     if (d != buf) {
         rv = shuftiRevSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi, buf, d);
-        // rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end, low4bits);
+        // rv = shortShufti(wide_mask_lo, wide_mask_hi, buf_end - S, buf_end);
         DEBUG_PRINTF("rv %p \n", rv);
         if (rv) return rv;
     }
@@ -220,18 +206,29 @@ const u8 *rshuftiExecReal(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *b
 template <uint16_t S>
 static really_inline
 const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, SuperVector<S> mask2_lo, SuperVector<S> mask2_hi,
-                    SuperVector<S> chars, const SuperVector<S> low4bits, const u8 *buf) {
+                    SuperVector<S> chars, const u8 *buf) {
 
+    const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
     SuperVector<S> chars_lo = chars & low4bits;
+    chars_lo.print8("chars_lo");
     SuperVector<S> chars_hi = chars.rshift64(4) & low4bits;
+    chars_hi.print8("chars_hi");
     SuperVector<S> c1_lo = mask1_lo.pshufb(chars_lo);
+    c1_lo.print8("c1_lo");
     SuperVector<S> c1_hi = mask1_hi.pshufb(chars_hi);
+    c1_hi.print8("c1_hi");
     SuperVector<S> t1 = c1_lo | c1_hi;
+    t1.print8("t1");
 
     SuperVector<S> c2_lo = mask2_lo.pshufb(chars_lo);
+    c2_lo.print8("c2_lo");
     SuperVector<S> c2_hi = mask2_hi.pshufb(chars_hi);
+    c2_hi.print8("c2_hi");
     SuperVector<S> t2 = c2_lo | c2_hi;
+    t2.print8("t2");
+    t2.rshift128(1).print8("t2.rshift128(1)");
     SuperVector<S> t = t1 | (t2.rshift128(1));
+    t.print8("t");
 
     typename SuperVector<S>::movemask_type z = t.eqmask(SuperVector<S>::Ones());
     DEBUG_PRINTF(" z: 0x%016llx\n", (u64a)z);
@@ -239,15 +236,13 @@ const u8 *fwdBlockDouble(SuperVector<S> mask1_lo, SuperVector<S> mask1_hi, Super
 }
 
 template <uint16_t S>
-const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi,
-                           m128 mask2_lo, m128 mask2_hi,
+const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi, m128 mask2_lo, m128 mask2_hi,
                            const u8 *buf, const u8 *buf_end) {
         assert(buf && buf_end);
     assert(buf < buf_end);
     DEBUG_PRINTF("shufti %p len %zu\n", buf, buf_end - buf);
     DEBUG_PRINTF("b %s\n", buf);
 
-    const SuperVector<S> low4bits = SuperVector<S>::dup_u8(0xf);
     const SuperVector<S> wide_mask1_lo(mask1_lo);
     const SuperVector<S> wide_mask1_hi(mask1_hi);
     const SuperVector<S> wide_mask2_lo(mask2_lo);
@@ -264,7 +259,7 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi,
         DEBUG_PRINTF("until aligned %p \n", d1);
         if (d1 != d) {
             SuperVector<S> chars = SuperVector<S>::loadu(d);
-            rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, low4bits, d);
+            rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d);
             DEBUG_PRINTF("rv %p \n", rv);
             if (rv) return rv;
             d = d1;
@@ -274,25 +269,25 @@ const u8 *shuftiDoubleExecReal(m128 mask1_lo, m128 mask1_hi,
         DEBUG_PRINTF("loops %ld \n", loops);
 
         for (size_t i = 0; i < loops; i++, d+= S) {
-            DEBUG_PRINTF("d %p \n", d);
+            DEBUG_PRINTF("it = %ld, d %p \n", i, d);
             const u8 *base = ROUNDUP_PTR(d, S);
             // On large packet buffers, this prefetch appears to get us about 2%.
             __builtin_prefetch(base + 256);
 
             SuperVector<S> chars = SuperVector<S>::load(d);
-            rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, low4bits, d);
+            rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, d);
             if (rv) return rv;
         }
     }
 
-    DEBUG_PRINTF("d %p e %p \n", d, buf_end);
+    DEBUG_PRINTF("tail d %p e %p \n", d, buf_end);
     // finish off tail
 
     if (d != buf_end) {
         SuperVector<S> chars = SuperVector<S>::loadu(buf_end - S);
-        rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, low4bits, buf_end - S);
+        rv = fwdBlockDouble(wide_mask1_lo, wide_mask1_hi, wide_mask2_lo, wide_mask2_hi, chars, buf_end - S);
         DEBUG_PRINTF("rv %p \n", rv);
-        if (rv) return rv;
+        if (rv >= buf && rv < buf_end) return rv;
     }
     
     return buf_end;

diff --git a/src/nfa/truffle_simd.hpp b/src/nfa/truffle_simd.hpp
index 439d94f957876d9cbd68d39acd7fb1caa4dcb3a4..73017578632b262252eb35723405d3f8ef76e887 100644 (file)
--- a/src/nfa/truffle_simd.hpp
+++ b/src/nfa/truffle_simd.hpp
@@ -236,7 +236,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
     if (d != buf) {
         rv = truffleRevMini(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, buf, d);
         DEBUG_PRINTF("rv %p \n", rv);
-        if (rv) return rv;
+        if (rv >= buf && rv < buf_end) return rv;
     }
     
     return buf - 1;