complete refactoring and unification of Vermicelli functions

diff --git a/src/nfa/vermicelli.hpp b/src/nfa/vermicelli.hpp
index 0b4686e1a28faa46b8793516be58d77469f3eee7..83eb2335e46dbe86141f7040d264fa21c2ffc470 100644 (file)
--- a/src/nfa/vermicelli.hpp
+++ b/src/nfa/vermicelli.hpp
@@ -75,4 +75,12 @@ const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, con
 }
 #endif
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end);
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* VERMICELLI_HPP */
\ No newline at end of file

diff --git a/src/nfa/vermicelli_simd.cpp b/src/nfa/vermicelli_simd.cpp
index 6348e6f302ffb93bf52c3fdba93f3bd3dd3faac1..cd818dfbce55effd022ac21dc2735946942539a6 100644 (file)
--- a/src/nfa/vermicelli_simd.cpp
+++ b/src/nfa/vermicelli_simd.cpp
@@ -41,85 +41,75 @@
 
 template <uint16_t S>
 static really_inline
-const u8 *vermicelliSingleBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
+const u8 *vermicelliBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
 
     SuperVector<S> mask = chars.eq(casemask & data);
     return first_non_zero_match<S>(buf, mask);
 }
 
+
 template <uint16_t S>
 static really_inline
-const u8 *rvermicelliSingleBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
+const u8 *vermicelliBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
 
     SuperVector<S> mask = chars.eq(casemask & data);
-    return last_non_zero_match<S>(buf, mask);
+    return first_zero_match_inverted<S>(buf, mask);
 }
 
 template <uint16_t S>
 static really_inline
-const u8 *vermicelliDoubleBlock(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask,
-                                const u8 *buf/*, SuperVector<S> *lastmask1, size_t len = S*/) {
-
-    // lastmask1->print8("lastmask1");
-    data.print8("data");
-    chars1.print8("chars1");
-    chars2.print8("chars2");
-    casemask.print8("casemask");
-    SuperVector<S> v = casemask & data;
-    v.print8("v");
-    SuperVector<S> mask1 = chars1.eq(v);
-    mask1.print8("mask1");
-    SuperVector<S> mask2 = chars2.eq(v);
-    mask2.print8("mask2");
-    SuperVector<S> mask = (mask1 & (mask2 >> 1));
-    mask.print8("mask");
-    DEBUG_PRINTF("len = %ld\n", len);
-    // *lastmask1 = mask1 >> (len -1);
-    // lastmask1->print8("lastmask1");
+const u8 *rvermicelliBlock(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
 
-    return first_non_zero_match<S>(buf, mask);
+    SuperVector<S> mask = chars.eq(casemask & data);
+    return last_non_zero_match<S>(buf, mask);
 }
 
+
 template <uint16_t S>
 static really_inline
-const u8 *vermicelliSingleBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
+const u8 *rvermicelliBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
 
     SuperVector<S> mask = chars.eq(casemask & data);
-    return first_zero_match_inverted<S>(buf, mask);
+    return last_zero_match_inverted<S>(buf, mask);
 }
 
 template <uint16_t S>
 static really_inline
-const u8 *rvermicelliSingleBlockNeg(SuperVector<S> data, SuperVector<S> chars, SuperVector<S> casemask, const u8 *buf) {
+const u8 *vermicelliDoubleBlock(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask,
+                                u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) {
 
-    SuperVector<S> mask = chars.eq(casemask & data);
-    return last_zero_match_inverted<S>(buf, mask);
+    SuperVector<S> v = casemask & data;
+    SuperVector<S> mask1 = chars1.eq(v);
+    SuperVector<S> mask2 = chars2.eq(v);
+    SuperVector<S> mask = mask1 & (mask2 >> 1);
+
+    DEBUG_PRINTF("rv[0] = %02hhx, rv[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) return buf - 1;
+
+    return first_non_zero_match<S>(buf, mask);
 }
-/*
+
 template <uint16_t S>
 static really_inline
-const u8 *vermicelliDoubleBlockNeg(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask,
-                                const u8 *buf, size_t len = S) {
-
-    // lastmask1.print8("lastmask1");
-    data.print8("data");
-    chars1.print8("chars1");
-    chars2.print8("chars2");
-    casemask.print8("casemask");
+const u8 *rvermicelliDoubleBlock(SuperVector<S> data, SuperVector<S> chars1, SuperVector<S> chars2, SuperVector<S> casemask,
+                                 u8 const c1, u8 const c2, u8 const casechar, const u8 *buf) {
+
     SuperVector<S> v = casemask & data;
-    v.print8("v");
     SuperVector<S> mask1 = chars1.eq(v);
-    mask1.print8("mask1");
     SuperVector<S> mask2 = chars2.eq(v);
-    mask2.print8("mask2");
-    SuperVector<S> mask = (mask1 & (mask2 >> 1));// | lastmask1;
-    mask.print8("mask");
-    DEBUG_PRINTF("len = %ld\n", len);
-    // lastmask1 = mask << (len -1);
-    // lastmask1.print8("lastmask1");
+    SuperVector<S> mask = (mask1 << 1)& mask2;
 
-    return last_zero_match_inverted<S>(buf, mask);
-}*/
+    DEBUG_PRINTF("buf[0] = %02hhx, buf[-1] = %02hhx\n", buf[0], buf[-1]);
+    bool partial_match = (((buf[0] & casechar) == c2) && ((buf[-1] & casechar) == c1));
+    DEBUG_PRINTF("partial = %d\n", partial_match);
+    if (partial_match) {
+        mask = mask | (SuperVector<S>::Ones() >> (S-1));
+    }
+
+    return last_non_zero_match<S>(buf, mask);
+}
 
 template <uint16_t S>
 static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
@@ -142,7 +132,7 @@ static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> c
         DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
         if (!ISALIGNED_N(d, S)) {
             SuperVector<S> data = SuperVector<S>::loadu(d);
-            rv = vermicelliSingleBlock(data, chars, casemask, d);
+            rv = vermicelliBlock(data, chars, casemask, d);
             if (rv) return rv;
             d = ROUNDUP_PTR(d, S);
         }
@@ -151,7 +141,7 @@ static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> c
             __builtin_prefetch(d + 64);
             DEBUG_PRINTF("d %p \n", d);
             SuperVector<S> data = SuperVector<S>::load(d);
-            rv = vermicelliSingleBlock(data, chars, casemask, d);
+            rv = vermicelliBlock(data, chars, casemask, d);
             if (rv) return rv;
             d += S;
         }
@@ -162,7 +152,7 @@ static const u8 *vermicelliExecReal(SuperVector<S> const chars, SuperVector<S> c
 
     if (d != buf_end) {
         SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
-        rv = vermicelliSingleBlock(data, chars, casemask, d);
+        rv = vermicelliBlock(data, chars, casemask, d);
         DEBUG_PRINTF("rv %p \n", rv);
         if (rv && rv < buf_end) return rv;
     }
@@ -180,8 +170,6 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
     const u8 *d = buf;
     const u8 *rv;
 
-    
-
     __builtin_prefetch(d +   64);
     __builtin_prefetch(d + 2*64);
     __builtin_prefetch(d + 3*64);
@@ -193,7 +181,7 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
         DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
         if (!ISALIGNED_N(d, S)) {
             SuperVector<S> data = SuperVector<S>::loadu(d);
-            rv = vermicelliSingleBlockNeg(data, chars, casemask, d);
+            rv = vermicelliBlockNeg(data, chars, casemask, d);
             if (rv) return rv;
             d = ROUNDUP_PTR(d, S);
         }
@@ -202,7 +190,7 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
             __builtin_prefetch(d + 64);
             DEBUG_PRINTF("d %p \n", d);
             SuperVector<S> data = SuperVector<S>::load(d);
-            rv = vermicelliSingleBlockNeg(data, chars, casemask, d);
+            rv = vermicelliBlockNeg(data, chars, casemask, d);
             if (rv) return rv;
             d += S;
         }
@@ -213,7 +201,7 @@ static const u8 *nvermicelliExecReal(SuperVector<S> const chars, SuperVector<S>
 
     if (d != buf_end) {
         SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
-        rv = vermicelliSingleBlockNeg(data, chars, casemask, d);
+        rv = vermicelliBlockNeg(data, chars, casemask, d);
         DEBUG_PRINTF("rv %p \n", rv);
         if (rv && rv < buf_end) return rv;
     }
@@ -244,7 +232,7 @@ const u8 *rvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const c
         DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
         if (!ISALIGNED_N(d, S)) {
             SuperVector<S> data = SuperVector<S>::loadu(d - S);
-            rv = rvermicelliSingleBlock(data, chars, casemask, d - S);
+            rv = rvermicelliBlock(data, chars, casemask, d - S);
             DEBUG_PRINTF("rv %p \n", rv);
             if (rv) return rv;
             d = ROUNDDOWN_PTR(d, S);
@@ -257,7 +245,7 @@ const u8 *rvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const c
 
             d -= S;
             SuperVector<S> data = SuperVector<S>::load(d);
-            rv = rvermicelliSingleBlock(data, chars, casemask, d);
+            rv = rvermicelliBlock(data, chars, casemask, d);
             if (rv) return rv;
         }
     }
@@ -267,7 +255,7 @@ const u8 *rvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const c
 
     if (d != buf) {
         SuperVector<S> data = SuperVector<S>::loadu(buf);
-        rv = rvermicelliSingleBlock(data, chars, casemask, buf);
+        rv = rvermicelliBlock(data, chars, casemask, buf);
         DEBUG_PRINTF("rv %p \n", rv);
         if (rv && rv < buf_end) return rv;
     }
@@ -298,7 +286,7 @@ const u8 *rnvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const
         DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
         if (!ISALIGNED_N(d, S)) {
             SuperVector<S> data = SuperVector<S>::loadu(d - S);
-            rv = rvermicelliSingleBlockNeg(data, chars, casemask, d - S);
+            rv = rvermicelliBlockNeg(data, chars, casemask, d - S);
             DEBUG_PRINTF("rv %p \n", rv);
             if (rv) return rv;
             d = ROUNDDOWN_PTR(d, S);
@@ -311,7 +299,7 @@ const u8 *rnvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const
 
             d -= S;
             SuperVector<S> data = SuperVector<S>::load(d);
-            rv = rvermicelliSingleBlockNeg(data, chars, casemask, d);
+            rv = rvermicelliBlockNeg(data, chars, casemask, d);
             if (rv) return rv;
         }
     }
@@ -321,7 +309,7 @@ const u8 *rnvermicelliExecReal(SuperVector<S> const chars, SuperVector<S> const
 
     if (d != buf) {
         SuperVector<S> data = SuperVector<S>::loadu(buf);
-        rv = rvermicelliSingleBlockNeg(data, chars, casemask, buf);
+        rv = rvermicelliBlockNeg(data, chars, casemask, buf);
         DEBUG_PRINTF("rv %p \n", rv);
         if (rv && rv < buf_end) return rv;
     }
@@ -355,7 +343,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
         DEBUG_PRINTF("until aligned %p \n", ROUNDUP_PTR(d, S));
         if (!ISALIGNED_N(d, S)) {
             SuperVector<S> data = SuperVector<S>::loadu(d);
-            rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1);
+            rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1);
             if (rv) return rv;
             d = ROUNDUP_PTR(d, S);
         }
@@ -364,11 +352,8 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
             __builtin_prefetch(d + 64);
             DEBUG_PRINTF("d %p \n", d);
             SuperVector<S> data = SuperVector<S>::load(d);
-            rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, &lastmask1);
-            if (rv) {
-                bool partial_match = (((rv[0] & casechar) == c2) && ((rv[-1] & casechar) == c1));
-                return rv - partial_match;
-            }
+            rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, &lastmask1);
+            if (rv) return rv;
             d += S;
         }
     }
@@ -378,7 +363,7 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
 
     if (d != buf_end) {
         SuperVector<S> data = SuperVector<S>::loadu_maskz(d, buf_end - d);
-        rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, d);//, buf_end - d);
+        rv = vermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);//, buf_end - d);
         DEBUG_PRINTF("rv %p \n", rv);
         if (rv && rv < buf_end) return rv;
     }
@@ -396,60 +381,63 @@ static const u8 *vermicelliDoubleExecReal(u8 const c1, u8 const c2, SuperVector<
 }
 
 // /* returns highest offset of c2 (NOTE: not c1) */
-// static really_inline
-// const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf,
-//                                 const u8 *buf_end) {
-//     DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n",
-//                  nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf));
-//     assert(buf < buf_end);
-
-//     VERM_TYPE chars1 = VERM_SET_FN(c1); /* nocase already uppercase */
-//     VERM_TYPE chars2 = VERM_SET_FN(c2); /* nocase already uppercase */
-
-// #ifdef HAVE_AVX512
-//     if (buf_end - buf <= VERM_BOUNDARY) {
-//         const u8 *ptr = nocase
-//                       ? rdvermMiniNocase(chars1, chars2, buf, buf_end)
-//                       : rdvermMini(chars1, chars2, buf, buf_end);
-
-//         if (ptr) {
-//             return ptr;
-//         }
-
-//         // check for partial match at end ???
-//         return buf - 1;
-//     }
-// #endif
-
-//     assert((buf_end - buf) >= VERM_BOUNDARY);
-//     size_t min = (size_t)buf_end % VERM_BOUNDARY;
-//     if (min) {
-//         // input not aligned, so we need to run one iteration with an unaligned
-//         // load, then skip buf forward to the next aligned address. There's
-//         // some small overlap here, but we don't mind scanning it twice if we
-//         // can do it quickly, do we?
-//         const u8 *ptr = nocase ? rdvermPreconditionNocase(chars1, chars2,
-//                                                           buf_end - VERM_BOUNDARY)
-//                                : rdvermPrecondition(chars1, chars2,
-//                                                     buf_end - VERM_BOUNDARY);
-
-//         if (ptr) {
-//             return ptr;
-//         }
-
-//         buf_end -= min;
-//         if (buf >= buf_end) {
-//             return buf_end;
-//         }
-//     }
-
-//     // Aligned loops from here on in
-//     if (nocase) {
-//         return rdvermSearchAlignedNocase(chars1, chars2, c1, c2, buf, buf_end);
-//     } else {
-//         return rdvermSearchAligned(chars1, chars2, c1, c2, buf, buf_end);
-//     }
-// }
+template <uint16_t S>
+const u8 *rvermicelliDoubleExecReal(char c1, char c2, SuperVector<S> const casemask, const u8 *buf, const u8 *buf_end) {
+    assert(buf && buf_end);
+    assert(buf < buf_end);
+    DEBUG_PRINTF("rverm %p len %zu\n", buf, buf_end - buf);
+    DEBUG_PRINTF("b %s\n", buf);
+    char s[255];
+    snprintf(s, buf_end - buf + 1, "%s", buf);
+    DEBUG_PRINTF("b %s\n", s);
+
+    const u8 *d = buf_end;
+    const u8 *rv;
+    const SuperVector<VECTORSIZE> chars1 = SuperVector<VECTORSIZE>::dup_u8(c1);
+    const SuperVector<VECTORSIZE> chars2 = SuperVector<VECTORSIZE>::dup_u8(c2);
+    const u8 casechar = casemask.u.u8[0];
+
+    __builtin_prefetch(d -   64);
+    __builtin_prefetch(d - 2*64);
+    __builtin_prefetch(d - 3*64);
+    __builtin_prefetch(d - 4*64);
+    DEBUG_PRINTF("start %p end %p \n", buf, d);
+    assert(d > buf);
+    if (d - S >= buf) {
+        // Reach vector aligned boundaries
+        DEBUG_PRINTF("until aligned %p \n", ROUNDDOWN_PTR(d, S));
+        if (!ISALIGNED_N(d, S)) {
+            SuperVector<S> data = SuperVector<S>::loadu(d - S);
+            rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d - S);
+            DEBUG_PRINTF("rv %p \n", rv);
+            if (rv && rv < buf_end) return rv;
+            d = ROUNDDOWN_PTR(d, S);
+        }
+
+        while (d - S >= buf) {
+            DEBUG_PRINTF("aligned %p \n", d);
+            // On large packet buffers, this prefetch appears to get us about 2%.
+            __builtin_prefetch(d - 64);
+
+            d -= S;
+            SuperVector<S> data = SuperVector<S>::load(d);
+            rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, d);
+            if (rv) return rv;
+        }
+    }
+
+    DEBUG_PRINTF("tail d %p e %p \n", buf, d);
+    // finish off head
+
+    if (d != buf) {
+        SuperVector<S> data = SuperVector<S>::loadu(buf);
+        rv = rvermicelliDoubleBlock(data, chars1, chars2, casemask, c1, c2, casechar, buf);
+        DEBUG_PRINTF("rv %p \n", rv);
+        if (rv && rv < buf_end) return rv;
+    }
+
+    return buf - 1;
+}
 
 extern "C" const u8 *vermicelliExec(char c, char nocase, const u8 *buf, const u8 *buf_end) {
     DEBUG_PRINTF("verm scan %s\\x%02hhx over %zu bytes\n",
@@ -505,4 +493,14 @@ extern "C" const u8 *vermicelliDoubleExec(char c1, char c2, char nocase, const u
     const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
 
     return vermicelliDoubleExecReal<VECTORSIZE>(c1, c2, casemask, buf, buf_end);
+}
+
+extern "C" const u8 *rvermicelliDoubleExec(char c1, char c2, char nocase, const u8 *buf, const u8 *buf_end) {
+    DEBUG_PRINTF("rev double verm scan %s\\x%02hhx%02hhx over %zu bytes\n",
+                 nocase ? "nocase " : "", c1, c2, (size_t)(buf_end - buf));
+    assert(buf < buf_end);
+
+    const SuperVector<VECTORSIZE> casemask{nocase ? getCaseMask<VECTORSIZE>() : SuperVector<VECTORSIZE>::Ones()};
+
+    return rvermicelliDoubleExecReal<VECTORSIZE>(c1, c2, casemask, buf, buf_end);
 }
\ No newline at end of file