From: Ken Steele <ken@tilera.com>
Date: Tue, 30 Sep 2014 14:58:35 +0000 (-0400)
Subject: Implement MPM opt for b2g, b3g, wumanber
X-Git-Tag: suricata-2.1beta3~42
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=d03f1244457e65831ab72f70f8ea606f1a5ea7a0;p=thirdparty%2Fsuricata.git

Implement MPM opt for b2g, b3g, wumanber

Found problems in b2gm and b2gc, so those are removed.
---

diff --git a/src/util-mpm-ac-tile-small.c b/src/util-mpm-ac-tile-small.c
index 8a0498654d..3390a917cb 100644
--- a/src/util-mpm-ac-tile-small.c
+++ b/src/util-mpm-ac-tile-small.c
@@ -37,8 +37,8 @@ uint32_t FUNC_NAME(SCACTileSearchCtx *ctx, MpmThreadCtx *mpm_thread_ctx,
     int i = 0;
     int matches = 0;
 
-    uint8_t bitarray[pmq->pattern_id_array_size];
-    memset(&bitarray, 0, pmq->pattern_id_array_size);
+    uint8_t bitarray[pmq->pattern_id_bitarray_size];
+    memset(bitarray, 0, pmq->pattern_id_bitarray_size);
 
     uint8_t* restrict xlate = ctx->translate_table;
     STYPE *state_table = (STYPE*)ctx->state_table;
diff --git a/src/util-mpm-ac-tile.c b/src/util-mpm-ac-tile.c
index 18d2d68e4c..e34911de96 100644
--- a/src/util-mpm-ac-tile.c
+++ b/src/util-mpm-ac-tile.c
@@ -1548,8 +1548,8 @@ uint32_t SCACTileSearchLarge(SCACTileSearchCtx *ctx, MpmThreadCtx *mpm_thread_ct
 
     SCACTilePatternList *pid_pat_list = ctx->pid_pat_list;
 
-    uint8_t bitarray[pmq->pattern_id_array_size];
-    memset(&bitarray, 0, pmq->pattern_id_array_size);
+    uint8_t bitarray[pmq->pattern_id_bitarray_size];
+    memset(bitarray, 0, pmq->pattern_id_bitarray_size);
 
     uint8_t* restrict xlate = ctx->translate_table;
     register int state = 0;
diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index 7219542035..f6c462ea25 100644
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -1307,8 +1307,8 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
     /* \todo Change it for stateful MPM.  Supply the state using mpm_thread_ctx */
     SCACPatternList *pid_pat_list = ctx->pid_pat_list;
 
-    uint8_t bitarray[pmq->pattern_id_array_size];
-    memset(&bitarray, 0, pmq->pattern_id_array_size);
+    uint8_t bitarray[pmq->pattern_id_bitarray_size];
+    memset(bitarray, 0, pmq->pattern_id_bitarray_size);
 
     if (ctx->state_count < 32767) {
         register SC_AC_STATE_TYPE_U16 state = 0;
diff --git a/src/util-mpm-b2g.c b/src/util-mpm-b2g.c
index 31ef46b525..b4e2fcb77b 100644
--- a/src/util-mpm-b2g.c
+++ b/src/util-mpm-b2g.c
@@ -282,6 +282,12 @@ void B2gFreePattern(MpmCtx *mpm_ctx, B2gPattern *p)
         mpm_ctx->memory_size -= p->len;
     }
 
+    if (p && p->sids) {
+        SCFree(p->sids);
+        mpm_ctx->memory_cnt--;
+        mpm_ctx->memory_size -= p->sids_size * sizeof(uint32_t);
+    }
+
     if (p) {
         SCFree(p);
         mpm_ctx->memory_cnt--;
@@ -357,6 +363,13 @@ static int B2gAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_
             }
         }
 
+        p->sids_size = 1;
+        p->sids = SCMalloc(p->sids_size * sizeof(uint32_t));
+        BUG_ON(p->sids == NULL);
+        p->sids[0] = sid;
+        mpm_ctx->memory_cnt++;
+        mpm_ctx->memory_size += sizeof(uint32_t);
+
         //printf("B2gAddPattern: ci \""); prt(p->ci,p->len);
         //printf("\" cs \""); prt(p->cs,p->len);
         //printf("\"\n");
@@ -373,6 +386,27 @@ static int B2gAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_
         if (mpm_ctx->maxlen < patlen) mpm_ctx->maxlen = patlen;
         if (mpm_ctx->minlen == 0) mpm_ctx->minlen = patlen;
         else if (mpm_ctx->minlen > patlen) mpm_ctx->minlen = patlen;
+    } else {
+        /* Multiple sids for the same pid, so keep an array of sids. */
+
+        /* TODO figure out how we can be called multiple times for the
+         * same CTX with the same sid */
+        int found = 0;
+        uint32_t x = 0;
+        for (x = 0; x < p->sids_size; x++) {
+            if (p->sids[x] == sid) {
+                found = 1;
+                break;
+            }
+        }
+        if (!found) {
+            uint32_t *sids = SCRealloc(p->sids, (sizeof(uint32_t) * (p->sids_size + 1)));
+            BUG_ON(sids == NULL);
+            p->sids = sids;
+            p->sids[p->sids_size] = sid;
+            p->sids_size++;
+            mpm_ctx->memory_size += sizeof(uint32_t);
+        }
     }
 
     return 0;
@@ -458,6 +492,8 @@ static void B2gPrepareHash(MpmCtx *mpm_ctx)
                 hi->id = ctx->parray[i]->id;
                 hi->ci = ctx->parray[i]->ci;
                 hi->cs = ctx->parray[i]->cs;
+                hi->sids = ctx->parray[i]->sids;
+                hi->sids_size = ctx->parray[i]->sids_size;
 
             } else {
                 B2gPattern *hi = B2gAllocHashItem(mpm_ctx);
@@ -470,6 +506,8 @@ static void B2gPrepareHash(MpmCtx *mpm_ctx)
                 hi->id = ctx->parray[i]->id;
                 hi->ci = ctx->parray[i]->ci;
                 hi->cs = ctx->parray[i]->cs;
+                hi->sids = ctx->parray[i]->sids;
+                hi->sids_size = ctx->parray[i]->sids_size;
 
                 /* Append this HashItem to the list */
                 B2gPattern *thi = &ctx->hash1[idx8];
@@ -518,6 +556,9 @@ static void B2gPrepareHash(MpmCtx *mpm_ctx)
                 hi->id = ctx->parray[i]->id;
                 hi->ci = ctx->parray[i]->ci;
                 hi->cs = ctx->parray[i]->cs;
+                hi->sids = ctx->parray[i]->sids;
+                hi->sids_size = ctx->parray[i]->sids_size;
+
                 ctx->hash[idx] = hi;
             } else {
                 B2gPattern *hi = B2gAllocHashItem(mpm_ctx);
@@ -531,6 +572,9 @@ static void B2gPrepareHash(MpmCtx *mpm_ctx)
                 hi->id = ctx->parray[i]->id;
                 hi->ci = ctx->parray[i]->ci;
                 hi->cs = ctx->parray[i]->cs;
+                hi->sids = ctx->parray[i]->sids;
+                hi->sids_size = ctx->parray[i]->sids_size;
+
                 if (ctx->parray[i]->len < ctx->pminlen[idx])
                     ctx->pminlen[idx] = ctx->parray[i]->len;
 
@@ -948,6 +992,12 @@ uint32_t B2gSearchBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
     if (buflen < ctx->m)
         return 0;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (pos <= (uint32_t)(buflen - B2G_Q + 1)) {
         uint16_t h = B2G_HASH16(u8_tolower(buf[pos - 1]),u8_tolower(buf[pos]));
         d = ctx->B2G[h];
@@ -1001,7 +1051,8 @@ uint32_t B2gSearchBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
 #endif
                                     COUNT(tctx->stat_loop_match++);
 
-                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id);
+                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id,
+                                                              bitarray, thi->sids, thi->sids_size);
                                 } else {
                                     COUNT(tctx->stat_loop_no_match++);
                                 }
@@ -1013,7 +1064,8 @@ uint32_t B2gSearchBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
 #endif
                                     COUNT(tctx->stat_loop_match++);
 
-                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id);
+                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id,
+                                                              bitarray, thi->sids, thi->sids_size);
                                 } else {
                                     COUNT(tctx->stat_loop_no_match++);
                                 }
@@ -1061,6 +1113,12 @@ uint32_t B2gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
     if (buflen < ctx->m)
         return 0;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (pos <= (buflen - ctx->m)) {
         j = ctx->m - 1;
         d = ~0;
@@ -1110,7 +1168,8 @@ uint32_t B2gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
                     if (SCMemcmpLowercase(thi->ci, buf+pos, thi->len) == 0) {
                         COUNT(tctx->stat_loop_match++);
 
-                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id);
+                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id,
+                                                  bitarray, thi->sids, thi->sids_size);
                     } else {
                         COUNT(tctx->stat_loop_no_match++);
                     }
@@ -1118,7 +1177,8 @@ uint32_t B2gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
                     if (SCMemcmp(thi->cs, buf+pos, thi->len) == 0) {
                         COUNT(tctx->stat_loop_match++);
 
-                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id);
+                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id,
+                                                              bitarray, thi->sids, thi->sids_size);
                     } else {
                         COUNT(tctx->stat_loop_no_match++);
                     }
@@ -1154,6 +1214,12 @@ uint32_t B2gSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
 
     //printf("BUF "); prt(buf,buflen); printf("\n");
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         uint8_t h8 = u8_tolower(*buf);
         hi = &ctx->hash1[h8];
@@ -1164,11 +1230,13 @@ uint32_t B2gSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
 
                 if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
                     if (h8 == p->ci[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id,
+                                              bitarray, thi->sids, thi->sids_size);
                     }
                 } else {
                     if (*buf == p->cs[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id,
+                                              bitarray, thi->sids, thi->sids_size);
                     }
                 }
             }
@@ -1185,15 +1253,15 @@ uint32_t B2gSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
                 if (h8 == p->ci[0] && u8_tolower(*(buf+1)) == p->ci[1]) {
                     //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n");
 //                    for (em = p->em; em; em = em->next) {
-                        if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id))
-                            cnt++;
+                    if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size))
+                        cnt++;
 //                    }
                 }
             } else {
                 if (*buf == p->cs[0] && *(buf+1) == p->cs[1]) {
                     //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n");
 //                    for (em = p->em; em; em = em->next) {
-                        if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id))
+                        if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size))
                             cnt++;
 //                    }
                 }
@@ -1229,6 +1297,12 @@ uint32_t B2gSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
 
     //printf("BUF "); prt(buf,buflen); printf("\n");
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         uint8_t h = u8_tolower(*buf);
         hi = &ctx->hash1[h];
@@ -1240,11 +1314,11 @@ uint32_t B2gSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
 
                 if (thi->flags & MPM_PATTERN_FLAG_NOCASE) {
                     if (u8_tolower(*buf) == thi->ci[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id, bitarray, thi->sids, thi->sids_size);
                     }
                 } else {
                     if (*buf == thi->cs[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, thi->id, bitarray, thi->sids, thi->sids_size);
                     }
                 }
             }
diff --git a/src/util-mpm-b2g.h b/src/util-mpm-b2g.h
index 2d22b0ef58..8c002b6345 100644
--- a/src/util-mpm-b2g.h
+++ b/src/util-mpm-b2g.h
@@ -59,6 +59,11 @@ typedef struct B2gPattern_ {
     uint8_t *original_pat;
     uint8_t *ci; /* case INsensitive */
     uint8_t *cs; /* case sensitive */
+
+    /* sid(s) for this pattern */
+    uint32_t sids_size;
+    uint32_t *sids;
+
     struct B2gPattern_ *next;
 } B2gPattern;
 
diff --git a/src/util-mpm-b3g.c b/src/util-mpm-b3g.c
index 12472528dc..03b7fd6927 100644
--- a/src/util-mpm-b3g.c
+++ b/src/util-mpm-b3g.c
@@ -245,6 +245,12 @@ void B3gFreePattern(MpmCtx *mpm_ctx, B3gPattern *p)
         mpm_ctx->memory_size -= p->len;
     }
 
+    if (p && p->sids) {
+        SCFree(p->sids);
+        mpm_ctx->memory_cnt--;
+        mpm_ctx->memory_size -= p->sids_size * sizeof(uint32_t);
+    }
+
     if (p) {
         SCFree(p);
         mpm_ctx->memory_cnt--;
@@ -310,6 +316,13 @@ static int B3gAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_
         //printf("\" cs \""); prt(p->cs,p->len);
         //printf("\" prefix_ci %" PRIu32 ", prefix_cs %" PRIu32 "\n", p->prefix_ci, p->prefix_cs);
 
+        p->sids_size = 1;
+        p->sids = SCMalloc(p->sids_size * sizeof(uint32_t));
+        BUG_ON(p->sids == NULL);
+        p->sids[0] = sid;
+        mpm_ctx->memory_cnt++;
+        mpm_ctx->memory_size += sizeof(uint32_t);
+
         /* put in the pattern hash */
         B3gInitHashAdd(ctx, p);
 
@@ -322,6 +335,27 @@ static int B3gAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_
         if (mpm_ctx->maxlen < patlen) mpm_ctx->maxlen = patlen;
         if (mpm_ctx->minlen == 0) mpm_ctx->minlen = patlen;
         else if (mpm_ctx->minlen > patlen) mpm_ctx->minlen = patlen;
+    } else {
+        /* Multiple sids for the same pid, so keep an array of sids. */
+
+        /* TODO figure out how we can be called multiple times for the
+         * same CTX with the same sid */
+        int found = 0;
+        uint32_t x = 0;
+        for (x = 0; x < p->sids_size; x++) {
+            if (p->sids[x] == sid) {
+                found = 1;
+                break;
+            }
+        }
+        if (!found) {
+            uint32_t *sids = SCRealloc(p->sids, (sizeof(uint32_t) * (p->sids_size + 1)));
+            BUG_ON(sids == NULL);
+            p->sids = sids;
+            p->sids[p->sids_size] = sid;
+            p->sids_size++;
+            mpm_ctx->memory_size += sizeof(uint32_t);
+        }
     }
 
     return 0;
@@ -882,6 +916,12 @@ uint32_t B3gSearchBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
     if (buflen < ctx->m)
         return 0;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (pos <= (uint32_t)(buflen - B3G_Q + 1)) {
         uint16_t h = B3G_HASH(u8_tolower(buf[pos - 1]), u8_tolower(buf[pos]),u8_tolower(buf[pos + 1]));
         d = ctx->B3G[h];
@@ -929,7 +969,7 @@ uint32_t B3gSearchBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
                                 if (memcmp_lowercase(p->ci, buf+j, p->len) == 0) {
                                     COUNT(tctx->stat_loop_match++);
 
-                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                                 } else {
                                     COUNT(tctx->stat_loop_no_match++);
                                 }
@@ -940,7 +980,7 @@ uint32_t B3gSearchBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
                                 if (memcmp(p->cs, buf+j, p->len) == 0) {
                                     COUNT(tctx->stat_loop_match++);
 
-                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                                    matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                                 } else {
                                     COUNT(tctx->stat_loop_no_match++);
                                 }
@@ -982,6 +1022,12 @@ uint32_t B3gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
     if (buflen < ctx->m)
         return 0;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (pos <= (buflen - ctx->m)) {
         j = ctx->m - 2;
         d = ~0;
@@ -1030,7 +1076,7 @@ uint32_t B3gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
                     if (memcmp_lowercase(p->ci, buf+pos, p->len) == 0) {
                         COUNT(tctx->stat_loop_match++);
 
-                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     } else {
                         COUNT(tctx->stat_loop_no_match++);
                     }
@@ -1041,7 +1087,7 @@ uint32_t B3gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
                     if (memcmp(p->cs, buf+pos, p->len) == 0) {
                         COUNT(tctx->stat_loop_match++);
 
-                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        matches += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     } else {
                         COUNT(tctx->stat_loop_no_match++);
                     }
@@ -1073,6 +1119,12 @@ uint32_t B3gSearch12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatch
 
     //printf("BUF "); prt(buf,buflen); printf("\n");
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         uint8_t h8 = u8_tolower(*buf);
         hi = &ctx->hash1[h8];
@@ -1083,11 +1135,11 @@ uint32_t B3gSearch12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatch
 
                 if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
                     if (h8 == p->ci[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     }
                 } else {
                     if (*buf == p->cs[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     }
                 }
             }
@@ -1103,11 +1155,11 @@ uint32_t B3gSearch12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatch
 
                 if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
                     if (h8 == p->ci[0] && u8_tolower(*(buf+1)) == p->ci[1]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     }
                 } else {
                     if (*buf == p->cs[0] && *(buf+1) == p->cs[1]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     }
                 }
             }
@@ -1139,6 +1191,12 @@ uint32_t B3gSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
 
     //printf("BUF "); prt(buf,buflen); printf("\n");
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         uint16_t h = u8_tolower(*buf) << b3g_hash_shift | u8_tolower(*(buf+1));
         hi = ctx->hash2[h];
@@ -1153,13 +1211,13 @@ uint32_t B3gSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
                 if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
                     if (u8_tolower(*buf) == p->ci[0] && u8_tolower(*(buf+1)) == p->ci[1]) {
                         //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n");
-                        if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id))
+                        if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size))
                             cnt++;
                     }
                 } else {
                     if (*buf == p->cs[0] && *(buf+1) == p->cs[1]) {
                         //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n");
-                        if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id))
+                        if (MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size))
                             cnt++;
                     }
                 }
@@ -1191,6 +1249,12 @@ uint32_t B3gSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
 
     //printf("BUF "); prt(buf,buflen); printf("\n");
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         uint8_t h = u8_tolower(*buf);
         hi = &ctx->hash1[h];
@@ -1204,11 +1268,11 @@ uint32_t B3gSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche
 
                 if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
                     if (u8_tolower(*buf) == p->ci[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     }
                 } else {
                     if (*buf == p->cs[0]) {
-                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                        cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                     }
                 }
             }
diff --git a/src/util-mpm-b3g.h b/src/util-mpm-b3g.h
index 60107ac5f3..2d75caa335 100644
--- a/src/util-mpm-b3g.h
+++ b/src/util-mpm-b3g.h
@@ -58,9 +58,15 @@ typedef struct B3gPattern_ {
     uint8_t *cs; /* case sensitive */
     uint8_t *ci; /* case INsensitive */
     uint16_t len;
-    struct B3gPattern_ *next;
     uint8_t flags;
     uint32_t id;
+
+    /* sid(s) for this pattern */
+    uint32_t sids_size;
+    uint32_t *sids;
+
+    struct B3gPattern_ *next;
+
 } B3gPattern;
 
 typedef struct B3gHashItem_ {
diff --git a/src/util-mpm-wumanber.c b/src/util-mpm-wumanber.c
index 15d7a23ccd..4966821f6e 100644
--- a/src/util-mpm-wumanber.c
+++ b/src/util-mpm-wumanber.c
@@ -283,6 +283,12 @@ void WmFreePattern(MpmCtx *mpm_ctx, WmPattern *p)
         mpm_ctx->memory_size -= p->len;
     }
 
+    if (p && p->sids) {
+        SCFree(p->sids);
+        mpm_ctx->memory_cnt--;
+        mpm_ctx->memory_size -= p->sids_size * sizeof(uint32_t);
+    }
+
     if (p) {
         SCFree(p);
         mpm_ctx->memory_cnt--;
@@ -369,6 +375,33 @@ static int WmAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t
         if (mpm_ctx->maxlen < patlen) mpm_ctx->maxlen = patlen;
         if (mpm_ctx->minlen == 0) mpm_ctx->minlen = patlen;
         else if (mpm_ctx->minlen > patlen) mpm_ctx->minlen = patlen;
+
+        p->sids_size = 1;
+        p->sids = SCMalloc(p->sids_size * sizeof(uint32_t));
+        BUG_ON(p->sids == NULL);
+        p->sids[0] = sid;
+        mpm_ctx->memory_cnt++;
+        mpm_ctx->memory_size += sizeof(uint32_t);
+
+    } else {
+        /* TODO figure out how we can be called multiple times for the same CTX with the same sid */
+
+        int found = 0;
+        uint32_t x = 0;
+        for (x = 0; x < p->sids_size; x++) {
+            if (p->sids[x] == sid) {
+                found = 1;
+                break;
+            }
+        }
+        if (!found) {
+            uint32_t *sids = SCRealloc(p->sids, (sizeof(uint32_t) * (p->sids_size + 1)));
+            BUG_ON(sids == NULL);
+            p->sids = sids;
+            p->sids[p->sids_size] = sid;
+            p->sids_size++;
+            mpm_ctx->memory_size += sizeof(uint32_t);
+        }
     }
 
     return 0;
@@ -763,6 +796,12 @@ uint32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
 
     buf+=(sl-1);
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         h = HASH9(wm_tolower(*buf),(wm_tolower(*(buf-1))));
         shift = ctx->shifttable[h];
@@ -808,7 +847,7 @@ uint32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
                             //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n");
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
 
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
@@ -820,7 +859,7 @@ uint32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa
                             //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n");
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
 
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
@@ -865,6 +904,12 @@ uint32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
     buf+=(sl-1);
     //buf++;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1)));
         h = HASH12(wm_tolower(*buf),(wm_tolower(*(buf-1))));
@@ -911,7 +956,7 @@ uint32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                             //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n");
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
 
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
@@ -923,7 +968,7 @@ uint32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                             //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n");
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
 
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
@@ -967,6 +1012,12 @@ uint32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
     buf+=(sl-1);
     //buf++;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1)));
         h = HASH14(wm_tolower(*buf),(wm_tolower(*(buf-1))));
@@ -1012,7 +1063,7 @@ uint32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                         if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) {
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
                         }
@@ -1022,7 +1073,7 @@ uint32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                         if (memcmp(p->cs, buf-sl+1, p->len) == 0) {
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
                         }
@@ -1065,6 +1116,12 @@ uint32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
     buf+=(sl-1);
     //buf++;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1)));
         h = HASH15(wm_tolower(*buf),(wm_tolower(*(buf-1))));
@@ -1111,7 +1168,7 @@ uint32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                             //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n");
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
                         }
@@ -1122,7 +1179,7 @@ uint32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                             //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n");
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
                         }
@@ -1165,6 +1222,12 @@ uint32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
     buf+=(sl-1);
     //buf++;
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     while (buf <= bufend) {
         //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1)));
         h = HASH16(wm_tolower(*buf),(wm_tolower(*(buf-1))));
@@ -1210,7 +1273,7 @@ uint32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                         if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) {
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
                         }
@@ -1220,7 +1283,7 @@ uint32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM
                         if (memcmp(p->cs, buf-sl+1, p->len) == 0) {
                             COUNT(tctx->stat_loop_match++);
 
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         } else {
                             COUNT(tctx->stat_loop_no_match++);
                         }
@@ -1253,6 +1316,12 @@ uint32_t WmSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
 
     //printf("BUF "); prt(buf,buflen); printf("\n");
 
+    uint8_t *bitarray = NULL;
+    if (pmq) {
+        bitarray = alloca(pmq->pattern_id_bitarray_size);
+        memset(bitarray, 0, pmq->pattern_id_bitarray_size);
+    }
+
     if (mpm_ctx->minlen == 1) {
         while (buf <= bufend) {
             uint8_t h = wm_tolower(*buf);
@@ -1267,11 +1336,11 @@ uint32_t WmSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
 
                     if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
                         if (wm_tolower(*buf) == p->ci[0]) {
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         }
                     } else {
                         if (*buf == p->cs[0]) {
-                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id);
+                            cnt += MpmVerifyMatch(mpm_thread_ctx, pmq, p->id, bitarray, p->sids, p->sids_size);
                         }
                     }
                 }
diff --git a/src/util-mpm-wumanber.h b/src/util-mpm-wumanber.h
index 3d826536c2..a8b0bdef5d 100644
--- a/src/util-mpm-wumanber.h
+++ b/src/util-mpm-wumanber.h
@@ -38,6 +38,11 @@ typedef struct WmPattern_ {
     uint16_t prefix_cs;
     uint8_t flags;
     uint32_t id; /* global pattern id */
+
+    /* sid(s) for this pattern */
+    uint32_t sids_size;
+    uint32_t *sids;
+
 } WmPattern;
 
 typedef struct WmHashItem_ {
diff --git a/src/util-mpm.c b/src/util-mpm.c
index cc40e0edc8..cc466177f4 100644
--- a/src/util-mpm.c
+++ b/src/util-mpm.c
@@ -31,8 +31,6 @@
 #include "util-mpm-wumanber.h"
 #include "util-mpm-b2g.h"
 #include "util-mpm-b3g.h"
-#include "util-mpm-b2gc.h"
-#include "util-mpm-b2gm.h"
 #include "util-mpm-ac.h"
 #include "util-mpm-ac-gfbs.h"
 #include "util-mpm-ac-bs.h"
@@ -481,7 +479,8 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid)
  *  \retval 1 (new) match
  */
 int
-MpmVerifyMatch(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, uint32_t patid)
+MpmVerifyMatch(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, uint32_t patid,
+               uint8_t *bitarray, uint32_t *sids, uint32_t sids_size)
 {
     SCEnter();
 
@@ -489,13 +488,22 @@ MpmVerifyMatch(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, uint32_t pati
     if (pmq != NULL && pmq->pattern_id_bitarray != NULL) {
         SCLogDebug("using pattern id arrays, storing %"PRIu32, patid);
 
-        if (!(pmq->pattern_id_bitarray[(patid / 8)] & (1<<(patid % 8)))) {
+        if ((bitarray[(patid / 8)] & (1<<(patid % 8))) == 0) {
+            bitarray[(patid / 8)] |= (1<<(patid % 8));
             /* flag this pattern id as being added now */
             pmq->pattern_id_bitarray[(patid / 8)] |= (1<<(patid % 8));
             /* append the pattern_id to the array with matches */
             pmq->pattern_id_array[pmq->pattern_id_array_cnt] = patid;
             pmq->pattern_id_array_cnt++;
             SCLogDebug("pattern_id_array_cnt %u", pmq->pattern_id_array_cnt);
+
+            SCLogDebug("Adding %u sids", sids_size);
+            // Add SIDs for this pattern
+            uint32_t x;
+            for (x = 0; x < sids_size; x++) {
+                pmq->rule_id_array[pmq->rule_id_array_cnt++] = sids[x];
+            }
+
         }
     }
 
@@ -606,8 +614,6 @@ void MpmTableSetup(void)
     MpmWuManberRegister();
     MpmB2gRegister();
     MpmB3gRegister();
-    MpmB2gcRegister();
-    MpmB2gmRegister();
     MpmACRegister();
     MpmACBSRegister();
     MpmACGfbsRegister();
diff --git a/src/util-mpm.h b/src/util-mpm.h
index 395c5cc8fb..cf5a7335c9 100644
--- a/src/util-mpm.h
+++ b/src/util-mpm.h
@@ -249,7 +249,8 @@ void PmqFree(PatternMatcherQueue *);
 void MpmTableSetup(void);
 void MpmRegisterTests(void);
 
-int MpmVerifyMatch(MpmThreadCtx *, PatternMatcherQueue *, uint32_t);
+int MpmVerifyMatch(MpmThreadCtx *, PatternMatcherQueue *, uint32_t,
+                   uint8_t *bitarray, uint32_t *sids, uint32_t sid_size);
 void MpmInitCtx(MpmCtx *mpm_ctx, uint16_t matcher);
 void MpmInitThreadCtx(MpmThreadCtx *mpm_thread_ctx, uint16_t, uint32_t);
 uint32_t MpmGetHashSize(const char *);