MPM: build sid list from MPM matches

Pmq add rule list: Array of uint32_t's to store (internal) sids from the MPM.

AC: store sids in the pattern list, append to Pmq::rule_id_array on match.

Detect: sort rule_id_array after it was set up by the MPM. Rule id's
(Signature::num) are ordered, and the rule's with the lowest id are to
be inspected first. As the MPM doesn't fill the array in order, but instead
'randomly' we need this sort step to assure proper inspection order.

diff --git a/src/detect.c b/src/detect.c
index 54d035af6ce89dc947345f28e2849eac5c66e1b3..9dd4b4e444f5bc4837374ce73990b599e7fab346 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -1019,6 +1019,18 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
             }
         }
     }
+
+    /* Sort the rule list to lets look at pmq.
+     * NOTE due to merging of 'stream' pmqs we *MAY* have duplicate entries */
+    if (det_ctx->pmq.rule_id_array_cnt) {
+        int DoSort(const void *a, const void *b) {
+            uint32_t x = *(uint32_t *)a;
+            uint32_t y = *(uint32_t *)b;
+            return x - y;
+        }
+        qsort(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt,
+                sizeof(uint32_t), DoSort);
+    }
 }
 
 #ifdef DEBUG

diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index f8e2199a3961f16c957025c6f62c51fe48951e92..ec16e415cfd72685913a1312c81de1e9026d195d 100644 (file)
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -357,6 +357,34 @@ static int SCACAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
         /* we need the max pat id */
         if (pid > ctx->max_pat_id)
             ctx->max_pat_id = pid;
+
+        p->sids_size = 1;
+        p->sids = SCMalloc(p->sids_size * sizeof(uint32_t));
+        BUG_ON(p->sids == NULL);
+        p->sids[0] = sid;
+        //SCLogInfo("MPM added %u:%u", pid, sid);
+    } else {
+        /* TODO figure out how we can be called multiple times for the same CTX with the same sid */
+
+        int found = 0;
+        uint32_t x = 0;
+        for (x = 0; x < p->sids_size; x++) {
+            if (p->sids[x] == sid) {
+                found = 1;
+                break;
+            }
+        }
+        if (!found) {
+            uint32_t *sids = SCRealloc(p->sids, (sizeof(uint32_t) * (p->sids_size + 1)));
+            BUG_ON(sids == NULL);
+            p->sids = sids;
+            p->sids[p->sids_size] = sid;
+            p->sids_size++;
+            //SCLogInfo("p->sids_size %u", p->sids_size);
+            //SCLogInfo("MPM added %u:%u (append)", pid, sid);
+        } else {
+            //SCLogInfo("rule %u already part of pid %u", sid, pid);
+        }
     }
 
     return 0;
@@ -1056,6 +1084,11 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx)
                    ctx->parray[i]->original_pat, ctx->parray[i]->len);
             ctx->pid_pat_list[ctx->parray[i]->id].patlen = ctx->parray[i]->len;
         }
+
+        /* ACPatternList now owns this memory */
+        //SCLogInfo("ctx->parray[i]->sids_size %u", ctx->parray[i]->sids_size);
+        ctx->pid_pat_list[ctx->parray[i]->id].sids_size = ctx->parray[i]->sids_size;
+        ctx->pid_pat_list[ctx->parray[i]->id].sids = ctx->parray[i]->sids;
     }
 
     /* prepare the state table required by AC */
@@ -1237,6 +1270,8 @@ void SCACDestroyCtx(MpmCtx *mpm_ctx)
         for (i = 0; i < (ctx->max_pat_id + 1); i++) {
             if (ctx->pid_pat_list[i].cs != NULL)
                 SCFree(ctx->pid_pat_list[i].cs);
+            if (ctx->pid_pat_list[i].sids != NULL)
+                SCFree(ctx->pid_pat_list[i].sids);
         }
         SCFree(ctx->pid_pat_list);
     }
@@ -1294,6 +1329,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF;
+
+                            uint32_t x;
+                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
+                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
+                            }
                         }
                         matches++;
                     } else {
@@ -1302,6 +1342,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
+
+                            uint32_t x;
+                            for (x = 0; x < pid_pat_list[pids[k]].sids_size; x++) {
+                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k]].sids[x];
+                            }
                         }
                         matches++;
                     }
@@ -1333,6 +1378,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF;
+
+                            uint32_t x;
+                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
+                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
+                            }
                         }
                         matches++;
                     } else {
@@ -1341,6 +1391,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
+
+                            uint32_t x;
+                            for (x = 0; x < pid_pat_list[pids[k]].sids_size; x++) {
+                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k]].sids[x];
+                            }
                         }
                         matches++;
                     }

diff --git a/src/util-mpm-ac.h b/src/util-mpm-ac.h
index 760fb70495c15749caafabf40e27236365ee1a1c..7519fffd620f2abc96e6e5e1a4229d5de343a3ea 100644 (file)
--- a/src/util-mpm-ac.h
+++ b/src/util-mpm-ac.h
@@ -52,12 +52,20 @@ typedef struct SCACPattern_ {
     /* pattern id */
     uint32_t id;
 
+    /* sid(s) for this pattern */
+    uint32_t sids_size;
+    uint32_t *sids;
+
     struct SCACPattern_ *next;
 } SCACPattern;
 
 typedef struct SCACPatternList_ {
     uint8_t *cs;
     uint16_t patlen;
+
+    /* sid(s) for this pattern */
+    uint32_t sids_size;
+    uint32_t *sids;
 } SCACPatternList;
 
 typedef struct SCACOutputTable_ {

diff --git a/src/util-mpm.c b/src/util-mpm.c
index df81e87a260210fd77629734a59e9b0054af51d3..cc40e0edc8c46868168d88448c8e8d81a7e31e76 100644 (file)
--- a/src/util-mpm.c
+++ b/src/util-mpm.c
@@ -452,6 +452,16 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid)
 
         SCLogDebug("pmq->pattern_id_array %p, pmq->pattern_id_bitarray %p",
                 pmq->pattern_id_array, pmq->pattern_id_bitarray);
+
+        pmq->rule_id_array_size = 65536 * sizeof(uint32_t);
+
+        pmq->rule_id_array = SCMalloc(pmq->rule_id_array_size);
+        if (pmq->rule_id_array == NULL) {
+            SCReturnInt(-1);
+        }
+        memset(pmq->rule_id_array, 0, pmq->rule_id_array_size);
+        pmq->rule_id_array_cnt = 0;
+
     }
 
     SCReturnInt(0);
@@ -510,6 +520,12 @@ void PmqMerge(PatternMatcherQueue *src, PatternMatcherQueue *dst)
     }
 
     /** \todo now set merged flag? */
+
+    if (src->rule_id_array && dst->rule_id_array) {
+        for (u = 0; u < src->rule_id_array_cnt; u++) {
+            dst->rule_id_array[dst->rule_id_array_cnt++] = src->rule_id_array[u];
+        }
+    }
 }
 
 /** \brief Reset a Pmq for reusage. Meant to be called after a single search.
@@ -532,6 +548,7 @@ void PmqReset(PatternMatcherQueue *pmq)
     }
     pmq->pattern_id_array_cnt = 0;
 */
+    pmq->rule_id_array_cnt = 0;
 }
 
 /** \brief Cleanup a Pmq
@@ -552,6 +569,11 @@ void PmqCleanup(PatternMatcherQueue *pmq)
         pmq->pattern_id_bitarray = NULL;
     }
 
+    if (pmq->rule_id_array != NULL) {
+        SCFree(pmq->rule_id_array);
+        pmq->rule_id_array = NULL;
+    }
+
     pmq->pattern_id_array_cnt = 0;
 }
 

diff --git a/src/util-mpm.h b/src/util-mpm.h
index f45ecfa4cb22e91265d37ea271898559d45eafc1..395c5cc8fbfeee3632dd2e829241d69c548161d4 100644 (file)
--- a/src/util-mpm.h
+++ b/src/util-mpm.h
@@ -106,6 +106,12 @@ typedef struct PatternMatcherQueue_ {
 
     uint8_t *pattern_id_bitarray;   /** bitarray with pattern id matches */
     uint32_t pattern_id_bitarray_size; /**< size in bytes */
+
+    /* used for storing rule id's */
+    uint32_t rule_id_array_size;
+    uint32_t *rule_id_array;
+    uint32_t rule_id_array_cnt;
+
 } PatternMatcherQueue;
 
 typedef struct MpmCtx_ {