detect grouping: multiple whitelist conditions

Instead of the binary yes/no whitelisting used so far, use different
values for different sorts of whitelist reasons. The port list will
be sorted by whitelist value first, then by rule count.

The goal is to whitelist groups that have weak sigs:

 - 1 byte pattern groups

 - SYN sigs

    Rules that check for SYN packets are mostly scan detection rules.
    They will be checked often as SYN packets are very common.

    e.g. alert tcp any any -> any 22 (flags:S,12; sid:123;)

    This patch adds whitelisting for SYN-sigs, so that the sigs end up
    in as unique groups as possible.

 - negated mpm sigs

    Currently negated mpm sigs are inspected often, so they are quite
    expensive. For this reason, try to whitelist them.

These values are set during 'stage 1', rule preprocessing.

diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index 86afa4e7962f07cf915314ecef1526ea95680467..96a6775ad862c6c5f5331db331f4f706fbfad812 100644 (file)
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -630,7 +630,7 @@ int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHea
         (*dst)->init->sig_array[idx] = (*dst)->init->sig_array[idx] | src->init->sig_array[idx];
 
     if (src->init->whitelist)
-        (*dst)->init->whitelist = 1;
+        (*dst)->init->whitelist = MAX((*dst)->init->whitelist, src->init->whitelist);
 
     if (src->mpm_content_minlen != 0) {
         if ((*dst)->mpm_content_minlen == 0)

diff --git a/src/detect.c b/src/detect.c
index 681fc082161dde0559049752d7f7912024709590..2353969244757b7e795d60cdb16dd8ec405945ef 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -2859,6 +2859,40 @@ static void SigParseApplyDsizeToContent(Signature *s)
     }
 }
 
+/** \brief Pure-PCRE or bytetest rule */
+int RuleInspectsPayloadHasNoMpm(const Signature *s)
+{
+    if (s->mpm_sm == NULL && s->sm_lists[DETECT_SM_LIST_PMATCH] != NULL)
+        return 1;
+    return 0;
+}
+
+int RuleGetMpmPatternSize(const Signature *s)
+{
+    if (s->mpm_sm == NULL)
+        return -1;
+    int mpm_list = SigMatchListSMBelongsTo(s, s->mpm_sm);
+    if (mpm_list < 0)
+        return -1;
+    const DetectContentData *cd = (const DetectContentData *)s->mpm_sm->ctx;
+    if (cd == NULL)
+        return -1;
+    return (int)cd->content_len;
+}
+
+int RuleMpmIsNegated(const Signature *s)
+{
+    if (s->mpm_sm == NULL)
+        return 0;
+    int mpm_list = SigMatchListSMBelongsTo(s, s->mpm_sm);
+    if (mpm_list < 0)
+        return 0;
+    const DetectContentData *cd = (const DetectContentData *)s->mpm_sm->ctx;
+    if (cd == NULL)
+        return 0;
+    return (cd->flags & DETECT_CONTENT_NEGATED);
+}
+
 int RulesGroupByProto(DetectEngineCtx *de_ctx)
 {
     Signature *s = de_ctx->sig_list;
@@ -2998,6 +3032,49 @@ static int PortIsWhitelisted(const DetectPort *a, int ipproto)
     return 0;
 }
 
+static int RuleSetWhitelist(Signature *s)
+{
+    DetectPort *p = NULL;
+    if (s->flags & SIG_FLAG_TOSERVER)
+        p = s->dp;
+    else if (s->flags & SIG_FLAG_TOCLIENT)
+        p = s->sp;
+    else
+        return 0;
+
+    /* for sigs that don't use 'any' as port, see if we want to
+     * whitelist poor sigs */
+    int wl = 0;
+    if (!(p->port == 0 && p->port2 == 65535)) {
+        /* pure pcre, bytetest, etc rules */
+        if (RuleInspectsPayloadHasNoMpm(s)) {
+            SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id);
+            wl = 99;
+
+        } else if (RuleMpmIsNegated(s)) {
+            SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id);
+            wl = 77;
+
+            /* one byte pattern in packet/stream payloads */
+        } else if (s->mpm_sm != NULL &&
+                   SigMatchListSMBelongsTo(s, s->mpm_sm) == DETECT_SM_LIST_PMATCH &&
+                   RuleGetMpmPatternSize(s) == 1)
+        {
+            SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id);
+            wl = 55;
+
+        } else if (DetectFlagsSignatureNeedsSynPackets(s) &&
+                   DetectFlagsSignatureNeedsSynOnlyPackets(s))
+        {
+            SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id);
+            wl = 33;
+        }
+    }
+
+    s->whitelist = wl;
+    return wl;
+}
+
 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx);
 int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b);
 
@@ -3040,11 +3117,14 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint3
             goto next;
         }
 
+        int wl = s->whitelist;
         while (p) {
             DetectPort *tmp = DetectPortCopySingle(de_ctx, p);
             BUG_ON(tmp == NULL);
             SigGroupHeadAppendSig(de_ctx, &tmp->sh, s);
-            tmp->sh->init->whitelist = PortIsWhitelisted(tmp, ipproto);
+
+            int pwl = PortIsWhitelisted(tmp, ipproto) ? 111 : 0;
+            tmp->sh->init->whitelist = MAX(wl, pwl);
             if (tmp->sh->init->whitelist) {
                 SCLogDebug("%s/%s Rule %u whitelisted port group %u:%u",
                         direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
@@ -3107,10 +3187,11 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint3
     }
 #if 0
     for (iter = list ; iter != NULL; iter = iter->next) {
-        SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s)",
+        SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)",
                 iter->port, iter->port2, iter->sh,
                 iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own",
-                iter->sh->init->whitelist ? "true" : "false");
+                iter->sh->init->whitelist ? "true" : "false",
+                iter->sh->init->whitelist);
     }
 #endif
     SCLogInfo("%s %s: %u port groups, %u unique SGH's, %u copies",
@@ -3223,6 +3304,8 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
         SignatureCreateMask(tmp_s);
         SigParseApplyDsizeToContent(tmp_s);
 
+        RuleSetWhitelist(tmp_s);
+
         de_ctx->sig_cnt++;
     }
 
@@ -3246,23 +3329,40 @@ error:
     return -1;
 }
 
-static int PortGroupIsWhitelisted(const DetectPort *a)
+static int PortGroupWhitelist(const DetectPort *a)
 {
     return a->sh->init->whitelist;
 }
 
 int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
 {
-    if (PortGroupIsWhitelisted(a) && !PortGroupIsWhitelisted(b))
+    if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) {
+        SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
+                a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
+                b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
         return 1;
-    if (!PortGroupIsWhitelisted(a) && PortGroupIsWhitelisted(b))
+    } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) {
+        SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
+                a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
+                b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
         return 0;
-    if (a->sh->sig_cnt > b->sh->sig_cnt) {
-        SCLogDebug("pg %u:%u %u > %u:%u %u",
-                a->port, a->port2, a->sh->sig_cnt,
-                b->port, b->port2, b->sh->sig_cnt);
+    } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) {
+        SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
+                a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
+                b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
         return 1;
+    } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) {
+        if (a->sh->sig_cnt > b->sh->sig_cnt) {
+            SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
+                    a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
+                    b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
+            return 1;
+        }
     }
+
+    SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
+            a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
+            b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
     return 0;
 }
 

diff --git a/src/detect.h b/src/detect.h
index 11c2c19339605bde504db03aa0a40f32263c69a7..03c354da8d1d6b9ac2de6c35bb104ba3125e0e18 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -460,6 +460,11 @@ typedef struct Signature_ {
     /* SigMatch list used for adding content and friends. E.g. file_data; */
     int list;
 
+    /** score to influence rule grouping. A higher value leads to a higher
+     *  likelyhood of a rulegroup with this sig ending up as a contained
+     *  group. */
+    int whitelist;
+
     /* Be careful, this pointer is only valid while parsing the sig,
      * to warn the user about any possible problem */
     char *sig_str;