mpm: implement prefiltering for smtp

diff --git a/src/detect-engine-filedata-smtp.c b/src/detect-engine-filedata-smtp.c
index 6cd1adb8684ad0bd9cbe61e24dfb1aa27537e6b7..7f0ff13e7f3226cff9c024ed200a56b3cc3ff6bf 100644 (file)
--- a/src/detect-engine-filedata-smtp.c
+++ b/src/detect-engine-filedata-smtp.c
@@ -215,3 +215,34 @@ void DetectEngineCleanSMTPBuffers(DetectEngineThreadCtx *det_ctx)
 
     return;
 }
+
+int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
+                           DetectEngineThreadCtx *det_ctx, Flow *f,
+                           SMTPState *smtp_state, uint8_t flags,
+                           void *tx, uint64_t idx)
+{
+    FileContainer *ffc = smtp_state->files_ts;
+    uint32_t cnt = 0;
+    uint32_t buffer_len = 0;
+    uint32_t stream_start_offset = 0;
+    uint8_t *buffer = 0;
+
+    if (ffc != NULL) {
+        File *file = ffc->head;
+        for (; file != NULL; file = file->next) {
+            buffer = DetectEngineSMTPGetBufferForTX(idx,
+                                                    de_ctx, det_ctx,
+                                                    f, file,
+                                                    flags,
+                                                    &buffer_len,
+                                                    &stream_start_offset);
+
+            if (buffer_len == 0)
+                goto end;
+
+            cnt = SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags);
+        }
+    }
+end:
+    return cnt;
+}

diff --git a/src/detect-engine-filedata-smtp.h b/src/detect-engine-filedata-smtp.h
index b574b974c2a01e85b7051b7389d517e875665ed5..666c9e56b471f50adc696cf1785b6d6cef5be864 100644 (file)
--- a/src/detect-engine-filedata-smtp.h
+++ b/src/detect-engine-filedata-smtp.h
@@ -23,6 +23,8 @@
 #ifndef __DETECT_ENGINE_FILEDATA_SMTP_H__
 #define __DETECT_ENGINE_FILEDATA_SMTP_H__
 
+#include "app-layer-smtp.h"
+
 int DetectEngineInspectSMTPFiledata(ThreadVars *tv,
                                     DetectEngineCtx *de_ctx,
                                     DetectEngineThreadCtx *det_ctx,
@@ -31,4 +33,9 @@ int DetectEngineInspectSMTPFiledata(ThreadVars *tv,
                                     void *tx, uint64_t tx_id);
 void DetectEngineCleanSMTPBuffers(DetectEngineThreadCtx *det_ctx);
 
+int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
+                           DetectEngineThreadCtx *det_ctx, Flow *f,
+                           SMTPState *smtp_state, uint8_t flags,
+                           void *tx, uint64_t idx);
+
 #endif /* __DETECT_ENGINE_FILEDATA_SMTP_H__ */

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 78f7f8a0c35a190f215a453a154f3e4ba7c1af50..1cbf2770975a5f11463aa8fd09386e0bf079b43f 100644 (file)
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -737,6 +737,36 @@ uint32_t StreamPatternSearch(DetectEngineThreadCtx *det_ctx, Packet *p,
     SCReturnInt(ret);
 }
 
+/**
+ * \brief SMTP Filedata match -- searches for one pattern per signature.
+ *
+ * \param det_ctx    Detection engine thread ctx.
+ * \param buffer     Buffer to inspect.
+ * \param buffer_len buffer length.
+ * \param flags      Flags
+ *
+ *  \retval ret Number of matches.
+ */
+uint32_t SMTPFiledataPatternSearch(DetectEngineThreadCtx *det_ctx,
+                              uint8_t *buffer, uint32_t buffer_len,
+                              uint8_t flags)
+{
+    SCEnter();
+
+    uint32_t ret = 0;
+
+    if (flags & STREAM_TOSERVER) {
+        if (det_ctx->sgh->mpm_smtp_filedata_ctx_ts == NULL)
+            SCReturnUInt(0);
+
+        ret = mpm_table[det_ctx->sgh->mpm_smtp_filedata_ctx_ts->mpm_type].
+            Search(det_ctx->sgh->mpm_smtp_filedata_ctx_ts, &det_ctx->mtcu,
+                   &det_ctx->pmq, buffer, buffer_len);
+    }
+
+    SCReturnUInt(ret);
+}
+
 /** \brief cleans up the mpm instance after a match */
 void PacketPatternCleanup(ThreadVars *t, DetectEngineThreadCtx *det_ctx)
 {
@@ -944,6 +974,16 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
         }
     }
 
+    if (sh->mpm_smtp_filedata_ctx_ts != NULL) {
+        if (sh->mpm_smtp_filedata_ctx_ts != NULL) {
+            if (!sh->mpm_smtp_filedata_ctx_ts->global) {
+                mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].DestroyCtx(sh->mpm_smtp_filedata_ctx_ts);
+                SCFree(sh->mpm_smtp_filedata_ctx_ts);
+            }
+            sh->mpm_smtp_filedata_ctx_ts = NULL;
+        }
+    }
+
     if (sh->mpm_hhd_ctx_ts != NULL || sh->mpm_hhd_ctx_tc != NULL) {
         if (sh->mpm_hhd_ctx_ts != NULL) {
             if (!sh->mpm_hhd_ctx_ts->global) {
@@ -1478,9 +1518,14 @@ static void PopulateMpmAddPatternToMpm(DetectEngineCtx *de_ctx,
                 if (cd->flags & DETECT_CONTENT_NEGATED)
                     s->flags |= SIG_FLAG_MPM_APPLAYER_NEG;
             } else if (sm_list == DETECT_SM_LIST_FILEDATA) {
-                if (s->flags & SIG_FLAG_TOCLIENT)
+                if (s->flags & SIG_FLAG_TOCLIENT) {
                     mpm_ctx_tc = sgh->mpm_hsbd_ctx_tc;
-                sgh->flags |= SIG_GROUP_HEAD_MPM_HSBD;
+                    sgh->flags |= SIG_GROUP_HEAD_MPM_HSBD;
+                }
+                if (s->flags & SIG_FLAG_TOSERVER) {
+                    mpm_ctx_ts = sgh->mpm_smtp_filedata_ctx_ts;
+                    sgh->flags |= SIG_GROUP_HEAD_MPM_FD_SMTP;
+                }
                 s->flags |= SIG_FLAG_MPM_APPLAYER;
                 if (cd->flags & DETECT_CONTENT_NEGATED)
                     s->flags |= SIG_FLAG_MPM_APPLAYER_NEG;
@@ -1934,6 +1979,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
     uint32_t has_co_hcbd = 0;
     /* used to indicate if sgh has atleast one sig with http_server_body */
     uint32_t has_co_hsbd = 0;
+    /* used to indicate if sgh has smtp file_data inspecting content */
+    uint32_t has_co_smtp = 0;
     /* used to indicate if sgh has atleast one sig with http_header */
     uint32_t has_co_hhd = 0;
     /* used to indicate if sgh has atleast one sig with http_raw_header */
@@ -1981,7 +2028,14 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         }
 
         if (s->sm_lists[DETECT_SM_LIST_FILEDATA] != NULL) {
-            has_co_hsbd = 1;
+            if (s->alproto == ALPROTO_SMTP)
+                has_co_smtp = 1;
+            else if (s->alproto == ALPROTO_HTTP)
+                has_co_hsbd = 1;
+            else if (s->alproto == ALPROTO_UNKNOWN) {
+                has_co_smtp = 1;
+                has_co_hsbd = 1;
+            }
         }
 
         if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) {
@@ -2131,6 +2185,20 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         MpmInitCtx(sh->mpm_hsbd_ctx_tc, de_ctx->mpm_matcher);
     }
 
+    if (has_co_smtp) {
+        if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE) {
+            sh->mpm_smtp_filedata_ctx_ts = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_smtp, 0);
+        } else {
+            sh->mpm_smtp_filedata_ctx_ts = MpmFactoryGetMpmCtxForProfile(de_ctx, MPM_CTX_FACTORY_UNIQUE_CONTEXT, 0);
+        }
+        if (sh->mpm_smtp_filedata_ctx_ts == NULL) {
+            SCLogDebug("sh->mpm_smtp_filedata_ctx_ts == NULL. This should never happen");
+            exit(EXIT_FAILURE);
+        }
+
+        MpmInitCtx(sh->mpm_smtp_filedata_ctx_ts, de_ctx->mpm_matcher);
+    }
+
     if (has_co_hhd) {
         if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE) {
             sh->mpm_hhd_ctx_ts = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_hhd, 0);
@@ -2299,6 +2367,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         has_co_uri ||
         has_co_hcbd ||
         has_co_hsbd ||
+        has_co_smtp ||
         has_co_hhd ||
         has_co_hrhd ||
         has_co_hmd ||
@@ -2442,6 +2511,19 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
              }
          }
 
+         if (sh->mpm_smtp_filedata_ctx_ts != NULL) {
+             if (sh->mpm_smtp_filedata_ctx_ts->pattern_cnt == 0) {
+                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts);
+                 sh->mpm_smtp_filedata_ctx_ts = NULL;
+             } else {
+                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                     if (mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare != NULL) {
+                         mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare(sh->mpm_smtp_filedata_ctx_ts);
+                     }
+                 }
+             }
+         }
+
          if (sh->mpm_hhd_ctx_ts != NULL) {
              if (sh->mpm_hhd_ctx_ts->pattern_cnt == 0) {
                  MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_ts);
@@ -2639,6 +2721,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         sh->mpm_hrhhd_ctx_ts = NULL;
         MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_dnsquery_ctx_ts);
         sh->mpm_dnsquery_ctx_ts = NULL;
+        MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts);
+        sh->mpm_smtp_filedata_ctx_ts = NULL;
 
         MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_tc);
         sh->mpm_proto_tcp_ctx_tc = NULL;

diff --git a/src/detect-engine-mpm.h b/src/detect-engine-mpm.h
index 05178e46fc197b366fb0a53490628a731b5ca58f..02df56f47bdc4fae3711111df1c535318f10234e 100644 (file)
--- a/src/detect-engine-mpm.h
+++ b/src/detect-engine-mpm.h
@@ -52,6 +52,7 @@ uint32_t HttpUAPatternSearch(DetectEngineThreadCtx *, uint8_t *, uint32_t, uint8
 uint32_t HttpHHPatternSearch(DetectEngineThreadCtx *, uint8_t *, uint32_t, uint8_t);
 uint32_t HttpHRHPatternSearch(DetectEngineThreadCtx *, uint8_t *, uint32_t, uint8_t);
 uint32_t DnsQueryPatternSearch(DetectEngineThreadCtx *det_ctx, uint8_t *buffer, uint32_t buffer_len, uint8_t flags);
+uint32_t SMTPFiledataPatternSearch(DetectEngineThreadCtx *det_ctx, uint8_t *buffer, uint32_t buffer_len, uint8_t flags);
 
 void PacketPatternCleanup(ThreadVars *, DetectEngineThreadCtx *);
 void StreamPatternCleanup(ThreadVars *t, DetectEngineThreadCtx *det_ctx, StreamMsg *smsg);

diff --git a/src/detect.c b/src/detect.c
index ec665944e1c76c40bc83276ef2d86a4ac08928ca..9b08e8852a0e6017dc35ff6b441759e9f2c14a5e 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -49,6 +49,7 @@
 #include "detect-dns-query.h"
 #include "detect-engine-state.h"
 #include "detect-engine-analyzer.h"
+#include "detect-engine-filedata-smtp.h"
 
 #include "detect-http-cookie.h"
 #include "detect-http-method.h"
@@ -160,6 +161,7 @@
 #include "app-layer.h"
 #include "app-layer-protos.h"
 #include "app-layer-htp.h"
+#include "app-layer-smtp.h"
 #include "detect-tls.h"
 #include "detect-tls-version.h"
 #include "detect-ssh-proto-version.h"
@@ -988,6 +990,32 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
                     FLOWLOCK_UNLOCK(p->flow);
                 }
             }
+        } else if (alproto == ALPROTO_SMTP && has_state) {
+            if (p->flowflags & FLOW_PKT_TOSERVER) {
+                if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_FD_SMTP) {
+                    FLOWLOCK_RDLOCK(p->flow);
+                    void *alstate = FlowGetAppState(p->flow);
+                    if (alstate == NULL) {
+                        SCLogDebug("no alstate");
+                        FLOWLOCK_UNLOCK(p->flow);
+                        return;
+                    }
+
+                    SMTPState *smtp_state = (SMTPState *)alstate;
+                    uint64_t idx = AppLayerParserGetTransactionInspectId(p->flow->alparser, flags);
+                    uint64_t total_txs = AppLayerParserGetTxCnt(p->flow->proto, alproto, alstate);
+                    for (; idx < total_txs; idx++) {
+                        void *tx = AppLayerParserGetTx(p->flow->proto, alproto, alstate, idx);
+                        if (tx == NULL)
+                            continue;
+
+                        PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_FD_SMTP);
+                        DetectEngineRunSMTPMpm(de_ctx, det_ctx, p->flow, smtp_state, flags, tx, idx);
+                        PACKET_PROFILING_DETECT_END(p, PROF_DETECT_MPM_FD_SMTP);
+                    }
+                    FLOWLOCK_UNLOCK(p->flow);
+                }
+            }
         }
 
         if (smsg != NULL && (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_STREAM)) {
@@ -2333,8 +2361,13 @@ static int SignatureCreateMask(Signature *s)
     }
 
     if (s->sm_lists[DETECT_SM_LIST_FILEDATA] != NULL) {
-        s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-        SCLogDebug("sig requires http app state");
+        /* set the state depending from the protocol */
+        if (s->alproto == ALPROTO_HTTP)
+            s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
+        else if (s->alproto == ALPROTO_SMTP)
+            s->mask |= SIG_MASK_REQUIRE_SMTP_STATE;
+
+        SCLogDebug("sig requires http or smtp app state");
     }
 
     if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) {
@@ -2561,6 +2594,9 @@ static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
     de_ctx->sgh_mpm_context_hsbd =
         MpmFactoryRegisterMpmCtxProfile(de_ctx, "hsbd",
                                         MPM_CTX_FACTORY_FLAGS_PREPARE_WITH_SIG_GROUP_BUILD);
+    de_ctx->sgh_mpm_context_smtp =
+        MpmFactoryRegisterMpmCtxProfile(de_ctx, "smtp",
+                                        MPM_CTX_FACTORY_FLAGS_PREPARE_WITH_SIG_GROUP_BUILD);
     de_ctx->sgh_mpm_context_hhd =
         MpmFactoryRegisterMpmCtxProfile(de_ctx, "hhd",
                                         MPM_CTX_FACTORY_FLAGS_PREPARE_WITH_SIG_GROUP_BUILD);
@@ -4698,6 +4734,16 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
         }
         //printf("hsbd- %d\n", mpm_ctx->pattern_cnt);
 
+        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_smtp, 0);
+        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
+            mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+        }
+        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_smtp, 1);
+        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
+            mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+        }
+        //printf("smtp- %d\n"; mpm_ctx->pattern_cnt);
+
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_hhd, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
             mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);

diff --git a/src/detect.h b/src/detect.h
index f34b46c7d0fb706069f96975f849d3580eeffe74..3b5ff6005cfb77720b4aa605b6da00f1a0670ad3 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -705,6 +705,7 @@ typedef struct DetectEngineCtx_ {
     int32_t sgh_mpm_context_hrhhd;
     int32_t sgh_mpm_context_app_proto_detect;
     int32_t sgh_mpm_context_dnsquery;
+    int32_t sgh_mpm_context_smtp;
 
     /* the max local id used amongst all sigs */
     int32_t byte_extract_max_local_id;
@@ -962,6 +963,7 @@ typedef struct SigTableElmt_ {
 #define SIG_GROUP_HEAD_HAVEFILEMD5      (1 << 21)
 #define SIG_GROUP_HEAD_HAVEFILESIZE     (1 << 22)
 #define SIG_GROUP_HEAD_MPM_DNSQUERY     (1 << 23)
+#define SIG_GROUP_HEAD_MPM_FD_SMTP      (1 << 24)
 
 typedef struct SigGroupHeadInitData_ {
     /* list of content containers
@@ -1024,6 +1026,7 @@ typedef struct SigGroupHead_ {
     MpmCtx *mpm_hhhd_ctx_ts;
     MpmCtx *mpm_hrhhd_ctx_ts;
     MpmCtx *mpm_dnsquery_ctx_ts;
+    MpmCtx *mpm_smtp_filedata_ctx_ts;
 
     MpmCtx *mpm_proto_tcp_ctx_tc;
     MpmCtx *mpm_proto_udp_ctx_tc;

diff --git a/src/suricata-common.h b/src/suricata-common.h
index d7126a874f8217308ffa016bd5a489b1eb1ab27c..98545fd80397add538330b42f5111978b7169b63 100644 (file)
--- a/src/suricata-common.h
+++ b/src/suricata-common.h
@@ -313,6 +313,7 @@ typedef enum PacketProfileDetectId_ {
     PROF_DETECT_ALERT,
     PROF_DETECT_CLEANUP,
     PROF_DETECT_GETSGH,
+    PROF_DETECT_MPM_FD_SMTP,
 
     PROF_DETECT_SIZE,
 } PacketProfileDetectId;