From: Anoop Saldanha <anoopsaldanha@gmail.com>
Date: Mon, 15 Oct 2012 19:44:47 +0000 (+0530)
Subject: feature #558.
X-Git-Tag: suricata-1.4beta3~52
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7a7cd6999ec211cce2922a35bc77a915b3515c69;p=thirdparty%2Fsuricata.git

feature #558.

Print FP info in rule analysis + other cleanup.
---

diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c
index 60efe1284b..bcf548cda4 100644
--- a/src/detect-engine-analyzer.c
+++ b/src/detect-engine-analyzer.c
@@ -26,18 +26,177 @@
 #include "suricata-common.h"
 #include "suricata.h"
 #include "detect.h"
+#include "detect-parse.h"
 #include "detect-engine-analyzer.h"
+#include "detect-engine-mpm.h"
 #include "conf.h"
 #include "detect-content.h"
 #include "detect-flow.h"
 #include "detect-flags.h"
+#include "util-print.h"
 
 static int rule_warnings_only = 0;
 static FILE *rule_engine_analysis_FD = NULL;
+static FILE *fp_engine_analysis_FD = NULL;
 static pcre *percent_re = NULL;
 static pcre_extra *percent_re_study = NULL;
 static char log_path[PATH_MAX];
 
+void EngineAnalysisFP(Signature *s, char *line)
+{
+    int fast_pattern_set = 0;
+    int fast_pattern_only_set = 0;
+    int fast_pattern_chop_set = 0;
+    DetectContentData *fp_cd = NULL;
+    SigMatch *mpm_sm = s->mpm_sm;
+
+    if (mpm_sm != NULL) {
+        fp_cd = (DetectContentData *)mpm_sm->ctx;
+        if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN) {
+            fast_pattern_set = 1;
+            if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
+                fast_pattern_only_set = 1;
+            } else if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
+                fast_pattern_chop_set = 1;
+            }
+        }
+    }
+
+    fprintf(fp_engine_analysis_FD, "== Sid: %u ==\n", s->id);
+    fprintf(fp_engine_analysis_FD, "%s\n", line);
+
+    fprintf(fp_engine_analysis_FD, "    Fast Pattern analysis:\n");
+    if (fp_cd == NULL) {
+        fprintf(fp_engine_analysis_FD, "        No content present\n");
+        fprintf(fp_engine_analysis_FD, "\n");
+        return;
+    }
+
+    fprintf(fp_engine_analysis_FD, "        Fast pattern matcher: ");
+    int list_type = SigMatchListSMBelongsTo(s, mpm_sm);
+    if (list_type == DETECT_SM_LIST_PMATCH)
+        fprintf(fp_engine_analysis_FD, "content\n");
+    else if (list_type == DETECT_SM_LIST_UMATCH)
+        fprintf(fp_engine_analysis_FD, "http uri content\n");
+    else if (list_type == DETECT_SM_LIST_HRUDMATCH)
+        fprintf(fp_engine_analysis_FD, "http raw uri content\n");
+    else if (list_type == DETECT_SM_LIST_HHDMATCH)
+        fprintf(fp_engine_analysis_FD, "http header content\n");
+    else if (list_type == DETECT_SM_LIST_HRHDMATCH)
+        fprintf(fp_engine_analysis_FD, "http raw header content\n");
+    else if (list_type == DETECT_SM_LIST_HMDMATCH)
+        fprintf(fp_engine_analysis_FD, "http method content\n");
+    else if (list_type == DETECT_SM_LIST_HCDMATCH)
+        fprintf(fp_engine_analysis_FD, "http cookie content\n");
+    else if (list_type == DETECT_SM_LIST_HCBDMATCH)
+        fprintf(fp_engine_analysis_FD, "http client body content\n");
+    else if (list_type == DETECT_SM_LIST_HSBDMATCH)
+        fprintf(fp_engine_analysis_FD, "http server body content\n");
+    else if (list_type == DETECT_SM_LIST_HSCDMATCH)
+        fprintf(fp_engine_analysis_FD, "http stat code content\n");
+    else if (list_type == DETECT_SM_LIST_HSMDMATCH)
+        fprintf(fp_engine_analysis_FD, "http stat msg content\n");
+    else if (list_type == DETECT_SM_LIST_HUADMATCH)
+        fprintf(fp_engine_analysis_FD, "http user agent content\n");
+
+    fprintf(fp_engine_analysis_FD, "        Fast pattern set: %s\n", fast_pattern_set ? "yes" : "no");
+    fprintf(fp_engine_analysis_FD, "        Fast pattern only set: %s\n",
+            fast_pattern_only_set ? "yes" : "no");
+    fprintf(fp_engine_analysis_FD, "        Fast pattern chop set: %s\n",
+            fast_pattern_chop_set ? "yes" : "no");
+    if (fast_pattern_chop_set) {
+        fprintf(fp_engine_analysis_FD, "        Fast pattern offset, length: %u, %u\n",
+                fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
+    }
+    fprintf(fp_engine_analysis_FD, "        Content negated: %s\n",
+            (fp_cd->flags & DETECT_CONTENT_NEGATED) ? "yes" : "no");
+
+    uint16_t patlen = fp_cd->content_len;
+    uint8_t *pat = SCMalloc(fp_cd->content_len + 1);
+    if (unlikely(pat == NULL)) {
+        SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
+        exit(EXIT_FAILURE);
+    }
+    memcpy(pat, fp_cd->content, fp_cd->content_len);
+    pat[fp_cd->content_len] = '\0';
+    fprintf(fp_engine_analysis_FD, "        Original content: ");
+    PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
+    fprintf(fp_engine_analysis_FD, "\n");
+
+    if (fast_pattern_chop_set) {
+        SCFree(pat);
+        patlen = fp_cd->fp_chop_len;
+        pat = SCMalloc(fp_cd->fp_chop_len + 1);
+        if (unlikely(pat == NULL)) {
+            exit(EXIT_FAILURE);
+        }
+        memcpy(pat, fp_cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
+        pat[fp_cd->fp_chop_len] = '\0';
+        fprintf(fp_engine_analysis_FD, "        Final content: ");
+        PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
+        fprintf(fp_engine_analysis_FD, "\n");
+    } else {
+        fprintf(fp_engine_analysis_FD, "        Final content: ");
+        PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
+        fprintf(fp_engine_analysis_FD, "\n");
+    }
+    SCFree(pat);
+
+    fprintf(fp_engine_analysis_FD, "\n");
+    return;
+}
+
+/**
+ * \brief Sets up the fast pattern analyzer according to the config.
+ *
+ * \retval 1 If rule analyzer successfully enabled.
+ * \retval 0 If not enabled.
+ */
+int SetupFPAnalyzer(void)
+{
+    int fp_engine_analysis_set = 0;
+
+    if ((ConfGetBool("engine-analysis.rules-fast-pattern",
+                     &fp_engine_analysis_set)) == 0) {
+        return 0;
+    }
+
+    if (fp_engine_analysis_set == 0)
+        return 0;
+
+    char *log_dir;
+    if (ConfGet("default-log-dir", &log_dir) != 1)
+        log_dir = DEFAULT_LOG_DIR;
+    snprintf(log_path, sizeof(log_path), "%s/%s", log_dir,
+             "rules_fast_pattern.txt");
+
+    fp_engine_analysis_FD = fopen(log_path, "w");
+    if (fp_engine_analysis_FD == NULL) {
+        SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", log_path,
+                   strerror(errno));
+        return 0;
+    }
+
+    SCLogInfo("Engine-Analyis for fast_pattern printed to file - %s",
+              log_path);
+
+    struct timeval tval;
+    struct tm *tms;
+    gettimeofday(&tval, NULL);
+    struct tm local_tm;
+    tms = (struct tm *)SCLocalTime(tval.tv_sec, &local_tm);
+    fprintf(fp_engine_analysis_FD, "----------------------------------------------"
+            "---------------------\n");
+    fprintf(fp_engine_analysis_FD, "Date: %" PRId32 "/%" PRId32 "/%04d -- "
+            "%02d:%02d:%02d\n",
+            tms->tm_mday, tms->tm_mon + 1, tms->tm_year + 1900, tms->tm_hour,
+            tms->tm_min, tms->tm_sec);
+    fprintf(fp_engine_analysis_FD, "----------------------------------------------"
+            "---------------------\n");
+
+    return 1;
+}
+
 /**
  * \brief Sets up the rule analyzer according to the config
  * \retval 1 if rule analyzer successfully enabled
@@ -66,6 +225,9 @@ int SetupRuleAnalyzer(void)
                 return 0;
             }
 
+            SCLogInfo("Engine-Analyis for rules printed to file - %s",
+                      log_path);
+
             struct timeval tval;
             struct tm *tms;
             gettimeofday(&tval, NULL);
@@ -97,6 +259,17 @@ int SetupRuleAnalyzer(void)
     return 1;
 }
 
+void CleanupFPAnalyzer(void)
+{
+    if (fp_engine_analysis_FD != NULL) {
+        fclose(fp_engine_analysis_FD);
+        fp_engine_analysis_FD = NULL;
+    }
+
+    return;
+}
+
+
 void CleanupRuleAnalyzer(void) {
     if (rule_engine_analysis_FD != NULL) {
          SCLogInfo("Engine-Analyis for rules printed to file - %s", log_path);
@@ -156,6 +329,86 @@ int PerCentEncodingMatch (uint8_t *content, uint8_t content_len)
     return ret;
 }
 
+static void EngineAnalysisRulesPrintFP(Signature *s)
+{
+    DetectContentData *fp_cd = NULL;
+    SigMatch *mpm_sm = s->mpm_sm;
+
+    if (mpm_sm != NULL) {
+        fp_cd = (DetectContentData *)mpm_sm->ctx;
+    }
+
+    if (fp_cd == NULL) {
+        return;
+    }
+
+    uint16_t patlen = fp_cd->content_len;
+    uint8_t *pat = SCMalloc(fp_cd->content_len + 1);
+    if (unlikely(pat == NULL)) {
+        SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
+        exit(EXIT_FAILURE);
+    }
+    memcpy(pat, fp_cd->content, fp_cd->content_len);
+    pat[fp_cd->content_len] = '\0';
+
+    if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
+        SCFree(pat);
+        patlen = fp_cd->fp_chop_len;
+        pat = SCMalloc(fp_cd->fp_chop_len + 1);
+        if (unlikely(pat == NULL)) {
+            exit(EXIT_FAILURE);
+        }
+        memcpy(pat, fp_cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
+        pat[fp_cd->fp_chop_len] = '\0';
+        fprintf(rule_engine_analysis_FD, "    Fast Pattern \"");
+        PrintRawUriFp(rule_engine_analysis_FD, pat, patlen);
+    } else {
+        fprintf(rule_engine_analysis_FD, "    Fast Pattern \"");
+        PrintRawUriFp(rule_engine_analysis_FD, pat, patlen);
+    }
+    SCFree(pat);
+
+    fprintf(rule_engine_analysis_FD, "\" on \"");
+
+    int list_type = SigMatchListSMBelongsTo(s, mpm_sm);
+    if (list_type == DETECT_SM_LIST_PMATCH) {
+        int payload = 0;
+        int stream = 0;
+        if (SignatureHasPacketContent(s))
+            payload = 1;
+        if (SignatureHasStreamContent(s))
+            stream = 1;
+        fprintf(rule_engine_analysis_FD, "%s",
+                payload ? (stream ? "payload and reassembled stream" : "payload") : "reassembled stream");
+    }
+    else if (list_type == DETECT_SM_LIST_UMATCH)
+        fprintf(rule_engine_analysis_FD, "http uri content");
+    else if (list_type == DETECT_SM_LIST_HRUDMATCH)
+        fprintf(rule_engine_analysis_FD, "http raw uri content");
+    else if (list_type == DETECT_SM_LIST_HHDMATCH)
+        fprintf(rule_engine_analysis_FD, "http header content");
+    else if (list_type == DETECT_SM_LIST_HRHDMATCH)
+        fprintf(rule_engine_analysis_FD, "http raw header content");
+    else if (list_type == DETECT_SM_LIST_HMDMATCH)
+        fprintf(rule_engine_analysis_FD, "http method content");
+    else if (list_type == DETECT_SM_LIST_HCDMATCH)
+        fprintf(rule_engine_analysis_FD, "http cookie content");
+    else if (list_type == DETECT_SM_LIST_HCBDMATCH)
+        fprintf(rule_engine_analysis_FD, "http client body content");
+    else if (list_type == DETECT_SM_LIST_HSBDMATCH)
+        fprintf(rule_engine_analysis_FD, "http server body content");
+    else if (list_type == DETECT_SM_LIST_HSCDMATCH)
+        fprintf(rule_engine_analysis_FD, "http stat code content");
+    else if (list_type == DETECT_SM_LIST_HSMDMATCH)
+        fprintf(rule_engine_analysis_FD, "http stat msg content");
+    else if (list_type == DETECT_SM_LIST_HUADMATCH)
+        fprintf(rule_engine_analysis_FD, "http user agent content");
+
+    fprintf(rule_engine_analysis_FD, "\" buffer.\n");
+
+    return;
+}
+
 /**
  * \brief Prints analysis of loaded rules.
  *
@@ -214,6 +467,7 @@ void EngineAnalysisRules(Signature *s, char *line)
     uint32_t warn_encoding_norm_http_buf = 0;
     uint32_t warn_offset_depth_pkt_stream = 0;
     uint32_t warn_offset_depth_alproto = 0;
+    uint32_t warn_non_alproto_fp_for_alproto_sig = 0;
 
     if (s->init_flags & SIG_FLAG_INIT_BIDIREC) {
         rule_bidirectional = 1;
@@ -454,6 +708,11 @@ void EngineAnalysisRules(Signature *s, char *line)
         rule_warning += 1;
         warn_offset_depth_alproto = 1;
     }
+    if (s->mpm_sm != NULL && s->alproto == ALPROTO_HTTP &&
+        SigMatchListSMBelongsTo(s, s->mpm_sm) == DETECT_SM_LIST_PMATCH) {
+        rule_warning += 1;
+        warn_non_alproto_fp_for_alproto_sig = 1;
+    }
 
     if (!rule_warnings_only || (rule_warnings_only && rule_warning > 0)) {
         fprintf(rule_engine_analysis_FD, "== Sid: %u ==\n", s->id);
@@ -481,6 +740,11 @@ void EngineAnalysisRules(Signature *s, char *line)
         if (rule_content || rule_content_http || rule_pcre || rule_pcre_http) {
             fprintf(rule_engine_analysis_FD, "    Rule contains %d content options, %d http content options, %d pcre options, and %d pcre options with http modifiers.\n", rule_content, rule_content_http, rule_pcre, rule_pcre_http);
         }
+
+        /* print fast pattern info */
+        EngineAnalysisRulesPrintFP(s);
+
+        /* this is where the warnings start */
         if (warn_pcre_no_content /*rule_pcre > 0 && rule_content == 0 && rule_content_http == 0*/) {
             fprintf(rule_engine_analysis_FD, "    Warning: Rule uses pcre without a content option present.\n"
                                              "             -Consider adding a content to improve performance of this rule.\n");
@@ -548,6 +812,12 @@ void EngineAnalysisRules(Signature *s, char *line)
                     "before we can detect the app layer protocol for the "
                     "flow.\n", s->alproto);
         }
+        if (warn_non_alproto_fp_for_alproto_sig) {
+            fprintf(rule_engine_analysis_FD, "    Warning: Rule app layer "
+                    "protocol is http, but the fast_pattern is set on the raw "
+                    "stream.  Consider adding fast_pattern over a http "
+                    "buffer for increased performance.");
+        }
         if (rule_warning == 0) {
             fprintf(rule_engine_analysis_FD, "    No warnings for this rule.\n");
         }
diff --git a/src/detect-engine-analyzer.h b/src/detect-engine-analyzer.h
index 93e491609a..2b1fc54a64 100644
--- a/src/detect-engine-analyzer.h
+++ b/src/detect-engine-analyzer.h
@@ -26,10 +26,16 @@
 
 #include <stdint.h>
 
+int SetupFPAnalyzer(void);
+void CleanupFPAnalyzer(void);
+
 int SetupRuleAnalyzer(void);
 void CleanupRuleAnalyzer (void);
+
 int PerCentEncodingSetup ();
 int PerCentEncodingMatch (uint8_t *content, uint8_t content_len);
+
+void EngineAnalysisFP(Signature *s, char *line);
 void EngineAnalysisRules(Signature *s, char *line);
 
 #endif /* __DETECT_ENGINE_ANALYZER_H__ */
diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 139b2b15fd..aded083915 100644
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -1657,218 +1657,140 @@ static void PopulateMpmAddPatternToMpm(DetectEngineCtx *de_ctx,
     return;
 }
 
-/**
- * \internal
- * \brief Setup the mpm content.
- *
- * \param de_ctx Pointer to the detect engine context.
- * \param sgh    Pointer to the signature group head against which we are
- *               adding patterns to the mpm ctx.
- *
- * \retval  0 Always.
- */
-static int PatternMatchPreparePopulateMpm(DetectEngineCtx *de_ctx,
-                                          SigGroupHead *sgh)
+SigMatch *RetrieveFPForSig(Signature *s)
 {
-    uint32_t sig;
-    uint8_t *fast_pattern = NULL;
-    uint8_t *has_non_negated_non_stream_pattern = NULL;
-
-    fast_pattern = (uint8_t *)SCMalloc(sgh->sig_cnt * sizeof(uint8_t));
-    if (fast_pattern == NULL) {
-        SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
-        exit(EXIT_FAILURE);
-    }
-    memset(fast_pattern, 0, sgh->sig_cnt * sizeof(uint8_t));
-    has_non_negated_non_stream_pattern = (uint8_t *)SCMalloc(sgh->sig_cnt * sizeof(uint8_t));
-    if (has_non_negated_non_stream_pattern == NULL) {
-        SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
-        exit(EXIT_FAILURE);
-    }
-    memset(has_non_negated_non_stream_pattern, 0, sgh->sig_cnt * sizeof(uint8_t));
+    SigMatch *mpm_sm = NULL;
+    uint8_t has_non_negated_non_stream_pattern = 0;
 
-    /* add all mpm candidates to a hash */
-    for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        Signature *s = sgh->match_array[sig];
-        if (s == NULL)
-            continue;
+    if (s->mpm_sm != NULL)
+        return s->mpm_sm;
 
-        /* we already have a sm set as fp for this sig.  Add it to the current
-         * mpm context */
-        if (s->mpm_sm != NULL) {
-            PopulateMpmAddPatternToMpm(de_ctx, sgh, s, s->mpm_sm);
+    for (int list_id = 0 ; list_id < DETECT_SM_LIST_MAX; list_id++) {
+        /* we have no keywords that support fp in this Signature sm list */
+        if (!FastPatternSupportEnabledForSigMatchList(list_id))
             continue;
-        }
 
-        int list_id = 0;
-        for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) {
-            /* we have no keywords that support fp in this Signature sm list */
-            if (!FastPatternSupportEnabledForSigMatchList(list_id))
+        for (SigMatch *sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
+            /* this keyword isn't registered for fp support */
+            if (sm->type != DETECT_CONTENT)
                 continue;
 
-            SigMatch *sm = NULL;
-            /* get the total no of patterns in this Signature, as well as find out
-             * if we have a fast_pattern set in this Signature */
-            for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
-                /* this keyword isn't registered for fp support */
-                if (sm->type != DETECT_CONTENT)
-                    continue;
-
-                //if (PopulateMpmSkipContent(sgh, s, sm)) {
-                //    continue;
-                //}
-
-                DetectContentData *cd = (DetectContentData *)sm->ctx;
-                if (!(cd->flags & DETECT_CONTENT_NEGATED) &&
-                    list_id != DETECT_SM_LIST_PMATCH &&
-                    /* don't consider http_method, http_stat_msg, http_stat_code
-                     * to automatically override longest stream match */
-                    list_id != DETECT_SM_LIST_HMDMATCH &&
-                    list_id != DETECT_SM_LIST_HSMDMATCH &&
-                    list_id != DETECT_SM_LIST_HSCDMATCH) {
-                    has_non_negated_non_stream_pattern[sig] = 1;
-                }
-
-                if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
-                    fast_pattern[sig] = 1;
-                    break;
-                }
-            } /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
-
-            /* found a fast pattern for the sig.  Let's get outta here */
-            if (fast_pattern[sig])
-                break;
-        } /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
-    } /* for (sig = 0; sig < sgh->sig_cnt; sig++) { */
+            DetectContentData *cd = (DetectContentData *)sm->ctx;
+            if (cd->flags & DETECT_CONTENT_FAST_PATTERN)
+                return sm;
+            if (!(cd->flags & DETECT_CONTENT_NEGATED) &&
+                (list_id != DETECT_SM_LIST_PMATCH) &&
+                (list_id != DETECT_SM_LIST_HMDMATCH) &&
+                (list_id != DETECT_SM_LIST_HSMDMATCH) &&
+                (list_id != DETECT_SM_LIST_HSCDMATCH)) {
+                has_non_negated_non_stream_pattern = 1;
+            }
+        }
+    }
 
-    /* now determine which one to add to the mpm phase */
-    for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        Signature *s = sgh->match_array[sig];
-        if (s == NULL)
+    int max_len = 0;
+    int max_len_negated = 0;
+    int max_len_non_negated = 0;
+    for (int list_id = 0; list_id < DETECT_SM_LIST_MAX; list_id++) {
+        if (!FastPatternSupportEnabledForSigMatchList(list_id))
             continue;
-        /* have taken care of this in the previous loop.  move on to the next sig */
-        if (s->mpm_sm != NULL) {
+
+        if (has_non_negated_non_stream_pattern &&
+            ((list_id == DETECT_SM_LIST_PMATCH) ||
+             (list_id == DETECT_SM_LIST_HMDMATCH) ||
+             (list_id == DETECT_SM_LIST_HSMDMATCH) ||
+             (list_id == DETECT_SM_LIST_HSCDMATCH))) {
             continue;
         }
 
-        int max_len = 0;
-        int max_len_negated = 0;
-        int max_len_non_negated = 0;
-        /* get the longest pattern in the sig */
-        if (!fast_pattern[sig]) {
-            SigMatch *sm = NULL;
-            int list_id = 0;
-            for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) {
-                if (!FastPatternSupportEnabledForSigMatchList(list_id))
-                    continue;
+        for (SigMatch *sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
+            if (sm->type != DETECT_CONTENT)
+                continue;
 
-                if (list_id == DETECT_SM_LIST_PMATCH &&
-                    !fast_pattern[sig] &&
-                    has_non_negated_non_stream_pattern[sig]) {
-                    continue;
-                }
+            DetectContentData *cd = (DetectContentData *)sm->ctx;
+            if (cd->flags & DETECT_CONTENT_NEGATED) {
+                if (max_len_negated < cd->content_len)
+                    max_len_negated = cd->content_len;
+            } else {
+                if (max_len_non_negated < cd->content_len)
+                    max_len_non_negated = cd->content_len;
+            }
+        }
+    }
 
-                for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
-                    if (sm->type != DETECT_CONTENT)
-                        continue;
+    int skip_negated_content = 0;
+    if (max_len_non_negated == 0) {
+        max_len = max_len_negated;
+        skip_negated_content = 0;
+    } else {
+        max_len = max_len_non_negated;
+        skip_negated_content = 1;
+    }
 
-                    //if (PopulateMpmSkipContent(sgh, s, sm)) {
-                    //    continue;
-                    //}
+    for (int list_id = 0; list_id < DETECT_SM_LIST_MAX; list_id++) {
+        if (!FastPatternSupportEnabledForSigMatchList(list_id))
+            continue;
 
-                    DetectContentData *cd = (DetectContentData *)sm->ctx;
-                    if (cd->flags & DETECT_CONTENT_NEGATED) {
-                        if (max_len_negated < cd->content_len)
-                            max_len_negated = cd->content_len;
-                    } else {
-                        if (max_len_non_negated < cd->content_len)
-                            max_len_non_negated = cd->content_len;
-                    }
-                } /* for ( ; list_id.. */
-            } /* for (sm = s->sm_lists.. */
-        } /* if */
-
-        int skip_negated_content = 0;
-        if (max_len_non_negated == 0) {
-            max_len = max_len_negated;
-            skip_negated_content = 0;
-        } else {
-            max_len = max_len_non_negated;
-            skip_negated_content = 1;
+        if (has_non_negated_non_stream_pattern &&
+            ((list_id == DETECT_SM_LIST_PMATCH) ||
+             (list_id == DETECT_SM_LIST_HMDMATCH) ||
+             (list_id == DETECT_SM_LIST_HSMDMATCH) ||
+             (list_id == DETECT_SM_LIST_HSCDMATCH))) {
+            continue;
         }
 
-        SigMatch *mpm_sm = NULL;
-        SigMatch *sm = NULL;
-        int list_id = 0;
-        for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) {
-            if (!FastPatternSupportEnabledForSigMatchList(list_id))
+        for (SigMatch *sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
+            if (sm->type != DETECT_CONTENT)
                 continue;
 
-            if (list_id == DETECT_SM_LIST_PMATCH &&
-                !fast_pattern[sig] &&
-                has_non_negated_non_stream_pattern[sig]) {
+            DetectContentData *cd = (DetectContentData *)sm->ctx;
+            if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
+                continue;
+            if (cd->content_len < max_len)
                 continue;
-            }
-
-            for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
-                if (sm->type != DETECT_CONTENT)
-                    continue;
-
-                /* skip in case of:
-                 * 1. we expect a fastpattern but this isn't it */
-                if (fast_pattern[sig]) {
-                    /* can be any content based keyword since all of them
-                     * now use a unified structure - DetectContentData */
-                    DetectContentData *cd = (DetectContentData *)sm->ctx;
-                    if (!(cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
-                        SCLogDebug("not a fast pattern %"PRIu32"", cd->id);
-                        continue;
-                    }
-                    SCLogDebug("fast pattern %"PRIu32"", cd->id);
-                } else {
-                    //if (PopulateMpmSkipContent(sgh, s, sm)) {
-                    //    continue;
-                    //}
-
-                    DetectContentData *cd = (DetectContentData *)sm->ctx;
-                    if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
-                        continue;
-                    if (cd->content_len < max_len)
-                        continue;
-
-                } /* else - if (fast_pattern[sig] == 1) */
 
-                if (mpm_sm == NULL) {
+            if (mpm_sm == NULL) {
+                mpm_sm = sm;
+            } else {
+                DetectContentData *data1 = (DetectContentData *)sm->ctx;
+                DetectContentData *data2 = (DetectContentData *)mpm_sm->ctx;
+                uint32_t ls = PatternStrength(data1->content, data1->content_len);
+                uint32_t ss = PatternStrength(data2->content, data2->content_len);
+                if (ls > ss) {
                     mpm_sm = sm;
-                    if (fast_pattern[sig])
-                        break;
-                } else {
-                    DetectContentData *data1 = (DetectContentData *)sm->ctx;
-                    DetectContentData *data2 = (DetectContentData *)mpm_sm->ctx;
-                    uint32_t ls = PatternStrength(data1->content, data1->content_len);
-                    uint32_t ss = PatternStrength(data2->content, data2->content_len);
-                    if (ls > ss) {
+                } else if (ls == ss) {
+                    /* if 2 patterns are of equal strength, we pick the longest */
+                    if (data1->content_len > data2->content_len)
                         mpm_sm = sm;
-                    } else if (ls == ss) {
-                        /* if 2 patterns are of equal strength, we pick the longest */
-                        if (data1->content_len > data2->content_len)
-                            mpm_sm = sm;
-                    } else {
-                        SCLogDebug("sticking with mpm_sm");
-                    }
-                } /* else - if (mpm == NULL) */
-            } /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
-            if (mpm_sm != NULL && fast_pattern[sig])
-                break;
-        } /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
+                } else {
+                    SCLogDebug("sticking with mpm_sm");
+                }
+            } /* else - if (mpm == NULL) */
+        } /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
+    } /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
 
-        PopulateMpmAddPatternToMpm(de_ctx, sgh, s, mpm_sm);
-    } /* for (sig = 0; sig < sgh->sig_cnt; sig++) */
+    return mpm_sm;
+}
 
-    if (fast_pattern != NULL)
-        SCFree(fast_pattern);
-    if (has_non_negated_non_stream_pattern != NULL)
-        SCFree(has_non_negated_non_stream_pattern);
+/**
+ * \internal
+ * \brief Setup the mpm content.
+ *
+ * \param de_ctx Pointer to the detect engine context.
+ * \param sgh    Pointer to the signature group head against which we are
+ *               adding patterns to the mpm ctx.
+ *
+ * \retval  0 Always.
+ */
+static int PatternMatchPreparePopulateMpm(DetectEngineCtx *de_ctx,
+                                          SigGroupHead *sgh)
+{
+    for (uint32_t sig = 0; sig < sgh->sig_cnt; sig++) {
+        Signature *s = sgh->match_array[sig];
+        if (s == NULL)
+            continue;
+        PopulateMpmAddPatternToMpm(de_ctx, sgh, s, RetrieveFPForSig(s));
+    } /* for (sig = 0; sig < sgh->sig_cnt; sig++) */
 
     return 0;
 }
diff --git a/src/detect-engine-mpm.h b/src/detect-engine-mpm.h
index d89b510a01..3033ab2a1d 100644
--- a/src/detect-engine-mpm.h
+++ b/src/detect-engine-mpm.h
@@ -26,6 +26,7 @@
 
 #include "tm-threads.h"
 
+#include "detect.h"
 #include "detect-content.h"
 #include "detect-uricontent.h"
 
@@ -78,5 +79,7 @@ uint32_t DetectPatternGetId(MpmPatternIdStore *, void *, uint8_t);
 int SignatureHasPacketContent(Signature *);
 int SignatureHasStreamContent(Signature *);
 
+SigMatch *RetrieveFPForSig(Signature *s);
+
 #endif /* __DETECT_ENGINE_MPM_H__ */
 
diff --git a/src/detect.c b/src/detect.c
index 8fbbafa77b..dab006c030 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -194,7 +194,6 @@ extern int rule_reload;
 extern int engine_analysis;
 static int fp_engine_analysis_set = 0;
 static int rule_engine_analysis_set = 0;
-static FILE *fp_engine_analysis_FD = NULL;
 
 SigMatch *SigMatchAlloc(void);
 void DetectExitPrintStats(ThreadVars *tv, void *data);
@@ -268,236 +267,6 @@ char *DetectLoadCompleteSigPath(char *sig_file)
     return path;
 }
 
-static inline void EngineAnalysisWriteFastPattern(Signature *s, SigMatch *mpm_sm)
-{
-    int fast_pattern_set = 0;
-    int fast_pattern_only_set = 0;
-    int fast_pattern_chop_set = 0;
-    DetectContentData *fp_cd = NULL;
-
-    if (mpm_sm != NULL) {
-        fp_cd = (DetectContentData *)mpm_sm->ctx;
-        if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN) {
-            fast_pattern_set = 1;
-            if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
-                fast_pattern_only_set = 1;
-            } else if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
-                fast_pattern_chop_set = 1;
-            }
-        }
-    }
-
-    if (fp_cd == NULL) {
-        fprintf(fp_engine_analysis_FD, "== Sid: %u ==\n", s->id);
-        fprintf(fp_engine_analysis_FD, "    No content present\n");
-        return;
-    }
-
-    fprintf(fp_engine_analysis_FD, "== Sid: %u ==\n", s->id);
-    fprintf(fp_engine_analysis_FD, "    Fast pattern matcher: ");
-    int list_type = SigMatchListSMBelongsTo(s, mpm_sm);
-    if (list_type == DETECT_SM_LIST_PMATCH)
-        fprintf(fp_engine_analysis_FD, "content\n");
-    else if (list_type == DETECT_SM_LIST_UMATCH)
-        fprintf(fp_engine_analysis_FD, "http uri content\n");
-    else if (list_type == DETECT_SM_LIST_HRUDMATCH)
-        fprintf(fp_engine_analysis_FD, "http raw uri content\n");
-    else if (list_type == DETECT_SM_LIST_HHDMATCH)
-        fprintf(fp_engine_analysis_FD, "http header content\n");
-    else if (list_type == DETECT_SM_LIST_HRHDMATCH)
-        fprintf(fp_engine_analysis_FD, "http raw header content\n");
-    else if (list_type == DETECT_SM_LIST_HMDMATCH)
-        fprintf(fp_engine_analysis_FD, "http method content\n");
-    else if (list_type == DETECT_SM_LIST_HCDMATCH)
-        fprintf(fp_engine_analysis_FD, "http cookie content\n");
-    else if (list_type == DETECT_SM_LIST_HCBDMATCH)
-        fprintf(fp_engine_analysis_FD, "http client body content\n");
-    else if (list_type == DETECT_SM_LIST_HSBDMATCH)
-        fprintf(fp_engine_analysis_FD, "http server body content\n");
-    else if (list_type == DETECT_SM_LIST_HSCDMATCH)
-        fprintf(fp_engine_analysis_FD, "http stat code content\n");
-    else if (list_type == DETECT_SM_LIST_HSMDMATCH)
-        fprintf(fp_engine_analysis_FD, "http stat msg content\n");
-
-    fprintf(fp_engine_analysis_FD, "    Fast pattern set: %s\n", fast_pattern_set ? "yes" : "no");
-    fprintf(fp_engine_analysis_FD, "    Fast pattern only set: %s\n",
-            fast_pattern_only_set ? "yes" : "no");
-    fprintf(fp_engine_analysis_FD, "    Fast pattern chop set: %s\n",
-            fast_pattern_chop_set ? "yes" : "no");
-    if (fast_pattern_chop_set) {
-        fprintf(fp_engine_analysis_FD, "    Fast pattern offset, length: %u, %u\n",
-                fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
-    }
-    fprintf(fp_engine_analysis_FD, "    Content negated: %s\n",
-            (fp_cd->flags & DETECT_CONTENT_NEGATED) ? "yes" : "no");
-
-    uint16_t patlen = fp_cd->content_len;
-    uint8_t *pat = SCMalloc(fp_cd->content_len + 1);
-    if (unlikely(pat == NULL)) {
-        SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
-        exit(EXIT_FAILURE);
-    }
-    memcpy(pat, fp_cd->content, fp_cd->content_len);
-    pat[fp_cd->content_len] = '\0';
-    fprintf(fp_engine_analysis_FD, "    Original content: ");
-    PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
-    fprintf(fp_engine_analysis_FD, "\n");
-
-    if (fast_pattern_chop_set) {
-        SCFree(pat);
-        patlen = fp_cd->fp_chop_len;
-        pat = SCMalloc(fp_cd->fp_chop_len + 1);
-        if (unlikely(pat == NULL)) {
-            exit(EXIT_FAILURE);
-        }
-        memcpy(pat, fp_cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
-        pat[fp_cd->fp_chop_len] = '\0';
-        fprintf(fp_engine_analysis_FD, "    Final content: ");
-        PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
-        fprintf(fp_engine_analysis_FD, "\n");
-    } else {
-        fprintf(fp_engine_analysis_FD, "    Final content: ");
-        PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
-        fprintf(fp_engine_analysis_FD, "\n");
-    }
-    SCFree(pat);
-
-    return;
-}
-
-/**
- * \brief Prints analysis of fast pattern for a signature.
- *
- *        The code here mimics the logic to select fast_pattern from staging.
- *        If any changes are made to the staging logic, this should follow suit.
- *
- * \param s Pointer to the signature.
- */
-void EngineAnalysisFastPattern(Signature *s)
-{
-    SigMatch *mpm_sm = NULL;
-    uint32_t fast_pattern = 0;
-    int list_id = 0;
-
-    for (list_id = 0; list_id < DETECT_SM_LIST_MAX; list_id++) {
-        /* we have no keywords that support fp in this Signature sm list */
-        if (!FastPatternSupportEnabledForSigMatchList(list_id))
-            continue;
-
-        SigMatch *sm = NULL;
-        /* get the total no of patterns in this Signature, as well as find out
-         * if we have a fast_pattern set in this Signature */
-        for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
-            /* this keyword isn't registered for fp support */
-            if (sm->type != DETECT_CONTENT)
-                continue;
-
-            DetectContentData *cd = (DetectContentData *)sm->ctx;
-            if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
-                fast_pattern = 1;
-                break;
-            }
-        } /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
-
-        /* found a fast pattern for the sig.  Let's get outta here */
-        if (fast_pattern)
-            break;
-    } /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
-
-    int max_len = 0;
-    int max_len_negated = 0;
-    int max_len_non_negated = 0;
-    /* get the longest pattern in the sig */
-    if (!fast_pattern) {
-        SigMatch *sm = NULL;
-        for (list_id = 0; list_id < DETECT_SM_LIST_MAX; list_id++) {
-            if (!FastPatternSupportEnabledForSigMatchList(list_id))
-                continue;
-
-            for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
-                if (sm->type != DETECT_CONTENT)
-                    continue;
-
-                DetectContentData *cd = (DetectContentData *)sm->ctx;
-                if (cd->flags & DETECT_CONTENT_NEGATED) {
-                    if (max_len_negated < cd->content_len)
-                        max_len_negated = cd->content_len;
-                } else {
-                    if (max_len_non_negated < cd->content_len)
-                        max_len_non_negated = cd->content_len;
-                }
-            }
-        }
-    }
-
-    int skip_negated_content = 0;
-    if (max_len_non_negated == 0) {
-        max_len = max_len_negated;
-        skip_negated_content = 0;
-    } else {
-        max_len = max_len_non_negated;
-        skip_negated_content = 1;
-    }
-
-    SigMatch *sm = NULL;
-    for (list_id = 0; list_id < DETECT_SM_LIST_MAX; list_id++) {
-        if (!FastPatternSupportEnabledForSigMatchList(list_id))
-            continue;
-
-        for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
-            if (sm->type != DETECT_CONTENT)
-                continue;
-
-            /* skip in case of:
-             * 1. we expect a fastpattern but this isn't it */
-            if (fast_pattern) {
-                /* can be any content based keyword since all of them
-                 * now use a unified structure - DetectContentData */
-                DetectContentData *cd = (DetectContentData *)sm->ctx;
-                if (!(cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
-                    SCLogDebug("not a fast pattern %"PRIu32"", cd->id);
-                    continue;
-                }
-                SCLogDebug("fast pattern %"PRIu32"", cd->id);
-            } else {
-                DetectContentData *cd = (DetectContentData *)sm->ctx;
-                if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
-                    continue;
-                if (cd->content_len < max_len)
-                    continue;
-
-            } /* else - if (fast_pattern[sig] == 1) */
-
-            if (mpm_sm == NULL) {
-                mpm_sm = sm;
-                if (fast_pattern)
-                    break;
-            } else {
-                DetectContentData *data1 = (DetectContentData *)sm->ctx;
-                DetectContentData *data2 = (DetectContentData *)mpm_sm->ctx;
-                uint32_t ls = PatternStrength(data1->content, data1->content_len);
-                uint32_t ss = PatternStrength(data2->content, data2->content_len);
-                if (ls > ss) {
-                    mpm_sm = sm;
-                } else if (ls == ss) {
-                    /* if 2 patterns are of equal strength, we pick the longest */
-                    if (data1->content_len > data2->content_len)
-                        mpm_sm = sm;
-                } else {
-                    SCLogDebug("sticking with mpm_sm");
-                }
-            } /* else - if (mpm == NULL) */
-        } /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
-        if (mpm_sm != NULL && fast_pattern)
-            break;
-    } /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
-
-    /* output result to file */
-    EngineAnalysisWriteFastPattern(s, mpm_sm);
-
-    return;
-}
-
 /**
  *  \brief Load a file with signatures
  *  \param de_ctx Pointer to the detection engine context
@@ -560,11 +329,14 @@ int DetectLoadSigFile(DetectEngineCtx *de_ctx, char *sig_file, int *sigs_tot) {
         sig = DetectEngineAppendSig(de_ctx, line);
         (*sigs_tot)++;
         if (sig != NULL) {
-            if (fp_engine_analysis_set) {
-                EngineAnalysisFastPattern(sig);
-            }
-            if (rule_engine_analysis_set) {
-                EngineAnalysisRules(sig, line);
+            if (rule_engine_analysis_set || fp_engine_analysis_set) {
+                sig->mpm_sm = RetrieveFPForSig(sig);
+                if (fp_engine_analysis_set) {
+                    EngineAnalysisFP(sig, line);
+                }
+                if (rule_engine_analysis_set) {
+                    EngineAnalysisRules(sig, line);
+                }
             }
             SCLogDebug("signature %"PRIu32" loaded", sig->id);
             good++;
@@ -603,46 +375,8 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, int sig_file_excl
     int sigtotal = 0;
     char *sfile = NULL;
 
-    /* needed by engine_analysis */
-    char log_path[PATH_MAX];
-
     if (engine_analysis) {
-        if ((ConfGetBool("engine-analysis.rules-fast-pattern",
-                         &fp_engine_analysis_set)) == 0) {
-            SCLogInfo("Conf parameter \"engine-analysis.rules-fast-pattern\" not "
-                      "found.  Defaulting to not printing the fast_pattern "
-                      "report.");
-            fp_engine_analysis_set = 0;
-        }
-        if (fp_engine_analysis_set) {
-            char *log_dir;
-            if (ConfGet("default-log-dir", &log_dir) != 1)
-                log_dir = DEFAULT_LOG_DIR;
-            snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, "rules_fast_pattern.txt");
-
-            fp_engine_analysis_FD = fopen(log_path, "w");
-            if (fp_engine_analysis_FD == NULL) {
-                SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", log_path,
-                           strerror(errno));
-                return -1;
-            }
-            struct timeval tval;
-            struct tm *tms;
-            gettimeofday(&tval, NULL);
-            struct tm local_tm;
-            tms = (struct tm *)SCLocalTime(tval.tv_sec, &local_tm);
-            fprintf(fp_engine_analysis_FD, "----------------------------------------------"
-                    "---------------------\n");
-            fprintf(fp_engine_analysis_FD, "Date: %" PRId32 "/%" PRId32 "/%04d -- "
-                    "%02d:%02d:%02d\n",
-                    tms->tm_mday, tms->tm_mon + 1, tms->tm_year + 1900, tms->tm_hour,
-                    tms->tm_min, tms->tm_sec);
-            fprintf(fp_engine_analysis_FD, "----------------------------------------------"
-                    "---------------------\n");
-        }
-        else {
-            SCLogInfo("Engine-Analysis for fast_pattern disabled in conf file.");
-        }
+        fp_engine_analysis_set = SetupFPAnalyzer();
         rule_engine_analysis_set = SetupRuleAnalyzer();
     }
 
@@ -734,17 +468,12 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, int sig_file_excl
 
  end:
     if (engine_analysis) {
-        if (fp_engine_analysis_set) {
-            if (fp_engine_analysis_FD != NULL) {
-                SCLogInfo("Engine-Analyis for fast_pattern printed to file - %s",
-                          log_path);
-                fclose(fp_engine_analysis_FD);
-                fp_engine_analysis_FD = NULL;
-            }
-        }
         if (rule_engine_analysis_set) {
             CleanupRuleAnalyzer();
         }
+        if (fp_engine_analysis_set) {
+            CleanupFPAnalyzer();
+        }
     }
 
     DetectParseDupSigHashFree(de_ctx);