From: Russ Combs (rucombs) <rucombs@cisco.com>
Date: Thu, 10 Dec 2015 12:39:48 +0000 (-0500)
Subject: Merge pull request #184 in SNORT/snort3 from crc/fp4 to master
X-Git-Tag: 3.0.0-233~684
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=2e5fa1ac084e971aa1b9e592846c09b5ec439f3b;p=thirdparty%2Fsnort3.git

Merge pull request #184 in SNORT/snort3 from crc/fp4 to master

Squashed commit of the following:

commit c8f7d4651a71920da0818c914ee8e167b8b058fc
Author: Russ Combs <rucombs@cisco.com>
Date:   Wed Dec 9 08:34:35 2015 -0500

    regex fast pattern support
    merge rule option types for content, pcre, and regex
---

diff --git a/extra/src/search_engines/lowmem.cc b/extra/src/search_engines/lowmem.cc
index 7bcb9c174..fdd29c62a 100644
--- a/extra/src/search_engines/lowmem.cc
+++ b/extra/src/search_engines/lowmem.cc
@@ -58,9 +58,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return KTrieAddPattern(obj, P, m, noCase, negative, user);
+        return KTrieAddPattern(obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc
index fa73bfa68..3b83ceb16 100644
--- a/src/detection/detection_options.cc
+++ b/src/detection/detection_options.cc
@@ -319,7 +319,6 @@ static const char* const option_type_str[] =
     "RULE_OPTION_TYPE_LEAF_NODE",
     "RULE_OPTION_TYPE_CONTENT",
     "RULE_OPTION_TYPE_FLOWBIT",
-    "RULE_OPTION_TYPE_PCRE",
     "RULE_OPTION_TYPE_OTHER"
 };
 
@@ -340,7 +339,6 @@ void print_option_tree(detection_option_tree_node_t* node, int level)
             print_option_tree(node->children[i], level+1);
         );
 }
-
 #endif
 
 int add_detection_option_tree(
@@ -627,21 +625,21 @@ int detection_option_node_evaluate(
                     {
                         if ( child_state->result == DETECTION_OPTION_NO_MATCH )
                         {
-                            if ( (child_node->option_type == RULE_OPTION_TYPE_CONTENT ||
-                                  child_node->option_type == RULE_OPTION_TYPE_PCRE) &&
-                                 !child_node->is_relative )
+                            if ( !child_node->is_relative )
                             {
-                                // If it's a non-relative content or pcre, no reason
-                                // to check again.  Only increment result once.
-                                // Should hit this condition on first loop iteration.
-                                if ( loop_count == 1 )
-                                    ++result;
+                                if ( child_node->option_type == RULE_OPTION_TYPE_CONTENT )
+                                {
+                                    // If it's a non-relative content or pcre, no reason
+                                    // to check again.  Only increment result once.
+                                    // Should hit this condition on first loop iteration.
+                                    if ( loop_count == 1 )
+                                        ++result;
 
-                                continue;
+                                    continue;
+                                }
                             }
 
-                            else if ( child_node->option_type == RULE_OPTION_TYPE_CONTENT &&
-                                      child_node->is_relative )
+                            else if ( child_node->option_type == RULE_OPTION_TYPE_CONTENT )
                             {
                                 // Check for an unbounded relative search.  If this
                                 // failed before, it's going to fail again so don't
diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc
index 1d2972d00..cb3e30bae 100644
--- a/src/detection/fp_create.cc
+++ b/src/detection/fp_create.cc
@@ -70,7 +70,7 @@ static unsigned mpse_count = 0;
 static void fpDeletePMX(void* data);
 
 static int fpGetFinalPattern(
-    FastPatternConfig*, PatternMatchData*, char** ret_pattern, int* ret_bytes);
+    FastPatternConfig*, PatternMatchData*, const char*& ret_pattern, int& ret_bytes);
 
 static void PrintFastPatternInfo(
     OptTreeNode*, PatternMatchData*, const char* pattern, int pattern_length);
@@ -345,14 +345,14 @@ static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree)
 }
 
 /* FLP_Trim
-  *
   * Trim zero byte prefixes, this increases uniqueness
+  * will not alter regex since they can't contain bald \0
   *
   * returns
   *   length - of trimmed pattern
   *   buff - ptr to new beggining of trimmed buffer
   */
-static int FLP_Trim(char* p, int plen, char** buff)
+static int FLP_Trim(const char* p, int plen, const char** buff)
 {
     int i;
     int size = 0;
@@ -561,7 +561,7 @@ static int fpFinishPortGroupRule(
     SnortConfig* sc, PortGroup* pg,
     OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp)
 {
-    char* pattern;
+    const char* pattern;
     int pattern_length;
 
     if ( !pmd )
@@ -577,7 +577,7 @@ static int fpFinishPortGroupRule(
         else
             pg->add_rule();
 
-        if (fpGetFinalPattern(fp, pmd, &pattern, &pattern_length) == -1)
+        if (fpGetFinalPattern(fp, pmd, pattern, pattern_length) == -1)
             return -1;
 
         /* create pmx */
@@ -610,8 +610,8 @@ static int fpFinishPortGroupRule(
                 pg->mpse[pmd->pm_type]->set_opt(1);
         }
 
-        pg->mpse[pmd->pm_type]->add_pattern(
-            sc, (uint8_t*)pattern, pattern_length, pmd->no_case, pmd->negated, pmx);
+        Mpse::PatternDescriptor desc(pmd->no_case, pmd->negated, pmd->literal);
+        pg->mpse[pmd->pm_type]->add_pattern(sc, (uint8_t*)pattern, pattern_length, desc, pmx);
     }
 
     return 0;
@@ -933,32 +933,34 @@ static void fpFreeRuleMaps(SnortConfig* sc)
 
 static int fpGetFinalPattern(
     FastPatternConfig* fp, PatternMatchData* pmd,
-    char** ret_pattern, int* ret_bytes)
+    const char*& ret_pattern, int& ret_bytes)
 {
-    char* pattern;
-    int bytes;
-
-    if ((fp == NULL) || (pmd == NULL)
-        || (ret_pattern == NULL) || (ret_bytes == NULL))
+    if ( !fp or !pmd )
     {
         return -1;
     }
 
-    pattern = pmd->pattern_buf;
-    bytes = pmd->pattern_size;
+    const char* pattern = pmd->pattern_buf;
+    int bytes = pmd->pattern_size;
 
-    /* Don't mess with fast pattern only contents - they should be inserted
-     * into the pattern matcher as is since the content won't be evaluated
-     * as a rule option.
-     * Don't mess with negated contents since truncating them could
-     * inadvertantly disable evaluation of a rule - the shorter pattern
-     * may be found, while the unaltered pattern may not be found,
-     * disabling inspection of a rule we should inspect */
-    if (pmd->fp_only > 0 || pmd->negated)
-    {
-        *ret_pattern = pattern;
-        *ret_bytes = bytes;
+    // Don't mess with:
+    //
+    // 1. fast pattern only contents - they should be inserted into the
+    // pattern matcher as is since the content won't be evaluated as a rule
+    // option.
+    //
+    // 2. negated contents since truncating them could inadvertantly
+    // disable evaluation of a rule - the shorter pattern may be found,
+    // while the unaltered pattern may not be found, disabling inspection
+    // of a rule we should inspect.
+    //
+    // 3. non-literals like regex - truncation could invalidate the
+    // expression.
 
+    if ( pmd->fp_only > 0 or pmd->negated or !pmd->literal )
+    {
+        ret_pattern = pattern;
+        ret_bytes = bytes;
         return 0;
     }
 
@@ -1003,8 +1005,8 @@ static int fpGetFinalPattern(
         }
     }
 
-    *ret_pattern = pattern;
-    *ret_bytes = fp->set_max(bytes);
+    ret_pattern = pattern;
+    ret_bytes = fp->set_max(bytes);
 
     return 0;
 }
diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc
index 8cdbf01dd..a6a3625c3 100644
--- a/src/detection/fp_detect.cc
+++ b/src/detection/fp_detect.cc
@@ -466,10 +466,6 @@ static int rule_tree_match(
     PMX* pmx = (PMX*)user;
     OTNX_MATCH_DATA* pomd = (OTNX_MATCH_DATA*)context;
 
-    //unsigned sz = pmx->pmd->pattern_size;
-    //assert(sz <= (unsigned)index and (unsigned)index <= pomd->size);
-    //assert(!strncasecmp((char*)pmx->pmd->pattern_buf, (char*)pomd->data+index-sz, sz));
-
     detection_option_tree_root_t* root = (detection_option_tree_root_t*)tree;
     detection_option_eval_data_t eval_data;
     NCListNode* ncl;
diff --git a/src/detection/pattern_match_data.h b/src/detection/pattern_match_data.h
index 9c52bd667..8959fb4da 100644
--- a/src/detection/pattern_match_data.h
+++ b/src/detection/pattern_match_data.h
@@ -37,26 +37,28 @@ struct PmdLastCheck
 struct PatternMatchData
 {
     // used by both
-    uint8_t negated;        /* search for "not this pattern" */
-    uint8_t fp;             /* For fast_pattern arguments */
-    uint8_t no_case;        /* Toggle case sensitivity */
-    uint8_t relative;       /* do relative pattern searching */
+    bool negated;        // search for "not this pattern"
+    bool fp;             // For fast_pattern arguments
+    bool no_case;        // Toggle case sensitivity
+    bool relative;       // do relative pattern searching
 
     uint16_t fp_offset;
     uint16_t fp_length;
 
-    int offset;             /* pattern search start offset */
-    int depth;              /* pattern search depth */
+    int offset;              // pattern search start offset
+    int depth;               // pattern search depth
 
-    unsigned pattern_size;  /* size of app layer pattern */
-    char* pattern_buf;      /* app layer pattern to match on */
+    unsigned pattern_size;   // size of app layer pattern
+    const char* pattern_buf; // app layer pattern to match on
+
+    bool literal;            // set to plain contents
 
     // not used by ips_content
     int8_t fp_only;
     uint8_t pm_type;
 
-    unsigned replace_size;  /* size of app layer replace pattern */
-    char* replace_buf;      /* app layer pattern to replace with */
+    unsigned replace_size;   // size of app layer replace pattern
+    const char* replace_buf; // app layer pattern to replace with
 
     // FIXIT-L wasting some memory here:
     // - this is not used by content option logic directly
diff --git a/src/detection/rule_option_types.h b/src/detection/rule_option_types.h
index a8da9894f..c9806b9a0 100644
--- a/src/detection/rule_option_types.h
+++ b/src/detection/rule_option_types.h
@@ -31,7 +31,6 @@ enum option_type_t
     RULE_OPTION_TYPE_LEAF_NODE,
     RULE_OPTION_TYPE_CONTENT,
     RULE_OPTION_TYPE_FLOWBIT,
-    RULE_OPTION_TYPE_PCRE,  // pcre and regex
     RULE_OPTION_TYPE_OTHER
 };
 
diff --git a/src/framework/mpse.h b/src/framework/mpse.h
index fb735f742..64dd16d39 100644
--- a/src/framework/mpse.h
+++ b/src/framework/mpse.h
@@ -51,9 +51,19 @@ public:
 public:
     virtual ~Mpse() { }
 
+    struct PatternDescriptor
+    {
+        bool no_case;
+        bool negated;
+        bool literal;
+
+        PatternDescriptor(bool noc = false, bool neg = false, bool lit = false)
+        { no_case = noc; negated = neg; literal = lit; }
+    };
+
     virtual int add_pattern(
         SnortConfig* sc, const uint8_t* pat, unsigned len,
-        bool noCase, bool negate, void* user) = 0;
+        const PatternDescriptor&, void* user) = 0;
 
     virtual int prep_patterns(SnortConfig*) = 0;
 
diff --git a/src/ips_options/ips_content.cc b/src/ips_options/ips_content.cc
index 300b87ec3..1beda0c08 100644
--- a/src/ips_options/ips_content.cc
+++ b/src/ips_options/ips_content.cc
@@ -161,7 +161,7 @@ ContentOption::~ContentOption()
         return;
 
     if ( cd->pmd.pattern_buf )
-        free(cd->pmd.pattern_buf);
+        free((char*)cd->pmd.pattern_buf);
 
     if ( cd->pmd.last_check )
         free(cd->pmd.last_check);
@@ -439,10 +439,12 @@ static void parse_content(ContentData* cd, const char* rule)
     unsigned dummy_size = buf.size();
 
     cd->pmd.pattern_buf = (char*)SnortAlloc(dummy_size+1);
-    memcpy(cd->pmd.pattern_buf, tmp_buf, dummy_size);
+    memcpy((char*)cd->pmd.pattern_buf, tmp_buf, dummy_size);
 
     cd->pmd.pattern_size = dummy_size;
     cd->pmd.negated = negated;
+    cd->pmd.literal = true;
+
     cd->set_max_jump_size();
 }
 
@@ -685,8 +687,10 @@ bool ContentModule::end(const char*, int, SnortConfig*)
     }
     if ( cd->pmd.no_case )
     {
+        char* s = (char*)cd->pmd.pattern_buf;
+
         for ( unsigned i = 0; i < cd->pmd.pattern_size; i++ )
-            cd->pmd.pattern_buf[i] = toupper(cd->pmd.pattern_buf[i]);
+            s[i] = toupper(cd->pmd.pattern_buf[i]);
     }
     cd->setup_bm();
     return true;
diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc
index b6b09f81e..b93bff667 100644
--- a/src/ips_options/ips_pcre.cc
+++ b/src/ips_options/ips_pcre.cc
@@ -352,7 +352,7 @@ syntax:
     if ( !pcre_data->expression )
         pcre_data->expression = SnortStrdup("");
 
-    ParseError("unable to parse pcre regex %s", data);
+    ParseError("unable to parse pcre %s", data);
 }
 
 /**
@@ -454,7 +454,7 @@ class PcreOption : public IpsOption
 {
 public:
     PcreOption(PcreData* c) :
-        IpsOption(s_name, RULE_OPTION_TYPE_PCRE)
+        IpsOption(s_name, RULE_OPTION_TYPE_CONTENT)
     { config = c; }
 
     ~PcreOption();
@@ -659,14 +659,14 @@ void pcre_cleanup(SnortConfig* sc)
 
 static const Parameter s_params[] =
 {
-    { "~regex", Parameter::PT_STRING, nullptr, nullptr,
+    { "~re", Parameter::PT_STRING, nullptr, nullptr,
       "Snort regular expression" },
 
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
 #define s_help \
-    "rule option for matching payload data with regex"
+    "rule option for matching payload data with pcre"
 
 class PcreModule : public Module
 {
@@ -704,7 +704,7 @@ bool PcreModule::begin(const char*, int, SnortConfig*)
 
 bool PcreModule::set(const char*, Value& v, SnortConfig*)
 {
-    if ( v.is("~regex") )
+    if ( v.is("~re") )
         pcre_parse(v.get_string(), data);
 
     else
diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc
index 0de6aebba..7407129f2 100644
--- a/src/ips_options/ips_regex.cc
+++ b/src/ips_options/ips_regex.cc
@@ -19,6 +19,8 @@
 // ips_regex.cc author Russ Combs <rucombs@cisco.com>
 // FIXIT-H add ! and anchor support like pcre and update retry
 
+#include <string.h>
+
 #include <assert.h>
 #include <string>
 
@@ -29,6 +31,7 @@
 #include "framework/ips_option.h"
 #include "framework/module.h"
 #include "detection/detection_defines.h"
+#include "detection/pattern_match_data.h"
 #include "hash/sfhashfcn.h"
 #include "main/snort_config.h"
 #include "main/thread.h"
@@ -42,6 +45,7 @@
 
 struct RegexConfig
 {
+    PatternMatchData pmd;
     std::string re;
     hs_database_t* db;
     unsigned flags;
@@ -52,10 +56,10 @@ struct RegexConfig
 
     void reset()
     {
+        memset(&pmd, 0, sizeof(pmd));
         re.clear();
         db = nullptr;
         flags = 0;
-        relative = false;
     }
 };
 
@@ -90,16 +94,21 @@ public:
     { return CAT_ADJUST; }
 
     bool is_relative() override
-    { return config.relative; }
+    { return config.pmd.relative; }
 
-    int eval(Cursor&, Packet*) override;
     bool retry() override;
 
+    PatternMatchData* get_pattern() override
+    { return &config.pmd; }
+
+    int eval(Cursor&, Packet*) override;
+
 private:
     RegexConfig config;
 };
 
-RegexOption::RegexOption(const RegexConfig& c) : IpsOption(s_name, RULE_OPTION_TYPE_PCRE)
+RegexOption::RegexOption(const RegexConfig& c) :
+    IpsOption(s_name, RULE_OPTION_TYPE_CONTENT)
 {
     config = c;
 
@@ -109,6 +118,10 @@ RegexOption::RegexOption(const RegexConfig& c) : IpsOption(s_name, RULE_OPTION_T
         //ParseError("can't initialize regex for '%s' (%d) %p",
         //    config.re.c_str(), err, s_scratch);
     }
+    config.pmd.pattern_buf = config.re.c_str();
+    config.pmd.pattern_size = config.re.size();
+    config.pmd.fp_length = config.pmd.pattern_size;
+    config.pmd.fp_offset = 0;
 }
 
 RegexOption::~RegexOption()
@@ -119,7 +132,7 @@ RegexOption::~RegexOption()
 
 uint32_t RegexOption::hash() const
 {
-    uint32_t a = config.flags, b = config.relative, c = 0;
+    uint32_t a = config.flags, b = config.pmd.relative, c = 0;
     mix_str(a, b, c, config.re.c_str());
     mix_str(a, b, c, get_name());
     finalize(a, b, c);
@@ -135,7 +148,7 @@ bool RegexOption::operator==(const IpsOption& ips) const
 
     if ( config.re == rhs.config.re and 
          config.flags == rhs.config.flags and
-         config.relative == rhs.config.relative )
+         config.pmd.relative == rhs.config.pmd.relative )
         return true;
 
     return false;
@@ -190,7 +203,7 @@ bool RegexOption::retry()
 
 static const Parameter s_params[] =
 {
-    { "~", Parameter::PT_STRING, nullptr, nullptr,
+    { "~re", Parameter::PT_STRING, nullptr, nullptr,
       "hyperscan regular expression" },
 
     { "nocase", Parameter::PT_IMPLIED, nullptr, nullptr,
@@ -245,7 +258,7 @@ bool RegexModule::begin(const char*, int, SnortConfig*)
 
 bool RegexModule::set(const char*, Value& v, SnortConfig*)
 {
-    if ( v.is("~") )
+    if ( v.is("~re") )
     {
         config.re = v.get_string();
         // remove quotes
@@ -263,7 +276,7 @@ bool RegexModule::set(const char*, Value& v, SnortConfig*)
         config.flags |= HS_FLAG_MULTILINE;
 
     else if ( v.is("relative") )
-        config.relative = true;
+        config.pmd.relative = true;
 
     else
         return false;
diff --git a/src/parser/parse_stream.cc b/src/parser/parse_stream.cc
index 71ba4991d..13c41e075 100644
--- a/src/parser/parse_stream.cc
+++ b/src/parser/parse_stream.cc
@@ -83,7 +83,7 @@ static uint8_t to_hex(char c)
 }
 
 static TokenType get_token(
-    istream& is, string& s, const char* punct, bool esc)
+    istream& is, string& s, const char* punct, int esc)
 {
     static int prev = EOF;
     int c, list = 0, state = 0;
@@ -205,7 +205,7 @@ static TokenType get_token(
                 return TT_STRING;
             }
             else if ( c == '\\' )
-                state = esc ? 4 : 16;
+                state = (esc > 0) ? 4 : 16;
             else if ( c == '\n' )
                 ParseWarning(WARN_RULES, "line break in string on line %d\n", lines-1);
             else
@@ -605,6 +605,20 @@ static bool exec(
     return false;
 }
 
+// FIXIT-L this should not be by name
+// probably should remove content escaping except for \" so
+// that individual rule options can do whatever
+static int get_escape(const string& s)
+{
+    if ( s == "pcre" )
+        return 0;  // no escape, option goes to ;
+
+    else if ( s == "regex" )
+        return -1; // no escape, option goes to "
+
+    return 1;      // escape, option goes to "
+}
+
 // parse_body() is called at the end of a stub rule to parse the detection
 // options in an so rule.  similar to parse_stream() except we start in a
 // different state.
@@ -614,7 +628,7 @@ static void parse_body(const char* extra, RuleParseState& rps, struct SnortConfi
 
     string tok;
     TokenType type;
-    bool esc = true;
+    int esc = 1;
 
     int num = 8;
     const char* punct = "(:,;)";
@@ -627,7 +641,7 @@ static void parse_body(const char* extra, RuleParseState& rps, struct SnortConfi
         exec(s->action, tok, rps, sc);
 
         num = s->next;
-        esc = (rps.key != "pcre");
+        esc = get_escape(rps.key);
 
         if ( s->punct )
             punct = s->punct;
@@ -638,7 +652,7 @@ void parse_stream(istream& is, struct SnortConfig* sc)
 {
     string tok;
     TokenType type;
-    bool esc = true;
+    int esc = 1;
 
     int num = 0;
     const char* punct = fsm[0].punct;
@@ -658,7 +672,7 @@ void parse_stream(istream& is, struct SnortConfig* sc)
             break;
 
         num = s->next;
-        esc = (rps.key != "pcre");
+        esc = get_escape(rps.key);
 
         if ( s->punct )
             punct = s->punct;
diff --git a/src/search_engines/CMakeLists.txt b/src/search_engines/CMakeLists.txt
index 1d8195047..450f86017 100644
--- a/src/search_engines/CMakeLists.txt
+++ b/src/search_engines/CMakeLists.txt
@@ -41,33 +41,31 @@ if ( HAVE_HYPERSCAN )
     )
 endif ()
 
+# FIXIT ideally don't include pat_stats.cc separately
+
 set (SEARCH_ENGINE_SOURCES
+    pat_stats.cc
     search_engines.cc
     search_engines.h
     search_tool.cc
     search_tool.h
     ${BNFA_SOURCES}
+    ${HYPER_SOURCES}
 )
 
-# FIXIT ideally don't include pat_stats.cc separately
-
 if ( STATIC_SEARCH_ENGINES )
     add_library(search_engines STATIC
         ${ACSMX_SOURCES}
         ${ACSMX2_SOURCES}
         ${INTEL_SOURCES}
-        ${HYPER_SOURCES}
         ${SEARCH_ENGINE_SOURCES}
         ${SEARCH_ENGINE_INCLUDES}
-        pat_stats.cc
     )
 
 else ( STATIC_SEARCH_ENGINES)
     add_library(search_engines STATIC
         ${SEARCH_ENGINE_SOURCES}
         ${SEARCH_ENGINE_INCLUDES}
-        ${BNFA_SOURCES}
-        pat_stats.cc
     )
 
     add_shared_library(acsmx search_engines ${ACSMX_SOURCES} pat_stats.cc)
diff --git a/src/search_engines/Makefile.am b/src/search_engines/Makefile.am
index 71d5fa3c7..56e679ea5 100644
--- a/src/search_engines/Makefile.am
+++ b/src/search_engines/Makefile.am
@@ -41,8 +41,7 @@ endif
 plugin_list = \
 $(acsmx_sources) \
 $(acsmx2_sources) \
-$(intel_sources) \
-$(hyper_sources)
+$(intel_sources)
 
 libsearch_engines_a_SOURCES = \
 search_engines.cc \
@@ -50,7 +49,8 @@ search_engines.h \
 search_tool.cc \
 search_tool.h \
 pat_stats.cc \
-$(bnfa_sources)
+$(bnfa_sources) \
+$(hyper_sources)
 
 # FIXIT ideally don't include pat_stats.cc separately
 
diff --git a/src/search_engines/ac_banded.cc b/src/search_engines/ac_banded.cc
index 5973c5822..e718dc9d3 100644
--- a/src/search_engines/ac_banded.cc
+++ b/src/search_engines/ac_banded.cc
@@ -53,9 +53,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return acsmAddPattern2(obj, P, m, noCase, negative, user);
+        return acsmAddPattern2(obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/search_engines/ac_bnfa.cc b/src/search_engines/ac_bnfa.cc
index 57b0717e8..a196c4409 100644
--- a/src/search_engines/ac_bnfa.cc
+++ b/src/search_engines/ac_bnfa.cc
@@ -66,9 +66,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return bnfaAddPattern(obj, P, m, noCase, negative, user);
+        return bnfaAddPattern(obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/search_engines/ac_full.cc b/src/search_engines/ac_full.cc
index 585c54921..69bc011ac 100644
--- a/src/search_engines/ac_full.cc
+++ b/src/search_engines/ac_full.cc
@@ -58,9 +58,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return acsmAddPattern2(obj, P, m, noCase, negative, user);
+        return acsmAddPattern2(obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/search_engines/ac_sparse.cc b/src/search_engines/ac_sparse.cc
index a7af7f7b2..9c8b160a6 100644
--- a/src/search_engines/ac_sparse.cc
+++ b/src/search_engines/ac_sparse.cc
@@ -49,9 +49,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return acsmAddPattern2(obj, P, m, noCase, negative, user);
+        return acsmAddPattern2(obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/search_engines/ac_sparse_bands.cc b/src/search_engines/ac_sparse_bands.cc
index 24e397e7f..a9595c6ef 100644
--- a/src/search_engines/ac_sparse_bands.cc
+++ b/src/search_engines/ac_sparse_bands.cc
@@ -52,9 +52,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return acsmAddPattern2(obj, P, m, noCase, negative, user);
+        return acsmAddPattern2(obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/search_engines/ac_std.cc b/src/search_engines/ac_std.cc
index 2b58f389a..b6e70178d 100644
--- a/src/search_engines/ac_std.cc
+++ b/src/search_engines/ac_std.cc
@@ -53,9 +53,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return acsmAddPattern(obj, P, m, noCase, negative, user);
+        return acsmAddPattern(obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/search_engines/acsmx2_api.cc b/src/search_engines/acsmx2_api.cc
index e3f99ec27..3cf7f5728 100644
--- a/src/search_engines/acsmx2_api.cc
+++ b/src/search_engines/acsmx2_api.cc
@@ -25,11 +25,15 @@
 #include "framework/mpse.h"
 
 #ifdef BUILDING_SO
+extern const BaseApi* se_ac_banded;
+extern const BaseApi* se_ac_full;
+extern const BaseApi* se_ac_sparse;
+extern const BaseApi* se_ac_sparse_bands;
+
 SO_PUBLIC const BaseApi* snort_plugins[] =
 {
     se_ac_banded,
     se_ac_full,
-    se_ac_full_q,
     se_ac_sparse,
     se_ac_sparse_bands,
     nullptr
diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc
index 00b2da373..399c996c2 100644
--- a/src/search_engines/hyperscan.cc
+++ b/src/search_engines/hyperscan.cc
@@ -44,37 +44,43 @@ struct Pattern
     void* user_tree;
     void* user_list;
 
-    Pattern(const uint8_t* s, unsigned n, bool bc, bool bn, void* u)
-    {
-        // FIXIT-H only escape content patterns, not regex patterns
-        escape(s, n);
-        len = n;
-        no_case = bc;
-        negate = bn;
-        user = u;
-        user_tree = user_list = nullptr;
-    }
-    void escape(const uint8_t* s, unsigned n)
+    Pattern(const uint8_t*, unsigned, const Mpse::PatternDescriptor&, void*);
+    void escape(const uint8_t*, unsigned, bool);
+};
+
+Pattern::Pattern(
+    const uint8_t* s, unsigned n, const Mpse::PatternDescriptor& d, void* u)
+{
+    escape(s, n, d.literal);
+
+    len = n;
+    no_case = d.no_case;
+    negate = d.negated;
+    user = u;
+    user_tree = user_list = nullptr;
+}
+
+void Pattern::escape(const uint8_t* s, unsigned n, bool literal)
+{
+    for ( unsigned i = 0; i < n; ++i )
     {
-        for ( unsigned i = 0; i < n; ++i )
+        if ( !isprint(s[i]) )
         {
-            if ( !isprint(s[i]) )
-            {
-                char hex[5];
-                snprintf(hex, sizeof(hex), "\\x%02X", s[i]);
-                pat += hex;
-            }
-            else
-            {
-                const char* special = ".^$*+?()[]{}\\|";
+            char hex[5];
+            snprintf(hex, sizeof(hex), "\\x%02X", s[i]);
+            pat += hex;
+        }
+        else
+        {
+            const char* special = ".^$*+?()[]{}\\|";
+
+            if ( literal and strchr(special, s[i]) )
+                pat += '\\';
 
-                if ( strchr(special, s[i]) )
-                    pat += '\\';
-                pat += s[i];
-            }
+            pat += s[i];
         }
     }
-};
+}
 
 typedef std::vector<Pattern> PatternVector;
 
@@ -108,9 +114,9 @@ public:
 
     int add_pattern(
         SnortConfig*, const uint8_t* pat, unsigned len,
-        bool no_case, bool negate, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        Pattern p(pat, len, no_case, negate, user);
+        Pattern p(pat, len, desc, user);
         pvector.push_back(p);
         ++patterns;
         return 0;
@@ -342,13 +348,13 @@ static const MpseApi hs_api =
     hs_print,
 };
 
-#ifdef BUILDING_SO
-SO_PUBLIC const BaseApi* snort_plugins[] =
-{
-    &hs_api.base,
-    nullptr
-};
-#else
+//#ifdef BUILDING_SO
+//SO_PUBLIC const BaseApi* snort_plugins[] =
+//{
+//    &hs_api.base,
+//    nullptr
+//};
+//#else
 const BaseApi* se_hyperscan = &hs_api.base;
-#endif
+//#endif
 
diff --git a/src/search_engines/intel_cpm.cc b/src/search_engines/intel_cpm.cc
index eefe000b4..09bc9726b 100644
--- a/src/search_engines/intel_cpm.cc
+++ b/src/search_engines/intel_cpm.cc
@@ -55,9 +55,9 @@ public:
 
     int add_pattern(
         SnortConfig* sc, const uint8_t* P, unsigned m,
-        bool noCase, bool negative, void* user) override
+        const PatternDescriptor& desc, void* user) override
     {
-        return IntelPmAddPattern(sc, obj, P, m, noCase, negative, user);
+        return IntelPmAddPattern(sc, obj, P, m, desc.no_case, desc.negated, user);
     }
 
     int prep_patterns(SnortConfig* sc) override
diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc
index cd04e50ef..f2b493f9b 100644
--- a/src/search_engines/search_tool.cc
+++ b/src/search_engines/search_tool.cc
@@ -45,8 +45,10 @@ void SearchTool::add(const char* pat, unsigned len, int id, bool no_case)
 
 void SearchTool::add(const uint8_t* pat, unsigned len, int id, bool no_case)
 {
+    Mpse::PatternDescriptor desc(no_case);
+
     if ( mpse )
-        mpse->add_pattern(nullptr,  pat, len, no_case, false, (void*)(long)id);
+        mpse->add_pattern(nullptr,  pat, len, desc, (void*)(long)id);
 
     if ( len > max_len )
         max_len = len;
diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc
index fe9a4dd9d..56b374b7e 100644
--- a/src/search_engines/test/hyperscan_test.cc
+++ b/src/search_engines/test/hyperscan_test.cc
@@ -166,7 +166,9 @@ TEST(mpse_hs_match, empty)
 
 TEST(mpse_hs_match, single)
 {
-    CHECK(hs->add_pattern(nullptr, (uint8_t*)"foo", 3, false, false, s_user) == 0);
+    Mpse::PatternDescriptor desc;
+
+    CHECK(hs->add_pattern(nullptr, (uint8_t*)"foo", 3, desc, s_user) == 0);
     CHECK(hs->prep_patterns(snort_conf) == 0);
     CHECK(hs->get_pattern_count() == 1);
 
@@ -179,9 +181,11 @@ TEST(mpse_hs_match, single)
 
 TEST(mpse_hs_match, other)
 {
-    CHECK(hs->add_pattern(nullptr, (uint8_t*)"foo", 3, false, true, s_user) == 0);
-    CHECK(hs->add_pattern(nullptr, (uint8_t*)"\rbar\n", 3, false, true, s_user) == 0);
-    CHECK(hs->add_pattern(nullptr, (uint8_t*)"(baz)", 3, false, true, s_user) == 0);
+    Mpse::PatternDescriptor desc(false, true, false);
+
+    CHECK(hs->add_pattern(nullptr, (uint8_t*)"foo", 3, desc, s_user) == 0);
+    CHECK(hs->add_pattern(nullptr, (uint8_t*)"\rbar\n", 3, desc, s_user) == 0);
+    CHECK(hs->add_pattern(nullptr, (uint8_t*)"(baz)", 3, desc, s_user) == 0);
 
     CHECK(hs->prep_patterns(snort_conf) == 0);
     CHECK(hs->get_pattern_count() == 3);
@@ -195,9 +199,11 @@ TEST(mpse_hs_match, other)
 
 TEST(mpse_hs_match, multi)
 {
-    CHECK(hs->add_pattern(nullptr, (uint8_t*)"foo", 3, false, false, s_user) == 0);
-    CHECK(hs->add_pattern(nullptr, (uint8_t*)"bar", 3, false, false, s_user) == 0);
-    CHECK(hs->add_pattern(nullptr, (uint8_t*)"baz", 3, false, false, s_user) == 0);
+    Mpse::PatternDescriptor desc;
+
+    CHECK(hs->add_pattern(nullptr, (uint8_t*)"foo", 3, desc, s_user) == 0);
+    CHECK(hs->add_pattern(nullptr, (uint8_t*)"bar", 3, desc, s_user) == 0);
+    CHECK(hs->add_pattern(nullptr, (uint8_t*)"baz", 3, desc, s_user) == 0);
 
     CHECK(hs->prep_patterns(snort_conf) == 0);
     CHECK(hs->get_pattern_count() == 3);
@@ -211,8 +217,10 @@ TEST(mpse_hs_match, multi)
 #if 0
 TEST(mpse_hs_match, regex)
 {
+    Mpse::PatternDescriptor desc;
+
     CHECK(hs->add_pattern(
-            nullptr, (uint8_t*)"(foo)|(bar)|(baz)", 17, false, false, s_user) == 0);
+            nullptr, (uint8_t*)"(foo)|(bar)|(baz)", 17, desc, s_user) == 0);
 
     CHECK(hs->prep_patterns(snort_conf) == 0);
     CHECK(hs->get_pattern_count() == 1);
@@ -225,9 +233,11 @@ TEST(mpse_hs_match, regex)
 
 TEST(mpse_hs_match, pcre)
 {
+    Mpse::PatternDescriptor desc;
+
     // from sid 23286
     CHECK(hs->add_pattern(
-            nullptr, (uint8_t*)"\\.definition\\s*\\(", 21, false, false, s_user) == 0);
+            nullptr, (uint8_t*)"\\.definition\\s*\\(", 21, desc, s_user) == 0);
 
     CHECK(hs->prep_patterns(snort_conf) == 0);
     CHECK(hs->get_pattern_count() == 1);
@@ -275,8 +285,10 @@ TEST_GROUP(mpse_hs_multi)
 
 TEST(mpse_hs_multi, single)
 {
-    CHECK(hs1->add_pattern(nullptr, (uint8_t*)"uba", 3, false, false, s_user) == 0);
-    CHECK(hs2->add_pattern(nullptr, (uint8_t*)"tuba", 4, false, false, s_user) == 0);
+    Mpse::PatternDescriptor desc;
+
+    CHECK(hs1->add_pattern(nullptr, (uint8_t*)"uba", 3, desc, s_user) == 0);
+    CHECK(hs2->add_pattern(nullptr, (uint8_t*)"tuba", 4, desc, s_user) == 0);
 
     CHECK(hs1->prep_patterns(snort_conf) == 0);
     CHECK(hs2->prep_patterns(snort_conf) == 0);
diff --git a/src/stream/tcp/test/CMakeLists.txt b/src/stream/tcp/test/CMakeLists.txt
index 5d04d91b3..ad48bea56 100644
--- a/src/stream/tcp/test/CMakeLists.txt
+++ b/src/stream/tcp/test/CMakeLists.txt
@@ -9,5 +9,5 @@ add_library( stream_tcp_test
 )
 
 # this test is broken, uncomment below when fixed
-#add_cpputest( tcp_normalizer_test stream_tcp_test )
+add_cpputest( tcp_normalizer_test stream_tcp_test )
 
diff --git a/src/utils/boyer_moore.cc b/src/utils/boyer_moore.cc
index 849645a69..021521860 100644
--- a/src/utils/boyer_moore.cc
+++ b/src/utils/boyer_moore.cc
@@ -95,7 +95,7 @@ int main()
  *      int * - the skip table
  *
  ****************************************************************/
-int* make_skip(char* ptrn, int plen)
+int* make_skip(const char* ptrn, int plen)
 {
     int i;
     int* skip = (int*)SnortAlloc(256* sizeof(int));
@@ -123,11 +123,11 @@ int* make_skip(char* ptrn, int plen)
  *      int * - the shift table
  *
  ****************************************************************/
-int* make_shift(char* ptrn, int plen)
+int* make_shift(const char* ptrn, int plen)
 {
     int* shift = (int*)SnortAlloc(plen * sizeof(int));
     int* sptr = shift + plen - 1;
-    char* pptr = ptrn + plen - 1;
+    const char* pptr = ptrn + plen - 1;
     char c;
 
     c = ptrn[plen - 1];
@@ -136,7 +136,7 @@ int* make_shift(char* ptrn, int plen)
 
     while (sptr-- != shift)
     {
-        char* p1 = ptrn + plen - 2, * p2, * p3;
+        const char* p1 = ptrn + plen - 2, * p2, * p3;
 
         do
         {
diff --git a/src/utils/boyer_moore.h b/src/utils/boyer_moore.h
index 291415463..183d28cdc 100644
--- a/src/utils/boyer_moore.h
+++ b/src/utils/boyer_moore.h
@@ -26,8 +26,8 @@
 #include "main/snort_types.h"
 
 // FIXIT-M: No associated resource destructor for make_skip & make_shift :(
-SO_PUBLIC int* make_skip(char*, int);
-SO_PUBLIC int* make_shift(char*, int);
+SO_PUBLIC int* make_skip(const char*, int);
+SO_PUBLIC int* make_shift(const char*, int);
 
 SO_PUBLIC int mSearch(const char*, int, const char*, int, int*, int*);
 SO_PUBLIC int mSearchCI(const char*, int, const char*, int, int*, int*);