From: russ <rucombs@cisco.com>
Date: Mon, 24 Jun 2019 04:22:04 +0000 (-0400)
Subject: Squashed commit of the following:
X-Git-Tag: 3.0.0-258~20
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=929c607971d13d54e2e052fb38b35a708b29e536;p=thirdparty%2Fsnort3.git

Squashed commit of the following:

commit 46b75614846523b09bc3f0381aa23c74c4b4037c
Author: russ <rucombs@cisco.com>
Date:   Fri Jun 21 22:17:05 2019 -0400

    ips: refactor fast pattern searching

commit ca549ab88276c9c1032be231ce6ab4be331c9920
Author: russ <rucombs@cisco.com>
Date:   Fri Jun 21 22:16:22 2019 -0400

    detection: allocate scratch after configuration

commit 1db4b7941c9b0e700f6b8c76a4718649d546678a
Author: russ <rucombs@cisco.com>
Date:   Wed Jun 19 12:21:49 2019 -0400

    detection: immediately onload after offloading when running regression tests

commit aecdde54894b4e2f9eddf1e641964ef1c1dac749
Author: russ <rucombs@cisco.com>
Date:   Tue Jun 4 09:44:36 2019 -0400

    detection: use offload_threads = N with -z = 1

commit bbe6eb1f255d190b6fa08fe6d9471681a430a165
Author: russ <rucombs@cisco.com>
Date:   Tue Jun 4 21:26:34 2019 -0400

    analyzer: 1024 contexts max is a better default until configurable

commit 45c29b39d7bdbdd3f7271d120899e14f67f8d40a
Author: russ <rucombs@cisco.com>
Date:   Tue Jun 4 09:45:08 2019 -0400

    detection: start offload threads before packet threads are pinned

commit f5788a9b17cea3545c05932d365c5736c1de5b54
Author: russ <rucombs@cisco.com>
Date:   Tue Jun 4 09:41:41 2019 -0400

    mpse: api init and print methods are optional

commit 619b7846de7cbd1d5962c92850ba855e3ce586d6
Author: russ <rucombs@cisco.com>
Date:   Sat Jun 1 13:48:43 2019 -0400

    ips: add missing non-fast-pattern warning

commit 05fd308f43484b2ed79a6a9d646aa203d2d1ffdd
Author: russ <rucombs@cisco.com>
Date:   Sat Jun 1 13:47:59 2019 -0400

    stream_tcp: fix non-deep detect profile exclusion

commit d141982727775c23eb0503550b4b89e77d3971a3
Author: russ <rucombs@cisco.com>
Date:   Fri May 31 16:32:29 2019 -0400

    snort: remove out-of-date Snort 2 version from -V
---

diff --git a/src/detection/detection_engine.cc b/src/detection/detection_engine.cc
index 5fa4dcf7f..98f02c7da 100644
--- a/src/detection/detection_engine.cc
+++ b/src/detection/detection_engine.cc
@@ -369,52 +369,31 @@ bool DetectionEngine::do_offload(Packet* p)
 
     p->context->conf = SnortConfig::get_conf();
 
-    // Build the searches list in the packet context
-    fp_partial(p);
-
-    if (p->context->searches.items.size() > 0)
-    {
-        trace_logf(detection, TRACE_DETECTION_ENGINE, "%" PRIu64 " de::offload %" PRIu64
-                " (r=%d)\n", p->context->packet_number, p->context->context_num,
-                offloader->count());
+    trace_logf(detection, TRACE_DETECTION_ENGINE, "%" PRIu64 " de::offload %" PRIu64
+        " (r=%d)\n", p->context->packet_number, p->context->context_num,
+        offloader->count());
 
-        sw->suspend();
-        p->set_offloaded();
+    sw->suspend();
+    p->set_offloaded();
 
-        offloader->put(p);
-        pc.offloads++;
+    offloader->put(p);
+    pc.offloads++;
 
-        return true;
-    }
-    else
-    {
-        bool depends_on_suspended = p->flow ? p->flow->context_chain.front() :
-            sw->non_flow_chain.front();
-
-        if (!depends_on_suspended)
-        {
-            fp_complete(p);
-            return false;
-        }
-        else
-        {
-            sw->suspend();
-            pc.offload_suspends++;
-            return true;
-        }
-    }
+#ifdef REG_TEST
+    onload();
+    return false;
+#else
+    return true;
+#endif
 }
 
 bool DetectionEngine::offload(Packet* p)
 {
     ContextSwitcher* sw = Analyzer::get_switcher();
+    fp_partial(p);
 
-    bool depends_on_suspended = 
-        p->flow ? p->flow->context_chain.front() : sw->non_flow_chain.front();
-
-    bool should_offload = p->dsize >= SnortConfig::get_conf()->offload_limit;
-
-    if ( should_offload )
+    if ( p->dsize >= SnortConfig::get_conf()->offload_limit and
+        p->context->searches.items.size() > 0 )
     {
         if ( offloader->available() )
             return do_offload(p);
@@ -422,9 +401,8 @@ bool DetectionEngine::offload(Packet* p)
         pc.offload_busy++;
     }
 
-    if ( depends_on_suspended )
+    if ( p->flow ? p->flow->context_chain.front() : sw->non_flow_chain.front() )
     {
-        fp_partial(p);
         p->context->searches.search_sync();
         sw->suspend();
         pc.offload_suspends++;
@@ -432,7 +410,7 @@ bool DetectionEngine::offload(Packet* p)
     }
 
     assert(p->flow ? !p->flow->is_suspended() : true);
-    fp_full(p);
+    fp_complete(p, true);
     return false;
 }
 
@@ -489,10 +467,16 @@ void DetectionEngine::onload()
 void DetectionEngine::resume_ready_suspends(IpsContextChain& chain)
 {
     while ( chain.front() and !chain.front()->packet->is_offloaded() )
+    {
+#ifdef REG_TEST
+        complete(chain.front()->packet);
+#else
         resume(chain.front()->packet);
+#endif
+    }
 }
 
-void DetectionEngine::resume(Packet* p)
+void DetectionEngine::complete(Packet* p)
 {
     trace_logf(detection, TRACE_DETECTION_ENGINE, "%" PRIu64 " de::resume %" PRIu64 " (r=%d)\n",
         p->context->packet_number, p->context->context_num, offloader->count());
@@ -501,6 +485,12 @@ void DetectionEngine::resume(Packet* p)
     sw->resume(p->context);
     
     fp_complete(p);
+}
+
+void DetectionEngine::resume(Packet* p)
+{
+    complete(p);
+
     finish_inspect_with_latency(p); // FIXIT-L should latency be evaluated here?
     finish_inspect(p, true);
     finish_packet(p);
diff --git a/src/detection/detection_engine.h b/src/detection/detection_engine.h
index c533a25da..b22683d0c 100644
--- a/src/detection/detection_engine.h
+++ b/src/detection/detection_engine.h
@@ -108,6 +108,7 @@ private:
     static struct SF_EVENTQ* get_event_queue();
     static bool do_offload(snort::Packet*);
     static void offload_thread(IpsContext*);
+    static void complete(snort::Packet*);
     static void resume(snort::Packet*);
     static void resume_ready_suspends(IpsContextChain&);
 
diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc
index c75320108..4e34f3b9d 100644
--- a/src/detection/fp_create.cc
+++ b/src/detection/fp_create.cc
@@ -513,9 +513,7 @@ static int fpAddPortGroupRule(
 {
     const MpseApi* search_api = nullptr;
     const MpseApi* offload_search_api = nullptr;
-    PatternMatchVector pmv;
     OptFpList* next = nullptr;
-    bool only_literal;
     bool exclude;
 
     // skip builtin rules, continue for text and so rules
@@ -528,9 +526,8 @@ static int fpAddPortGroupRule(
     search_api = fp->get_search_api();
     assert(search_api);
 
-    only_literal = !MpseManager::is_regex_capable(search_api);
-
-    pmv = get_fp_content(otn, next, srvc, only_literal, exclude);
+    bool only_literal = !MpseManager::is_regex_capable(search_api);
+    PatternMatchVector pmv = get_fp_content(otn, next, srvc, only_literal, exclude);
 
     if ( !pmv.empty() )
     {
@@ -565,8 +562,8 @@ static int fpAddPortGroupRule(
             pmv.pop_back();
 
             if ( !main_pmd->is_relative() && !main_pmd->is_negated() && main_pmd->fp_only >= 0 &&
-                    // FIXIT-L no_case consideration is mpse specific, delegate
-                    !main_pmd->offset && !main_pmd->depth && main_pmd->is_no_case() )
+                // FIXIT-L no_case consideration is mpse specific, delegate
+                !main_pmd->offset && !main_pmd->depth && main_pmd->is_no_case() )
             {
                 if ( !next || !next->ips_opt || !next->ips_opt->is_relative() )
                     main_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_NORMAL);
@@ -592,7 +589,8 @@ static int fpAddPortGroupRule(
             {
                 if (!pg->mpsegrp[main_pmd->pm_type]->create_normal_mpse(sc, &agent))
                 {
-                    ParseError("Failed to create normal pattern matcher for %d", main_pmd->pm_type);
+                    ParseError("Failed to create normal pattern matcher for %d",
+                        main_pmd->pm_type);
                     return -1;
                 }
 
@@ -607,8 +605,8 @@ static int fpAddPortGroupRule(
                 pmv_ol.pop_back();
 
                 if ( !ol_pmd->is_relative() && !ol_pmd->is_negated() && ol_pmd->fp_only >= 0 &&
-                        // FIXIT-L no_case consideration is mpse specific, delegate
-                        !ol_pmd->offset && !ol_pmd->depth && ol_pmd->is_no_case() )
+                    // FIXIT-L no_case consideration is mpse specific, delegate
+                    !ol_pmd->offset && !ol_pmd->depth && ol_pmd->is_no_case() )
                 {
                     if ( !next_ol || !next_ol->ips_opt || !next_ol->ips_opt->is_relative() )
                         ol_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_OFFLOAD);
@@ -626,7 +624,7 @@ static int fpAddPortGroupRule(
                     if (!pg->mpsegrp[main_pmd->pm_type]->create_offload_mpse(sc, &agent_offload))
                     {
                         ParseError("Failed to create offload pattern matcher for %d",
-                                main_pmd->pm_type);
+                            main_pmd->pm_type);
                         return -1;
                     }
 
@@ -647,7 +645,7 @@ static int fpAddPortGroupRule(
 
                 // Now add patterns
                 if (fpFinishPortGroupRule(sc, pg->mpsegrp[main_pmd->pm_type]->normal_mpse,
-                            otn, main_pmd, fp, Mpse::MPSE_TYPE_NORMAL, true) == 0)
+                        otn, main_pmd, fp, Mpse::MPSE_TYPE_NORMAL, true) == 0)
                 {
                     if (main_pmd->pattern_size > otn->longestPatternLen)
                         otn->longestPatternLen = main_pmd->pattern_size;
@@ -655,7 +653,7 @@ static int fpAddPortGroupRule(
                     // Add Alternative patterns
                     for (auto p : pmv)
                         fpAddAlternatePatterns(sc, pg->mpsegrp[main_pmd->pm_type]->normal_mpse,
-                                otn, p, fp, Mpse::MPSE_TYPE_NORMAL);
+                            otn, p, fp, Mpse::MPSE_TYPE_NORMAL);
                 }
             }
 
@@ -667,7 +665,7 @@ static int fpAddPortGroupRule(
 
                 // Now add patterns
                 if (fpFinishPortGroupRule(sc, pg->mpsegrp[main_pmd->pm_type]->offload_mpse,
-                            otn, ol_pmd, fp, Mpse::MPSE_TYPE_OFFLOAD, true) == 0)
+                        otn, ol_pmd, fp, Mpse::MPSE_TYPE_OFFLOAD, true) == 0)
                 {
                     if (ol_pmd->pattern_size > otn->longestPatternLen)
                         otn->longestPatternLen = ol_pmd->pattern_size;
@@ -675,18 +673,20 @@ static int fpAddPortGroupRule(
                     // Add Alternative patterns
                     for (auto p : pmv_ol)
                         fpAddAlternatePatterns(sc, pg->mpsegrp[main_pmd->pm_type]->offload_mpse,
-                                otn, p, fp, Mpse::MPSE_TYPE_OFFLOAD);
+                            otn, p, fp, Mpse::MPSE_TYPE_OFFLOAD);
                 }
             }
 
-            if (add_rule)
+            if ( add_rule )
             {
-                if (add_nfp_rule)
-                    pg->add_nfp_rule(otn);
-                else
+                if ( !add_nfp_rule )
                     pg->add_rule();
+                else
+                {
+                    pg->add_nfp_rule(otn);
+                    print_nfp_info(s_group, otn);
+                }
             }
-
             return 0;
         }
     }
@@ -1775,8 +1775,10 @@ static void print_nfp_info(const char* group, OptTreeNode* otn)
     if ( otn->warned_fp() )
         return;
 
-    ParseWarning(WARN_RULES, "%s rule %u:%u:%u has no fast pattern",
-        group, otn->sigInfo.gid, otn->sigInfo.sid, otn->sigInfo.rev);
+    const char* type = otn->longestPatternLen ? "negated" : "no";
+
+    ParseWarning(WARN_RULES, "%s rule %u:%u:%u has %s fast pattern",
+        group, otn->sigInfo.gid, otn->sigInfo.sid, otn->sigInfo.rev, type);
 
     otn->set_warned_fp();
 }
diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc
index e5db0d3dc..2020dc336 100644
--- a/src/detection/fp_detect.cc
+++ b/src/detection/fp_detect.cc
@@ -93,6 +93,8 @@ THREAD_LOCAL ProfileStats ruleRTNEvalPerfStats;
 THREAD_LOCAL ProfileStats ruleOTNEvalPerfStats;
 THREAD_LOCAL ProfileStats ruleNFPEvalPerfStats;
 
+static void fp_immediate(Packet*);
+
 // Initialize the OtnxMatchData structure.  We do this for
 // every packet so this only sets the necessary counters to
 // zero which saves us time.
@@ -873,8 +875,8 @@ static inline int batch_search(
     assert(so->get_normal_mpse()->get_pattern_count() > 0);
     cnt++;
 
-    //FIXIT-P Batch outer UDP payload searches for teredo set and the outer header
-    //during any signature evaluation
+    // FIXIT-P Batch outer UDP payload searches for teredo set and the outer header
+    // during any signature evaluation
     if ( omd->p->ptrs.udph && (omd->p->proto_bits & (PROTO_BIT__TEREDO | PROTO_BIT__GTP)) )
     {
         int start_state = 0;
@@ -985,110 +987,143 @@ static int fp_search(
     return 0;
 }
 
-//  This function does a set-wise match on content, and walks an otn list
-//  for non-content.  The otn list search will eventually be redone for
-//  for performance purposes.
-
-static inline int fpEvalHeaderSW(PortGroup* port_group, Packet* p,
-    int check_ports, char ip_rule, int type, OtnxMatchData* omd, FPTask task)
+static inline void eval_fp(
+    PortGroup* port_group, Packet* p, OtnxMatchData* omd, char ip_rule,
+    int check_ports, int type)
 {
-    if ( !p->is_detection_enabled(p->packet_flags & PKT_FROM_CLIENT) )
-        return 0;
-
     const uint8_t* tmp_payload = nullptr;
-    int8_t curr_ip_layer = 0;
-    bool repeat = false;
     uint16_t tmp_dsize = 0;
-    FastPatternConfig* fp = SnortConfig::get_conf()->fast_pattern_config;
 
-    print_pkt_info(p);
+    if ( !ip_rule )
+        p->packet_flags &= ~PKT_IP_RULE;
 
-    if ( task & FPTask::FP )
+    else
     {
-        if (ip_rule)
-        {
-            tmp_payload = p->data;
-            tmp_dsize = p->dsize;
+        int8_t curr_ip_layer = 0;
 
-            if (layer::set_outer_ip_api(p, p->ptrs.ip_api, curr_ip_layer))
-            {
-                p->data = p->ptrs.ip_api.ip_data();
-                p->dsize = p->ptrs.ip_api.pay_len();
-                p->packet_flags |= PKT_IP_RULE;
-                repeat = true;
-            }
-        }
-        else
+        tmp_payload = p->data;  // FIXIT-H restore even with offload
+        tmp_dsize = p->dsize;
+
+        if (layer::set_outer_ip_api(p, p->ptrs.ip_api, curr_ip_layer))
         {
-            p->packet_flags &= ~PKT_IP_RULE;
+            p->data = p->ptrs.ip_api.ip_data();
+            p->dsize = p->ptrs.ip_api.pay_len();
+            p->packet_flags |= PKT_IP_RULE;
         }
+    }
+
+    if ( DetectionEngine::content_enabled(p) )
+    {
+        FastPatternConfig* fp = SnortConfig::get_conf()->fast_pattern_config;
 
-        if ( DetectionEngine::content_enabled(p) )
+        if ( fp->get_stream_insert() || !(p->packet_flags & PKT_STREAM_INSERT) )
+            if ( fp_search(port_group, p, check_ports, type, omd) )
+                return;
+    }
+    if ( ip_rule )
+    {
+        p->data = tmp_payload;
+        p->dsize = tmp_dsize;
+    }
+}
+
+static inline void eval_nfp(
+    PortGroup* port_group, Packet* p, OtnxMatchData* omd, char ip_rule)
+{
+    bool repeat = false;
+    int8_t curr_ip_layer = 0;
+
+    const uint8_t* tmp_payload = nullptr;
+    uint16_t tmp_dsize = 0;
+
+    FastPatternConfig* fp = SnortConfig::get_conf()->fast_pattern_config;
+
+    if (ip_rule)
+    {
+        tmp_payload = p->data;
+        tmp_dsize = p->dsize;
+
+        if (layer::set_outer_ip_api(p, p->ptrs.ip_api, curr_ip_layer))
         {
-            if ( fp->get_stream_insert() || !(p->packet_flags & PKT_STREAM_INSERT) )
-                if ( fp_search(port_group, p, check_ports, type, omd) )
-                    return 0;
+            p->data = p->ptrs.ip_api.ip_data();
+            p->dsize = p->ptrs.ip_api.pay_len();
+            p->packet_flags |= PKT_IP_RULE;
+            repeat = true;
         }
     }
-
-    if ( task & FPTask::NON_FP )
+    do
     {
-        do
+        if (port_group->nfp_rule_count)
         {
-            if (port_group->nfp_rule_count)
-            {
-                // walk and test the nfp OTNs
-                if ( fp->get_debug_print_nc_rules() )
-                    LogMessage("NC-testing %u rules\n", port_group->nfp_rule_count);
+            // walk and test the nfp OTNs
+            if ( fp->get_debug_print_nc_rules() )
+                LogMessage("NC-testing %u rules\n", port_group->nfp_rule_count);
 
-                detection_option_eval_data_t eval_data;
+            detection_option_eval_data_t eval_data;
 
-                eval_data.pomd = omd;
-                eval_data.p = p;
-                eval_data.pmd = nullptr;
-                eval_data.flowbit_failed = 0;
-                eval_data.flowbit_noalert = 0;
+            eval_data.pomd = omd;
+            eval_data.p = p;
+            eval_data.pmd = nullptr;
+            eval_data.flowbit_failed = 0;
+            eval_data.flowbit_noalert = 0;
 
-                int rval = 0;
-                {
-                    DeepProfile rule_profile(rulePerfStats);
-                    DeepProfile rule_nfp_eval_profile(ruleNFPEvalPerfStats);
-                    trace_log(detection, TRACE_RULE_EVAL, "Testing non-content rules\n");
-                    rval = detection_option_tree_evaluate(
-                        (detection_option_tree_root_t*)port_group->nfp_tree, &eval_data);
-                }
+            int rval = 0;
+            {
+                DeepProfile rule_profile(rulePerfStats);
+                DeepProfile rule_nfp_eval_profile(ruleNFPEvalPerfStats);
+                trace_log(detection, TRACE_RULE_EVAL, "Testing non-content rules\n");
+                rval = detection_option_tree_evaluate(
+                    (detection_option_tree_root_t*)port_group->nfp_tree, &eval_data);
+            }
 
-                if (rval)
-                    pmqs.qualified_events++;
-                else
-                    pmqs.non_qualified_events++;
+            if (rval)
+                pmqs.qualified_events++;
+            else
+                pmqs.non_qualified_events++;
 
-                pc.hard_evals++;
-            }
+            pc.hard_evals++;
+        }
 
-            // FIXIT-L need to eval all IP layers, etc.
-            // FIXIT-L why run only nfp rules?
-            if (ip_rule)
+        // FIXIT-L should really be logging any events based on curr_ip_layer
+        if (ip_rule)
+        {
+            /* Evaluate again with the next IP layer */
+            if (layer::set_outer_ip_api(p, p->ptrs.ip_api, curr_ip_layer))
             {
-                /* Evaluate again with the next IP layer */
-                if (layer::set_outer_ip_api(p, p->ptrs.ip_api, curr_ip_layer))
-                {
-                    p->data = p->ptrs.ip_api.ip_data();
-                    p->dsize = p->ptrs.ip_api.pay_len();
-                    p->packet_flags |= PKT_IP_RULE_2ND | PKT_IP_RULE;
-                }
-                else
-                {
-                    /* Set the data & dsize back to original values. */
-                    p->data = tmp_payload;
-                    p->dsize = tmp_dsize;
-                    p->packet_flags &= ~(PKT_IP_RULE| PKT_IP_RULE_2ND);
-                    repeat = false;
-                }
+                p->data = p->ptrs.ip_api.ip_data();
+                p->dsize = p->ptrs.ip_api.pay_len();
+                p->packet_flags |= PKT_IP_RULE_2ND | PKT_IP_RULE;
+            }
+            else
+            {
+                /* Set the data & dsize back to original values. */
+                p->data = tmp_payload;
+                p->dsize = tmp_dsize;
+                p->packet_flags &= ~(PKT_IP_RULE| PKT_IP_RULE_2ND);
+                repeat = false;
             }
         }
-        while (repeat);
     }
+    while (repeat);
+}
+
+//  This function does a set-wise match on content, and walks an otn list
+//  for non-content.  The otn list search will eventually be redone for
+//  for performance purposes.
+
+static inline int fpEvalHeaderSW(PortGroup* port_group, Packet* p,
+    int check_ports, char ip_rule, int type, OtnxMatchData* omd, FPTask task)
+{
+    if ( !p->is_detection_enabled(p->packet_flags & PKT_FROM_CLIENT) )
+        return 0;
+
+    print_pkt_info(p);
+
+    if ( task & FPTask::FP )
+        eval_fp(port_group, p, omd, ip_rule, check_ports, type);
+
+    if ( task & FPTask::NON_FP )
+        eval_nfp(port_group, p, omd, ip_rule);
 
     return 0;
 }
@@ -1220,6 +1255,10 @@ static void fpEvalPacketUdp(Packet* p, OtnxMatchData* omd, FPTask task)
 
     fpEvalHeaderUdp(p, omd, task);
 
+    // FIXIT-P Batch outer UDP payload searches for teredo set and the outer header
+    // during any signature evaluation
+    fp_immediate(p);
+
     p->ptrs.sp = tmp_sp;
     p->ptrs.dp = tmp_dp;
     p->ptrs.udph = tmp_udph;
@@ -1297,44 +1336,48 @@ static void fpEvalPacket(Packet* p, FPTask task)
     }
 }
 
-void fp_full(Packet* p)
+void fp_partial(Packet* p)
 {
     IpsContext* c = p->context;
     MpseStash* stash = c->stash;
     stash->enable_process();
     stash->init();
+    stash->disable_process();
     init_match_info(c->otnx);
-
     c->searches.mf = rule_tree_queue;
     c->searches.context = c;
-    fpEvalPacket(p, FPTask::BOTH);
-
-    if ( c->searches.search_sync() )
-        stash->process(rule_tree_match, c);
-
-    fpFinalSelectEvent(c->otnx, p);
+    assert(!c->searches.items.size());
+    fpEvalPacket(p, FPTask::FP);
 }
 
-void fp_partial(Packet* p)
+void fp_complete(Packet* p, bool search)
 {
     IpsContext* c = p->context;
     MpseStash* stash = c->stash;
     stash->enable_process();
-    stash->init();
-    stash->disable_process();
-    init_match_info(c->otnx);
-    c->searches.mf = rule_tree_queue;
-    c->searches.context = c;
-    fpEvalPacket(p, FPTask::FP);
+
+    if ( search )
+        c->searches.search_sync();
+
+    stash->process(rule_tree_match, c);
+    fpEvalPacket(p, FPTask::NON_FP);
+    fpFinalSelectEvent(c->otnx, p);
+    c->searches.items.clear();
 }
 
-void fp_complete(Packet* p)
+void fp_full(Packet* p)
+{
+    fp_partial(p);
+    fp_complete(p, true);
+}
+
+static void fp_immediate(Packet* p)
 {
     IpsContext* c = p->context;
     MpseStash* stash = c->stash;
     stash->enable_process();
+    c->searches.search_sync();
     stash->process(rule_tree_match, c);
-    fpEvalPacket(p, FPTask::NON_FP);
-    fpFinalSelectEvent(c->otnx, p);
+    c->searches.items.clear();
 }
 
diff --git a/src/detection/fp_detect.h b/src/detection/fp_detect.h
index e4e62f1d9..432bb80fa 100644
--- a/src/detection/fp_detect.h
+++ b/src/detection/fp_detect.h
@@ -99,7 +99,7 @@ void fp_clear_context(snort::IpsContext&);
 
 void fp_full(snort::Packet*);
 void fp_partial(snort::Packet*);
-void fp_complete(snort::Packet*);
+void fp_complete(snort::Packet*, bool search = false);
 
 #endif
 
diff --git a/src/detection/regex_offload.cc b/src/detection/regex_offload.cc
index 8bea6ca57..7dcea543f 100644
--- a/src/detection/regex_offload.cc
+++ b/src/detection/regex_offload.cc
@@ -38,6 +38,8 @@
 #include "latency/packet_latency.h"
 #include "latency/rule_latency.h"
 #include "main/snort_config.h"
+#include "main/thread.h"
+#include "main/thread_config.h"
 #include "managers/module_manager.h"
 #include "utils/stats.h"
 
@@ -52,6 +54,12 @@ struct RegexRequest
     std::mutex mutex;
     std::condition_variable cond;
 
+#ifdef REG_TEST
+    // used to make main thread wait for results to get predictable behavior
+    std::mutex sync_mutex;
+    std::condition_variable sync_cond;
+#endif
+
     std::atomic<bool> offload { false };
 
     bool go = true;
@@ -170,8 +178,10 @@ bool MpseRegexOffload::get(snort::Packet*& p)
 
 ThreadRegexOffload::ThreadRegexOffload(unsigned max) : RegexOffload(max)
 {
+    unsigned i = ThreadConfig::get_instance_max();
+
     for ( auto* req : idle )
-        req->thread = new std::thread(worker, req, snort::SnortConfig::get_conf());
+        req->thread = new std::thread(worker, req, snort::SnortConfig::get_conf(), i++);
 }
 
 ThreadRegexOffload::~ThreadRegexOffload()
@@ -207,11 +217,21 @@ void ThreadRegexOffload::put(snort::Packet* p)
     busy.emplace_back(req);
     p->context->regex_req_it = std::prev(busy.end());
 
-    std::unique_lock<std::mutex> lock(req->mutex);
-    req->packet = p;
+    {
+        std::unique_lock<std::mutex> lock(req->mutex);
+        req->packet = p;
+
+        req->offload = true;
+        req->cond.notify_one();
+    }
 
-    req->offload = true;
-    req->cond.notify_one();
+#ifdef REG_TEST
+    {
+        std::unique_lock<std::mutex> sync_lock(req->sync_mutex);
+        while ( req->offload and req->sync_cond.wait_for(sync_lock, std::chrono::seconds(1))
+            == std::cv_status::timeout );
+    }
+#endif
 }
 
 bool ThreadRegexOffload::get(snort::Packet*& p)
@@ -239,8 +259,10 @@ bool ThreadRegexOffload::get(snort::Packet*& p)
     return false;
 }
 
-void ThreadRegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_config)
+void ThreadRegexOffload::worker(
+    RegexRequest* req, snort::SnortConfig* initial_config, unsigned id)
 {
+    set_instance_id(id);
     snort::SnortConfig::set_conf(initial_config);
 
     while ( true )
@@ -249,11 +271,6 @@ void ThreadRegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_c
             std::unique_lock<std::mutex> lock(req->mutex);
             req->cond.wait_for(lock, std::chrono::seconds(1));
 
-            // setting conf is somewhat expensive, checking the conf is not
-            // this occurs here to take advantage if idling
-            if ( req->packet and req->packet->context->conf != snort::SnortConfig::get_conf() )
-                snort::SnortConfig::set_conf(req->packet->context->conf);
-
             if ( !req->go )
                 break;
 
@@ -265,10 +282,12 @@ void ThreadRegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_c
         assert(req->packet->is_offloaded());
         assert(req->packet->context->searches.items.size() > 0);
 
+        snort::SnortConfig::set_conf(req->packet->context->conf);
         snort::IpsContext* c = req->packet->context;
         snort::Mpse::MpseRespType resp_ret;
 
         c->searches.offload_search();
+
         do
         {
             resp_ret = c->searches.receive_offload_responses();
@@ -287,6 +306,13 @@ void ThreadRegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_c
 
         c->searches.items.clear();
         req->offload = false;
+
+#ifdef REG_TEST
+        {
+            std::unique_lock<std::mutex> lock(req->sync_mutex);
+            req->sync_cond.notify_one();
+        }
+#endif
     }
     snort::ModuleManager::accumulate_offload("search_engine");
     snort::ModuleManager::accumulate_offload("detection");
diff --git a/src/detection/regex_offload.h b/src/detection/regex_offload.h
index fdc5faf8b..782fb1609 100644
--- a/src/detection/regex_offload.h
+++ b/src/detection/regex_offload.h
@@ -89,7 +89,7 @@ public:
     bool get(snort::Packet*&) override;
 
 private:
-    static void worker(RegexRequest*, snort::SnortConfig*);
+    static void worker(RegexRequest*, snort::SnortConfig*, unsigned id);
 };
 
 #endif
diff --git a/src/framework/mpse.cc b/src/framework/mpse.cc
index e23529fdd..0158aaa8f 100644
--- a/src/framework/mpse.cc
+++ b/src/framework/mpse.cc
@@ -90,17 +90,12 @@ void Mpse::_search(MpseBatch& batch, MpseType mpse_type)
         for ( auto& so : item.second.so )
         {
             start_state = 0;
-            switch (mpse_type)
-            {
-                case MPSE_TYPE_NORMAL:
-                    item.second.matches += so->get_normal_mpse()->search(item.first.buf,
-                            item.first.len, batch.mf, batch.context, &start_state);
-                    break;
-                case MPSE_TYPE_OFFLOAD:
-                    item.second.matches += so->get_offload_mpse()->search(item.first.buf,
-                            item.first.len, batch.mf, batch.context, &start_state);
-                    break;
-            }
+
+            Mpse* mpse = (mpse_type == MPSE_TYPE_OFFLOAD) ?
+                so->get_offload_mpse() : so->get_normal_mpse();
+
+            item.second.matches += mpse->search(
+                item.first.buf, item.first.len, batch.mf, batch.context, &start_state);
         }
         item.second.done = true;
     }
@@ -108,21 +103,23 @@ void Mpse::_search(MpseBatch& batch, MpseType mpse_type)
 
 Mpse::MpseRespType Mpse::poll_responses(MpseBatch*& batch, MpseType mpse_type)
 {
+    FastPatternConfig* fp = SnortConfig::get_conf()->fast_pattern_config;
+    assert(fp);
+
     const MpseApi* search_api = nullptr;
 
-    if ( SnortConfig::get_conf()->fast_pattern_config )
+    switch (mpse_type)
     {
-        switch (mpse_type)
-        {
-            case MPSE_TYPE_NORMAL:
-                search_api = SnortConfig::get_conf()->fast_pattern_config->get_search_api();
-                break;
-            case MPSE_TYPE_OFFLOAD:
-                search_api = SnortConfig::get_conf()->fast_pattern_config->get_offload_search_api();
-                if (!search_api)
-                    search_api = SnortConfig::get_conf()->fast_pattern_config->get_search_api();
-                break;
-        }
+    case MPSE_TYPE_NORMAL:
+        search_api = fp->get_search_api();
+        break;
+
+    case MPSE_TYPE_OFFLOAD:
+        search_api = fp->get_offload_search_api();
+
+        if (!search_api)
+            search_api = fp->get_search_api();
+        break;
     }
 
     if (search_api)
diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc
index 892e8e905..00f79c4db 100644
--- a/src/ips_options/ips_regex.cc
+++ b/src/ips_options/ips_regex.cc
@@ -176,8 +176,8 @@ IpsOption::EvalStatus RegexOption::eval(Cursor& c, Packet*)
     if ( pos > c.size() )
         return NO_MATCH;
 
-    hs_scratch_t *ss =
-        (hs_scratch_t *) SnortConfig::get_conf()->state[get_instance_id()][scratch_index];
+    hs_scratch_t* ss =
+        (hs_scratch_t*)SnortConfig::get_conf()->state[get_instance_id()][scratch_index];
 
     s_to = 0;
 
diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc
index e304e0134..6d028f110 100644
--- a/src/main/analyzer.cc
+++ b/src/main/analyzer.cc
@@ -478,7 +478,11 @@ void Analyzer::idle()
 void Analyzer::init_unprivileged()
 {
     // using dummy values until further integration
+#ifdef REG_TEST
     const unsigned max_contexts = 20;
+#else
+    const unsigned max_contexts = 1024;
+#endif
 
     switcher = new ContextSwitcher;
 
@@ -486,7 +490,6 @@ void Analyzer::init_unprivileged()
         switcher->push(new IpsContext);
 
     SnortConfig* sc = SnortConfig::get_conf();
-
     CodecManager::thread_init(sc);
 
     // this depends on instantiated daq capabilities
@@ -496,7 +499,6 @@ void Analyzer::init_unprivileged()
     InitTag();
     EventTrace_Init();
     detection_filter_init(sc->detection_filter_config);
-    DetectionEngine::thread_init();
 
     EventManager::open_outputs();
     IpsManager::setup_options();
@@ -606,6 +608,9 @@ void Analyzer::operator()(Swapper* ps, uint16_t run_num)
     if (SnortConfig::pcap_show())
         show_source();
 
+    // init here to pin separately from packet threads
+    DetectionEngine::thread_init();
+
     // Perform all packet thread initialization actions that need to be taken with escalated
     // privileges prior to starting the DAQ module.
     SnortConfig::get_conf()->thread_config->implement_thread_affinity(STHREAD_TYPE_PACKET,
diff --git a/src/main/modules.cc b/src/main/modules.cc
index fed81a21e..796df16da 100644
--- a/src/main/modules.cc
+++ b/src/main/modules.cc
@@ -110,7 +110,9 @@ class DetectionModule : public Module
 public:
     DetectionModule() :
         Module("detection", detection_help, detection_params, false, &TRACE_NAME(detection)) {}
+
     bool set(const char*, Value&, SnortConfig*) override;
+    bool end(const char*, int, SnortConfig*) override;
 
     const PegInfo* get_pegs() const override
     { return pc_names; }
@@ -122,6 +124,14 @@ public:
     { return GLOBAL; }
 };
 
+bool DetectionModule::end(const char*, int, SnortConfig* sc)
+{
+    if ( sc->offload_threads and ThreadConfig::get_instance_max() != 1 )
+        ParseError("You can not enable experimental offload with more than one packet thread.");
+
+    return true;
+}
+
 bool DetectionModule::set(const char* fqn, Value& v, SnortConfig* sc)
 {
     if ( v.is("asn1") )
diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index 059dc7fa7..d4ba8f0b2 100644
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -194,8 +194,8 @@ void SnortConfig::init(const SnortConfig* const other_conf, ProtocolReference* p
         ActionManager::new_config(this);
         InspectorManager::new_config(this);
 
-        num_slots = ThreadConfig::get_instance_max();
-        state = new std::vector<void*>[num_slots];
+        num_slots = 0;
+        state = nullptr;
 
         profiler = new ProfilerConfig;
         latency = new LatencyConfig();
@@ -250,7 +250,7 @@ SnortConfig::~SnortConfig()
     FreeReferences(references);
 
     // Only call scratch cleanup if we actually called scratch setup
-    if ( state[0].size() > 0 )
+    if ( state and state[0].size() > 0 )
     {
         for ( unsigned i = scratch_handlers.size(); i > 0; i-- )
         {
@@ -489,8 +489,8 @@ void SnortConfig::merge(SnortConfig* cmd_line)
     // FIXIT-M should cmd_line use the same var list / table?
     var_list = nullptr;
 
-    delete[] state;
-    num_slots = ThreadConfig::get_instance_max();
+    assert(!state);
+    num_slots = offload_threads + ThreadConfig::get_instance_max();
     state = new std::vector<void*>[num_slots];
 }
 
diff --git a/src/main/snort_module.cc b/src/main/snort_module.cc
index 73eb83901..3d994140e 100644
--- a/src/main/snort_module.cc
+++ b/src/main/snort_module.cc
@@ -279,7 +279,9 @@ static const Parameter s_params[] =
     { "-y", Parameter::PT_IMPLIED, nullptr, nullptr,
       "include year in timestamp in the alert and log files" },
 
-    { "-z", Parameter::PT_INT, "0:max32", "1",
+    // do not provide parameter default as it will cause the value to change
+    // after allocations in SnortConfig if snort = { } is set in Lua
+    { "-z", Parameter::PT_INT, "0:max32", nullptr,
       "<count> maximum number of packet threads (same as --max-packet-threads); "
       "0 gets the number of CPU cores reported by the system; default is 1" },
 
@@ -407,7 +409,7 @@ static const Parameter s_params[] =
     { "--markup", Parameter::PT_IMPLIED, nullptr, nullptr,
       "output help in asciidoc compatible format" },
 
-    { "--max-packet-threads", Parameter::PT_INT, "0:max32", "1",
+    { "--max-packet-threads", Parameter::PT_INT, "0:max32", nullptr,
       "<count> configure maximum number of packet threads (same as -z)" },
 
     { "--mem-check", Parameter::PT_IMPLIED, nullptr, nullptr,
@@ -595,6 +597,7 @@ public:
 
     bool begin(const char*, int, SnortConfig*) override;
     bool set(const char*, Value&, SnortConfig*) override;
+    bool end(const char*, int, SnortConfig*) override;
 
     const PegInfo* get_pegs() const override
     { return proc_names; }
@@ -1041,6 +1044,14 @@ bool SnortModule::set(const char*, Value& v, SnortConfig* sc)
     return true;
 }
 
+bool SnortModule::end(const char*, int, SnortConfig* sc)
+{
+    if ( sc->offload_threads and ThreadConfig::get_instance_max() != 1 )
+        ParseError("You can not enable experimental offload with more than one packet thread.");
+
+    return true;
+}
+
 //-------------------------------------------------------------------------
 // singleton
 //-------------------------------------------------------------------------
diff --git a/src/main/thread.cc b/src/main/thread.cc
index da1459cc3..4abc65b20 100644
--- a/src/main/thread.cc
+++ b/src/main/thread.cc
@@ -73,7 +73,8 @@ SThreadType get_thread_type()
 const char* get_instance_file(std::string& file, const char* name)
 {
     bool sep = false;
-    file = !snort::SnortConfig::get_conf()->log_dir.empty() ? snort::SnortConfig::get_conf()->log_dir : "./";
+    file = !snort::SnortConfig::get_conf()->log_dir.empty() ?
+        snort::SnortConfig::get_conf()->log_dir : "./";
 
     if ( file.back() != '/' )
         file += '/';
@@ -87,7 +88,8 @@ const char* get_instance_file(std::string& file, const char* name)
     if ( (ThreadConfig::get_instance_max() > 1) || snort::SnortConfig::get_conf()->id_zero )
     {
         char id[8];
-        snprintf(id, sizeof(id), "%u", get_instance_id() + snort::SnortConfig::get_conf()->id_offset);
+        snprintf(id, sizeof(id), "%u",
+            get_instance_id() + snort::SnortConfig::get_conf()->id_offset);
         file += id;
         sep = true;
     }
diff --git a/src/managers/mpse_manager.cc b/src/managers/mpse_manager.cc
index 656e9d2df..2e851fcf4 100644
--- a/src/managers/mpse_manager.cc
+++ b/src/managers/mpse_manager.cc
@@ -82,7 +82,7 @@ const MpseApi* MpseManager::get_search_api(const char* name)
 {
     const MpseApi* api = ::get_api(name);
 
-    if ( api )
+    if ( api and api->init )
         api->init();
 
     return api;
@@ -134,6 +134,7 @@ void MpseManager::delete_search_engine(Mpse* eng)
 void MpseManager::print_mpse_summary(const MpseApi* api)
 {
     assert(api);
+    if ( api->print )
     api->print();
 }
 
diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc
index 15c1201f8..a506e2193 100644
--- a/src/search_engines/hyperscan.cc
+++ b/src/search_engines/hyperscan.cc
@@ -280,7 +280,8 @@ int HyperscanMpse::_search(
     match_cb = mf;
     match_ctx = pv;
 
-    hs_scratch_t *ss = (hs_scratch_t *) SnortConfig::get_conf()->state[get_instance_id()][scratch_index];
+    hs_scratch_t* ss =
+        (hs_scratch_t*)SnortConfig::get_conf()->state[get_instance_id()][scratch_index];
 
     // scratch is null for the degenerate case w/o patterns
     assert(!hs_db or ss);
diff --git a/src/stream/tcp/tcp_reassembler.cc b/src/stream/tcp/tcp_reassembler.cc
index 6c78b2e77..3241b996e 100644
--- a/src/stream/tcp/tcp_reassembler.cc
+++ b/src/stream/tcp/tcp_reassembler.cc
@@ -583,7 +583,11 @@ int TcpReassembler::_flush_to_seq(
             tcpStats.rebuilt_packets++;
             tcpStats.rebuilt_bytes += flushed_bytes;
 
+#ifdef DEEP_PROFILING
             NoProfile exclude(s5TcpFlushPerfStats);
+#else
+            NoProfile exclude(s5TcpPerfStats);
+#endif
 
             if ( !Analyzer::get_local_analyzer()->inspect_rebuilt(pdu) )
                 last_pdu = pdu;
@@ -672,7 +676,13 @@ int TcpReassembler::do_zero_byte_flush(TcpReassemblerState& trs, Packet* p, uint
         trs.flush_count++;
 
         show_rebuilt_packet(trs, pdu);
-        NoProfile profile_exclude(s5TcpFlushPerfStats);
+
+#ifdef DEEP_PROFILING
+        NoProfile exclude(s5TcpFlushPerfStats);
+#else
+        NoProfile exclude(s5TcpPerfStats);
+#endif
+
         Analyzer::get_local_analyzer()->inspect_rebuilt(pdu);
 
         if ( trs.tracker->splitter )
diff --git a/src/utils/util.cc b/src/utils/util.cc
index 1bc4a6493..1dd8c4792 100644
--- a/src/utils/util.cc
+++ b/src/utils/util.cc
@@ -95,19 +95,13 @@ void StoreSnortInfoStrings()
  ****************************************************************************/
 int DisplayBanner()
 {
-    const char* info = getenv("HOSTTYPE");
-
-    if ( !info )
-        info="from 2.9.11";  // last sync with head
-
     const char* ljv = LUAJIT_VERSION;
     while ( *ljv && !isdigit(*ljv) )
         ++ljv;
 
     LogMessage("\n");
     LogMessage("   ,,_     -*> Snort++ <*-\n");
-    LogMessage("  o\"  )~   Version %s (Build %s) %s\n",
-        VERSION, BUILD, info);
+    LogMessage("  o\"  )~   Version %s (Build %s)\n", VERSION, BUILD);
     LogMessage("   ''''    By Martin Roesch & The Snort Team\n");
     LogMessage("           http://snort.org/contact#team\n");
     LogMessage("           Copyright (C) 2014-2019 Cisco and/or its affiliates."