From: Steve Chew (stechew) <stechew@cisco.com>
Date: Mon, 28 Oct 2019 23:56:09 +0000 (-0400)
Subject: Merge pull request #1813 in SNORT/snort3 from ~RUCOMBS/snort3:conf_loading to master
X-Git-Tag: 3.0.0-263~5
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4f08a9e15ccf4adce6b4537eeed9124105677c2b;p=thirdparty%2Fsnort3.git

Merge pull request #1813 in SNORT/snort3 from ~RUCOMBS/snort3:conf_loading to master

Squashed commit of the following:

commit 6ccd7795e4be8bd78c937316a7733326676e9f7b
Author: russ <rucombs@cisco.com>
Date:   Mon Oct 28 16:05:22 2019 -0400

    rule_state: use more accurate error message and other internal fixes

commit 17249d6de69dd6d4f7361052eb3328fae497b2ac
Author: russ <rucombs@cisco.com>
Date:   Tue Oct 22 19:30:42 2019 -0400

    ips: add states member to allow separate configs for rules and states

commit ea165938df076947a3afb2c21649536404d66027
Author: russ <rucombs@cisco.com>
Date:   Tue Oct 22 19:03:56 2019 -0400

    rule_state: restore to facilitate continuous delivery

commit a05f4fb5201833fb1aff644a2cce0007c72b04fc
Author: russ <rucombs@cisco.com>
Date:   Mon Oct 21 08:09:44 2019 -0400

    ips_option::enable: default gid = 1 as with text rules

commit 815cb7d64a3261ad22d38d8c3c4ebf692911813b
Author: russ <rucombs@cisco.com>
Date:   Mon Oct 21 07:54:32 2019 -0400

    ips_option::enable: invalid gid, sid is just a warning

commit 32c5ee376908c23b1dafb87c20b6103e7f902382
Author: russ <rucombs@cisco.com>
Date:   Sun Oct 20 09:12:14 2019 -0400

    snort: dump gids and sids in sorted order

commit 4106d2784a59f7e2077dcc43966f571d70a48971
Author: russ <rucombs@cisco.com>
Date:   Sat Oct 19 10:11:11 2019 -0400

    rule_state: replace with rule option enable

    The enable option is used in rule stubs that set the state of a rule in the
    current ips policy.  The rule may have been loaded in the current or any other
    ips policy.  Text rule stubs are preferred over large LuaJIT tables since the
    latter have limit of 64K constants per function and each row consumes one
    constant.  The stubs have the same rule syntax as text rules and can be defined
    in alll the same places (ips.rules, ips.include, snort -R, snort
    --stdin-rules).  enable may be set to no, yes, or inherit and defaults to yes.

commit ba221295e607fa5a89dd3ff59d688f43bcddf8e7
Author: russ <rucombs@cisco.com>
Date:   Fri Oct 18 10:58:40 2019 -0400

    framework: map parameters for faster lookup

commit 39eed8eb4700a4dbca41381b438d48b4a441af55
Author: russ <rucombs@cisco.com>
Date:   Thu Oct 17 20:28:12 2019 -0400

    rule_state: ensure later entries override first

commit 2094997a2d7c55de3f0af390dc415fb9e6ffa4b9
Author: russ <rucombs@cisco.com>
Date:   Wed Oct 16 23:45:31 2019 -0400

    style: miscellaneous fixups

commit d98beb407148807943771e5ff13774bb1f6f2899
Author: Steve Chew <stechew@cisco.com>
Date:   Wed Oct 9 11:28:40 2019 -0400

    managers: Improve performance by using map instead of list for IPS options.

commit 590e5743d25952dc2259344c9a62df7b7e148d06
Author: Steve Chew <stechew@cisco.com>
Date:   Wed Oct 9 01:23:19 2019 -0400

    managers: Improve performance by using hash table instead of list for modules.

commit df0530c469c50383d9da4976c073b02f4d20d051
Author: russ <rucombs@cisco.com>
Date:   Mon Oct 14 11:08:39 2019 -0400

    help: remove obsoleted require(snort_config) from --dump-defaults output

commit 740b16e3b0d89649f0cf3236a0fcbfc996356235
Author: russ <rucombs@cisco.com>
Date:   Mon Oct 14 08:16:51 2019 -0400

    rule_state: switch back to standard syntax

commit 8b44fc699329a64409ccb558be8ddc8b23133a54
Author: russ <rucombs@cisco.com>
Date:   Mon Oct 14 08:16:21 2019 -0400

    lua: do not traverse tables needlessly
---

diff --git a/src/detection/rules.cc b/src/detection/rules.cc
index d00d408f0..2510b2934 100644
--- a/src/detection/rules.cc
+++ b/src/detection/rules.cc
@@ -36,34 +36,35 @@
 
 #include "treenodes.h"
 
-#ifdef REG_TEST
-#define ParseError(...) ParseWarning(WARN_RULES, __VA_ARGS__)
-#endif
-
 using namespace snort;
 
-void RuleState::apply(SnortConfig* sc)
+void RuleStateMap::apply(SnortConfig* sc)
 {
-    OptTreeNode* otn = OtnLookup(sc->otn_map, gid, sid);
-
-    if ( otn == nullptr )
-        ParseError("Rule state specified for invalid rule %u:%u", gid, sid);
-    else
+    for ( auto it : map )
     {
-        if ( sc->global_rule_state )
+        const RuleKey& k = it.first;
+        OptTreeNode* otn = OtnLookup(sc->otn_map, k.gid, k.sid);
+
+        if ( !otn )
+            ParseWarning(WARN_RULES, "Rule state specified for unknown rule %u:%u", k.gid, k.sid);
+        else
         {
-            for ( unsigned i = 0; i < sc->policy_map->ips_policy_count(); i++ )
+            if ( sc->global_rule_state )
             {
-                if ( sc->policy_map->get_ips_policy(i) )
-                    apply(sc, otn, i);
+                for ( unsigned i = 0; i < sc->policy_map->ips_policy_count(); i++ )
+                {
+                    if ( sc->policy_map->get_ips_policy(i) )
+                        apply(sc, otn, i, it.second);
+                }
             }
+            else
+                apply(sc, otn, it.second.policy_id, it.second);
         }
-        else
-            apply(sc, otn, policy);
     }
 }
 
-void RuleState::apply(SnortConfig* sc, OptTreeNode* otn, unsigned ips_num)
+void RuleStateMap::apply(
+    SnortConfig* sc, OptTreeNode* otn, unsigned ips_num, RuleState& s)
 {
     RuleTreeNode* rtn = getRtnFromOtn(otn, ips_num);
 
@@ -74,11 +75,11 @@ void RuleState::apply(SnortConfig* sc, OptTreeNode* otn, unsigned ips_num)
         return;
 
     rtn = dup_rtn(rtn);
-    update_rtn(rtn);
+    update_rtn(rtn, s);
     addRtnToOtn(sc, otn, rtn, ips_num);
 }
 
-RuleTreeNode* RuleState::dup_rtn(RuleTreeNode* rtn)
+RuleTreeNode* RuleStateMap::dup_rtn(RuleTreeNode* rtn)
 {
     RuleTreeNode* ret = new RuleTreeNode(*rtn);
 
@@ -105,23 +106,14 @@ RuleTreeNode* RuleState::dup_rtn(RuleTreeNode* rtn)
     return ret;
 }
 
-void RuleState::update_rtn(RuleTreeNode* rtn)
+void RuleStateMap::update_rtn(RuleTreeNode* rtn, RuleState& s)
 {
-    switch ( action )
-    {
-        case IpsPolicy::LOG: rtn->action = Actions::Type::LOG; break;
-        case IpsPolicy::PASS: rtn->action = Actions::Type::PASS; break;
-        case IpsPolicy::ALERT: rtn->action = Actions::Type::ALERT; break;
-        case IpsPolicy::DROP: rtn->action = Actions::Type::DROP; break;
-        case IpsPolicy::BLOCK: rtn->action = Actions::Type::BLOCK; break;
-        case IpsPolicy::RESET: rtn->action = Actions::Type::RESET; break;
-        case IpsPolicy::INHERIT_ACTION: break;
-    }
-    switch ( enable )
+    switch ( s.enable )
     {
-        case IpsPolicy::DISABLED: rtn->clear_enabled(); break;
-        case IpsPolicy::ENABLED: rtn->set_enabled(); break;
-        case IpsPolicy::INHERIT_ENABLE: break;
+    case IpsPolicy::DISABLED: rtn->clear_enabled(); break;
+    case IpsPolicy::ENABLED: rtn->set_enabled(); break;
+    case IpsPolicy::INHERIT_ENABLE: break;
     }
+    rtn->action = s.action;
 }
 
diff --git a/src/detection/rules.h b/src/detection/rules.h
index be76f1944..1a65ee121 100644
--- a/src/detection/rules.h
+++ b/src/detection/rules.h
@@ -24,6 +24,8 @@
 // misc rule and rule list support
 // FIXIT-L refactor this header
 
+#include <map>
+
 #include "actions/actions.h"
 #include "main/policy.h"
 
@@ -62,28 +64,37 @@ struct RuleListNode
     RuleListNode* next;   /* the next RuleListNode */
 };
 
-class RuleState
+struct RuleKey
 {
-public:
-    RuleState(unsigned g, unsigned s, IpsPolicy::Action a, IpsPolicy::Enable e) :
-        gid(g), sid(s), action(a), enable(e)
-    { policy = snort::get_ips_policy()->policy_id; }
-
-    virtual ~RuleState() = default;
-
-    void apply(snort::SnortConfig*);
-    void update_rtn(RuleTreeNode*);
-
-private:
     unsigned gid;
     unsigned sid;
-    unsigned policy;
 
-    IpsPolicy::Action action;
+    friend bool operator< (const RuleKey& lhs, const RuleKey& rhs)
+    { return lhs.gid < rhs.gid ? true : (lhs.gid > rhs.gid ? false : (lhs.sid < rhs.sid)); }
+};
+
+struct RuleState
+{
+    unsigned policy_id;
+    snort::Actions::Type action;
     IpsPolicy::Enable enable;
+};
 
-    void apply(snort::SnortConfig*, OptTreeNode* otn, unsigned ips_num);
+class RuleStateMap
+{
+public:
+    void add(RuleKey& key, RuleState& state)
+    { map[key] = state; }
+
+    void apply(snort::SnortConfig*);
+
+private:
     RuleTreeNode* dup_rtn(RuleTreeNode*);
+    void update_rtn(RuleTreeNode*, RuleState&);
+    void apply(snort::SnortConfig*, OptTreeNode*, unsigned ips_num, RuleState&);
+
+private:
+    std::map<RuleKey, RuleState> map;
 };
 
 #endif
diff --git a/src/detection/treenodes.h b/src/detection/treenodes.h
index c582f1be9..f99d9338a 100644
--- a/src/detection/treenodes.h
+++ b/src/detection/treenodes.h
@@ -25,6 +25,7 @@
 #include "actions/actions.h"
 #include "detection/signature.h"
 #include "detection/rule_option_types.h"
+#include "main/policy.h"
 #include "main/snort_types.h"
 #include "time/clock_defs.h"
 
@@ -139,8 +140,9 @@ struct RuleTreeNode
 struct OptTreeNode
 {
     using Flag = uint8_t;
-    static constexpr Flag WARNED_FP = 0x01;
-    static constexpr Flag STATELESS = 0x02;
+    static constexpr Flag WARNED_FP  = 0x01;
+    static constexpr Flag STATELESS  = 0x02;
+    static constexpr Flag RULE_STATE = 0x04;
 
     /* metadata about signature */
     SigInfo sigInfo;
@@ -173,6 +175,7 @@ struct OptTreeNode
     unsigned short proto_node_num = 0;
     uint16_t longestPatternLen = 0;
 
+    IpsPolicy::Enable enable;
     Flag flags = 0;
 
     void set_warned_fp()
@@ -187,6 +190,12 @@ struct OptTreeNode
     bool stateless() const
     { return flags & STATELESS; }
 
+    void set_enabled(IpsPolicy::Enable e)
+    { enable = e; flags |= RULE_STATE; }
+
+    bool is_rule_state_stub()
+    { return flags & RULE_STATE; }
+
     bool enabled_somewhere() const
     {
         for ( unsigned i = 0; i < proto_node_num; i++ )
diff --git a/src/ips_options/CMakeLists.txt b/src/ips_options/CMakeLists.txt
index d6ff245e7..affe4d62b 100644
--- a/src/ips_options/CMakeLists.txt
+++ b/src/ips_options/CMakeLists.txt
@@ -15,6 +15,7 @@ SET( PLUGIN_LIST
     ips_byte_math.cc
     ips_byte_test.cc
     ips_cvs.cc
+    ips_enable.cc
     ips_file_type.cc
     ips_flags.cc
     ips_fragbits.cc
diff --git a/src/ips_options/ips_enable.cc b/src/ips_options/ips_enable.cc
new file mode 100644
index 000000000..3c2eba32e
--- /dev/null
+++ b/src/ips_options/ips_enable.cc
@@ -0,0 +1,137 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// ips_enable.cc author Russ Combs <rucombs@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "detection/treenodes.h"
+#include "detection/rules.h"
+#include "framework/decode_data.h"
+#include "framework/ips_option.h"
+#include "framework/module.h"
+#include "main/snort_config.h"
+
+using namespace snort;
+
+#define s_name "enable"
+
+//-------------------------------------------------------------------------
+// module
+//-------------------------------------------------------------------------
+
+static const Parameter s_params[] =
+{
+    { "~enable", Parameter::PT_ENUM, "no | yes | inherit", "yes",
+      "enable or disable rule in current ips policy or use default defined by ips policy" },
+
+    { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
+};
+
+#define s_help \
+    "stub rule option to enable or disable full rule"
+
+class EnableModule : public Module
+{
+public:
+    EnableModule() : Module(s_name, s_help, s_params) { }
+    bool begin(const char*, int, SnortConfig*) override;
+    bool set(const char*, Value&, SnortConfig*) override;
+
+    Usage get_usage() const override
+    { return DETECT; }
+
+public:
+    IpsPolicy::Enable enable;
+};
+
+bool EnableModule::begin(const char*, int, SnortConfig* sc)
+{
+    if ( !sc->rule_states )
+        sc->rule_states = new RuleStateMap;
+
+    enable = IpsPolicy::Enable::ENABLED;
+    return true;
+}
+
+bool EnableModule::set(const char*, Value& v, SnortConfig*)
+{
+    if ( !v.is("~enable") )
+        return false;
+
+    enable = IpsPolicy::Enable(v.get_uint8());
+    return true;
+}
+
+//-------------------------------------------------------------------------
+// api methods
+//-------------------------------------------------------------------------
+
+static Module* mod_ctor()
+{
+    return new EnableModule;
+}
+
+static void mod_dtor(Module* m)
+{
+    delete m;
+}
+
+static IpsOption* enable_ctor(Module* p, OptTreeNode* otn)
+{
+    EnableModule* m = (EnableModule*)p;
+    otn->set_enabled(m->enable);
+    return nullptr;
+}
+
+static const IpsApi enable_api =
+{
+    {
+        PT_IPS_OPTION,
+        sizeof(IpsApi),
+        IPSAPI_VERSION,
+        0,
+        API_RESERVED,
+        API_OPTIONS,
+        s_name,
+        s_help,
+        mod_ctor,
+        mod_dtor
+    },
+    OPT_TYPE_META,
+    1, PROTO_BIT__NONE,
+    nullptr,
+    nullptr,
+    nullptr,
+    nullptr,
+    enable_ctor,
+    nullptr,
+    nullptr
+};
+
+#ifdef BUILDING_SO
+SO_PUBLIC const BaseApi* snort_plugins[] =
+#else
+const BaseApi* ips_enable[] =
+#endif
+{
+    &enable_api.base,
+    nullptr
+};
+
diff --git a/src/ips_options/ips_options.cc b/src/ips_options/ips_options.cc
index 42649cb6d..e26906597 100644
--- a/src/ips_options/ips_options.cc
+++ b/src/ips_options/ips_options.cc
@@ -55,6 +55,7 @@ extern const BaseApi* ips_byte_jump[];
 extern const BaseApi* ips_byte_math[];
 extern const BaseApi* ips_byte_test[];
 extern const BaseApi* ips_cvs[];
+extern const BaseApi* ips_enable[];
 extern const BaseApi* ips_file_type[];
 extern const BaseApi* ips_flags[];
 extern const BaseApi* ips_fragbits[];
@@ -127,6 +128,7 @@ void load_ips_options()
     PluginManager::load_plugins(ips_byte_math);
     PluginManager::load_plugins(ips_byte_test);
     PluginManager::load_plugins(ips_cvs);
+    PluginManager::load_plugins(ips_enable);
     PluginManager::load_plugins(ips_file_type);
     PluginManager::load_plugins(ips_flags);
     PluginManager::load_plugins(ips_fragbits);
diff --git a/src/main/modules.cc b/src/main/modules.cc
index 5f59de895..f033f2bb7 100644
--- a/src/main/modules.cc
+++ b/src/main/modules.cc
@@ -28,6 +28,7 @@
 
 #include "codecs/codec_module.h"
 #include "detection/fp_config.h"
+#include "detection/rules.h"
 #include "filters/detection_filter.h"
 #include "filters/rate_filter.h"
 #include "filters/sfrf.h"
@@ -1305,12 +1306,15 @@ static const Parameter ips_params[] =
     { "mode", Parameter::PT_ENUM, "tap | inline | inline-test", nullptr,
       "set policy mode" },
 
-    { "rules", Parameter::PT_STRING, nullptr, nullptr,
-      "snort rules and includes" },
-
     { "obfuscate_pii", Parameter::PT_BOOL, nullptr, "false",
       "mask all but the last 4 characters of credit card and social security numbers" },
 
+    { "rules", Parameter::PT_STRING, nullptr, nullptr,
+      "snort rules and includes (may contain states too)" },
+
+    { "states", Parameter::PT_STRING, nullptr, nullptr,
+      "snort rule states and includes (may contain rules too)" },
+
 #ifdef HAVE_UUID
     { "uuid", Parameter::PT_STRING, nullptr, "00000000-0000-0000-0000-000000000000",
       "IPS policy uuid" },
@@ -1357,12 +1361,15 @@ bool IpsModule::set(const char*, Value& v, SnortConfig* sc)
     else if ( v.is("mode") )
         p->policy_mode = (PolicyMode)v.get_uint8();
 
-    else if ( v.is("rules") )
-        p->rules = v.get_string();
-
     else if ( v.is("obfuscate_pii") )
         p->obfuscate_pii = v.get_bool();
 
+    else if ( v.is("rules") )
+        p->rules += v.get_string();
+
+    else if ( v.is("states") )
+        p->states += v.get_string();
+
 #ifdef HAVE_UUID
     else if ( v.is("uuid") )
     {
@@ -1838,7 +1845,7 @@ bool RateFilterModule::end(const char*, int idx, SnortConfig* sc)
 static const Parameter single_rule_state_params[] =
 {
     { "action", Parameter::PT_ENUM,
-      "log | pass | alert | drop | block | reset | react | reject | rewrite | inherit", "inherit",
+      "log | pass | alert | drop | block | reset", "alert",
       "apply action if rule matches or inherit from rule definition" },
 
     { "enable", Parameter::PT_ENUM, "no | yes | inherit", "inherit",
@@ -1856,7 +1863,8 @@ static const Parameter rule_state_params[] =
 };
 
 #define rule_state_help \
-    "enable/disable and set actions for specific IPS rules"
+    "enable/disable and set actions for specific IPS rules; " \
+    "deprecated, use rule state stubs with enable instead"
 
 class RuleStateModule : public Module
 {
@@ -1873,9 +1881,8 @@ public:
     { return DETECT; }
 
 private:
-    unsigned gid, sid;
-    IpsPolicy::Action action;
-    IpsPolicy::Enable enable;
+    RuleKey key;
+    RuleState state;
 };
 
 bool RuleStateModule::matches(const char* param, std::string& name)
@@ -1886,9 +1893,9 @@ bool RuleStateModule::matches(const char* param, std::string& name)
     std::stringstream ss(name);
     char sep;
 
-    ss >> gid >> sep >> sid;
+    ss >> key.gid >> sep >> key.sid;
 
-    if ( gid and sid and sep == ':' )
+    if ( key.gid and key.sid and sep == ':' )
         return true;
 
     return false;
@@ -1900,16 +1907,13 @@ bool RuleStateModule::set(const char* fqn, Value& v, SnortConfig*)
     if ( !strcmp(fqn, "$gid_sid") )
         return true;
 
-    if ( gid and sid )
+    if ( key.gid and key.sid )
     {
         if ( v.is("action") )
-            action = IpsPolicy::Action(v.get_uint8());
+            state.action = snort::Actions::Type(v.get_uint8() + 1);
 
         else if ( v.is("enable") )
-            enable = IpsPolicy::Enable(v.get_uint8());
-
-        else
-            return false;
+            state.enable = IpsPolicy::Enable(v.get_uint8());
     }
     else
         return false;
@@ -1917,19 +1921,28 @@ bool RuleStateModule::set(const char* fqn, Value& v, SnortConfig*)
     return true;
 }
 
-bool RuleStateModule::begin(const char*, int, SnortConfig*)
+bool RuleStateModule::begin(const char*, int, SnortConfig* sc)
 {
-    gid = sid = 0;
-    action = IpsPolicy::Action::INHERIT_ACTION;
-    enable = IpsPolicy::Enable::INHERIT_ENABLE;
+    if ( !sc->rule_states )
+        sc->rule_states = new RuleStateMap;
+
+    else
+    {
+        key = { 0, 0 };
+        state.action = snort::Actions::Type::ALERT;
+        state.enable = IpsPolicy::Enable::INHERIT_ENABLE;
+    }
     return true;
 }
 
 bool RuleStateModule::end(const char*, int, SnortConfig* sc)
 {
-    if ( gid and sid )
-        sc->rule_states.emplace_back(new RuleState(gid, sid, action, enable));
-    gid = sid = 0;
+    if ( !key.gid or !key.sid )
+        return false;
+
+    state.policy_id = snort::get_ips_policy()->policy_id;
+    sc->rule_states->add(key, state);
+
     return true;
 }
 
diff --git a/src/main/policy.h b/src/main/policy.h
index 5351462e0..62b4c95c5 100644
--- a/src/main/policy.h
+++ b/src/main/policy.h
@@ -132,11 +132,9 @@ private:
 // detection stuff
 //-------------------------------------------------------------------------
 
-// this is the ips policy post ac-split
 struct IpsPolicy
 {
 public:
-    enum Action : uint8_t { LOG, PASS, ALERT, DROP, BLOCK, RESET, INHERIT_ACTION };
     enum Enable : uint8_t { DISABLED, ENABLED, INHERIT_ENABLE };
 
     IpsPolicy(PolicyId = 0);
@@ -152,7 +150,9 @@ public:
 
     std::string includer;
     std::string include;
+
     std::string rules;
+    std::string states;
 
     uint32_t var_id;
 
diff --git a/src/main/snort.cc b/src/main/snort.cc
index d467356c6..bb2075b70 100644
--- a/src/main/snort.cc
+++ b/src/main/snort.cc
@@ -137,6 +137,7 @@ void Snort::init(int argc, char** argv)
     ModuleManager::init();
     ScriptManager::load_scripts(snort_cmd_line_conf->script_paths);
     PluginManager::load_plugins(snort_cmd_line_conf->plugin_path);
+    ModuleManager::load_params();
 
     if ( snort_cmd_line_conf->logging_flags & LOGGING_FLAG__SHOW_PLUGINS )
     {
diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index 7de40be73..7447f8d33 100644
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -329,8 +329,12 @@ void SnortConfig::setup()
     ParseRules(this);
     OrderRuleLists(this);
 
-    for ( auto& state : rule_states )
-        state->apply(this);
+    if ( rule_states )
+    {
+        rule_states->apply(this);
+        delete rule_states;
+        rule_states = nullptr;
+    }
 
     /* Need to do this after dynamic detection stuff is initialized, too */
     IpsManager::verify(this);
diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index b6c092b9c..4ab41711c 100644
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -26,6 +26,7 @@
 #include <sys/types.h>
 
 #include <list>
+#include <vector>
 
 #include "events/event_queue.h"
 #include "framework/bits.h"
@@ -120,7 +121,7 @@ enum TunnelFlags
 };
 
 class FastPatternConfig;
-class RuleState;
+class RuleStateMap;
 class ThreadConfig;
 
 struct srmm_table_t;
@@ -331,7 +332,7 @@ public:
 
     int thiszone = 0;
 
-    std::vector<RuleState*> rule_states;
+    RuleStateMap* rule_states = nullptr;
     ClassType* classifications = nullptr;
     ReferenceSystemNode* references = nullptr;
     GHash* otn_map = nullptr;
diff --git a/src/managers/ips_manager.cc b/src/managers/ips_manager.cc
index ca515f61c..3a2eeaacd 100644
--- a/src/managers/ips_manager.cc
+++ b/src/managers/ips_manager.cc
@@ -24,7 +24,7 @@
 #include "ips_manager.h"
 
 #include <cassert>
-#include <list>
+#include <map>
 
 #include "detection/fp_detect.h"
 #include "detection/treenodes.h"
@@ -47,8 +47,8 @@ struct Option
     { api = p; init = false; count = 0; }
 };
 
-typedef list<Option*> OptionList;
-static OptionList s_options;
+typedef map<std::string, Option*> OptionMap;
+static OptionMap s_options;
 
 static std::string current_keyword = std::string();
 static Module* current_module = nullptr;
@@ -60,13 +60,14 @@ static const Parameter* current_params = nullptr;
 
 void IpsManager::add_plugin(const IpsApi* api)
 {
-    s_options.emplace_back(new Option(api));
+    assert(s_options.find(api->base.name) == s_options.end());
+    s_options[api->base.name] = new Option(api);
 }
 
 void IpsManager::release_plugins()
 {
-    for ( auto* p : s_options )
-        delete p;
+    for ( auto& p : s_options )
+        delete p.second;
 
     s_options.clear();
 }
@@ -75,8 +76,8 @@ void IpsManager::dump_plugins()
 {
     Dumper d("IPS Options");
 
-    for ( auto* p : s_options )
-        d.dump(p->api->base.name, p->api->base.version);
+    for ( auto& p : s_options )
+        d.dump(p.second->api->base.name, p.second->api->base.version);
 }
 
 //-------------------------------------------------------------------------
@@ -99,8 +100,10 @@ static bool set_arg(
     const char* opt, const char* val, SnortConfig* sc)
 {
     if ( !p->is_positional() )
-        p = Parameter::find(p, opt);
-
+    {
+        const Parameter* q = ModuleManager::get_parameter(m->get_name(), opt);
+        p = q ? q : Parameter::find(p, opt);
+    }
     else if ( *opt )  // must contain spaces like ip_proto:! 6;
         return false;
 
@@ -113,6 +116,13 @@ static bool set_arg(
     if ( p->type == Parameter::PT_IMPLIED )
         v.set(true);
 
+    else if ( p->type == Parameter::PT_BOOL )
+    {
+        if ( !val or !strcmp(val, "true") )
+            v.set(true);
+        else
+            v.set(false);
+    }
     else if ( p->type == Parameter::PT_INT )
     {
         char* end = nullptr;
@@ -154,9 +164,9 @@ static bool set_arg(
 
 static Option* get_opt(const char* keyword)
 {
-    for ( auto* p : s_options )
-        if ( !strcasecmp(p->api->base.name, keyword) )
-            return p;
+    auto opt = s_options.find(keyword);
+    if ( opt != s_options.end() )
+        return opt->second;
 
     return nullptr;
 }
@@ -320,18 +330,18 @@ void IpsManager::global_init(SnortConfig*)
 
 void IpsManager::global_term(SnortConfig* sc)
 {
-    for ( auto* p : s_options )
-        if ( p->init && p->api->pterm )
+    for ( auto& p : s_options )
+        if ( p.second->init && p.second->api->pterm )
         {
-            p->api->pterm(sc);
-            p->init = false;
+            p.second->api->pterm(sc);
+            p.second->init = false;
         }
 }
 
 void IpsManager::reset_options()
 {
-    for ( auto* p : s_options )
-        p->count = 0;
+    for ( auto& p : s_options )
+        p.second->count = 0;
 
     // this is the default when we start parsing a rule body
     IpsOption::set_buffer("pkt_data");
@@ -339,23 +349,23 @@ void IpsManager::reset_options()
 
 void IpsManager::setup_options()
 {
-    for ( auto* p : s_options )
-        if ( p->init && p->api->tinit )
-            p->api->tinit(SnortConfig::get_conf());
+    for ( auto& p : s_options )
+        if ( p.second->init && p.second->api->tinit )
+            p.second->api->tinit(SnortConfig::get_conf());
 }
 
 void IpsManager::clear_options()
 {
-    for ( auto* p : s_options )
-        if ( p->init && p->api->tterm )
-            p->api->tterm(SnortConfig::get_conf());
+    for ( auto& p : s_options )
+        if ( p.second->init && p.second->api->tterm )
+            p.second->api->tterm(SnortConfig::get_conf());
 }
 
 bool IpsManager::verify(SnortConfig* sc)
 {
-    for ( auto* p : s_options )
-        if ( p->init && p->api->verify )
-            p->api->verify(sc);
+    for ( auto& p : s_options )
+        if ( p.second->init && p.second->api->verify )
+            p.second->api->verify(sc);
 
     return true;
 }
@@ -364,9 +374,9 @@ bool IpsManager::verify(SnortConfig* sc)
 
 static const IpsApi* find_api(const char* name)
 {
-    for ( auto wrap : s_options )
-        if ( !strcmp(wrap->api->base.name, name) )
-            return wrap->api;
+    for ( auto& wrap : s_options )
+        if ( !strcmp(wrap.second->api->base.name, name) )
+            return wrap.second->api;
 
     return nullptr;
 }
diff --git a/src/managers/module_manager.cc b/src/managers/module_manager.cc
index af7b9974e..c97785b0d 100644
--- a/src/managers/module_manager.cc
+++ b/src/managers/module_manager.cc
@@ -26,10 +26,13 @@
 #include <libgen.h>
 #include <lua.hpp>
 
+#include <algorithm>
 #include <cassert>
 #include <iostream>
 #include <mutex>
 #include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "framework/base_api.h"
 #include "framework/module.h"
@@ -66,8 +69,9 @@ struct ModHook
     void init();
 };
 
-typedef std::list<ModHook*> ModuleList;
-static ModuleList s_modules;
+static std::unordered_map<std::string, ModHook*> s_modules;
+static std::unordered_map<std::string, const Parameter*> s_pmap;
+
 static unsigned s_errors = 0;
 
 std::set<uint32_t> ModuleManager::gids;
@@ -195,9 +199,9 @@ static void trace(const char* s, const char* fqn, Value& v)
 
 static ModHook* get_hook(const char* s)
 {
-    for ( auto p : s_modules )
-        if ( !strcmp(p->mod->get_name(), s) )
-            return p;
+    auto mh = s_modules.find(s);
+    if ( mh != s_modules.end() )
+        return mh->second;
 
     return nullptr;
 }
@@ -493,25 +497,21 @@ static bool set_value(const char* fqn, Value& v)
         return found;
     }
 
-    if ( mod->get_usage() == Module::GLOBAL &&
-         only_inspection_policy() && !default_inspection_policy() )
-        return true;
-
-    if ( mod->get_usage() != Module::INSPECT && only_inspection_policy() )
-        return true;
+    const Parameter* p;
+    auto a = s_pmap.find(t);
 
-    if ( mod->get_usage() != Module::DETECT && only_ips_policy() )
-        return true;
+    if ( a != s_pmap.end() )
+        p = a->second;
 
-    if ( mod->get_usage() != Module::CONTEXT && only_network_policy() )
-        return true;
-
-    // now we must traverse the mod params to get the leaf
-    string s = fqn;
-    const Parameter* p = get_params(s, mod, mod->get_parameters());
+    else
+    {
+        // now we must traverse the mod params to get the leaf
+        string s = fqn;
+        p = get_params(s, mod, mod->get_parameters());
 
-    if ( !p )
-        p = get_params(s, mod, mod->get_default_parameters());
+        if ( !p )
+            p = get_params(s, mod, mod->get_default_parameters());
+    }
 
     if ( !p )
     {
@@ -577,7 +577,7 @@ static bool begin(Module* m, const Parameter* p, const char* s, int idx, int dep
     else
     {
         assert(p);
-        if ((!idx and p->type == Parameter::PT_LIST) or
+        if ( (!idx and p->type == Parameter::PT_LIST) or
              (idx and p->type != Parameter::PT_LIST) )
         {
             if ( !m->verified_begin(s, idx, s_config) )
@@ -676,6 +676,24 @@ static bool end(Module* m, const Parameter* p, const char* s, int idx)
     return true;
 }
 
+static bool interested(Module* m)
+{
+    if ( m->get_usage() == Module::GLOBAL &&
+         only_inspection_policy() && !default_inspection_policy() )
+        return false;
+
+    if ( m->get_usage() != Module::INSPECT && only_inspection_policy() )
+        return false;
+
+    if ( m->get_usage() != Module::DETECT && only_ips_policy() )
+        return false;
+
+    if ( m->get_usage() != Module::CONTEXT && only_network_policy() )
+        return false;
+
+    return true;
+}
+
 //-------------------------------------------------------------------------
 // ffi methods
 //-------------------------------------------------------------------------
@@ -713,18 +731,8 @@ SO_PUBLIC bool open_table(const char* s, int idx)
     Module* m = h->mod;
     const Parameter* p = nullptr;
 
-    if ( m->get_usage() == Module::GLOBAL &&
-         only_inspection_policy() && !default_inspection_policy() )
-        return true;
-
-    if ( m->get_usage() != Module::INSPECT && only_inspection_policy() )
-        return true;
-
-    if ( m->get_usage() != Module::DETECT && only_ips_policy() )
-        return true;
-
-    if ( m->get_usage() != Module::CONTEXT && only_network_policy() )
-        return true;
+    if ( !interested(m) )
+        return false;
 
     if ( strcmp(m->get_name(), s) )
     {
@@ -739,7 +747,7 @@ SO_PUBLIC bool open_table(const char* s, int idx)
             ParseError("can't find %s", s);
             return false;
         }
-        else if ((idx > 0) && (p->type == Parameter::PT_TABLE))
+        else if ( (idx > 0) && (p->type == Parameter::PT_TABLE) )
         {
             ParseError("%s is a table; all elements must be named", s);
             return false;
@@ -760,6 +768,7 @@ SO_PUBLIC bool open_table(const char* s, int idx)
         ParseError("can't open %s", m->get_name());
         return false;
     }
+
     return true;
 }
 
@@ -774,19 +783,6 @@ SO_PUBLIC void close_table(const char* s, int idx)
 
     if ( ModHook* h = get_hook(key.c_str()) )
     {
-        if ( h->mod->get_usage() == Module::GLOBAL &&
-             only_inspection_policy() && !default_inspection_policy() )
-            return;
-
-        if ( h->mod->get_usage() != Module::INSPECT && only_inspection_policy() )
-            return;
-
-        if ( h->mod->get_usage() != Module::DETECT && only_ips_policy() )
-            return;
-
-        if ( h->mod->get_usage() != Module::CONTEXT && only_network_policy() )
-            return;
-
         if ( !end(h->mod, nullptr, s, idx) )
             ParseError("can't close %s", h->mod->get_name());
 
@@ -847,6 +843,10 @@ static bool comp_gids(const ModHook* l, const ModHook* r)
 {
     const Module* lm = l->mod;
     const Module* rm = r->mod;
+
+    if ( lm->get_gid() == rm->get_gid() )
+        return comp_mods(l, r);
+
     return ( lm->get_gid() < rm->get_gid() );
 }
 
@@ -861,8 +861,8 @@ void ModuleManager::init()
 
 void ModuleManager::term()
 {
-    for ( auto* p : s_modules )
-        delete p;
+    for ( auto& mh : s_modules )
+        delete mh.second;
 
     s_modules.clear();
 }
@@ -870,7 +870,10 @@ void ModuleManager::term()
 void ModuleManager::add_module(Module* m, const BaseApi* b)
 {
     ModHook* mh = new ModHook(m, b);
-    s_modules.emplace_back(mh);
+
+    assert(s_modules.find(mh->mod->get_name()) == s_modules.end());
+
+    s_modules[mh->mod->get_name()] = mh;
 
     Profiler::register_module(m);
 
@@ -880,9 +883,9 @@ void ModuleManager::add_module(Module* m, const BaseApi* b)
 
 Module* ModuleManager::get_module(const char* s)
 {
-    for ( auto p : s_modules )
-        if ( !strcmp(p->mod->get_name(), s) )
-            return p->mod;
+    auto mh = s_modules.find(s);
+    if ( mh != s_modules.end() )
+        return mh->second->mod;
 
     return nullptr;
 }
@@ -903,8 +906,18 @@ list<Module*> ModuleManager::get_all_modules()
 {
     list<Module*> ret;
 
-    for ( auto& m : s_modules )
-       ret.emplace_back(m->mod);
+    for ( auto& mh : s_modules )
+       ret.emplace_back(mh.second->mod);
+
+    return ret;
+}
+
+static list<ModHook*> get_all_modhooks()
+{
+    list<ModHook*> ret;
+
+    for ( auto& mh : s_modules )
+       ret.emplace_back(mh.second);
 
     return ret;
 }
@@ -921,18 +934,19 @@ unsigned ModuleManager::get_errors()
 void ModuleManager::list_modules(const char* s)
 {
     PlugType pt = s ? PluginManager::get_type(s) : PT_MAX;
-    s_modules.sort(comp_mods);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
     unsigned c = 0;
 
-    for ( auto* p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
         if (
             !s || !*s ||
-            (p->api && p->api->type == pt) ||
-            (!p->api && !strcmp(s, "basic"))
+            (mh->api && mh->api->type == pt) ||
+            (!mh->api && !strcmp(s, "basic"))
             )
         {
-            LogMessage("%s\n", p->mod->get_name());
+            LogMessage("%s\n", mh->mod->get_name());
             c++;
         }
     }
@@ -942,28 +956,30 @@ void ModuleManager::list_modules(const char* s)
 
 void ModuleManager::show_modules()
 {
-    s_modules.sort(comp_mods);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
 
-    for ( auto* p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        const char* t = p->api ? PluginManager::get_type_name(p->api->type) : "basic";
+        const char* t = mh->api ? PluginManager::get_type_name(mh->api->type) : "basic";
 
         cout << Markup::item();
-        cout << Markup::emphasis(p->mod->get_name());
+        cout << Markup::emphasis(mh->mod->get_name());
         cout << " (" << t;
-        cout << "): " << p->mod->get_help();
+        cout << "): " << mh->mod->get_help();
         cout << endl;
     }
 }
 
 void ModuleManager::dump_modules()
 {
-    s_modules.sort(comp_mods);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
     Dumper d("Modules");
 
-    for ( auto* p : s_modules )
-        if ( !p->api )
-            d.dump(p->mod->get_name());
+    for ( auto* mh : mod_hooks )
+        if ( !mh->api )
+            d.dump(mh->mod->get_name());
 }
 
 static const char* mod_type(const BaseApi* api)
@@ -994,12 +1010,13 @@ void ModuleManager::show_module(const char* name)
         cerr << "module name required" << endl;
         return;
     }
-    s_modules.sort(comp_gids);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_gids);
     unsigned c = 0;
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        const Module* m = p->mod;
+        const Module* m = mh->mod;
         assert(m);
 
         if ( strcmp(m->get_name(), name) )
@@ -1010,7 +1027,7 @@ void ModuleManager::show_module(const char* name)
         if ( const char* h = m->get_help() )
             cout << endl << "What: " << h << endl;
 
-        cout << endl << "Type: "  << mod_type(p->api) << endl;
+        cout << endl << "Type: "  << mod_type(mh->api) << endl;
         cout << endl << "Usage: "  << mod_use(m->get_usage()) << endl;
 
         const Parameter* params = m->get_parameters();
@@ -1086,12 +1103,13 @@ static bool selected(const Module* m, const char* pfx, bool exact)
 
 void ModuleManager::show_configs(const char* pfx, bool exact)
 {
-    s_modules.sort(comp_mods);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
     unsigned c = 0;
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        Module* m = p->mod;
+        Module* m = mh->mod;
         string s;
 
         if ( !selected(m, pfx, exact) )
@@ -1133,18 +1151,18 @@ void ModuleManager::show_configs(const char* pfx, bool exact)
 void ModuleManager::dump_defaults(const char* pfx)
 {
     dump_fmt = DF_LUA;
-    cout << "require('snort_config')" << endl;
     show_configs(pfx);
 }
 
 void ModuleManager::show_commands(const char* pfx, bool exact)
 {
-    s_modules.sort(comp_mods);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
     unsigned n = 0;
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        const Module* m = p->mod;
+        const Module* m = mh->mod;
 
         if ( !selected(m, pfx, exact) )
             continue;
@@ -1158,7 +1176,7 @@ void ModuleManager::show_commands(const char* pfx, bool exact)
         {
             cout << Markup::item();
             cout << Markup::emphasis_on();
-            cout << p->mod->get_name();
+            cout << mh->mod->get_name();
             cout << "." << c->name;
             cout << Markup::emphasis_off();
             cout << c->get_arg_list();
@@ -1179,12 +1197,13 @@ bool ModuleManager::gid_in_use(uint32_t gid)
 
 void ModuleManager::show_gids(const char* pfx, bool exact)
 {
-    s_modules.sort(comp_gids);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_gids);
     unsigned c = 0;
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        const Module* m = p->mod;
+        const Module* m = mh->mod;
         assert(m);
 
         if ( !selected(m, pfx, exact) )
@@ -1222,12 +1241,13 @@ static const char* peg_op(CountType ct)
 
 void ModuleManager::show_pegs(const char* pfx, bool exact)
 {
-    s_modules.sort(comp_gids);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_gids);
     unsigned c = 0;
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        const Module* m = p->mod;
+        const Module* m = mh->mod;
         assert(m);
 
         if ( !selected(m, pfx, exact) )
@@ -1242,7 +1262,7 @@ void ModuleManager::show_pegs(const char* pfx, bool exact)
         {
             cout << Markup::item();
             cout << Markup::emphasis_on();
-            cout << p->mod->get_name();
+            cout << mh->mod->get_name();
             cout << "." << pegs->name;
             cout << Markup::emphasis_off();
             cout << ": " << pegs->help;
@@ -1256,50 +1276,17 @@ void ModuleManager::show_pegs(const char* pfx, bool exact)
         cout << "no match" << endl;
 }
 
-void ModuleManager::show_rules(const char* pfx, bool exact)
-{
-    s_modules.sort(comp_gids);
-    unsigned c = 0;
-
-    for ( auto p : s_modules )
-    {
-        const Module* m = p->mod;
-
-        if ( !selected(m, pfx, exact) )
-            continue;
-
-        const RuleMap* r = m->get_rules();
-        unsigned gid = m->get_gid();
-
-        if ( !r )
-            continue;
-
-        while ( r->msg )
-        {
-            cout << Markup::item();
-            cout << Markup::emphasis_on();
-            cout << gid << ":" << r->sid;
-            cout << Markup::emphasis_off();
-            cout << " (" << m->get_name() << ")";
-            cout << " " << r->msg;
-            cout << endl;
-            r++;
-        }
-        c++;
-    }
-    if ( !c )
-        cout << "no match" << endl;
-}
-
 void ModuleManager::load_commands(Shell* sh)
 {
     // FIXIT-L ideally only install commands from configured modules
     // FIXIT-L install commands into working shell
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        if ( p->reg )
-            sh->install(p->mod->get_name(), p->reg);
+        if ( mh->reg )
+            sh->install(mh->mod->get_name(), mh->reg);
     }
 }
 
@@ -1327,11 +1314,12 @@ static void make_rule(ostream& os, const Module* m, const RuleMap* r)
 // (we don't want to suppress it because it could mean something is broken)
 void ModuleManager::load_rules(SnortConfig* sc)
 {
-    s_modules.sort(comp_gids);
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_gids);
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        const Module* m = p->mod;
+        const Module* m = mh->mod;
         const RuleMap* r = m->get_rules();
 
         if ( !r )
@@ -1353,113 +1341,225 @@ void ModuleManager::load_rules(SnortConfig* sc)
     }
 }
 
-void ModuleManager::dump_rules(const char* pfx)
+void ModuleManager::dump_stats(SnortConfig*, const char* skip, bool dynamic)
 {
-    s_modules.sort(comp_gids);
-    unsigned len = pfx ? strlen(pfx) : 0;
-    unsigned c = 0;
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
 
-    for ( auto p : s_modules )
+    for ( auto* mh : mod_hooks )
     {
-        const Module* m = p->mod;
+        if ( !skip || !strstr(skip, mh->mod->get_name()) )
+        {
+            std::lock_guard<std::mutex> lock(stats_mutex);
+            if ( dynamic )
+                mh->mod->show_dynamic_stats();
+            else
+                mh->mod->show_stats();
+        }
+    }
+}
 
-        if ( pfx && strncmp(m->get_name(), pfx, len) )
-            continue;
+void ModuleManager::accumulate(SnortConfig*)
+{
+    auto mod_hooks = get_all_modhooks();
 
-        const RuleMap* r = m->get_rules();
+    for ( auto* mh : mod_hooks )
+    {
+        std::lock_guard<std::mutex> lock(stats_mutex);
+        mh->mod->prep_counts();
+        mh->mod->sum_stats(true);
+    }
+}
 
-        if ( !r )
-            continue;
+void ModuleManager::accumulate_offload(const char* name)
+{
+    ModHook* mh = get_hook(name);
+    if ( mh )
+    {
+        std::lock_guard<std::mutex> lock(stats_mutex);
+        mh->mod->prep_counts();
+        mh->mod->sum_stats(true);
+    }
+}
 
-        ostream& ss = cout;
+void ModuleManager::reset_stats(SnortConfig*)
+{
+    auto mod_hooks = get_all_modhooks();
 
-        while ( r->msg )
-        {
-            make_rule(ss, m, r);
-            r++;
-        }
-        c++;
+    for ( auto* mh : mod_hooks )
+    {
+        std::lock_guard<std::mutex> lock(stats_mutex);
+        mh->mod->reset_stats();
     }
-    if ( !c )
-        cout << "no match" << endl;
 }
 
-void ModuleManager::dump_msg_map(const char* pfx)
+//-------------------------------------------------------------------------
+// parameter loading
+//-------------------------------------------------------------------------
+
+static void load_table(string&, const Parameter*);
+
+static void load_field(string& key, const Parameter* p)
 {
-    s_modules.sort(comp_gids);
-    unsigned len = pfx ? strlen(pfx) : 0;
-    unsigned c = 0;
+    unsigned n = key.size();
 
-    for ( auto p : s_modules )
+    if ( p->name )
     {
-        const Module* m = p->mod;
-
-        if ( pfx && strncmp(m->get_name(), pfx, len) )
-            continue;
+        if ( n )
+            key += ".";
+        key += p->name;
+    }
 
-        const RuleMap* r = m->get_rules();
+    if ( p->type == Parameter::PT_TABLE or p->type == Parameter::PT_LIST )
+        load_table(key, (const Parameter*)p->range);
 
-        if ( !r )
-            continue;
+    else
+        s_pmap[key] = p;
 
-        unsigned gid = m->get_gid();
+    key.erase(n);
+}
 
-        while ( r->msg )
-        {
-            cout << gid << " || " << r->sid << " || ";
-            cout << m->get_name() << ": ";
-            cout << r->msg << endl;
-            r++;
-        }
-        c++;
-    }
-    if ( !c )
-        cout << "no match" << endl;
+static void load_table(string& key, const Parameter* p)
+{
+    while ( p && p->name )
+        load_field(key, p++);
 }
 
-void ModuleManager::dump_stats(SnortConfig*, const char* skip, bool dynamic)
+void ModuleManager::load_params()
 {
-    for ( auto p : s_modules )
+    auto mod_hooks = get_all_modhooks();
+    mod_hooks.sort(comp_mods);
+
+    for ( auto* mh : mod_hooks )
     {
-        if ( !skip || !strstr(skip, p->mod->get_name()) )
+        Module* m = mh->mod;
+        string s;
+
+        if ( m->is_list() )
         {
-            std::lock_guard<std::mutex> lock(stats_mutex);
-            if (dynamic)
-                p->mod->show_dynamic_stats();
+            s = m->name;
+
+            if ( m->params->name )
+                load_table(s, m->params);
             else
-                p->mod->show_stats();
+                load_field(s, m->params);
         }
+        else if ( m->is_table() )
+        {
+            s = m->name;
+            load_table(s, m->params);
+        }
+        else
+        {
+            load_field(s, m->params);
+        }
+
+        s = m->name;
+
+        if ( m->default_params )
+            load_table(s, m->default_params);
     }
 }
 
-void ModuleManager::accumulate(SnortConfig*)
+const Parameter* ModuleManager::get_parameter(const char* table, const char* option)
+{
+    string key = table;
+    key += '.';
+    key += option;
+
+    auto a = s_pmap.find(key);
+
+    if (a != s_pmap.end() )
+        return a->second;
+
+    return nullptr;
+}
+
+//--------------------------------------------------------------------------
+// builtin rule outputs
+//--------------------------------------------------------------------------
+
+struct RulePtr
 {
-    for ( auto p : s_modules )
+    const Module* mod;
+    const RuleMap* rule;
+
+    RulePtr(const Module* m, const RuleMap* r) : mod(m), rule(r) { }
+
+    bool operator< (RulePtr& rhs) const
     {
-        std::lock_guard<std::mutex> lock(stats_mutex);
-        p->mod->prep_counts();
-        p->mod->sum_stats(true);
+        if ( mod->get_gid() != rhs.mod->get_gid() )
+            return mod->get_gid() < rhs.mod->get_gid();
+
+         return rule->sid < rhs.rule->sid;
+    }
+};
+
+static std::vector<RulePtr> get_rules(const char* pfx, bool exact = false)
+{
+    auto mod_hooks = get_all_modhooks();
+    std::vector<RulePtr> rule_set;
+
+    for ( auto* mh : mod_hooks )
+    {
+        const Module* m = mh->mod;
+
+        if ( !selected(m, pfx, exact) )
+            continue;
+
+        const RuleMap* r = m->get_rules();
+
+        if ( !r )
+            continue;
+
+        while ( r->msg )
+            rule_set.push_back(RulePtr(m, r++));
     }
-    std::lock_guard<std::mutex> lock(stats_mutex);
+    std::sort(rule_set.begin(), rule_set.end());
+    return rule_set;
 }
 
-void ModuleManager::accumulate_offload(const char* name)
+void ModuleManager::dump_rules(const char* pfx)
+{
+    std::vector<RulePtr> rule_set = get_rules(pfx);
+
+    for ( auto rp : rule_set )
+        make_rule(cout, rp.mod, rp.rule);
+
+    if ( !rule_set.size() )
+        cout << "no match" << endl;
+}
+
+void ModuleManager::show_rules(const char* pfx, bool exact)
 {
-    ModHook* p = get_hook(name);
-    if ( p )
+    std::vector<RulePtr> rule_set = get_rules(pfx, exact);
+
+    for ( auto rp : rule_set )
     {
-        std::lock_guard<std::mutex> lock(stats_mutex);
-        p->mod->prep_counts();
-        p->mod->sum_stats(true);
+        cout << Markup::item();
+        cout << Markup::emphasis_on();
+        cout << rp.mod->get_gid() << ":" << rp.rule->sid;
+        cout << Markup::emphasis_off();
+        cout << " (" << rp.mod->get_name() << ")";
+        cout << " " << rp.rule->msg;
+        cout << endl;
     }
-    std::lock_guard<std::mutex> lock(stats_mutex);
+    if ( !rule_set.size() )
+        cout << "no match" << endl;
 }
 
-void ModuleManager::reset_stats(SnortConfig*)
+void ModuleManager::dump_msg_map(const char* pfx)
 {
-    for ( auto p : s_modules )
+    std::vector<RulePtr> rule_set = get_rules(pfx);
+
+    for ( auto rp : rule_set )
     {
-        std::lock_guard<std::mutex> lock(stats_mutex);
-        p->mod->reset_stats();
+        cout << rp.mod->get_gid() << " || ";
+        cout << rp.rule->sid << " || ";
+        cout << rp.mod->get_name() << ": ";
+        cout << rp.rule->msg << endl;
     }
+    if ( !rule_set.size() )
+        cout << "no match" << endl;
 }
+
diff --git a/src/managers/module_manager.h b/src/managers/module_manager.h
index aebc83f17..a8c153d41 100644
--- a/src/managers/module_manager.h
+++ b/src/managers/module_manager.h
@@ -71,6 +71,9 @@ public:
     static void dump_rules(const char* = nullptr);
     static void dump_defaults(const char* = nullptr);
 
+    static void load_params();
+    static const struct Parameter* get_parameter(const char* table, const char* option);
+
     static void load_commands(Shell*);
     static void load_rules(SnortConfig*);
     static void set_config(SnortConfig*);
diff --git a/src/parser/parse_rule.cc b/src/parser/parse_rule.cc
index 11da6a321..f57f27408 100644
--- a/src/parser/parse_rule.cc
+++ b/src/parser/parse_rule.cc
@@ -1127,6 +1127,33 @@ OptTreeNode* parse_rule_open(SnortConfig* sc, RuleTreeNode& rtn, bool stub)
     return otn;
 }
 
+static void parse_rule_state(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* otn)
+{
+    if ( !otn->sigInfo.gid )
+        otn->sigInfo.gid = GID_DEFAULT;
+
+    if ( otn->num_detection_opts )
+    {
+        ParseError("%u:%u rule state stubs do not support detection options",
+            otn->sigInfo.gid, otn->sigInfo.sid);
+    }
+    RuleKey key = { otn->sigInfo.gid, otn->sigInfo.sid };
+    RuleState state =
+    {
+        snort::get_ips_policy()->policy_id, // FIXIT-H need parsing policy for reload
+        rtn.action,
+        otn->enable
+    };
+    sc->rule_states->add(key, state);
+
+    if ( rtn.sip )
+        sfvar_free(rtn.sip);
+    if ( rtn.dip )
+        sfvar_free(rtn.dip);
+
+    OtnFree(otn);
+}
+
 void parse_rule_close(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* otn)
 {
     if ( s_ignore )
@@ -1135,6 +1162,12 @@ void parse_rule_close(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* otn)
         return;
     }
 
+    if ( otn->is_rule_state_stub() )
+    {
+        parse_rule_state(sc, rtn, otn);
+        return;
+    }
+
     static bool entered = false;
 
     if ( entered )
diff --git a/src/parser/parser.cc b/src/parser/parser.cc
index 54ffc5d32..30c58d1f8 100644
--- a/src/parser/parser.cc
+++ b/src/parser/parser.cc
@@ -29,6 +29,7 @@
 #include <iostream>
 
 #include "detection/fp_config.h"
+#include "detection/rules.h"
 #include "detection/sfrim.h"
 #include "filters/detection_filter.h"
 #include "filters/rate_filter.h"
@@ -242,15 +243,11 @@ void parser_init()
         ParseAbort("failed to create rule index map.");
 }
 
-void parser_term(SnortConfig* sc)
+void parser_term(SnortConfig*)
 {
     parse_rule_term();
     RuleIndexMapFree(ruleIndexMap);
     ruleIndexMap = nullptr;
-
-    for ( auto& r : sc->rule_states )
-        delete r;
-    sc->rule_states.clear();
 }
 
 SnortConfig* ParseSnortConf(const SnortConfig* boot_conf, const char* fname, bool is_fatal)
@@ -372,6 +369,13 @@ void ParseRules(SnortConfig* sc)
             pop_parse_location();
         }
 
+        if ( !p->states.empty() )
+        {
+            push_parse_location("C", file.c_str(), "ips.states");
+            ParseConfigString(sc, p->states.c_str());
+            pop_parse_location();
+        }
+
         if ( !idx and !s_aux_rules.empty() )
         {
             p->includer.clear();