Pull request #3668: process: Watchdog to abort snort when multiple packet thread...

Merge in SNORT/snort3 from ~AMUTTUVA/snort3:snortWatchdogEnhancement to master

Squashed commit of the following:

commit 242c3a800c4c72a72c81db304e03e1254ac53eaf
Author: Akhilesh MY <amuttuva@cisco.com>
Date:   Wed Nov 16 06:39:52 2022 -0500

    process: Watchdog to abort snort when multiple packet thread becomes unresponsive

diff --git a/src/main.cc b/src/main.cc
index 6335a4483956c0cd23a0b6fd2be30beff0ddfb70..060af2d9194d34e6fc2b056e527c90af909c4772 100644 (file)
--- a/src/main.cc
+++ b/src/main.cc
@@ -345,6 +345,31 @@ int main_reset_stats(lua_State* L)
     return 0;
 }
 
+int main_set_watchdog_params(lua_State* L)
+{
+    ControlConn* ctrlcon = ControlConn::query_from_lua(L);
+    SnortConfig* sc = SnortConfig::get_main_conf();
+
+    if ( sc && L )
+    {
+        int seconds = luaL_optint(L, 1, -1);
+        int thread_count = luaL_optint(L, 2, -1);
+        // Timer and thread count are accessed only in main thread context
+        if ( seconds != -1 )
+            sc->set_watchdog(seconds);
+
+        if ( thread_count != -1 )
+            sc->set_watchdog_min_thread_count(thread_count);
+
+        std::ostringstream watchdog_timer_msg;
+        watchdog_timer_msg << "== setting watchdog timer to " << sc->watchdog_timer
+                           << ", min thread count to " << sc->watchdog_min_thread_count << "\n";
+        send_response(ctrlcon, watchdog_timer_msg.str().c_str());
+    }
+
+    return 0;
+}
+
 int main_rotate_stats(lua_State* L)
 {
     ControlConn* ctrlcon = ControlConn::query_from_lua(L);

diff --git a/src/main.h b/src/main.h
index 4c422dde62126bfd9b5edcb82738436c77f87211..b7fff84fb7eb56d00936a36a489bbdcff0fea0d9 100644 (file)
--- a/src/main.h
+++ b/src/main.h
@@ -29,6 +29,7 @@ const char* get_prompt();
 int main_delete_inspector(lua_State* = nullptr);
 int main_dump_stats(lua_State* = nullptr);
 int main_reset_stats(lua_State* = nullptr);
+int main_set_watchdog_params(lua_State* = nullptr);
 int main_rotate_stats(lua_State* = nullptr);
 int main_reload_config(lua_State* = nullptr);
 int main_reload_policy(lua_State* = nullptr);

diff --git a/src/main/modules.cc b/src/main/modules.cc
index 70489e09ceeaa65f667480a3fa8a8765938d9f3a..313a7e46b547cb209da36051030f65b9b05b2a5c 100644 (file)
--- a/src/main/modules.cc
+++ b/src/main/modules.cc
@@ -1329,6 +1329,9 @@ static const Parameter process_params[] =
     { "watchdog_timer", Parameter::PT_INT, "0:60", "0",
       "watchdog timer for packet threads (seconds, 0 to disable)" },
 
+    { "watchdog_min_thread_count", Parameter::PT_INT, "1:65535", "1",
+      "minimum unresponsive threads for watchdog to trigger" },
+
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
@@ -1396,6 +1399,9 @@ bool ProcessModule::set(const char*, Value& v, SnortConfig* sc)
     else if ( v.is("watchdog_timer") )
         sc->set_watchdog(v.get_uint16());
 
+    else if ( v.is("watchdog_min_thread_count") )
+        sc->set_watchdog_min_thread_count(v.get_uint16());    
+
     return true;
 }
 

diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index 595adc0374ceabbe7ca54aab8720b1e0fcaabe71..cafc1611a3a20b9f7737bc09a8388c86f18cf008 100644 (file)
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -645,6 +645,11 @@ void SnortConfig::set_watchdog(uint16_t n)
     watchdog_timer = n;
 }
 
+void SnortConfig::set_watchdog_min_thread_count(uint16_t n)
+{
+    watchdog_min_thread_count = n;
+}
+
 void SnortConfig::set_dirty_pig(bool enabled)
 {
     dirty_pig = enabled;

diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index dfeb7df97fa14d287b1f5277f770f05687b5297a..14035b5d99df883d28c3ea36b306060b11f86088 100644 (file)
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -245,6 +245,7 @@ public:
     int user_id = -1;
     int group_id = -1;
     uint16_t watchdog_timer = 0;
+    uint16_t watchdog_min_thread_count = 1;
     bool dirty_pig = false;
 
     std::string chroot_dir;        /* -t or config chroot */
@@ -462,6 +463,7 @@ public:
     void set_umask(uint32_t);
     void set_utc(bool);
     void set_watchdog(uint16_t);
+    void set_watchdog_min_thread_count(uint16_t);
     SO_PUBLIC bool set_latency_enable();
 
     //------------------------------------------------------

diff --git a/src/main/snort_module.cc b/src/main/snort_module.cc
index 0b33ea6009294826d187505d16e7d3b6fdd4285d..c71bb792785ed545ab1e35273cf96122f4a6bcd2 100644 (file)
--- a/src/main/snort_module.cc
+++ b/src/main/snort_module.cc
@@ -101,8 +101,19 @@ static const Parameter s_pktnum[] =
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
+static const Parameter s_watchdog[] =
+{
+    { "timer", Parameter::PT_INT, "0:max32", nullptr,
+      "timer for watchdog" },
+    { "min_thread_count", Parameter::PT_INT, "0:max32", nullptr,
+      "min thread count for watchdog" },
+
+    { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
+};
+
 static const Command snort_cmds[] =
 {
+    { "set_watchdog_params", main_set_watchdog_params, s_watchdog, "set watchdog parameters" },
     { "show_plugins", main_dump_plugins, nullptr, "show available plugins" },
 
     { "delete_inspector", main_delete_inspector, s_delete,

diff --git a/src/main/thread_config.cc b/src/main/thread_config.cc
index 36f63c96049993b6817977edf25c1ad076ab0949..5d16c870d6d295c37d79fb97613a3a9e78b2116a 100644 (file)
--- a/src/main/thread_config.cc
+++ b/src/main/thread_config.cc
@@ -275,13 +275,23 @@ void Watchdog::kick()
     unsigned max = ThreadConfig::get_instance_max();
     if ( waiting )
     {
-        WarningMessage("Packet processing thread is unresponsive, aborting Snort!\n");
+        uint16_t thread_count = 0;
+        WarningMessage("Packet processing threads are unresponsive\n");
         WarningMessage("Unresponsive thread ID: ");
         for ( unsigned i = 0; i < max; ++i )
+        {
             if ( !resp[i] )
+            {
+                ++thread_count;
                 WarningMessage("%d ", i);
+            }
+        }
         WarningMessage("\n");
-        abort();
+        if ( thread_count >= SnortConfig::get_conf()->watchdog_min_thread_count )
+        {
+            WarningMessage("Aborting Snort\n");
+            abort();
+        }
     }
 
     for ( unsigned i = 0; i < max; ++i )