From: Shanmugam S (shanms) Date: Wed, 30 Nov 2022 13:26:52 +0000 (+0000) Subject: Pull request #3668: process: Watchdog to abort snort when multiple packet thread... X-Git-Tag: 3.1.48.0~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7af8479fbfd7cb827dd5b58b11017ebe888abf82;p=thirdparty%2Fsnort3.git Pull request #3668: process: Watchdog to abort snort when multiple packet thread becomes unresponsive Merge in SNORT/snort3 from ~AMUTTUVA/snort3:snortWatchdogEnhancement to master Squashed commit of the following: commit 242c3a800c4c72a72c81db304e03e1254ac53eaf Author: Akhilesh MY Date: Wed Nov 16 06:39:52 2022 -0500 process: Watchdog to abort snort when multiple packet thread becomes unresponsive --- diff --git a/src/main.cc b/src/main.cc index 6335a4483..060af2d91 100644 --- a/src/main.cc +++ b/src/main.cc @@ -345,6 +345,31 @@ int main_reset_stats(lua_State* L) return 0; } +int main_set_watchdog_params(lua_State* L) +{ + ControlConn* ctrlcon = ControlConn::query_from_lua(L); + SnortConfig* sc = SnortConfig::get_main_conf(); + + if ( sc && L ) + { + int seconds = luaL_optint(L, 1, -1); + int thread_count = luaL_optint(L, 2, -1); + // Timer and thread count are accessed only in main thread context + if ( seconds != -1 ) + sc->set_watchdog(seconds); + + if ( thread_count != -1 ) + sc->set_watchdog_min_thread_count(thread_count); + + std::ostringstream watchdog_timer_msg; + watchdog_timer_msg << "== setting watchdog timer to " << sc->watchdog_timer + << ", min thread count to " << sc->watchdog_min_thread_count << "\n"; + send_response(ctrlcon, watchdog_timer_msg.str().c_str()); + } + + return 0; +} + int main_rotate_stats(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); diff --git a/src/main.h b/src/main.h index 4c422dde6..b7fff84fb 100644 --- a/src/main.h +++ b/src/main.h @@ -29,6 +29,7 @@ const char* get_prompt(); int main_delete_inspector(lua_State* = nullptr); int main_dump_stats(lua_State* = nullptr); int main_reset_stats(lua_State* = nullptr); +int main_set_watchdog_params(lua_State* = nullptr); int main_rotate_stats(lua_State* = nullptr); int main_reload_config(lua_State* = nullptr); int main_reload_policy(lua_State* = nullptr); diff --git a/src/main/modules.cc b/src/main/modules.cc index 70489e09c..313a7e46b 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -1329,6 +1329,9 @@ static const Parameter process_params[] = { "watchdog_timer", Parameter::PT_INT, "0:60", "0", "watchdog timer for packet threads (seconds, 0 to disable)" }, + { "watchdog_min_thread_count", Parameter::PT_INT, "1:65535", "1", + "minimum unresponsive threads for watchdog to trigger" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -1396,6 +1399,9 @@ bool ProcessModule::set(const char*, Value& v, SnortConfig* sc) else if ( v.is("watchdog_timer") ) sc->set_watchdog(v.get_uint16()); + else if ( v.is("watchdog_min_thread_count") ) + sc->set_watchdog_min_thread_count(v.get_uint16()); + return true; } diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 595adc037..cafc1611a 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -645,6 +645,11 @@ void SnortConfig::set_watchdog(uint16_t n) watchdog_timer = n; } +void SnortConfig::set_watchdog_min_thread_count(uint16_t n) +{ + watchdog_min_thread_count = n; +} + void SnortConfig::set_dirty_pig(bool enabled) { dirty_pig = enabled; diff --git a/src/main/snort_config.h b/src/main/snort_config.h index dfeb7df97..14035b5d9 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -245,6 +245,7 @@ public: int user_id = -1; int group_id = -1; uint16_t watchdog_timer = 0; + uint16_t watchdog_min_thread_count = 1; bool dirty_pig = false; std::string chroot_dir; /* -t or config chroot */ @@ -462,6 +463,7 @@ public: void set_umask(uint32_t); void set_utc(bool); void set_watchdog(uint16_t); + void set_watchdog_min_thread_count(uint16_t); SO_PUBLIC bool set_latency_enable(); //------------------------------------------------------ diff --git a/src/main/snort_module.cc b/src/main/snort_module.cc index 0b33ea600..c71bb7927 100644 --- a/src/main/snort_module.cc +++ b/src/main/snort_module.cc @@ -101,8 +101,19 @@ static const Parameter s_pktnum[] = { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; +static const Parameter s_watchdog[] = +{ + { "timer", Parameter::PT_INT, "0:max32", nullptr, + "timer for watchdog" }, + { "min_thread_count", Parameter::PT_INT, "0:max32", nullptr, + "min thread count for watchdog" }, + + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + static const Command snort_cmds[] = { + { "set_watchdog_params", main_set_watchdog_params, s_watchdog, "set watchdog parameters" }, { "show_plugins", main_dump_plugins, nullptr, "show available plugins" }, { "delete_inspector", main_delete_inspector, s_delete, diff --git a/src/main/thread_config.cc b/src/main/thread_config.cc index 36f63c960..5d16c870d 100644 --- a/src/main/thread_config.cc +++ b/src/main/thread_config.cc @@ -275,13 +275,23 @@ void Watchdog::kick() unsigned max = ThreadConfig::get_instance_max(); if ( waiting ) { - WarningMessage("Packet processing thread is unresponsive, aborting Snort!\n"); + uint16_t thread_count = 0; + WarningMessage("Packet processing threads are unresponsive\n"); WarningMessage("Unresponsive thread ID: "); for ( unsigned i = 0; i < max; ++i ) + { if ( !resp[i] ) + { + ++thread_count; WarningMessage("%d ", i); + } + } WarningMessage("\n"); - abort(); + if ( thread_count >= SnortConfig::get_conf()->watchdog_min_thread_count ) + { + WarningMessage("Aborting Snort\n"); + abort(); + } } for ( unsigned i = 0; i < max; ++i )