From: Tom Peters (thopeter) Date: Tue, 1 Mar 2022 21:30:30 +0000 (+0000) Subject: Pull request #3235: process: add watchdog to detect packet threads dead lock or dead... X-Git-Tag: 3.1.25.0~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5797a1fe4ef927b9f034518e6ad0a8d3f35ebb75;p=thirdparty%2Fsnort3.git Pull request #3235: process: add watchdog to detect packet threads dead lock or dead loop Merge in SNORT/snort3 from ~SBAIGAL/snort3:watchdog to master Squashed commit of the following: commit 8879f0f31b9ff1ad0b7b15f8650153ab9eecccbb Author: Steven Baigal (sbaigal) Date: Thu Jan 13 12:35:30 2022 -0500 process: add watchdog to detect packet threads dead lock or dead loop --- diff --git a/src/main.cc b/src/main.cc index 44b6ab670..0e67c224b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1103,6 +1103,7 @@ static void snort_main() max_pigs = ThreadConfig::get_instance_max(); assert(max_pigs > 0); + ThreadConfig::start_watchdog(); // maximum number of state change notifications per pig constexpr unsigned max_grunts = static_cast(Analyzer::State::NUM_STATES); diff --git a/src/main/modules.cc b/src/main/modules.cc index 07b71f33b..33ae97078 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -1432,6 +1432,9 @@ static const Parameter process_params[] = { "utc", Parameter::PT_BOOL, nullptr, "false", "use UTC instead of local time for timestamps" }, + { "watchdog_timer", Parameter::PT_INT, "0:60", "0", + "watchdog timer for packet threads (seconds, 0 to disable)" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -1496,6 +1499,9 @@ bool ProcessModule::set(const char*, Value& v, SnortConfig* sc) else if (v.is("name")) name = v.get_string(); + else if ( v.is("watchdog_timer") ) + sc->set_watchdog(v.get_uint16()); + return true; } diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 89d0e8c48..99f2436c5 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -618,6 +618,11 @@ void SnortConfig::set_log_dir(const char* directory) log_dir.clear(); } +void SnortConfig::set_watchdog(uint16_t n) +{ + watchdog_timer = n; +} + void SnortConfig::set_dirty_pig(bool enabled) { dirty_pig = enabled; diff --git a/src/main/snort_config.h b/src/main/snort_config.h index 75b18b299..f1eb85ecc 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -268,7 +268,7 @@ public: // chown() use this later, -1 means no change to user_id/group_id int user_id = -1; int group_id = -1; - + uint16_t watchdog_timer = 0; bool dirty_pig = false; std::string chroot_dir; /* -t or config chroot */ @@ -471,6 +471,7 @@ public: void set_no_logging_timestamps(bool); void set_obfuscate(bool); void set_obfuscation_mask(const char*); + void set_overlay_trace_config(TraceConfig*); void set_include_path(const char*); void set_process_all_events(bool); void set_rule_db_dir(const char*); @@ -480,7 +481,7 @@ public: void set_uid(const char*); void set_umask(uint32_t); void set_utc(bool); - void set_overlay_trace_config(TraceConfig*); + void set_watchdog(uint16_t); SO_PUBLIC bool set_latency_enable(); //------------------------------------------------------ diff --git a/src/main/thread_config.cc b/src/main/thread_config.cc index 7740d5dcd..f4609bd23 100644 --- a/src/main/thread_config.cc +++ b/src/main/thread_config.cc @@ -23,9 +23,13 @@ #include "thread_config.h" +#include #include +#include "analyzer_command.h" #include "log/messages.h" +#include "main/snort_config.h" +#include "time/periodic.h" #include "utils/util.h" #ifdef UNIT_TEST @@ -236,6 +240,77 @@ void ThreadConfig::implement_named_thread_affinity(const string& name) implement_thread_affinity(get_thread_type(), DEFAULT_THREAD_ID); } +// watchdog stuff +struct Watchdog +{ + Watchdog(uint16_t tm) : seconds_count(tm) + { + resp = new std::atomic_bool[ThreadConfig::get_instance_max()]; + } + ~Watchdog() { delete[] resp; } + void kick(); + bool waiting = false; + std::atomic_bool* resp; + uint16_t seconds_count; +}; + +class WatchdogKick : public AnalyzerCommand +{ +public: + WatchdogKick(Watchdog* d) : dog(d) { dog->waiting = true; } + bool execute(Analyzer&, void**) override + { + dog->resp[get_instance_id()] = true; + return true; + } + const char* stringify() override { return "WATCHDOG_KICK"; } + + ~WatchdogKick() override { dog->waiting = false; } +private: + Watchdog* dog; +}; + +void Watchdog::kick() +{ + unsigned max = ThreadConfig::get_instance_max(); + if ( waiting ) + { + int n = 0; + WarningMessage("Packet processing thread is unresponsive, aborting Snort!\n"); + WarningMessage("Unresponsive thread ID: "); + for ( unsigned i = 0; i < max; ++i ) + if ( !resp[i] ) + WarningMessage("%d ", i); + WarningMessage("\n"); + abort(); + } + + for ( unsigned i = 0; i < max; ++i ) + resp[i] = false; + + main_broadcast_command(new WatchdogKick(this), nullptr); +} + +static void s_watchdog_handler(void*) +{ + static Watchdog s_dog(SnortConfig::get_conf()->watchdog_timer); + if ( SnortConfig::get_conf()->watchdog_timer > 0 ) + { + if ( s_dog.seconds_count > 0 ) + s_dog.seconds_count--; + else + { + s_dog.kick(); + s_dog.seconds_count = SnortConfig::get_conf()->watchdog_timer; + } + } +} + +void ThreadConfig::start_watchdog() +{ + Periodic::register_handler(s_watchdog_handler, nullptr, 0, 1000); +} + // ----------------------------------------------------------------------------- // unit tests diff --git a/src/main/thread_config.h b/src/main/thread_config.h index 8c945d744..ca9e97076 100644 --- a/src/main/thread_config.h +++ b/src/main/thread_config.h @@ -38,6 +38,7 @@ public: static void set_instance_max(unsigned); static unsigned get_instance_max(); static void term(); + static void start_watchdog(); ~ThreadConfig(); void set_thread_affinity(SThreadType, unsigned id, CpuSet*);