Pull request #3235: process: add watchdog to detect packet threads dead lock or dead...

Merge in SNORT/snort3 from ~SBAIGAL/snort3:watchdog to master

Squashed commit of the following:

commit 8879f0f31b9ff1ad0b7b15f8650153ab9eecccbb
Author: Steven Baigal (sbaigal) <sbaigal@cisco.com>
Date:   Thu Jan 13 12:35:30 2022 -0500

    process: add watchdog to detect packet threads dead lock or dead loop

diff --git a/src/main.cc b/src/main.cc
index 44b6ab670504e2b8ed5afda8b0d6283f131845dc..0e67c224b4e9afd8b33ccc8fdd56e3e44f8fa5cd 100644 (file)
--- a/src/main.cc
+++ b/src/main.cc
@@ -1103,6 +1103,7 @@ static void snort_main()
 
     max_pigs = ThreadConfig::get_instance_max();
     assert(max_pigs > 0);
+    ThreadConfig::start_watchdog();
 
     // maximum number of state change notifications per pig
     constexpr unsigned max_grunts = static_cast<unsigned>(Analyzer::State::NUM_STATES);

diff --git a/src/main/modules.cc b/src/main/modules.cc
index 07b71f33bb49a977418084ed5006dd27ae640364..33ae970782f8f967b4dd50dd0fa68013c25c5012 100644 (file)
--- a/src/main/modules.cc
+++ b/src/main/modules.cc
@@ -1432,6 +1432,9 @@ static const Parameter process_params[] =
     { "utc", Parameter::PT_BOOL, nullptr, "false",
       "use UTC instead of local time for timestamps" },
 
+    { "watchdog_timer", Parameter::PT_INT, "0:60", "0",
+      "watchdog timer for packet threads (seconds, 0 to disable)" },
+
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
@@ -1496,6 +1499,9 @@ bool ProcessModule::set(const char*, Value& v, SnortConfig* sc)
     else if (v.is("name"))
         name = v.get_string();
 
+    else if ( v.is("watchdog_timer") )
+        sc->set_watchdog(v.get_uint16());
+
     return true;
 }
 

diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index 89d0e8c480bc87235c2035cfe6ec1733d802021d..99f2436c5c04e271cdf120362a50e2b4bbe397b7 100644 (file)
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -618,6 +618,11 @@ void SnortConfig::set_log_dir(const char* directory)
         log_dir.clear();
 }
 
+void SnortConfig::set_watchdog(uint16_t n)
+{
+    watchdog_timer = n;
+}
+
 void SnortConfig::set_dirty_pig(bool enabled)
 {
     dirty_pig = enabled;

diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index 75b18b299d7eaec168419b19092ebf1fe66fab4e..f1eb85ecc1e5103ce55869f0d09093901364c105 100644 (file)
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -268,7 +268,7 @@ public:
     // chown() use this later, -1 means no change to user_id/group_id
     int user_id = -1;
     int group_id = -1;
-
+    uint16_t watchdog_timer = 0;
     bool dirty_pig = false;
 
     std::string chroot_dir;        /* -t or config chroot */
@@ -471,6 +471,7 @@ public:
     void set_no_logging_timestamps(bool);
     void set_obfuscate(bool);
     void set_obfuscation_mask(const char*);
+    void set_overlay_trace_config(TraceConfig*);
     void set_include_path(const char*);
     void set_process_all_events(bool);
     void set_rule_db_dir(const char*);
@@ -480,7 +481,7 @@ public:
     void set_uid(const char*);
     void set_umask(uint32_t);
     void set_utc(bool);
-    void set_overlay_trace_config(TraceConfig*);
+    void set_watchdog(uint16_t);
     SO_PUBLIC bool set_latency_enable();
 
     //------------------------------------------------------

diff --git a/src/main/thread_config.cc b/src/main/thread_config.cc
index 7740d5dcd5d7435f8c95ce3efb2e14fc6e7aa82e..f4609bd237e19a2b8d5d6227d98b039a95f8a1e4 100644 (file)
--- a/src/main/thread_config.cc
+++ b/src/main/thread_config.cc
@@ -23,9 +23,13 @@
 
 #include "thread_config.h"
 
+#include <atomic>
 #include <hwloc.h>
 
+#include "analyzer_command.h"
 #include "log/messages.h"
+#include "main/snort_config.h"
+#include "time/periodic.h"
 #include "utils/util.h"
 
 #ifdef UNIT_TEST
@@ -236,6 +240,77 @@ void ThreadConfig::implement_named_thread_affinity(const string& name)
         implement_thread_affinity(get_thread_type(), DEFAULT_THREAD_ID);
 }
 
+// watchdog stuff
+struct Watchdog
+{
+    Watchdog(uint16_t tm) : seconds_count(tm)
+    {
+        resp = new std::atomic_bool[ThreadConfig::get_instance_max()];
+    }
+    ~Watchdog() { delete[] resp; }
+    void kick();
+    bool waiting = false;
+    std::atomic_bool* resp;
+    uint16_t seconds_count;
+};
+
+class WatchdogKick : public AnalyzerCommand
+{
+public:
+    WatchdogKick(Watchdog* d) : dog(d) { dog->waiting = true; }
+    bool execute(Analyzer&, void**) override
+    {
+        dog->resp[get_instance_id()] = true;
+        return true;
+    }
+    const char* stringify() override { return "WATCHDOG_KICK"; }
+
+    ~WatchdogKick() override { dog->waiting = false; }
+private:
+    Watchdog* dog;
+};
+
+void Watchdog::kick()
+{
+    unsigned max = ThreadConfig::get_instance_max();
+    if ( waiting )
+    {
+        int n = 0;
+        WarningMessage("Packet processing thread is unresponsive, aborting Snort!\n");
+        WarningMessage("Unresponsive thread ID: ");
+        for ( unsigned i = 0; i < max; ++i )
+            if ( !resp[i] )
+                WarningMessage("%d ", i);
+        WarningMessage("\n");
+        abort();
+    }
+
+    for ( unsigned i = 0; i < max; ++i )
+        resp[i] = false;
+
+    main_broadcast_command(new WatchdogKick(this), nullptr);
+}
+
+static void s_watchdog_handler(void*)
+{
+    static Watchdog s_dog(SnortConfig::get_conf()->watchdog_timer);
+    if ( SnortConfig::get_conf()->watchdog_timer > 0 )
+    {
+        if ( s_dog.seconds_count > 0 )
+            s_dog.seconds_count--;
+        else
+        {
+            s_dog.kick();
+            s_dog.seconds_count = SnortConfig::get_conf()->watchdog_timer;
+        }
+    }
+}
+
+void ThreadConfig::start_watchdog()
+{
+    Periodic::register_handler(s_watchdog_handler, nullptr, 0, 1000);
+}
+
 
 // -----------------------------------------------------------------------------
 // unit tests

diff --git a/src/main/thread_config.h b/src/main/thread_config.h
index 8c945d744044b33fcbf9f912ec5e435cfed95d71..ca9e9707677e15a35a9aeed1d5caff1c4f59bc05 100644 (file)
--- a/src/main/thread_config.h
+++ b/src/main/thread_config.h
@@ -38,6 +38,7 @@ public:
     static void set_instance_max(unsigned);
     static unsigned get_instance_max();
     static void term();
+    static void start_watchdog();
 
     ~ThreadConfig();
     void set_thread_affinity(SThreadType, unsigned id, CpuSet*);