Merge pull request #2254 in SNORT/snort3 from ~SMINUT/snort3:thread_pinning to master

Squashed commit of the following:

commit 19737811dc07397ac6d61328ac5d1b60b7df7ff6
Author: Silviu Minut <sminut@cisco.com>
Date:   Tue Jun 9 23:30:57 2020 -0400

    main: configure and set main thread affinity

diff --git a/src/main.cc b/src/main.cc
index 6726f02f7fb03a7ef644752beae8cd50b0157876..e9ac830400eff7d940d32aed388fe89c9efcee29 100644 (file)
--- a/src/main.cc
+++ b/src/main.cc
@@ -991,6 +991,9 @@ static void snort_main()
     ControlMgmt::socket_init();
 #endif
 
+    SnortConfig::get_conf()->thread_config->implement_thread_affinity(
+        STHREAD_TYPE_MAIN, get_instance_id());
+
     max_pigs = ThreadConfig::get_instance_max();
     assert(max_pigs > 0);
 

diff --git a/src/main/CMakeLists.txt b/src/main/CMakeLists.txt
index c5eaed25eb9f9f0563b831e662a88ee7e46fd59c..502e5710dc6951540aaa09681550ac78d3247e69 100644 (file)
--- a/src/main/CMakeLists.txt
+++ b/src/main/CMakeLists.txt
@@ -7,6 +7,7 @@ set (INCLUDES
     snort_debug.h
     snort_types.h
     thread.h
+    thread_config.h
 )
 
 if ( ENABLE_SHELL )

diff --git a/src/main/modules.cc b/src/main/modules.cc
index 1230faf711826648367c8ce3daecfcf69d2d93be..11a6bf74f5a80eccfb897d146d32baa47b0096b7 100644 (file)
--- a/src/main/modules.cc
+++ b/src/main/modules.cc
@@ -1272,9 +1272,15 @@ static const Parameter thread_pinning_params[] =
     { "cpuset", Parameter::PT_STRING, nullptr, nullptr,
       "pin the associated thread to this cpuset" },
 
-    { "thread", Parameter::PT_INT, "0:65535", "0",
+    { "thread", Parameter::PT_INT, "0:65535", nullptr,
       "set cpu affinity for the <cur_thread_num> thread that runs" },
 
+    { "type", Parameter::PT_ENUM, "other|packet|main", nullptr,
+      "define which threads will have specified affinity, by their type" },
+
+    { "name", Parameter::PT_STRING, nullptr, nullptr,
+      "define which threads will have specified affinity, by thread name" },
+
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
@@ -1324,6 +1330,8 @@ public:
 private:
     int thread;
     CpuSet* cpuset;
+    string type;
+    string name;
 };
 
 bool ProcessModule::set(const char*, Value& v, SnortConfig* sc)
@@ -1356,9 +1364,16 @@ bool ProcessModule::set(const char*, Value& v, SnortConfig* sc)
         if (!(cpuset = ThreadConfig::validate_cpuset_string(v.get_string())))
             return false;
     }
+
     else if (v.is("thread"))
         thread = v.get_uint16();
 
+    else if (v.is("type"))
+        type = v.get_string();
+
+    else if (v.is("name"))
+        name = v.get_string();
+
     else
         return false;
 
@@ -1369,6 +1384,8 @@ bool ProcessModule::begin(const char*, int, SnortConfig*)
 {
     thread = -1;
     cpuset = nullptr;
+    type.clear();
+    name.clear();
     return true;
 }
 
@@ -1379,19 +1396,61 @@ bool ProcessModule::end(const char* fqn, int idx, SnortConfig* sc)
 
     if (!strcmp(fqn, "process.threads"))
     {
-        if (thread == -1)
+        if (!cpuset)
         {
-            ParseError("%s - no thread ID specified", fqn);
-            if (cpuset)
+            ParseError("%s - no cpuset specified", fqn);
+            return false;
+        }
+
+        if (thread >= 0)
+        {
+            // Packet thread affinity.
+            if ( !(name.empty() && (type.empty() || type == "packet")) )
+            {
+                ParseError("%s - type or name specified for thread %d", fqn, thread);
                 ThreadConfig::destroy_cpuset(cpuset);
+                return false;
+            }
+
+            // Thread type is implicitly "packet".
+            sc->thread_config->set_thread_affinity(STHREAD_TYPE_PACKET, thread, cpuset);
+        }
+
+        else if (!type.empty() && name.empty())
+        {
+            // Type-based affinity: main, or other.
+            thread = ThreadConfig::DEFAULT_THREAD_ID;
+            if (type == "main")
+                sc->thread_config->set_thread_affinity(STHREAD_TYPE_MAIN, thread, cpuset);
+            else if (type == "other")
+                sc->thread_config->set_thread_affinity(STHREAD_TYPE_OTHER, thread, cpuset);
+            else
+            {
+                ParseError("%s - bad thread type %s", fqn, type.c_str());
+                ThreadConfig::destroy_cpuset(cpuset);
+                return false;
+            }
+        }
+
+        else if (type.empty() && !name.empty())
+        {
+            // name-based affinity
+            sc->thread_config->set_named_thread_affinity(name, cpuset);
+        }
+
+        else if (!type.empty() && !name.empty())
+        {
+            ParseError("%s - can't specify both type and name", fqn);
+            ThreadConfig::destroy_cpuset(cpuset);
             return false;
         }
-        if (!cpuset)
+
+        else
         {
-            ParseError("%s - no cpuset specified for thread %d", fqn, thread);
+            ParseError("%s - none of thread, type or name specified", fqn);
+            ThreadConfig::destroy_cpuset(cpuset);
             return false;
         }
-        sc->thread_config->set_thread_affinity(STHREAD_TYPE_PACKET, thread, cpuset);
     }
 
     return true;

diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index b10bed934e02589cd9d0390c3287e84da500b3b6..769f7ceeedc85447b0f8e995241938fcf8ff2d75 100644 (file)
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -127,7 +127,6 @@ enum TunnelFlags
 
 class FastPatternConfig;
 class RuleStateMap;
-class ThreadConfig;
 class TraceConfig;
 
 struct srmm_table_t;
@@ -155,6 +154,7 @@ namespace snort
 {
 class GHash;
 class ProtocolReference;
+class ThreadConfig;
 class XHash;
 struct ProfilerConfig;
 
@@ -703,7 +703,7 @@ public:
 
     // runtime access to mutable config - main thread only, and only special cases
     SO_PUBLIC static SnortConfig* get_main_conf();
-    
+
     static void set_conf(const SnortConfig*);
 
     SO_PUBLIC void register_reload_resource_tuner(ReloadResourceTuner& rrt)

diff --git a/src/main/test/stubs.h b/src/main/test/stubs.h
index ac572c840c66a92aec7d96a598cd98dfadda62f8..7d46a49ed72c91a418f2fb57064203a73f76b527 100644 (file)
--- a/src/main/test/stubs.h
+++ b/src/main/test/stubs.h
@@ -66,7 +66,6 @@ THREAD_LOCAL DAQStats daq_stats;
 void Profiler::start() { }
 void Profiler::stop(uint64_t) { }
 void Profiler::consolidate_stats() { }
-void ThreadConfig::implement_thread_affinity(SThreadType, unsigned) { }
 void Swapper::apply(Analyzer&) { }
 Swapper::~Swapper() { }
 void OopsHandler::tinit() { }
@@ -211,4 +210,5 @@ NetworkPolicy* get_network_policy() { return nullptr; }
 InspectionPolicy* get_inspection_policy() { return nullptr; }
 Flow::Flow() { }
 Flow::~Flow() { }
+void ThreadConfig::implement_thread_affinity(SThreadType, unsigned) { }
 }

diff --git a/src/main/thread.cc b/src/main/thread.cc
index d4ff0f838f53946095cae03f399c3f5749f3e74c..71386bca7387309ae5cfc9985a10610627109edd 100644 (file)
--- a/src/main/thread.cc
+++ b/src/main/thread.cc
@@ -36,7 +36,7 @@
 
 static THREAD_LOCAL uint16_t run_num = 0;
 static THREAD_LOCAL unsigned instance_id = 0;
-static THREAD_LOCAL SThreadType thread_type = STHREAD_TYPE_MAIN;
+static THREAD_LOCAL SThreadType thread_type = STHREAD_TYPE_OTHER;
 
 void set_run_num(uint16_t num)
 { run_num = num; }

diff --git a/src/main/thread.h b/src/main/thread.h
index d0b5410dea184951a7691d085bbedd75a56bbc6c..4705ea878e41bf66b70791dc38fd56b388d4d502 100644 (file)
--- a/src/main/thread.h
+++ b/src/main/thread.h
@@ -39,6 +39,7 @@
 
 enum SThreadType
 {
+    STHREAD_TYPE_OTHER,
     STHREAD_TYPE_PACKET,
     STHREAD_TYPE_MAIN
 };

diff --git a/src/main/thread_config.cc b/src/main/thread_config.cc
index e58441df586923d2ab082c22aa474e010a0dcce1..d44997ce1b3ac30eb17ff0988b92ad16004f1549 100644 (file)
--- a/src/main/thread_config.cc
+++ b/src/main/thread_config.cc
@@ -122,8 +122,11 @@ void ThreadConfig::term()
 
 ThreadConfig::~ThreadConfig()
 {
-    for (auto iter = thread_affinity.begin(); iter != thread_affinity.end(); iter++)
-        delete iter->second;
+    for (auto& iter : thread_affinity)
+        delete iter.second;
+
+    for (auto& iter : named_thread_affinity)
+        delete iter.second;
 }
 
 void ThreadConfig::set_thread_affinity(SThreadType type, unsigned id, CpuSet* cpuset)
@@ -141,6 +144,19 @@ void ThreadConfig::set_thread_affinity(SThreadType type, unsigned id, CpuSet* cp
         ParseWarning(WARN_CONF, "This platform does not support setting thread affinity.\n");
 }
 
+void ThreadConfig::set_named_thread_affinity(const std::string& name, CpuSet* cpuset)
+{
+    if (topology_support->cpubind->set_thisthread_cpubind)
+    {
+        auto iter = named_thread_affinity.find(name);
+        if (iter != named_thread_affinity.end())
+            delete iter->second;
+        named_thread_affinity[name] = cpuset;
+    }
+    else
+        ParseWarning(WARN_CONF, "This platform does not support setting thread affinity.\n");
+}
+
 void ThreadConfig::implement_thread_affinity(SThreadType type, unsigned id)
 {
     if (!topology_support->cpubind->set_thisthread_cpubind)
@@ -169,12 +185,43 @@ void ThreadConfig::implement_thread_affinity(SThreadType type, unsigned id)
     if (hwloc_set_cpubind(topology, desired_cpuset, HWLOC_CPUBIND_THREAD))
     {
         FatalError("Failed to pin thread %u (type %u) to %s: %s (%d)\n",
-                id, type, s, get_error(errno), errno);
+            id, type, s, get_error(errno), errno);
     }
 
     free(s);
 }
 
+void ThreadConfig::implement_named_thread_affinity(const std::string& name)
+{
+    if (!topology_support->cpubind->set_thisthread_cpubind)
+        return;
+
+    auto iter = named_thread_affinity.find(name);
+    if (iter != named_thread_affinity.end())
+    {
+        char* s;
+
+        auto desired_cpuset = iter->second->cpuset;
+        hwloc_bitmap_list_asprintf(&s, desired_cpuset);
+
+        auto current_cpuset = hwloc_bitmap_alloc();
+        hwloc_get_cpubind(topology, current_cpuset, HWLOC_CPUBIND_THREAD);
+        if (!hwloc_bitmap_isequal(current_cpuset, desired_cpuset))
+            LogMessage("Binding thread %s to %s.\n", name.c_str(), s);
+        hwloc_bitmap_free(current_cpuset);
+
+        if (hwloc_set_cpubind(topology, desired_cpuset, HWLOC_CPUBIND_THREAD))
+        {
+            FatalError("Failed to pin thread %s to %s: %s (%d)\n",
+                name.c_str(), s, get_error(errno), errno);
+        }
+
+        free(s);
+    }
+    else
+        implement_thread_affinity(get_thread_type(), DEFAULT_THREAD_ID);
+}
+
 
 // -----------------------------------------------------------------------------
 // unit tests
@@ -233,4 +280,54 @@ TEST_CASE("Set and implement thread affinity", "[ThreadConfig]")
     }
 }
 
+TEST_CASE("Named thread affinity configured", "[ThreadConfig]")
+{
+    if (topology_support->cpubind->set_thisthread_cpubind)
+    {
+        CpuSet* cpuset = new CpuSet(hwloc_bitmap_dup(process_cpuset));
+        ThreadConfig tc;
+
+        hwloc_cpuset_t thread_cpuset = hwloc_bitmap_alloc();
+
+        // Configure named thread.
+        hwloc_bitmap_singlify(cpuset->cpuset);
+        tc.set_named_thread_affinity("found", cpuset);
+
+        // The one in the named map, should have the specified cpuset.
+        tc.implement_named_thread_affinity("found");
+        hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD);
+        CHECK(hwloc_bitmap_isequal(thread_cpuset, cpuset->cpuset));
+
+        // The one not in the named map, should have the process cpuset
+        // if no type has been configured for it.
+        tc.implement_named_thread_affinity("not found, no type configured");
+        hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD);
+        CHECK(hwloc_bitmap_isequal(thread_cpuset, process_cpuset));
+
+        hwloc_bitmap_free(thread_cpuset);
+    }
+}
+
+TEST_CASE("Named thread affinity with type configured", "[ThreadConfig]")
+{
+    if (topology_support->cpubind->set_thisthread_cpubind)
+    {
+        CpuSet* type_cpuset = new CpuSet(hwloc_bitmap_dup(process_cpuset));
+        ThreadConfig tc;
+
+        hwloc_cpuset_t thread_cpuset = hwloc_bitmap_alloc();
+
+        // Configure type affinity, but not the named thread affinity.
+        hwloc_bitmap_singlify(type_cpuset->cpuset);
+        tc.set_thread_affinity(STHREAD_TYPE_OTHER, ThreadConfig::DEFAULT_THREAD_ID, type_cpuset);
+
+        // The named thread should inherit the type affinity.
+        tc.implement_named_thread_affinity("not found, type other");
+        hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD);
+        CHECK(hwloc_bitmap_isequal(thread_cpuset, type_cpuset->cpuset));
+
+        hwloc_bitmap_free(thread_cpuset);
+    }
+}
+
 #endif

diff --git a/src/main/thread_config.h b/src/main/thread_config.h
index a0cdbcc69be6506083c3c96b716663291ba636c9..58f3fa941832527e0e711948b8a4e0724863eca7 100644 (file)
--- a/src/main/thread_config.h
+++ b/src/main/thread_config.h
@@ -20,12 +20,16 @@
 #define THREAD_CONFIG_H
 
 #include <map>
+#include <string>
 
 #include "main/thread.h"
 
 struct CpuSet;
 
-class ThreadConfig
+namespace snort
+{
+
+class SO_PUBLIC ThreadConfig
 {
 public:
     static bool init();
@@ -37,7 +41,12 @@ public:
 
     ~ThreadConfig();
     void set_thread_affinity(SThreadType, unsigned id, CpuSet*);
+    void set_named_thread_affinity(const std::string&, CpuSet*);
     void implement_thread_affinity(SThreadType, unsigned id);
+    void implement_named_thread_affinity(const std::string& name);
+
+    static constexpr unsigned int DEFAULT_THREAD_ID = 0;
+
 private:
     struct TypeIdPair
     {
@@ -56,6 +65,8 @@ private:
         }
     };
     std::map<TypeIdPair, CpuSet*, TypeIdPairComparer> thread_affinity;
+    std::map<std::string, CpuSet*> named_thread_affinity;
 };
+}
 
 #endif