From: Michael Altizer (mialtize) Date: Wed, 24 Jun 2020 18:37:00 +0000 (+0000) Subject: Merge pull request #2254 in SNORT/snort3 from ~SMINUT/snort3:thread_pinning to master X-Git-Tag: 3.0.2-1~19 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d1942b76533d5ae56033b6e06e01d1dd3273a0d6;p=thirdparty%2Fsnort3.git Merge pull request #2254 in SNORT/snort3 from ~SMINUT/snort3:thread_pinning to master Squashed commit of the following: commit 19737811dc07397ac6d61328ac5d1b60b7df7ff6 Author: Silviu Minut Date: Tue Jun 9 23:30:57 2020 -0400 main: configure and set main thread affinity --- diff --git a/src/main.cc b/src/main.cc index 6726f02f7..e9ac83040 100644 --- a/src/main.cc +++ b/src/main.cc @@ -991,6 +991,9 @@ static void snort_main() ControlMgmt::socket_init(); #endif + SnortConfig::get_conf()->thread_config->implement_thread_affinity( + STHREAD_TYPE_MAIN, get_instance_id()); + max_pigs = ThreadConfig::get_instance_max(); assert(max_pigs > 0); diff --git a/src/main/CMakeLists.txt b/src/main/CMakeLists.txt index c5eaed25e..502e5710d 100644 --- a/src/main/CMakeLists.txt +++ b/src/main/CMakeLists.txt @@ -7,6 +7,7 @@ set (INCLUDES snort_debug.h snort_types.h thread.h + thread_config.h ) if ( ENABLE_SHELL ) diff --git a/src/main/modules.cc b/src/main/modules.cc index 1230faf71..11a6bf74f 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -1272,9 +1272,15 @@ static const Parameter thread_pinning_params[] = { "cpuset", Parameter::PT_STRING, nullptr, nullptr, "pin the associated thread to this cpuset" }, - { "thread", Parameter::PT_INT, "0:65535", "0", + { "thread", Parameter::PT_INT, "0:65535", nullptr, "set cpu affinity for the thread that runs" }, + { "type", Parameter::PT_ENUM, "other|packet|main", nullptr, + "define which threads will have specified affinity, by their type" }, + + { "name", Parameter::PT_STRING, nullptr, nullptr, + "define which threads will have specified affinity, by thread name" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -1324,6 +1330,8 @@ public: private: int thread; CpuSet* cpuset; + string type; + string name; }; bool ProcessModule::set(const char*, Value& v, SnortConfig* sc) @@ -1356,9 +1364,16 @@ bool ProcessModule::set(const char*, Value& v, SnortConfig* sc) if (!(cpuset = ThreadConfig::validate_cpuset_string(v.get_string()))) return false; } + else if (v.is("thread")) thread = v.get_uint16(); + else if (v.is("type")) + type = v.get_string(); + + else if (v.is("name")) + name = v.get_string(); + else return false; @@ -1369,6 +1384,8 @@ bool ProcessModule::begin(const char*, int, SnortConfig*) { thread = -1; cpuset = nullptr; + type.clear(); + name.clear(); return true; } @@ -1379,19 +1396,61 @@ bool ProcessModule::end(const char* fqn, int idx, SnortConfig* sc) if (!strcmp(fqn, "process.threads")) { - if (thread == -1) + if (!cpuset) { - ParseError("%s - no thread ID specified", fqn); - if (cpuset) + ParseError("%s - no cpuset specified", fqn); + return false; + } + + if (thread >= 0) + { + // Packet thread affinity. + if ( !(name.empty() && (type.empty() || type == "packet")) ) + { + ParseError("%s - type or name specified for thread %d", fqn, thread); ThreadConfig::destroy_cpuset(cpuset); + return false; + } + + // Thread type is implicitly "packet". + sc->thread_config->set_thread_affinity(STHREAD_TYPE_PACKET, thread, cpuset); + } + + else if (!type.empty() && name.empty()) + { + // Type-based affinity: main, or other. + thread = ThreadConfig::DEFAULT_THREAD_ID; + if (type == "main") + sc->thread_config->set_thread_affinity(STHREAD_TYPE_MAIN, thread, cpuset); + else if (type == "other") + sc->thread_config->set_thread_affinity(STHREAD_TYPE_OTHER, thread, cpuset); + else + { + ParseError("%s - bad thread type %s", fqn, type.c_str()); + ThreadConfig::destroy_cpuset(cpuset); + return false; + } + } + + else if (type.empty() && !name.empty()) + { + // name-based affinity + sc->thread_config->set_named_thread_affinity(name, cpuset); + } + + else if (!type.empty() && !name.empty()) + { + ParseError("%s - can't specify both type and name", fqn); + ThreadConfig::destroy_cpuset(cpuset); return false; } - if (!cpuset) + + else { - ParseError("%s - no cpuset specified for thread %d", fqn, thread); + ParseError("%s - none of thread, type or name specified", fqn); + ThreadConfig::destroy_cpuset(cpuset); return false; } - sc->thread_config->set_thread_affinity(STHREAD_TYPE_PACKET, thread, cpuset); } return true; diff --git a/src/main/snort_config.h b/src/main/snort_config.h index b10bed934..769f7ceee 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -127,7 +127,6 @@ enum TunnelFlags class FastPatternConfig; class RuleStateMap; -class ThreadConfig; class TraceConfig; struct srmm_table_t; @@ -155,6 +154,7 @@ namespace snort { class GHash; class ProtocolReference; +class ThreadConfig; class XHash; struct ProfilerConfig; @@ -703,7 +703,7 @@ public: // runtime access to mutable config - main thread only, and only special cases SO_PUBLIC static SnortConfig* get_main_conf(); - + static void set_conf(const SnortConfig*); SO_PUBLIC void register_reload_resource_tuner(ReloadResourceTuner& rrt) diff --git a/src/main/test/stubs.h b/src/main/test/stubs.h index ac572c840..7d46a49ed 100644 --- a/src/main/test/stubs.h +++ b/src/main/test/stubs.h @@ -66,7 +66,6 @@ THREAD_LOCAL DAQStats daq_stats; void Profiler::start() { } void Profiler::stop(uint64_t) { } void Profiler::consolidate_stats() { } -void ThreadConfig::implement_thread_affinity(SThreadType, unsigned) { } void Swapper::apply(Analyzer&) { } Swapper::~Swapper() { } void OopsHandler::tinit() { } @@ -211,4 +210,5 @@ NetworkPolicy* get_network_policy() { return nullptr; } InspectionPolicy* get_inspection_policy() { return nullptr; } Flow::Flow() { } Flow::~Flow() { } +void ThreadConfig::implement_thread_affinity(SThreadType, unsigned) { } } diff --git a/src/main/thread.cc b/src/main/thread.cc index d4ff0f838..71386bca7 100644 --- a/src/main/thread.cc +++ b/src/main/thread.cc @@ -36,7 +36,7 @@ static THREAD_LOCAL uint16_t run_num = 0; static THREAD_LOCAL unsigned instance_id = 0; -static THREAD_LOCAL SThreadType thread_type = STHREAD_TYPE_MAIN; +static THREAD_LOCAL SThreadType thread_type = STHREAD_TYPE_OTHER; void set_run_num(uint16_t num) { run_num = num; } diff --git a/src/main/thread.h b/src/main/thread.h index d0b5410de..4705ea878 100644 --- a/src/main/thread.h +++ b/src/main/thread.h @@ -39,6 +39,7 @@ enum SThreadType { + STHREAD_TYPE_OTHER, STHREAD_TYPE_PACKET, STHREAD_TYPE_MAIN }; diff --git a/src/main/thread_config.cc b/src/main/thread_config.cc index e58441df5..d44997ce1 100644 --- a/src/main/thread_config.cc +++ b/src/main/thread_config.cc @@ -122,8 +122,11 @@ void ThreadConfig::term() ThreadConfig::~ThreadConfig() { - for (auto iter = thread_affinity.begin(); iter != thread_affinity.end(); iter++) - delete iter->second; + for (auto& iter : thread_affinity) + delete iter.second; + + for (auto& iter : named_thread_affinity) + delete iter.second; } void ThreadConfig::set_thread_affinity(SThreadType type, unsigned id, CpuSet* cpuset) @@ -141,6 +144,19 @@ void ThreadConfig::set_thread_affinity(SThreadType type, unsigned id, CpuSet* cp ParseWarning(WARN_CONF, "This platform does not support setting thread affinity.\n"); } +void ThreadConfig::set_named_thread_affinity(const std::string& name, CpuSet* cpuset) +{ + if (topology_support->cpubind->set_thisthread_cpubind) + { + auto iter = named_thread_affinity.find(name); + if (iter != named_thread_affinity.end()) + delete iter->second; + named_thread_affinity[name] = cpuset; + } + else + ParseWarning(WARN_CONF, "This platform does not support setting thread affinity.\n"); +} + void ThreadConfig::implement_thread_affinity(SThreadType type, unsigned id) { if (!topology_support->cpubind->set_thisthread_cpubind) @@ -169,12 +185,43 @@ void ThreadConfig::implement_thread_affinity(SThreadType type, unsigned id) if (hwloc_set_cpubind(topology, desired_cpuset, HWLOC_CPUBIND_THREAD)) { FatalError("Failed to pin thread %u (type %u) to %s: %s (%d)\n", - id, type, s, get_error(errno), errno); + id, type, s, get_error(errno), errno); } free(s); } +void ThreadConfig::implement_named_thread_affinity(const std::string& name) +{ + if (!topology_support->cpubind->set_thisthread_cpubind) + return; + + auto iter = named_thread_affinity.find(name); + if (iter != named_thread_affinity.end()) + { + char* s; + + auto desired_cpuset = iter->second->cpuset; + hwloc_bitmap_list_asprintf(&s, desired_cpuset); + + auto current_cpuset = hwloc_bitmap_alloc(); + hwloc_get_cpubind(topology, current_cpuset, HWLOC_CPUBIND_THREAD); + if (!hwloc_bitmap_isequal(current_cpuset, desired_cpuset)) + LogMessage("Binding thread %s to %s.\n", name.c_str(), s); + hwloc_bitmap_free(current_cpuset); + + if (hwloc_set_cpubind(topology, desired_cpuset, HWLOC_CPUBIND_THREAD)) + { + FatalError("Failed to pin thread %s to %s: %s (%d)\n", + name.c_str(), s, get_error(errno), errno); + } + + free(s); + } + else + implement_thread_affinity(get_thread_type(), DEFAULT_THREAD_ID); +} + // ----------------------------------------------------------------------------- // unit tests @@ -233,4 +280,54 @@ TEST_CASE("Set and implement thread affinity", "[ThreadConfig]") } } +TEST_CASE("Named thread affinity configured", "[ThreadConfig]") +{ + if (topology_support->cpubind->set_thisthread_cpubind) + { + CpuSet* cpuset = new CpuSet(hwloc_bitmap_dup(process_cpuset)); + ThreadConfig tc; + + hwloc_cpuset_t thread_cpuset = hwloc_bitmap_alloc(); + + // Configure named thread. + hwloc_bitmap_singlify(cpuset->cpuset); + tc.set_named_thread_affinity("found", cpuset); + + // The one in the named map, should have the specified cpuset. + tc.implement_named_thread_affinity("found"); + hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD); + CHECK(hwloc_bitmap_isequal(thread_cpuset, cpuset->cpuset)); + + // The one not in the named map, should have the process cpuset + // if no type has been configured for it. + tc.implement_named_thread_affinity("not found, no type configured"); + hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD); + CHECK(hwloc_bitmap_isequal(thread_cpuset, process_cpuset)); + + hwloc_bitmap_free(thread_cpuset); + } +} + +TEST_CASE("Named thread affinity with type configured", "[ThreadConfig]") +{ + if (topology_support->cpubind->set_thisthread_cpubind) + { + CpuSet* type_cpuset = new CpuSet(hwloc_bitmap_dup(process_cpuset)); + ThreadConfig tc; + + hwloc_cpuset_t thread_cpuset = hwloc_bitmap_alloc(); + + // Configure type affinity, but not the named thread affinity. + hwloc_bitmap_singlify(type_cpuset->cpuset); + tc.set_thread_affinity(STHREAD_TYPE_OTHER, ThreadConfig::DEFAULT_THREAD_ID, type_cpuset); + + // The named thread should inherit the type affinity. + tc.implement_named_thread_affinity("not found, type other"); + hwloc_get_cpubind(topology, thread_cpuset, HWLOC_CPUBIND_THREAD); + CHECK(hwloc_bitmap_isequal(thread_cpuset, type_cpuset->cpuset)); + + hwloc_bitmap_free(thread_cpuset); + } +} + #endif diff --git a/src/main/thread_config.h b/src/main/thread_config.h index a0cdbcc69..58f3fa941 100644 --- a/src/main/thread_config.h +++ b/src/main/thread_config.h @@ -20,12 +20,16 @@ #define THREAD_CONFIG_H #include +#include #include "main/thread.h" struct CpuSet; -class ThreadConfig +namespace snort +{ + +class SO_PUBLIC ThreadConfig { public: static bool init(); @@ -37,7 +41,12 @@ public: ~ThreadConfig(); void set_thread_affinity(SThreadType, unsigned id, CpuSet*); + void set_named_thread_affinity(const std::string&, CpuSet*); void implement_thread_affinity(SThreadType, unsigned id); + void implement_named_thread_affinity(const std::string& name); + + static constexpr unsigned int DEFAULT_THREAD_ID = 0; + private: struct TypeIdPair { @@ -56,6 +65,8 @@ private: } }; std::map thread_affinity; + std::map named_thread_affinity; }; +} #endif