From: Michael Altizer (mialtize) Date: Thu, 22 Aug 2019 21:33:17 +0000 (-0400) Subject: Merge pull request #1715 in SNORT/snort3 from ~DAVMCPHE/snort3:reload_adjust_cleanup... X-Git-Tag: 3.0.0-260~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d53d0bce4fe3f586eea5d38e64181838c24206c3;p=thirdparty%2Fsnort3.git Merge pull request #1715 in SNORT/snort3 from ~DAVMCPHE/snort3:reload_adjust_cleanup to master Squashed commit of the following: commit e18fe7cb30171778f1fa02e8744f8d16913473e6 Author: davis mcpherson Date: Mon Aug 19 17:43:49 2019 -0400 reload: fix coding style issues, support multiple in progress analyzer commands, support associated AC state for execute method, move reload tune logic for ACSwap to the execute command --- diff --git a/src/framework/module.h b/src/framework/module.h index 5b0838b11..61e883e1d 100644 --- a/src/framework/module.h +++ b/src/framework/module.h @@ -75,17 +75,6 @@ struct RuleMap const char* msg; }; -class ReloadMemcapManager -{ -public: - virtual ~ReloadMemcapManager() = default; - - virtual bool tune_memcap() = 0; - virtual bool tune_memcap_idle() = 0; -protected: - ReloadMemcapManager() = default; -}; - class SO_PUBLIC Module { public: @@ -196,9 +185,6 @@ public: void enable_trace(); - const ReloadMemcapManager* get_reload_mcm() const - { return reload_mcm; } - protected: Module(const char* name, const char* help); Module(const char* name, const char* help, const Parameter*, @@ -220,7 +206,6 @@ private: int table_level = 0; Trace* trace; - ReloadMemcapManager* reload_mcm = nullptr; void set_peg_count(int index, PegCount value) { diff --git a/src/main/ac_shell_cmd.cc b/src/main/ac_shell_cmd.cc index 9a59bcd80..ce40e0335 100644 --- a/src/main/ac_shell_cmd.cc +++ b/src/main/ac_shell_cmd.cc @@ -41,14 +41,14 @@ ACShellCmd::ACShellCmd(int fd, AnalyzerCommand *ac) : ac(ac) } } -void ACShellCmd::execute(Analyzer& analyzer) +bool ACShellCmd::execute(Analyzer& analyzer, void** state) { ControlConn* control_conn = ControlMgmt::find_control(control_fd); if( control_conn ) control_conn->send_queued_response(); - ac->execute(analyzer); + return ac->execute(analyzer, state); } ACShellCmd::~ACShellCmd() diff --git a/src/main/ac_shell_cmd.h b/src/main/ac_shell_cmd.h index 29b21fc98..bf30db597 100644 --- a/src/main/ac_shell_cmd.h +++ b/src/main/ac_shell_cmd.h @@ -31,7 +31,7 @@ class ACShellCmd : public snort::AnalyzerCommand public: ACShellCmd() = delete; ACShellCmd(int fd, snort::AnalyzerCommand* ac_cmd); - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return ac->stringify(); } ~ACShellCmd() override; diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc index fe58a0ff7..24ed7b9d2 100644 --- a/src/main/analyzer.cc +++ b/src/main/analyzer.cc @@ -75,8 +75,6 @@ using namespace std; static MainHook_f main_hook = snort_ignore; THREAD_LOCAL ProfileStats daqPerfStats; -THREAD_LOCAL std::list *rel_managers; - static THREAD_LOCAL Analyzer* local_analyzer = nullptr; //------------------------------------------------------------------------- @@ -527,7 +525,6 @@ void Analyzer::reinit(SnortConfig* sc) { InspectorManager::thread_reinit(sc); ActionManager::thread_reinit(sc); - rel_managers = new std::list(sc->get_reload_memcap_managers()); } void Analyzer::term() @@ -581,8 +578,6 @@ void Analyzer::term() Active::thread_term(); delete switcher; - delete rel_managers; - sfthreshold_free(); RateFilter_Cleanup(); } @@ -667,22 +662,27 @@ bool Analyzer::handle_command() if (!ac) return false; - ac->execute(*this); - - add_command_to_completed_queue(ac); + void* ac_state = nullptr; + if ( ac->execute(*this, &ac_state) ) + add_command_to_completed_queue(ac); + else + add_command_to_uncompleted_queue(ac, ac_state); return true; } -void Analyzer::add_command_to_completed_queue(AnalyzerCommand *ac) +void Analyzer::add_command_to_uncompleted_queue(AnalyzerCommand* aci, void* acs) +{ + UncompletedAnalyzerCommand* cac = new UncompletedAnalyzerCommand(aci, acs); + + uncompleted_work_queue.push_back(cac); +} + +void Analyzer::add_command_to_completed_queue(AnalyzerCommand* ac) { - if (ac->is_complete()) - { completed_work_queue_mutex.lock(); completed_work_queue.push(ac); completed_work_queue_mutex.unlock(); - } else - cache_analyzer_command(ac); } void Analyzer::handle_commands() @@ -691,6 +691,24 @@ void Analyzer::handle_commands() ; } +void Analyzer::handle_uncompleted_commands() +{ + std::list::iterator it = uncompleted_work_queue.begin(); + while (it != uncompleted_work_queue.end() ) + { + UncompletedAnalyzerCommand* cac = *it; + + if (cac->command->execute(*this, &cac->state) ) + { + add_command_to_completed_queue(cac->command); + it = uncompleted_work_queue.erase(it); + delete cac; + } + else + ++it; + } +} + DAQ_RecvStatus Analyzer::process_messages() { // Max receive becomes the minimum of the configured batch size, the remaining exit_after @@ -726,21 +744,7 @@ DAQ_RecvStatus Analyzer::process_messages() process_daq_msg(msg, false); DetectionEngine::onload(); process_retry_queue(); - - if (rel_managers and rel_managers->size()) - { - auto manager = rel_managers->front(); - if (manager->tune_memcap()) - { - rel_managers->pop_front(); - } - } - else - { - if(ac) - add_command_to_completed_queue(ac); - } - + handle_uncompleted_commands(); } if (exit_after_cnt && (exit_after_cnt -= num_recv) == 0) diff --git a/src/main/analyzer.h b/src/main/analyzer.h index f024c2e3e..a77f4d569 100644 --- a/src/main/analyzer.h +++ b/src/main/analyzer.h @@ -27,10 +27,10 @@ #include #include +#include #include #include #include -#include #include "thread.h" @@ -42,15 +42,25 @@ class Swapper; namespace snort { class AnalyzerCommand; +class ReloadResourceTuner; class SFDAQInstance; struct Packet; struct SnortConfig; struct ProfileStats; -class ReloadMemcapManager; } typedef bool (* MainHook_f)(snort::Packet*); +class UncompletedAnalyzerCommand +{ +public: + UncompletedAnalyzerCommand(snort::AnalyzerCommand* ac, void* acs) : command(ac), state(acs) + { } + + snort::AnalyzerCommand* command = nullptr; + void* state = nullptr; +}; + class Analyzer { public: @@ -102,6 +112,7 @@ private: void analyze(); bool handle_command(); void handle_commands(); + void handle_uncompleted_commands(); DAQ_RecvStatus process_messages(); void process_daq_msg(DAQ_Msg_h, bool retry); void process_daq_pkt_msg(DAQ_Msg_h, bool retry); @@ -113,9 +124,9 @@ private: void init_unprivileged(); void term(); void show_source(); - void cache_analyzer_command(snort::AnalyzerCommand* aci) { ac = aci; } - void add_command_to_completed_queue(snort::AnalyzerCommand *ac); - snort::AnalyzerCommand* get_analyzer_command() { return ac; } + void add_command_to_uncompleted_queue(snort::AnalyzerCommand*, void*); + void add_command_to_completed_queue(snort::AnalyzerCommand*); + public: std::queue completed_work_queue; std::mutex completed_work_queue_mutex; @@ -136,9 +147,8 @@ private: RetryQueue* retry_queue = nullptr; OopsHandler* oops_handler = nullptr; ContextSwitcher* switcher = nullptr; - snort::AnalyzerCommand* ac = nullptr; - std::mutex pending_work_queue_mutex; + std::list uncompleted_work_queue; }; extern THREAD_LOCAL snort::ProfileStats daqPerfStats; diff --git a/src/main/analyzer_command.cc b/src/main/analyzer_command.cc index fb0fab7c0..eeb9c8908 100644 --- a/src/main/analyzer_command.cc +++ b/src/main/analyzer_command.cc @@ -25,6 +25,7 @@ #include +#include "framework/module.h" #include "log/messages.h" #include "managers/module_manager.h" #include "utils/stats.h" @@ -35,44 +36,51 @@ #include "snort_config.h" #include "swapper.h" -void ACStart::execute(Analyzer& analyzer) +bool ACStart::execute(Analyzer& analyzer, void**) { analyzer.start(); + return true; } -void ACRun::execute(Analyzer& analyzer) +bool ACRun::execute(Analyzer& analyzer, void**) { analyzer.run(paused); paused = false; + return true; } -void ACStop::execute(Analyzer& analyzer) +bool ACStop::execute(Analyzer& analyzer, void**) { analyzer.stop(); + return true; } -void ACPause::execute(Analyzer& analyzer) +bool ACPause::execute(Analyzer& analyzer, void**) { analyzer.pause(); + return true; } -void ACResume::execute(Analyzer& analyzer) +bool ACResume::execute(Analyzer& analyzer, void**) { analyzer.resume(msg_cnt); + return true; } -void ACRotate::execute(Analyzer& analyzer) +bool ACRotate::execute(Analyzer& analyzer, void**) { analyzer.rotate(); + return true; } -void ACGetStats::execute(Analyzer&) +bool ACGetStats::execute(Analyzer&, void**) { // FIXIT-P This incurs locking on all threads to retrieve stats. It // could be reimplemented to optimize for large thread counts by // retrieving stats in the command and accumulating in the main thread. snort::ModuleManager::accumulate(snort::SnortConfig::get_conf()); + return true; } ACGetStats::~ACGetStats() @@ -89,23 +97,64 @@ ACSwap::ACSwap(Swapper* ps, Request* req, bool from_shell) : ps(ps), request(req Swapper::set_reload_in_progress(true); } -void ACSwap::execute(Analyzer& analyzer) +bool ACSwap::execute(Analyzer& analyzer, void** ac_state) { if (ps) + { ps->apply(analyzer); + + snort::SnortConfig* sc = ps->get_new_conf(); + if ( sc ) + { + std::list* reload_tuners; + + if ( !*ac_state ) + { + reload_tuners = new std::list(sc->get_reload_resource_tuners()); + *ac_state = reload_tuners; + } + else + reload_tuners = (std::list*)*ac_state; + + if ( !reload_tuners->empty() ) + { + auto rrt = reload_tuners->front(); + if (rrt->tune_resources()) + reload_tuners->pop_front(); + } + + // check for empty again and free list instance if we are done + if ( reload_tuners->empty() ) + { + delete reload_tuners; + return true; + } + + return false; + } + } + + return true; } ACSwap::~ACSwap() { + if (ps) + { + snort::SnortConfig* sc = ps->get_new_conf(); + if ( sc ) + sc->clear_reload_resource_tuner_list(); + } delete ps; Swapper::set_reload_in_progress(false); snort::LogMessage("== reload complete\n"); request->respond("== reload complete\n", from_shell, true); } -void ACDAQSwap::execute(Analyzer& analyzer) +bool ACDAQSwap::execute(Analyzer& analyzer, void**) { analyzer.reload_daq(); + return true; } ACDAQSwap::~ACDAQSwap() diff --git a/src/main/analyzer_command.h b/src/main/analyzer_command.h index 3b0f1118b..8d36718f7 100644 --- a/src/main/analyzer_command.h +++ b/src/main/analyzer_command.h @@ -34,23 +34,21 @@ class AnalyzerCommand { public: virtual ~AnalyzerCommand() = default; - virtual void execute(Analyzer&) = 0; + virtual bool execute(Analyzer&, void**) = 0; virtual const char* stringify() = 0; unsigned get() { return ++ref_count; } unsigned put() { return --ref_count; } - bool is_complete() { return completion_status; } - void set_completion_status(bool status) { completion_status = status; } SO_PUBLIC static snort::SFDAQInstance* get_daq_instance(Analyzer& analyzer); + private: unsigned ref_count = 0; - bool completion_status = true; }; } class ACGetStats : public snort::AnalyzerCommand { public: - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "GET_STATS"; } ~ACGetStats() override; }; @@ -58,7 +56,7 @@ public: class ACPause : public snort::AnalyzerCommand { public: - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "PAUSE"; } }; @@ -66,7 +64,7 @@ class ACResume : public snort::AnalyzerCommand { public: ACResume(uint64_t msg_cnt): msg_cnt(msg_cnt) { } - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "RESUME"; } private: uint64_t msg_cnt; @@ -75,7 +73,7 @@ private: class ACRotate : public snort::AnalyzerCommand { public: - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "ROTATE"; } }; @@ -84,7 +82,7 @@ class ACRun : public snort::AnalyzerCommand public: ACRun() = delete; ACRun(bool is_paused = false ) { paused = is_paused; } - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "RUN"; } private: bool paused = false; @@ -93,14 +91,14 @@ private: class ACStart : public snort::AnalyzerCommand { public: - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "START"; } }; class ACStop : public snort::AnalyzerCommand { public: - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "STOP"; } }; @@ -109,7 +107,7 @@ class ACSwap : public snort::AnalyzerCommand public: ACSwap() = delete; ACSwap(Swapper* ps, Request* req, bool from_shell); - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "SWAP"; } ~ACSwap() override; private: @@ -121,7 +119,7 @@ private: class ACDAQSwap : public snort::AnalyzerCommand { public: - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "DAQ_SWAP"; } ~ACDAQSwap() override; }; diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 170b48e6b..9e6acc6b1 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -310,7 +310,7 @@ SnortConfig::~SnortConfig() delete daq_config; delete proto_ref; - reload_managers.clear(); + reload_tuners.clear(); trim_heap(); } @@ -1085,14 +1085,3 @@ void SnortConfig::set_conf(SnortConfig* sc) set_policies(sc, sh); } } - -SO_PUBLIC bool SnortConfig::register_reload_memcap_manager(ReloadMemcapManager *memcap_manager) -{ - reload_managers.push_back(memcap_manager); - return true; -} - -std::list SnortConfig::get_reload_memcap_managers() -{ - return reload_managers; -} diff --git a/src/main/snort_config.h b/src/main/snort_config.h index 580c7ce9e..04b698d6f 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -25,14 +25,14 @@ #include +#include + #include "events/event_queue.h" #include "framework/bits.h" #include "main/policy.h" #include "main/thread.h" #include "sfip/sf_cidr.h" -#include - #define DEFAULT_LOG_DIR "." enum RunFlag @@ -145,21 +145,36 @@ struct VarNode; namespace snort { -struct ProfilerConfig; class ProtocolReference; +class ReloadResourceTuner; +struct ProfilerConfig; struct GHash; struct XHash; - -class ReloadMemcapManager; - struct SnortConfig; + typedef void (* ScScratchFunc)(SnortConfig* sc); +class ReloadResourceTuner +{ +public: + static const unsigned RELOAD_MAX_WORK_PER_PACKET = 3; + static const unsigned RELOAD_MAX_WORK_WHEN_IDLE = 10; + + virtual ~ReloadResourceTuner() = default; + + virtual bool tune_resources() = 0; + virtual bool tune_resources_idle() = 0; + +protected: + ReloadResourceTuner() = default; + + unsigned max_work = RELOAD_MAX_WORK_PER_PACKET; + unsigned max_work_idle = RELOAD_MAX_WORK_WHEN_IDLE; +}; + struct SnortConfig { private: - std::list reload_managers; - void init(const SnortConfig* const, ProtocolReference*); bool verify_stream_inspectors(); @@ -170,9 +185,6 @@ public: SnortConfig(const SnortConfig&) = delete; - SO_PUBLIC bool register_reload_memcap_manager(ReloadMemcapManager *); - std::list get_reload_memcap_managers(); - void setup(); void post_setup(); bool verify(); @@ -389,6 +401,10 @@ public: bool cloned = false; +private: + std::list reload_tuners; + +public: //------------------------------------------------------ // decoding related uint8_t get_num_layers() const @@ -694,6 +710,15 @@ public: static void set_conf(SnortConfig*); SO_PUBLIC static SnortConfig* get_conf(); + + SO_PUBLIC void register_reload_resource_tuner(ReloadResourceTuner& rrt) + { reload_tuners.push_back(&rrt); } + + const std::list& get_reload_resource_tuners() const + { return reload_tuners; } + + void clear_reload_resource_tuner_list() + { reload_tuners.clear(); } }; } diff --git a/src/main/swapper.h b/src/main/swapper.h index 46a9aeeb8..f219c4e2e 100644 --- a/src/main/swapper.h +++ b/src/main/swapper.h @@ -40,6 +40,7 @@ public: ~Swapper(); void apply(Analyzer&); + snort::SnortConfig* get_new_conf() { return new_conf; } static bool get_reload_in_progress() { return reload_in_progress; } static void set_reload_in_progress(bool rip) { reload_in_progress = rip; } diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 7debbed92..6dc94b728 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -95,7 +95,7 @@ class AcAppIdDebug : public AnalyzerCommand { public: AcAppIdDebug(AppIdDebugSessionConstraints* cs); - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "APPID_DEBUG"; } private: @@ -112,7 +112,7 @@ AcAppIdDebug::AcAppIdDebug(AppIdDebugSessionConstraints* cs) } } -void AcAppIdDebug::execute(Analyzer&) +bool AcAppIdDebug::execute(Analyzer&, void**) { if (appidDebug) { @@ -122,6 +122,8 @@ void AcAppIdDebug::execute(Analyzer&) appidDebug->set_constraints("appid", nullptr); } // FIXIT-L Add a warning if command was called without appid configured? + + return true; } static int enable_debug(lua_State* L) diff --git a/src/network_inspectors/packet_capture/capture_module.cc b/src/network_inspectors/packet_capture/capture_module.cc index 7472a1d78..e37f2834e 100644 --- a/src/network_inspectors/packet_capture/capture_module.cc +++ b/src/network_inspectors/packet_capture/capture_module.cc @@ -72,7 +72,7 @@ class PacketCaptureDebug : public AnalyzerCommand { public: PacketCaptureDebug(const char* f); - void execute(Analyzer&) override; + bool execute(Analyzer&, void**) override; const char* stringify() override { return "PACKET_CAPTURE_DEBUG"; } private: bool enable = false; @@ -107,12 +107,14 @@ PacketCaptureDebug::PacketCaptureDebug(const char* f) } } -void PacketCaptureDebug::execute(Analyzer&) +bool PacketCaptureDebug::execute(Analyzer&, void**) { if (enable) packet_capture_enable(filter); else packet_capture_disable(); + + return true; } CaptureModule::CaptureModule() : diff --git a/src/network_inspectors/packet_tracer/packet_tracer_module.cc b/src/network_inspectors/packet_tracer/packet_tracer_module.cc index 5a56be16b..47cfaa404 100644 --- a/src/network_inspectors/packet_tracer/packet_tracer_module.cc +++ b/src/network_inspectors/packet_tracer/packet_tracer_module.cc @@ -70,8 +70,8 @@ static const Command packet_tracer_cmds[] = class PacketTracerDebug : public AnalyzerCommand { public: - PacketTracerDebug(PTSessionConstraints *cs); - void execute(Analyzer &) override; + PacketTracerDebug(PTSessionConstraints* cs); + bool execute(Analyzer&, void**) override; const char *stringify() override { return "PACKET_TRACER_DEBUG"; } private: @@ -79,7 +79,7 @@ class PacketTracerDebug : public AnalyzerCommand bool enable = false; }; -PacketTracerDebug::PacketTracerDebug(PTSessionConstraints *cs) +PacketTracerDebug::PacketTracerDebug(PTSessionConstraints* cs) { if (cs) { @@ -88,12 +88,14 @@ PacketTracerDebug::PacketTracerDebug(PTSessionConstraints *cs) } } -void PacketTracerDebug::execute(Analyzer &) +bool PacketTracerDebug::execute(Analyzer&, void**) { if (enable) PacketTracer::set_constraints(&constraints); else PacketTracer::set_constraints(nullptr); + + return true; } static int enable(lua_State* L) diff --git a/src/stream/CMakeLists.txt b/src/stream/CMakeLists.txt index 62c2dbb0f..19eb7f09e 100644 --- a/src/stream/CMakeLists.txt +++ b/src/stream/CMakeLists.txt @@ -10,7 +10,6 @@ add_subdirectory(file) add_subdirectory(test) set (STREAM_INCLUDES - flush_bucket.h paf.h stream.h stream_splitter.h