From: Tom Peters (thopeter) Date: Tue, 26 Oct 2021 17:19:13 +0000 (+0000) Subject: Merge pull request #3107 in SNORT/snort3 from ~SBAIGAL/snort3:reload_debug_logs to... X-Git-Tag: 3.1.16.0~17 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=19c7b97404533f80833c2cf4ec625bb9a0bc4f84;p=thirdparty%2Fsnort3.git Merge pull request #3107 in SNORT/snort3 from ~SBAIGAL/snort3:reload_debug_logs to master Squashed commit of the following: commit a3b8308a9465c46127a77588774e81fcc6eb6357 Author: Steven Baigal (sbaigal) Date: Fri Sep 24 16:11:09 2021 -0400 reload: add logs to track reload process swapper: moved out reload progress flag to reload tracker --- diff --git a/src/control/control.cc b/src/control/control.cc index 10190c57c..bed0bec33 100644 --- a/src/control/control.cc +++ b/src/control/control.cc @@ -92,6 +92,7 @@ int ControlConn::read_commands() { std::string command = next_command; next_command.append(buf, nl - p); + LogMessage("Control: received command, %s\n", next_command.c_str()); pending_commands.push(std::move(next_command)); next_command.clear(); p = nl + 1; diff --git a/src/control/control.h b/src/control/control.h index ca94b0ce6..d6f271774 100644 --- a/src/control/control.h +++ b/src/control/control.h @@ -53,6 +53,7 @@ public: bool is_removed() const { return removed; } bool has_pending_command() const { return !pending_commands.empty(); } time_t get_touched() const; + std::string get_current_command() const { return pending_commands.front(); } void configure() const; int read_commands(); diff --git a/src/main.cc b/src/main.cc index 389a0edce..8a0192dce 100644 --- a/src/main.cc +++ b/src/main.cc @@ -34,6 +34,7 @@ #include "lua/lua.h" #include "main/analyzer.h" #include "main/analyzer_command.h" +#include "main/reload_tracker.h" #include "main/shell.h" #include "main/snort.h" #include "main/snort_config.h" @@ -348,7 +349,7 @@ int main_rotate_stats(lua_State* L) int main_reload_config(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); - if ( Swapper::get_reload_in_progress() ) + if ( !ReloadTracker::start(ctrlcon) ) { send_response(ctrlcon, "== reload pending; retry\n"); return 0; @@ -370,6 +371,7 @@ int main_reload_config(lua_State* L) } send_response(ctrlcon, ".. reloading configuration\n"); + ReloadTracker::update(ctrlcon,"start loading ..."); const SnortConfig* old = SnortConfig::get_conf(); SnortConfig* sc = Snort::get_reload_config(fname, plugin_path, old); @@ -379,11 +381,16 @@ int main_reload_config(lua_State* L) { std::string response_message = "== reload failed - restart required - "; response_message += get_reload_errors_description() + "\n"; + ReloadTracker::failed(ctrlcon, "restart required"); send_response(ctrlcon, response_message.c_str()); reset_reload_errors(); } else + { + ReloadTracker::failed(ctrlcon, "bad config"); send_response(ctrlcon, "== reload failed - bad config\n"); + } + HostAttributesManager::load_failure_cleanup(); return 0; @@ -408,6 +415,7 @@ int main_reload_config(lua_State* L) TraceApi::thread_reinit(sc->trace_config); proc_stats.conf_reloads++; + ReloadTracker::update(ctrlcon, "start swapping configuration ..."); send_response(ctrlcon, ".. swapping configuration\n"); main_broadcast_command(new ACSwap(new Swapper(old, sc), ctrlcon), ctrlcon); @@ -417,7 +425,7 @@ int main_reload_config(lua_State* L) int main_reload_policy(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); - if ( Swapper::get_reload_in_progress() ) + if ( !ReloadTracker::start(ctrlcon) ) { send_response(ctrlcon, "== reload pending; retry\n"); return 0; @@ -434,6 +442,7 @@ int main_reload_policy(lua_State* L) send_response(ctrlcon, ".. reloading policy\n"); else { + ReloadTracker::failed(ctrlcon, "filename required"); send_response(ctrlcon, "== filename required\n"); return 0; } @@ -443,6 +452,7 @@ int main_reload_policy(lua_State* L) if ( !sc ) { + ReloadTracker::failed(ctrlcon, "failed to update policy"); send_response(ctrlcon, "== reload failed\n"); return 0; } @@ -450,6 +460,7 @@ int main_reload_policy(lua_State* L) SnortConfig::set_conf(sc); proc_stats.policy_reloads++; + ReloadTracker::update(ctrlcon, "start swapping configuration ..."); send_response(ctrlcon, ".. swapping policy\n"); main_broadcast_command(new ACSwap(new Swapper(old, sc), ctrlcon), ctrlcon); @@ -459,7 +470,7 @@ int main_reload_policy(lua_State* L) int main_reload_module(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); - if ( Swapper::get_reload_in_progress() ) + if ( !ReloadTracker::start(ctrlcon) ) { send_response(ctrlcon, "== reload pending; retry\n"); return 0; @@ -476,6 +487,7 @@ int main_reload_module(lua_State* L) send_response(ctrlcon, ".. reloading module\n"); else { + ReloadTracker::failed(ctrlcon, "module name required"); send_response(ctrlcon, "== module name required\n"); return 0; } @@ -485,6 +497,7 @@ int main_reload_module(lua_State* L) if ( !sc ) { + ReloadTracker::failed(ctrlcon, "failed to update module"); send_response(ctrlcon, "== reload failed\n"); return 0; } @@ -492,6 +505,7 @@ int main_reload_module(lua_State* L) SnortConfig::set_conf(sc); proc_stats.policy_reloads++; + ReloadTracker::update(ctrlcon, "start swapping configuration ..."); send_response(ctrlcon, ".. swapping module\n"); main_broadcast_command(new ACSwap(new Swapper(old, sc), ctrlcon), ctrlcon); @@ -511,9 +525,8 @@ int main_reload_daq(lua_State* L) int main_reload_hosts(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); - if ( Swapper::get_reload_in_progress() ) + if ( !ReloadTracker::start(ctrlcon) ) { - WarningMessage("Reload in progress. Cannot reload host attribute table.\n"); send_response(ctrlcon, "== reload pending; retry\n"); return 0; } @@ -531,19 +544,21 @@ int main_reload_hosts(lua_State* L) if ( fname and *fname ) { - LogMessage("Reloading Host attribute table from %s.\n", fname); + std::string msg = "Reloading Host attribute table from "; + msg += fname; + ReloadTracker::update(ctrlcon, msg.c_str()); send_response(ctrlcon, ".. reloading hosts table\n"); } else { - ErrorMessage("Reload failed. Host attribute table filename required.\n"); + ReloadTracker::failed(ctrlcon, "host attribute table filename required."); send_response(ctrlcon, "== filename required\n"); return 0; } if ( !HostAttributesManager::load_hosts_file(sc, fname) ) { - ErrorMessage("Host attribute table reload from %s failed.\n", fname); + ReloadTracker::failed(ctrlcon, "failed to load host table."); send_response(ctrlcon, "== reload failed\n"); return 0; } @@ -553,6 +568,7 @@ int main_reload_hosts(lua_State* L) assert( num_hosts >= 0 ); LogMessage("Host attribute table: %d hosts loaded successfully.\n", num_hosts); + ReloadTracker::update(ctrlcon, "start swapping configuration ..."); send_response(ctrlcon, ".. swapping hosts table\n"); main_broadcast_command(new ACHostAttributesSwap(ctrlcon), ctrlcon); @@ -562,7 +578,7 @@ int main_reload_hosts(lua_State* L) int main_delete_inspector(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); - if ( Swapper::get_reload_in_progress() ) + if ( !ReloadTracker::start(ctrlcon) ) { send_response(ctrlcon, "== delete pending; retry\n"); return 0; @@ -579,6 +595,7 @@ int main_delete_inspector(lua_State* L) send_response(ctrlcon, ".. deleting inspector\n"); else { + ReloadTracker::failed(ctrlcon, "inspector name required."); send_response(ctrlcon, "== inspector name required\n"); return 0; } @@ -588,12 +605,14 @@ int main_delete_inspector(lua_State* L) if ( !sc ) { + ReloadTracker::failed(ctrlcon, "failed to update policy"); send_response(ctrlcon, "== reload failed\n"); return 0; } SnortConfig::set_conf(sc); proc_stats.inspector_deletions++; + ReloadTracker::update(ctrlcon, "start swapping configuration ..."); send_response(ctrlcon, ".. deleted inspector\n"); main_broadcast_command(new ACSwap(new Swapper(old, sc), ctrlcon), ctrlcon); diff --git a/src/main/CMakeLists.txt b/src/main/CMakeLists.txt index 0d378c553..c26feb611 100644 --- a/src/main/CMakeLists.txt +++ b/src/main/CMakeLists.txt @@ -2,6 +2,7 @@ set (INCLUDES analyzer_command.h policy.h + reload_tracker.h snort.h snort_config.h snort_debug.h @@ -31,6 +32,7 @@ add_library (main OBJECT oops_handler.cc oops_handler.h policy.cc + reload_tracker.cc shell.h shell.cc snort.cc diff --git a/src/main/analyzer_command.cc b/src/main/analyzer_command.cc index 725e31abd..e32928a16 100644 --- a/src/main/analyzer_command.cc +++ b/src/main/analyzer_command.cc @@ -34,6 +34,7 @@ #include "utils/stats.h" #include "analyzer.h" +#include "reload_tracker.h" #include "snort.h" #include "snort_config.h" #include "swapper.h" @@ -108,10 +109,7 @@ ACResetStats::ACResetStats(clear_counter_type_t requested_type_l) : requested_ty requested_type_l) { } ACSwap::ACSwap(Swapper* ps, ControlConn *ctrlcon) : ps(ps), ctrlcon(ctrlcon) -{ - assert(Swapper::get_reload_in_progress() == false); - Swapper::set_reload_in_progress(true); -} +{ } bool ACSwap::execute(Analyzer& analyzer, void** ac_state) { @@ -181,7 +179,7 @@ ACSwap::~ACSwap() delete ps; HostAttributesManager::swap_cleanup(); - Swapper::set_reload_in_progress(false); + ReloadTracker::end(ctrlcon); LogMessage("== reload complete\n"); if (ctrlcon && !ctrlcon->is_local()) ctrlcon->respond("== reload complete\n"); @@ -189,10 +187,7 @@ ACSwap::~ACSwap() ACHostAttributesSwap::ACHostAttributesSwap(ControlConn *ctrlcon) : ctrlcon(ctrlcon) -{ - assert(Swapper::get_reload_in_progress() == false); - Swapper::set_reload_in_progress(true); -} +{ } bool ACHostAttributesSwap::execute(Analyzer&, void**) { @@ -203,7 +198,7 @@ bool ACHostAttributesSwap::execute(Analyzer&, void**) ACHostAttributesSwap::~ACHostAttributesSwap() { HostAttributesManager::swap_cleanup(); - Swapper::set_reload_in_progress(false); + ReloadTracker::end(ctrlcon); LogMessage("== reload host attributes complete\n"); if (ctrlcon && !ctrlcon->is_local()) ctrlcon->respond("== reload host attributes complete\n"); diff --git a/src/main/reload_tracker.cc b/src/main/reload_tracker.cc new file mode 100644 index 000000000..6c07c3285 --- /dev/null +++ b/src/main/reload_tracker.cc @@ -0,0 +1,82 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2021-2021 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// reload_tracker.cc author Steven Baigal + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "reload_tracker.h" + +#include + +#include "control/control.h" +#include "log/messages.h" + +using namespace snort; + +bool ReloadTracker::reload_in_progress = false; +std::string ReloadTracker::current_command; +ControlConn* ReloadTracker::ctrl = nullptr; + +bool ReloadTracker::start(ControlConn* ctrlcon) +{ + if (reload_in_progress) + { + LogMessage("Reload in progress [%s], attempting command: [%s]\n", + current_command.c_str(), + ctrlcon ? ctrlcon->get_current_command().substr(0, 50).c_str() : "signal"); + return false; + } + reload_in_progress = true; + current_command = (ctrlcon ? ctrlcon->get_current_command().substr(0, 50) : "signal"); + LogMessage("Reload started. [%s]\n", current_command.c_str()); + ctrl = ctrlcon; + return true; +} + +void ReloadTracker::end(ControlConn* ctrlcon) +{ + if (ctrl == ctrlcon and reload_in_progress) + LogMessage("Reload ended. [%s]\n", current_command.c_str()); + else + ErrorMessage("Reload session mismatched for ending tracker\n"); + current_command.clear(); + ctrl = nullptr; + reload_in_progress = false; +} + +void ReloadTracker::failed(ControlConn* ctrlcon, const char* reason) +{ + if (ctrl == ctrlcon and reload_in_progress) + LogMessage("Reload failed! %s [%s]\n", reason, current_command.c_str()); + else + ErrorMessage("Reload session mismatched for failing tracker\n"); + current_command.clear(); + ctrl = nullptr; + reload_in_progress = false; +} + +void ReloadTracker::update(ControlConn* ctrlcon, const char* status) +{ + if (ctrl == ctrlcon and reload_in_progress) + LogMessage("Reload update: %s [%s]\n", status, current_command.c_str()); + else + ErrorMessage("Reload session mismatched for updating tracker\n"); +} + diff --git a/src/main/reload_tracker.h b/src/main/reload_tracker.h new file mode 100644 index 000000000..043e0f8f3 --- /dev/null +++ b/src/main/reload_tracker.h @@ -0,0 +1,49 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2021-2021 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// reload_tracker.h author Steven Baigal + +#ifndef RELOAD_TRACKER_H +#define RELOAD_TRACKER_H + +#include + +#include "main/snort_types.h" + +class ControlConn; + +namespace snort +{ + +class SO_PUBLIC ReloadTracker +{ +public: + ReloadTracker() = delete; + static bool start(ControlConn* ctrlcon); + static void end(ControlConn* ctrlcon); + static void failed(ControlConn* ctrlcon, const char* reason); + static void update(ControlConn* ctrlcon, const char* status); + +private: + static bool reload_in_progress; + static std::string current_command; + static ControlConn* ctrl; +}; + +} + +#endif diff --git a/src/main/swapper.cc b/src/main/swapper.cc index 821728a7d..9325445f6 100644 --- a/src/main/swapper.cc +++ b/src/main/swapper.cc @@ -31,8 +31,6 @@ using namespace snort; -bool Swapper::reload_in_progress = false; - Swapper::Swapper(SnortConfig* s) { old_conf = nullptr; diff --git a/src/main/swapper.h b/src/main/swapper.h index 18500364f..722d90300 100644 --- a/src/main/swapper.h +++ b/src/main/swapper.h @@ -43,14 +43,9 @@ public: void finish(Analyzer&); snort::SnortConfig* get_new_conf() { return new_conf; } - static bool get_reload_in_progress() { return reload_in_progress; } - static void set_reload_in_progress(bool rip) { reload_in_progress = rip; } - private: const snort::SnortConfig* old_conf; snort::SnortConfig* new_conf; - - static bool reload_in_progress; }; #endif diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 08886116a..137572532 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -33,6 +33,7 @@ #include "log/messages.h" #include "main/analyzer.h" #include "main/analyzer_command.h" +#include "main/reload_tracker.h" #include "main/snort.h" #include "main/swapper.h" #include "managers/inspector_manager.h" @@ -149,7 +150,8 @@ class ACThirdPartyAppIdContextSwap : public AnalyzerCommand { public: bool execute(Analyzer&, void**) override; - ACThirdPartyAppIdContextSwap(const AppIdInspector& inspector): inspector(inspector) + ACThirdPartyAppIdContextSwap(const AppIdInspector& inspector, ControlConn* conn) + : inspector(inspector), tracker_ref(conn) { LogMessage("== swapping third-party configuration\n"); } @@ -158,6 +160,7 @@ public: const char* stringify() override { return "THIRD-PARTY_CONTEXT_SWAP"; } private: const AppIdInspector& inspector; + ControlConn* tracker_ref; }; bool ACThirdPartyAppIdContextSwap::execute(Analyzer&, void**) @@ -176,6 +179,7 @@ ACThirdPartyAppIdContextSwap::~ACThirdPartyAppIdContextSwap() std::string file_path = ctxt.get_tp_appid_ctxt()->get_user_config(); ctxt.get_odp_ctxt().get_app_info_mgr().dump_appid_configurations(file_path); LogMessage("== third-party configuration swap complete\n"); + ReloadTracker::end(tracker_ref); } class ACThirdPartyAppIdContextUnload : public AnalyzerCommand @@ -213,10 +217,11 @@ ACThirdPartyAppIdContextUnload::~ACThirdPartyAppIdContextUnload() delete tp_ctxt; AppIdContext& ctxt = inspector.get_ctxt(); ctxt.create_tp_appid_ctxt(); - main_broadcast_command(new ACThirdPartyAppIdContextSwap(inspector)); + main_broadcast_command(new ACThirdPartyAppIdContextSwap(inspector, ctrlcon)); LogMessage("== reload third-party complete\n"); - ctrlcon->respond("== reload third-party complete\n"); - Swapper::set_reload_in_progress(false); + if (ctrlcon && !ctrlcon->is_local()) + ctrlcon->respond("== reload third-party complete\n"); + ReloadTracker::update(ctrlcon, "unload old third-party complete, start swapping to new configuration."); } class ACOdpContextSwap : public AnalyzerCommand @@ -265,9 +270,8 @@ ACOdpContextSwap::~ACOdpContextSwap() file_path = std::string(ctxt.config.app_detector_dir) + "/../userappid.conf"; ctxt.get_odp_ctxt().get_app_info_mgr().dump_appid_configurations(file_path); } - LogMessage("== reload detectors complete\n"); + ReloadTracker::end(ctrlcon); ctrlcon->respond("== reload detectors complete\n"); - Swapper::set_reload_in_progress(false); } static int enable_debug(lua_State* L) @@ -320,14 +324,16 @@ static int disable_debug(lua_State* L) static int reload_third_party(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); - if (Swapper::get_reload_in_progress()) + if (!ReloadTracker::start(ctrlcon)) { ctrlcon->respond("== reload pending; retry\n"); return 0; } + AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME); if (!inspector) { + ReloadTracker::failed(ctrlcon, "appid not enabled"); ctrlcon->respond("== reload third-party failed - appid not enabled\n"); return 0; } @@ -335,10 +341,12 @@ static int reload_third_party(lua_State* L) ThirdPartyAppIdContext* old_ctxt = ctxt.get_tp_appid_ctxt(); if (!old_ctxt) { + ReloadTracker::failed(ctrlcon, "third-party module doesn't exist"); ctrlcon->respond("== reload third-party failed - third-party module doesn't exist\n"); return 0; } - Swapper::set_reload_in_progress(true); + + ReloadTracker::update(ctrlcon, "unloading old third-party configuration"); ctrlcon->respond("== unloading old third-party configuration\n"); main_broadcast_command(new ACThirdPartyAppIdContextUnload(*inspector, old_ctxt, ctrlcon), ctrlcon); return 0; @@ -356,7 +364,7 @@ static void clear_dynamic_host_cache_services() static int reload_detectors(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); - if (Swapper::get_reload_in_progress()) + if ( !ReloadTracker::start(ctrlcon) ) { ctrlcon->respond("== reload pending; retry\n"); return 0; @@ -365,9 +373,10 @@ static int reload_detectors(lua_State* L) if (!inspector) { ctrlcon->respond("== reload detectors failed - appid not enabled\n"); + ReloadTracker::failed(ctrlcon, "appid not enabled"); return 0; } - Swapper::set_reload_in_progress(true); + ctrlcon->respond(".. reloading detectors\n"); AppIdContext& ctxt = inspector->get_ctxt(); @@ -388,6 +397,7 @@ static int reload_detectors(lua_State* L) odp_ctxt.initialize(*inspector); ctrlcon->respond("== swapping detectors configuration\n"); + ReloadTracker::update(ctrlcon, "swapping detectors configuration"); main_broadcast_command(new ACOdpContextSwap(*inspector, old_odp_ctxt, ctrlcon), ctrlcon); return 0; }