From 8f082a6d39c2a8dcc736cfa5d27cae2cbdf9b06a Mon Sep 17 00:00:00 2001 From: "Umang Sharma (umasharm)" Date: Sun, 4 May 2025 16:19:14 +0000 Subject: [PATCH] Pull request #4721: AppID Third party sync events for Multiprocess Merge in SNORT/snort3 from ~UMASHARM/snort3:appid_tp_syncevents to master Squashed commit of the following: commit e9776d26a8d485b85ba3d99c37f8a841f8c960ee Author: Umang Sharma Date: Fri May 2 17:28:07 2025 -0400 appid: multiprocess init for appid tp syncevents --- src/framework/mp_data_bus.cc | 10 ++++++++-- src/framework/mp_data_bus.h | 4 +++- src/network_inspectors/appid/CMakeLists.txt | 2 ++ src/network_inspectors/appid/appid_config.cc | 4 ++++ src/network_inspectors/appid/appid_module.cc | 3 +++ .../appid/test/tp_lib_handler_test.cc | 17 ++++++++++++++++ src/network_inspectors/appid/test/tp_mock.cc | 11 ++++++++++ .../appid/tp_lib_handler.cc | 20 +++++++++++++++++++ src/network_inspectors/appid/tp_lib_handler.h | 3 +++ 9 files changed, 71 insertions(+), 3 deletions(-) diff --git a/src/framework/mp_data_bus.cc b/src/framework/mp_data_bus.cc index 8197afeb4..71418dc5c 100644 --- a/src/framework/mp_data_bus.cc +++ b/src/framework/mp_data_bus.cc @@ -119,7 +119,7 @@ unsigned MPDataBus::init(int max_procs) if (transport_layer == nullptr) { ErrorMessage("MPDataBus: Failed to get transport layer\n"); - return 0; + return 1; } transport_layer->register_receive_handler(std::bind(&MPDataBus::receive_message, this, std::placeholders::_1)); @@ -215,7 +215,7 @@ bool MPDataBus::publish(unsigned pub_id, unsigned evt_id, std::shared_ptrmp_dbus && !SnortConfig::get_conf()->mp_dbus->transport_layer) + if (!SnortConfig::get_conf()->mp_dbus or !SnortConfig::get_conf()->mp_dbus->transport_layer) { ErrorMessage("MPDataBus: MPDataBus or transport layer not initialized\n"); return; @@ -278,6 +278,12 @@ void MPDataBus::process_event_queue() MPDataBusLog("Processing event for publisher ID %u \n", event_info->pub_id); + if (!transport_layer){ + run_thread.store(false); + ErrorMessage("MPDataBus: Transport layer not initialized\n"); + return; + } + auto send_res = transport_layer->send_to_transport(*event_info); { diff --git a/src/framework/mp_data_bus.h b/src/framework/mp_data_bus.h index 06c9efe9b..6638b6431 100644 --- a/src/framework/mp_data_bus.h +++ b/src/framework/mp_data_bus.h @@ -160,8 +160,10 @@ public: // and the shared_ptr will handle the memory management by reference counting static bool publish(unsigned pub_id, unsigned evt_id, std::shared_ptr e, Flow* f = nullptr); + // The user needs to pass the MPSerializeFunc and MPDeserializeFunc function pointers + // to the register_event_helpers function, which will be used to serialize and deserialize + // before publishing any events to the MPDataBus static void register_event_helpers(const PubKey& key, unsigned evt_id, MPSerializeFunc& mp_serializer_helper, MPDeserializeFunc& mp_deserializer_helper); - // API for receiving the DataEvent and Event type from transport layer using EventInfo void receive_message(const MPEventInfo& event_info); diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt index 4faa99fb7..fd83fe82e 100644 --- a/src/network_inspectors/appid/CMakeLists.txt +++ b/src/network_inspectors/appid/CMakeLists.txt @@ -10,6 +10,8 @@ set (APPID_INCLUDES tp_appid_module_api.h tp_appid_session_api.h tp_appid_types.h + ../../framework/mp_data_bus.h + ../../framework/mp_data_bus.cc ) set ( APPID_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc index 20b6a5d42..ba7b09a2b 100644 --- a/src/network_inspectors/appid/appid_config.cc +++ b/src/network_inspectors/appid/appid_config.cc @@ -153,7 +153,11 @@ bool AppIdContext::init_appid(SnortConfig* sc, AppIdInspector& inspector) // do not reload third party on reload_config() if (!tp_appid_ctxt) + { tp_appid_ctxt = TPLibHandler::create_tp_appid_ctxt(config, *odp_ctxt); + if (sc->max_procs > 1) + TPLibHandler::tp_mp_init(*tp_appid_ctxt); + } once = true; } else diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 4a4d1a9d1..4c0acda50 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -52,6 +52,7 @@ #include "appid_peg_counts.h" #include "service_state.h" #include "appid_cpu_profile_table.h" +#include "tp_lib_handler.h" using namespace snort; using namespace std; @@ -233,6 +234,8 @@ ACThirdPartyAppIdContextUnload::~ACThirdPartyAppIdContextUnload() delete tp_ctxt; AppIdContext& ctxt = inspector.get_ctxt(); ctxt.create_tp_appid_ctxt(); + if (SnortConfig::get_conf()->max_procs > 1) + TPLibHandler::tp_mp_init(*ctxt.get_tp_appid_ctxt()); main_broadcast_command(new ACThirdPartyAppIdContextSwap(inspector, ctrlcon), ctrlcon); log_message("== unload old third-party complete\n"); ReloadTracker::update(ctrlcon, "unload old third-party complete, start swapping to new configuration."); diff --git a/src/network_inspectors/appid/test/tp_lib_handler_test.cc b/src/network_inspectors/appid/test/tp_lib_handler_test.cc index d3c22071a..f4c1deaf7 100644 --- a/src/network_inspectors/appid/test/tp_lib_handler_test.cc +++ b/src/network_inspectors/appid/test/tp_lib_handler_test.cc @@ -92,6 +92,8 @@ TEST(tp_lib_handler, load_unload) ThirdPartyAppIdContext* tp_appid_ctxt = TPLibHandler::create_tp_appid_ctxt(config, ctxt.get_odp_ctxt()); CHECK_TRUE(tp_appid_ctxt != nullptr); + TPLibHandler::tp_mp_init(*tp_appid_ctxt); + TpAppIdCreateSession asf = tph->tpsession_factory(); ThirdPartyAppIdSession* tpsession = asf(*tp_appid_ctxt); @@ -111,6 +113,21 @@ TEST(tp_lib_handler, tp_lib_handler_get) TPLibHandler::pfini(); } +TEST(tp_lib_handler, tp_mp_init) +{ + config.tp_appid_path="./libtp_mock.so"; + config.tp_appid_config="./tp.config"; + + tph = TPLibHandler::get(); + ThirdPartyAppIdContext* tp_appid_ctxt = TPLibHandler::create_tp_appid_ctxt(config, ctxt.get_odp_ctxt()); + + TPLibHandler::tp_mp_init(*tp_appid_ctxt); + CHECK_TRUE(tp_appid_ctxt != nullptr); + + delete tp_appid_ctxt; + TPLibHandler::pfini(); +} + TEST(tp_lib_handler, load_error) { // Trigger load error: diff --git a/src/network_inspectors/appid/test/tp_mock.cc b/src/network_inspectors/appid/test/tp_mock.cc index 1dcdce17f..6cd80f6a4 100644 --- a/src/network_inspectors/appid/test/tp_mock.cc +++ b/src/network_inspectors/appid/test/tp_mock.cc @@ -61,6 +61,7 @@ public: int tinit() override {return 0;} bool tfini(bool) override {return false;} + int tp_mp_init(ThirdPartyAppIdContext *tp_appid_ctxt) {return true;} const string& get_user_config() const override { return user_config; } private: @@ -105,6 +106,11 @@ extern "C" return new ThirdPartyAppIdSessionImpl(ctxt); } + SO_PUBLIC int tp_mp_init(ThirdPartyAppIdContext& ctxt) + { + return 0; + } + SO_PUBLIC int tp_appid_pfini() { return 0; @@ -114,5 +120,10 @@ extern "C" { return 0; } + + SO_PUBLIC int tp_appid_mp_init(ThirdPartyAppIdContext& ctxt) + { + return 0; + } } diff --git a/src/network_inspectors/appid/tp_lib_handler.cc b/src/network_inspectors/appid/tp_lib_handler.cc index e6b16000f..aedfdf2b3 100644 --- a/src/network_inspectors/appid/tp_lib_handler.cc +++ b/src/network_inspectors/appid/tp_lib_handler.cc @@ -56,6 +56,7 @@ bool TPLibHandler::load_callback(const char* const path) { { "tp_appid_create_ctxt", (dummyFunc*)&tp_appid_create_ctxt }, { "tp_appid_create_session", (dummyFunc*)&tp_appid_create_session }, + { "tp_appid_mp_init", (dummyFunc*)&tp_appid_mp_init }, { "tp_appid_pfini", (dummyFunc*)&tp_appid_pfini }, { "tp_appid_tfini", (dummyFunc*)&tp_appid_tfini }, { nullptr, nullptr } @@ -131,6 +132,25 @@ ThirdPartyAppIdContext* TPLibHandler::create_tp_appid_ctxt(const AppIdConfig& co return tp_appid_ctxt; } +void TPLibHandler::tp_mp_init(ThirdPartyAppIdContext& tp_appid_ctxt) +{ + assert(self != nullptr); + int ret = 0; + + if (self->tp_appid_mp_init) + { + ret = self->tp_appid_mp_init(tp_appid_ctxt); + } + + if (ret != 0) + { + APPID_LOG(nullptr, TRACE_ERROR_LEVEL, "Could not subscribe to the appid tp syncevent\n", ret); + return; + } + + return; +} + void TPLibHandler::tfini() { assert(self != nullptr); diff --git a/src/network_inspectors/appid/tp_lib_handler.h b/src/network_inspectors/appid/tp_lib_handler.h index bf0187b8f..15c862e38 100644 --- a/src/network_inspectors/appid/tp_lib_handler.h +++ b/src/network_inspectors/appid/tp_lib_handler.h @@ -31,6 +31,7 @@ class OdpContext; // Must return null if it fails to create the object. typedef ThirdPartyAppIdContext* (* TpAppIdCreateCtxt)(ThirdPartyConfig& ); typedef ThirdPartyAppIdSession* (* TpAppIdCreateSession)(ThirdPartyAppIdContext& ctxt); +typedef int (* TpMPInit)(ThirdPartyAppIdContext& ctxt); typedef int (* TpAppIdPfini)(); typedef int (* TpAppIdTfini)(); @@ -50,6 +51,7 @@ public: const OdpContext& odp_ctxt); static void tfini(); static void pfini(); + static void tp_mp_init(ThirdPartyAppIdContext& tp_appid_ctxt); TpAppIdCreateSession tpsession_factory() const { @@ -66,6 +68,7 @@ private: TpAppIdCreateSession tp_appid_create_session = nullptr; TpAppIdPfini tp_appid_pfini = nullptr; TpAppIdTfini tp_appid_tfini = nullptr; + TpMPInit tp_appid_mp_init = nullptr; bool load_callback(const char* path); }; -- 2.47.3