From dae748585681b5c7a10bbbf4c7dffeeec980baeb Mon Sep 17 00:00:00 2001 From: "Umang Sharma (umasharm)" Date: Fri, 3 May 2024 17:30:45 +0000 Subject: [PATCH] Pull request #4288: appid : Appid CPU Profiler Table and CLI Merge in SNORT/snort3 from ~UMASHARM/snort3:appid_cpu_profiling to master Squashed commit of the following: commit 9dfca5d8512eb3a899baaa397bab37dae320e004 Author: Umang Sharma Date: Fri Mar 29 15:31:08 2024 -0400 appid : Appid CPU Profiler Table and CLI --- src/network_inspectors/appid/CMakeLists.txt | 2 + .../appid/app_cpu_profile_table.cc | 204 ++++++++++++++++++ .../appid/app_cpu_profile_table.h | 65 ++++++ .../appid/app_info_table.cc | 9 +- src/network_inspectors/appid/appid_api.cc | 10 + src/network_inspectors/appid/appid_api.h | 1 + .../appid/appid_cip_event_handler.cc | 12 ++ src/network_inspectors/appid/appid_config.cc | 16 ++ src/network_inspectors/appid/appid_config.h | 11 + .../appid/appid_discovery.cc | 14 ++ .../appid/appid_http_event_handler.cc | 14 +- src/network_inspectors/appid/appid_module.cc | 55 +++++ src/network_inspectors/appid/appid_session.cc | 21 +- src/network_inspectors/appid/appid_session.h | 6 +- .../appid/appid_ssh_event_handler.cc | 13 ++ .../test/detector_plugins_mock.h | 3 + .../test/service_plugin_mock.h | 2 + .../appid/test/appid_api_test.cc | 6 + .../appid/test/appid_debug_test.cc | 2 + .../appid/test/appid_discovery_test.cc | 4 + .../appid/test/appid_http_event_test.cc | 4 + .../appid/test/appid_http_session_test.cc | 2 + .../appid/test/appid_mock_session.h | 3 +- .../appid/test/service_state_test.cc | 3 + .../appid/test/tp_lib_handler_test.cc | 1 + .../appid/tp_appid_module_api.cc | 20 ++ .../appid/tp_appid_utils.cc | 10 + src/profiler/profiler.cc | 2 + 28 files changed, 510 insertions(+), 5 deletions(-) create mode 100644 src/network_inspectors/appid/app_cpu_profile_table.cc create mode 100644 src/network_inspectors/appid/app_cpu_profile_table.h diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt index d386787d0..fc1db4456 100644 --- a/src/network_inspectors/appid/CMakeLists.txt +++ b/src/network_inspectors/appid/CMakeLists.txt @@ -207,6 +207,8 @@ set ( APPID_SOURCES tp_appid_session_api.h tp_appid_module_api.h tp_appid_module_api.cc + app_cpu_profile_table.cc + app_cpu_profile_table.h ) #if (STATIC_INSPECTORS) diff --git a/src/network_inspectors/appid/app_cpu_profile_table.cc b/src/network_inspectors/appid/app_cpu_profile_table.cc new file mode 100644 index 000000000..c4bd1cfa4 --- /dev/null +++ b/src/network_inspectors/appid/app_cpu_profile_table.cc @@ -0,0 +1,204 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// app_cpu_profiling_table.cc author Umang Sharma + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "log/text_log.h" +#include "time/packet_time.h" +#include +#include +#include + +#include "appid_session.h" +#include "app_cpu_profile_table.h" + +using namespace snort; + +const char* table_header = "AppId Performance Statistics (all)\n========================================================================================================================\n"; +const char* columns = " AppId App Name Microsecs Packets Avg/Packet Sessions Avg/Session \n"; +const char* partition = "------------------------------------------------------------------------------------------------------------------------\n"; + +std::string FormatWithCommas(uint64_t value) +{ + std::string numStr = std::to_string(value); + int insertPosition = numStr.length() - 3; + while (insertPosition > 0) + { + numStr.insert(insertPosition, ","); + insertPosition -= 3; + } + return numStr; +} + +// Comparator for priority queue based on avg_processing_time/session +struct CompareByAvgProcessingTime { + bool operator()(const std::pair& a, const std::pair& b) const { + if (a.second.processed_packets == 0 or b.second.processed_packets == 0) { + return false; + } + return a.second.processing_time/a.second.per_appid_sessions < b.second.processing_time/b.second.per_appid_sessions ; + } +}; + +void AppidCPUProfilingManager::display_appid_cpu_profiler_table(AppId appid) +{ + if (appid_cpu_profiling_table.empty()) + { + appid_log(nullptr, TRACE_INFO_LEVEL,"Appid CPU Profiler Table is empty\n", appid); + return; + } + auto bucket = appid_cpu_profiling_table.find(appid); + + if (bucket != appid_cpu_profiling_table.end()) + { + appid_log(nullptr, TRACE_INFO_LEVEL, table_header); + appid_log(nullptr, TRACE_INFO_LEVEL, columns); + appid_log(nullptr, TRACE_INFO_LEVEL, partition); + + appid_log(nullptr, TRACE_INFO_LEVEL, " %5d %-25.25s %18s %12s %15s %12s %12s\n", + appid, bucket->second.app_name.c_str(), FormatWithCommas(bucket->second.processing_time).c_str(), FormatWithCommas(bucket->second.processed_packets).c_str(), FormatWithCommas(bucket->second.processing_time/bucket->second.processed_packets).c_str(), + FormatWithCommas(bucket->second.per_appid_sessions).c_str(), FormatWithCommas(bucket->second.processing_time/bucket->second.per_appid_sessions).c_str()); + } + else + { + appid_log(nullptr, TRACE_INFO_LEVEL,"Appid %d not found in the table\n", appid); + } +} + +void AppidCPUProfilingManager::display_appid_cpu_profiler_table() +{ + if (appid_cpu_profiling_table.empty()) + { + appid_log(nullptr, TRACE_INFO_LEVEL,"Appid CPU Profiler Table is empty\n"); + return; + } + + std::priority_queue, std::vector>, CompareByAvgProcessingTime> sorted_appid_cpu_profiler_table; + + for (const auto& entry : appid_cpu_profiling_table) + { + sorted_appid_cpu_profiler_table.push(entry); // Push the pair (AppId, AppidCPUProfilerStats) + } + + appid_log(nullptr, TRACE_INFO_LEVEL, table_header); + appid_log(nullptr, TRACE_INFO_LEVEL, columns); + appid_log(nullptr, TRACE_INFO_LEVEL, partition); + + while (!sorted_appid_cpu_profiler_table.empty()) + { + auto entry = sorted_appid_cpu_profiler_table.top(); + sorted_appid_cpu_profiler_table.pop(); + if (!entry.second.processed_packets) + continue; + + appid_log(nullptr, TRACE_INFO_LEVEL, " %5d %-25.25s %18s %12s %15s %12s %12s\n", + entry.first, entry.second.app_name.c_str(), FormatWithCommas(entry.second.processing_time).c_str(), FormatWithCommas(entry.second.processed_packets).c_str(), FormatWithCommas(entry.second.processing_time/entry.second.processed_packets).c_str(), + FormatWithCommas(entry.second.per_appid_sessions).c_str(), FormatWithCommas(entry.second.processing_time/entry.second.per_appid_sessions).c_str()); + } +} + +AppidCPUProfilingManager::AppidCPUProfilingManager() +{ + appid_cpu_profiling_table.clear(); +} + +void AppidCPUProfilingManager::cleanup_appid_cpu_profiler_table() +{ + appid_cpu_profiling_table.clear(); +} + +void AppidCPUProfilingManager::insert_appid_cpu_profiler_record(AppId appId, const AppidCPUProfilerStats& stats) +{ + auto it = appid_cpu_profiling_table.find(appId); + if (it == appid_cpu_profiling_table.end()) + { + appid_cpu_profiling_table.emplace(appId, stats); + } + else + { + it->second.processing_time += stats.processing_time; + it->second.processed_packets += stats.processed_packets; + it->second.per_appid_sessions += 1; + } +} + +void AppidCPUProfilingManager::check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId payload_id) +{ + if (payload_id > APP_ID_NONE) + { + const char* app_name = asd->get_odp_ctxt().get_app_info_mgr().get_app_name(payload_id); + if (app_name == nullptr) + app_name = "unknown"; + + stats_bucket_insert(payload_id, app_name, asd->stats.prev_payload_processing_time, asd->stats.prev_payload_processing_packets); + } +} + +void AppidCPUProfilingManager::stats_bucket_insert(AppId appid, const char* app_name, uint64_t processing_time, uint64_t processed_packets) +{ + if (!processed_packets or !processing_time) + { + appid_log(nullptr, TRACE_INFO_LEVEL, "appid: processed packets/time are NULL for appid : %d , app_name : %s , processing time :%lu \n", appid, app_name, processing_time); + return; + } + + AppidCPUProfilerStats stats(app_name, processing_time, processed_packets, 1); + insert_appid_cpu_profiler_record(appid, stats); +} + +void AppidCPUProfilingManager::check_appid_cpu_profiler_table_entry(const AppIdSession* asd,AppId service_id, AppId client_id, AppId payload_id, AppId misc_id) +{ + if (!asd->stats.processing_time or !asd->stats.cpu_profiler_pkt_count) + return; + + if (service_id > APP_ID_NONE) + { + const char* app_name = asd->get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); + if (app_name == nullptr) + app_name = "unknown"; + + stats_bucket_insert(service_id, app_name, asd->stats.processing_time, asd->stats.cpu_profiler_pkt_count); + } + if (client_id > APP_ID_NONE and client_id != service_id){ + const char* app_name = asd->get_odp_ctxt().get_app_info_mgr().get_app_name(client_id); + if (app_name == nullptr) + app_name = "unknown"; + + stats_bucket_insert(client_id, app_name, asd->stats.processing_time, asd->stats.cpu_profiler_pkt_count); + } + if (payload_id > APP_ID_NONE and payload_id != service_id and payload_id != client_id) + { + const char* app_name = asd->get_odp_ctxt().get_app_info_mgr().get_app_name(payload_id); + if (app_name == nullptr) + app_name = "unknown"; + + stats_bucket_insert(payload_id, app_name, asd->stats.processing_time - asd->stats.prev_payload_processing_time, asd->stats.cpu_profiler_pkt_count - asd->stats.prev_payload_processing_packets); + } + if (misc_id > APP_ID_NONE and misc_id != service_id and misc_id != client_id and misc_id != payload_id) + { + const char* app_name = asd->get_odp_ctxt().get_app_info_mgr().get_app_name(misc_id); + if (app_name == nullptr) + app_name = "unknown"; + + stats_bucket_insert(misc_id, app_name, asd->stats.processing_time, asd->stats.cpu_profiler_pkt_count); + } +} diff --git a/src/network_inspectors/appid/app_cpu_profile_table.h b/src/network_inspectors/appid/app_cpu_profile_table.h new file mode 100644 index 000000000..827860819 --- /dev/null +++ b/src/network_inspectors/appid/app_cpu_profile_table.h @@ -0,0 +1,65 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// app_cpu_profile_table.h author Umang Sharma + +#ifndef APP_CPU_PROFILE_TABLE_H +#define APP_CPU_PROFILE_TABLE_H + +#include +#include + +#include "main/thread.h" +#include "utils/util.h" +#include "application_ids.h" +#include "application_ids.h" +#include "app_info_table.h" + +class AppIdSession; +class OdpContext; + +struct AppidCPUProfilerStats { + std::string app_name; + uint64_t processing_time = 0; + uint64_t processed_packets = 0; + uint32_t per_appid_sessions = 0; + + AppidCPUProfilerStats(const char* app_name, uint64_t processing_time, uint64_t processed_packets, uint32_t per_appid_sessions) : + app_name(app_name), processing_time(processing_time), processed_packets(processed_packets), per_appid_sessions (per_appid_sessions) + { } +}; + +class AppidCPUProfilingManager { +private: + typedef std::unordered_map AppidCPUProfilingTable; + AppidCPUProfilingTable appid_cpu_profiling_table; + +public: + AppidCPUProfilingManager(); + + void stats_bucket_insert(AppId appid, const char* app_name, uint64_t processing_time, uint64_t processed_packets); + void insert_appid_cpu_profiler_record(AppId appId, const AppidCPUProfilerStats& stats); + void check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId service_id, AppId client_id, AppId payload_id, AppId misc_id); + void check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId payload_id); + + void display_appid_cpu_profiler_table(); + void display_appid_cpu_profiler_table(AppId appid); + + void cleanup_appid_cpu_profiler_table(); +}; +#endif diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc index 120ab6684..c87c70fca 100644 --- a/src/network_inspectors/appid/app_info_table.cc +++ b/src/network_inspectors/appid/app_info_table.cc @@ -616,6 +616,13 @@ void AppInfoManager::load_odp_config(OdpContext& odp_ctxt, const char* path) { odp_ctxt.eve_http_client = atoi(conf_val) ? true : false; } + else if (!(strcasecmp(conf_key, "appid_cpu_profiling"))) + { + if (!(strcasecmp(conf_val, "disabled"))) + { + odp_ctxt.appid_cpu_profiler = false; + } + } else ParseWarning(WARN_CONF, "appid: unsupported configuration: %s\n", conf_key); } @@ -757,4 +764,4 @@ void AppInfoManager::init_appid_info_table(const AppIdConfig& config, load_odp_config(odp_ctxt, filepath); } } - + diff --git a/src/network_inspectors/appid/appid_api.cc b/src/network_inspectors/appid/appid_api.cc index 87b1a9ec1..65fb7ce48 100644 --- a/src/network_inspectors/appid/appid_api.cc +++ b/src/network_inspectors/appid/appid_api.cc @@ -275,3 +275,13 @@ const char* AppIdApi::get_appid_detector_directory() const return inspector->get_config().app_detector_dir; } + +void AppIdApi::reset_appid_cpu_profiler_stats() +{ + AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME); + if (!inspector) + return; + const AppIdContext& ctxt = inspector->get_ctxt(); + OdpContext& odp_ctxt = ctxt.get_odp_ctxt(); + odp_ctxt.get_appid_cpu_profiler_mgr().cleanup_appid_cpu_profiler_table(); +} diff --git a/src/network_inspectors/appid/appid_api.h b/src/network_inspectors/appid/appid_api.h index 7c87eadff..bdf56013b 100644 --- a/src/network_inspectors/appid/appid_api.h +++ b/src/network_inspectors/appid/appid_api.h @@ -53,6 +53,7 @@ public: const AppIdSessionApi* get_appid_session_api(const Flow& flow) const; bool is_inspection_needed(const Inspector& g) const; const char* get_appid_detector_directory() const; + void reset_appid_cpu_profiler_stats(); bool is_service_http_type(AppId service_id) const { diff --git a/src/network_inspectors/appid/appid_cip_event_handler.cc b/src/network_inspectors/appid/appid_cip_event_handler.cc index 554752968..16e11e3f2 100644 --- a/src/network_inspectors/appid/appid_cip_event_handler.cc +++ b/src/network_inspectors/appid/appid_cip_event_handler.cc @@ -68,6 +68,12 @@ void CipEventHandler::handle(DataEvent& event, Flow* flow) if (!asd) return; + + bool is_appid_cpu_profiling_running = (asd->get_odp_ctxt().is_appid_cpu_profiler_running()); + Stopwatch per_appid_event_cpu_timer; + + if (is_appid_cpu_profiling_running) + per_appid_event_cpu_timer.start(); if (!pkt_thread_odp_ctxt or (asd->get_odp_ctxt_version() != pkt_thread_odp_ctxt->get_version())) return; @@ -103,4 +109,10 @@ void CipEventHandler::handle(DataEvent& event, Flow* flow) } asd->publish_appid_event(change_bits, *p); + + if (is_appid_cpu_profiling_running) + { + per_appid_event_cpu_timer.stop(); + asd->stats.processing_time += TO_USECS(per_appid_event_cpu_timer.get()); + } } diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc index 73c7e65fd..9ef62ccc8 100644 --- a/src/network_inspectors/appid/appid_config.cc +++ b/src/network_inspectors/appid/appid_config.cc @@ -49,6 +49,7 @@ #include "service_plugins/service_ssl.h" #include "tp_appid_utils.h" #include "tp_lib_handler.h" +#include "profiler/profiler_defs.h" using namespace snort; @@ -110,6 +111,10 @@ void AppIdContext::pterm() if (odp_ctxt) { odp_ctxt->get_app_info_mgr().cleanup_appid_info_table(); + if (odp_ctxt->is_appid_cpu_profiler_running()) + odp_ctxt->get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(); + + odp_ctxt->get_appid_cpu_profiler_mgr().cleanup_appid_cpu_profiler_table(); delete odp_ctxt; odp_ctxt = nullptr; } @@ -219,8 +224,19 @@ void OdpContext::dump_appid_config() appid_log(nullptr, TRACE_INFO_LEVEL, "Appid Config: max_packet_before_service_fail %" PRIu16" \n", max_packet_before_service_fail); appid_log(nullptr, TRACE_INFO_LEVEL, "Appid Config: max_packet_service_fail_ignore_bytes %" PRIu16" \n", max_packet_service_fail_ignore_bytes); appid_log(nullptr, TRACE_INFO_LEVEL, "Appid Config: eve_http_client %s\n", (eve_http_client ? "True" : "False")); + appid_log(nullptr, TRACE_INFO_LEVEL, "Appid Config: appid_cpu_profiler %s\n", (appid_cpu_profiler ? "True" : "False")); } +bool OdpContext::is_appid_cpu_profiler_running() +{ + return (TimeProfilerStats::is_enabled() and appid_cpu_profiler); +} + +bool OdpContext::is_appid_cpu_profiler_enabled() +{ + return appid_cpu_profiler; +} + OdpContext::OdpContext(const AppIdConfig& config, SnortConfig* sc) { app_info_mgr.init_appid_info_table(config, sc, *this); diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index 66b947e98..d41566bda 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -45,6 +45,8 @@ #include "detector_plugins/ssh_patterns.h" #include "tp_appid_module_api.h" #include "utils/sflsq.h" +#include "app_cpu_profile_table.h" +#include "profiler/profiler_defs.h" #define APP_ID_PORT_ARRAY_SIZE 65536 @@ -142,11 +144,14 @@ public: uint16_t max_packet_service_fail_ignore_bytes = DEFAULT_MAX_PKT_BEFORE_SERVICE_FAIL_IGNORE_BYTES; FirstPktAppIdDiscovered first_pkt_appid_prefix = NO_APPID_FOUND; bool eve_http_client = true; + bool appid_cpu_profiler= true; OdpContext(const AppIdConfig&, snort::SnortConfig*); void initialize(AppIdInspector& inspector); void reload(); void dump_appid_config(); + bool is_appid_cpu_profiler_enabled(); + bool is_appid_cpu_profiler_running(); uint32_t get_version() const { @@ -250,6 +255,11 @@ public: return alpn_matchers; } + AppidCPUProfilingManager& get_appid_cpu_profiler_mgr() + { + return app_cpu_profiler_mgr; + } + unsigned get_pattern_count(); void add_port_service_id(IpProtocol, uint16_t, AppId); void add_protocol_service_id(IpProtocol, AppId); @@ -261,6 +271,7 @@ public: private: AppInfoManager app_info_mgr; + AppidCPUProfilingManager app_cpu_profiler_mgr; ClientDiscovery client_disco_mgr; HostPortCache host_port_cache; HostPortCache first_pkt_cache; diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index 02bc7c314..b766cbea9 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -39,6 +39,7 @@ #include "appid_http_session.h" #include "appid_inspector.h" #include "appid_session.h" +#include "app_cpu_profile_table.h" #include "appid_utils/ip_funcs.h" #include "client_plugins/client_discovery.h" #include "detector_plugins/detector_dns.h" @@ -140,6 +141,12 @@ void AppIdDiscovery::do_application_discovery(Packet* p, AppIdInspector& inspect if (!do_pre_discovery(p, asd, inspector, protocol, outer_protocol, direction, odp_ctxt)) return; + bool is_appid_cpu_profiling_running = (odp_ctxt.is_appid_cpu_profiler_running()); + Stopwatch per_appid_cpu_timer; + + if (is_appid_cpu_profiling_running) + per_appid_cpu_timer.start(); + AppId service_id = APP_ID_NONE; AppId client_id = APP_ID_NONE; AppId payload_id = APP_ID_NONE; @@ -150,6 +157,12 @@ void AppIdDiscovery::do_application_discovery(Packet* p, AppIdInspector& inspect do_post_discovery(p, *asd, is_discovery_done, service_id, client_id, payload_id, misc_id, change_bits); + + if (is_appid_cpu_profiling_running) + { + per_appid_cpu_timer.stop(); + asd->stats.processing_time += TO_USECS(per_appid_cpu_timer.get()); + } } static bool set_network_attributes(AppIdSession* asd, Packet* p, IpProtocol& protocol, @@ -260,6 +273,7 @@ bool AppIdDiscovery::do_pre_discovery(Packet* p, AppIdSession*& asd, AppIdInspec // refactor to pass this as ref and delete any checks for null appid_stats.processed_packets++; asd->session_packet_count++; + asd->stats.cpu_profiler_pkt_count++; if (direction == APP_ID_FROM_INITIATOR) { diff --git a/src/network_inspectors/appid/appid_http_event_handler.cc b/src/network_inspectors/appid/appid_http_event_handler.cc index 3a4380ad2..bab90299d 100644 --- a/src/network_inspectors/appid/appid_http_event_handler.cc +++ b/src/network_inspectors/appid/appid_http_event_handler.cc @@ -31,6 +31,7 @@ #include "detection/detection_engine.h" #include "app_info_table.h" +#include "app_cpu_profile_table.h" #include "appid_debug.h" #include "appid_discovery.h" #include "appid_http_session.h" @@ -76,6 +77,12 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow) HttpEvent* http_event = (HttpEvent*)&event; AppidChangeBits change_bits; + bool is_appid_cpu_profiling_running = (asd->get_odp_ctxt().is_appid_cpu_profiler_running()); + Stopwatch per_appid_event_cpu_timer; + + if (is_appid_cpu_profiling_running) + per_appid_event_cpu_timer.start(); + if ((asd->get_tp_appid_ctxt() or ThirdPartyAppIdContext::get_tp_reload_in_progress()) and !http_event->get_is_httpx()) return; @@ -199,5 +206,10 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow) asd->publish_appid_event(change_bits, *p, http_event->get_is_httpx(), asd->get_api().get_hsessions_size() - 1); -} + if (is_appid_cpu_profiling_running) + { + per_appid_event_cpu_timer.stop(); + asd->stats.processing_time += TO_USECS(per_appid_event_cpu_timer.get()); + } +} diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index a4c018dc5..abdcde5e7 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -268,6 +268,8 @@ bool ACOdpContextSwap::execute(Analyzer&, void**) ACOdpContextSwap::~ACOdpContextSwap() { odp_ctxt.get_app_info_mgr().cleanup_appid_info_table(); + odp_ctxt.get_appid_cpu_profiler_mgr().cleanup_appid_cpu_profiler_table(); + delete &odp_ctxt; AppIdContext& ctxt = inspector.get_ctxt(); LuaDetectorManager::cleanup_after_swap(); @@ -389,6 +391,49 @@ static void clear_dynamic_host_cache_services() } } + +static int show_cpu_profiler_stats(lua_State* L) +{ + int appid = luaL_optint(L, 1, 0); + ControlConn* ctrlcon = ControlConn::query_from_lua(L); + AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME); + if (!inspector) + { + ctrlcon->respond("== displaying appid cpu profiler failed - appid not enabled\n"); + return 0; + } + const AppIdContext& ctxt = inspector->get_ctxt(); + OdpContext& odp_ctxt = ctxt.get_odp_ctxt(); + if (odp_ctxt.is_appid_cpu_profiler_enabled()) + { + ctrlcon->respond("== showing appid cpu profiler table\n"); + if (!appid) + odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(); + else + odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(appid); + } + else + ctrlcon->respond("appid cpu profiler is disabled\n"); + + return 0; +} + +static int show_cpu_profiler_status(lua_State* L) +{ + ControlConn* ctrlcon = ControlConn::query_from_lua(L); + AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME); + if (!inspector) + { + ctrlcon->respond("== appid cpu profiler status check failed- appid not enabled\n"); + return 0; + } + const AppIdContext& ctxt = inspector->get_ctxt(); + OdpContext& odp_ctxt = ctxt.get_odp_ctxt(); + ctrlcon->respond("appid cpu profiler enabled: %s , running: %s \n", + odp_ctxt.is_appid_cpu_profiler_enabled() ? "yes" : "no", odp_ctxt.is_appid_cpu_profiler_running() ? "yes" : "no"); + return 0; +} + static int reload_detectors(lua_State* L) { ControlConn* ctrlcon = ControlConn::query_from_lua(L); @@ -470,6 +515,13 @@ static const Parameter enable_debug_params[] = { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; +static const Parameter appid_cpu_params[] = +{ + { "appid", Parameter::PT_INT, nullptr, nullptr, "show appid cpu profiling stats" }, + + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + static const Command appid_cmds[] = { { "enable_debug", enable_debug, enable_debug_params, "enable appid debugging"}, @@ -477,6 +529,9 @@ static const Command appid_cmds[] = { "reload_third_party", reload_third_party, nullptr, "reload appid third-party module" }, { "reload_detectors", reload_detectors, nullptr, "reload appid detectors" }, { "print_appid_config", print_appid_config, nullptr, "print appid configs" }, + { "show_cpu_profiler_stats", show_cpu_profiler_stats, appid_cpu_params, "show appid cpu profiling stats" }, + { "show_cpu_profiler_status", show_cpu_profiler_status, nullptr, "show appid cpu profiling status" }, + { nullptr, nullptr, nullptr, nullptr } }; diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc index 9b7927431..4a338e3b8 100644 --- a/src/network_inspectors/appid/appid_session.cc +++ b/src/network_inspectors/appid/appid_session.cc @@ -138,6 +138,12 @@ AppIdSession::AppIdSession(IpProtocol proto, const SfIp* ip, uint16_t port, AppIdSession::~AppIdSession() { + // Skip sessions using old odp context after reload detectors for appid cpu profiling + if ((pkt_thread_odp_ctxt->get_version() == api.asd->get_odp_ctxt_version()) and api.asd->get_odp_ctxt().is_appid_cpu_profiler_running()) + { + api.asd->get_odp_ctxt().get_appid_cpu_profiler_mgr().check_appid_cpu_profiler_table_entry(api.asd, api.service.get_id(), api.client.get_id(), api.payload.get_id(), api.get_misc_app_id()); + } + if (!in_expected_cache) { if (config.log_stats) @@ -384,6 +390,13 @@ void AppIdSession::check_ssl_detection_restart(AppidChangeBits& change_bits, encrypted.misc_id = pick_ss_misc_app_id(); encrypted.referred_id = pick_ss_referred_payload_app_id(); + if (odp_ctxt.is_appid_cpu_profiler_running()) + { + odp_ctxt.get_appid_cpu_profiler_mgr().check_appid_cpu_profiler_table_entry(api.asd, encrypted.service_id, encrypted.client_id, encrypted.payload_id, encrypted.misc_id); + this->stats.processing_time = 0; + this->stats.cpu_profiler_pkt_count = 0; + } + reinit_session_data(change_bits, curr_tp_appid_ctxt); appid_log(CURRENT_PACKET, TRACE_DEBUG_LEVEL, "SSL decryption is available, restarting app detection\n"); @@ -407,6 +420,13 @@ void AppIdSession::check_tunnel_detection_restart() appid_log(CURRENT_PACKET, TRACE_DEBUG_LEVEL, "Found HTTP Tunnel, restarting app Detection\n"); + if (odp_ctxt.is_appid_cpu_profiler_running()) + { + odp_ctxt.get_appid_cpu_profiler_mgr().check_appid_cpu_profiler_table_entry(api.asd, api.service.get_id(), api.client.get_id(), api.payload.get_id(), api.get_misc_app_id()); + this->stats.processing_time = 0; + this->stats.cpu_profiler_pkt_count = 0; + } + // service if (api.service.get_id() == api.service.get_port_service_id()) api.service.set_id(APP_ID_NONE, odp_ctxt); @@ -1215,4 +1235,3 @@ void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packe else appid_log(&p, TRACE_DEBUG_LEVEL, "Published event for changes: %s\n", str.c_str()); } - diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h index 0e6fd4ffe..715d3f20d 100644 --- a/src/network_inspectors/appid/appid_session.h +++ b/src/network_inspectors/appid/appid_session.h @@ -299,7 +299,11 @@ public: uint32_t last_packet_second; uint64_t initiator_bytes; uint64_t responder_bytes; - } stats = { 0, 0, 0, 0 }; + uint32_t cpu_profiler_pkt_count; + uint32_t prev_payload_processing_packets; + uint64_t processing_time; + uint64_t prev_payload_processing_time; + } stats = { 0, 0, 0, 0, 0, 0, 0, 0}; //appIds picked from encrypted session. struct diff --git a/src/network_inspectors/appid/appid_ssh_event_handler.cc b/src/network_inspectors/appid/appid_ssh_event_handler.cc index 68cd935f0..61e66a2a9 100644 --- a/src/network_inspectors/appid/appid_ssh_event_handler.cc +++ b/src/network_inspectors/appid/appid_ssh_event_handler.cc @@ -28,6 +28,7 @@ #include "appid_debug.h" #include "appid_detector.h" #include "appid_inspector.h" +#include "profiler/profiler_defs.h" using namespace snort; using namespace std; @@ -157,6 +158,12 @@ void SshEventHandler::handle(DataEvent& event, Flow* flow) if (!asd) return; + bool is_appid_cpu_profiling_running = (asd->get_odp_ctxt().is_appid_cpu_profiler_running()); + Stopwatch per_appid_event_cpu_timer; + + if (is_appid_cpu_profiling_running) + per_appid_event_cpu_timer.start(); + if (asd->get_odp_ctxt_version() != pkt_thread_odp_ctxt->get_version()) return; // Skip detection for sessions using old odp context after odp reload if (!asd->get_session_flags(APPID_SESSION_DISCOVER_APP | APPID_SESSION_SPECIAL_MONITORED)) @@ -239,4 +246,10 @@ void SshEventHandler::handle(DataEvent& event, Flow* flow) break; } + + if (is_appid_cpu_profiling_running) + { + per_appid_event_cpu_timer.stop(); + asd->stats.processing_time += TO_USECS(per_appid_event_cpu_timer.get()); + } } diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h index 99f58ff3d..3ec89ef90 100644 --- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h +++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h @@ -212,6 +212,9 @@ AppInfoTableEntry* AppInfoManager::get_app_info_entry(AppId, const AppInfoTable& return nullptr; } + +AppidCPUProfilingManager::AppidCPUProfilingManager() { } + bool AppIdReloadTuner::tinit() { return false; } bool AppIdReloadTuner::tune_resources(unsigned int) diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h index 6fb04cd84..bebe04351 100644 --- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h +++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h @@ -203,6 +203,8 @@ ServiceDiscoveryState* AppIdServiceState::add(SfIp const*, IpProtocol, { return nullptr; } +AppidCPUProfilingManager::AppidCPUProfilingManager() { } + void ServiceDiscoveryState::set_service_id_valid(ServiceDetector*) { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) diff --git a/src/network_inspectors/appid/test/appid_api_test.cc b/src/network_inspectors/appid/test/appid_api_test.cc index c9a887b64..c62636992 100644 --- a/src/network_inspectors/appid/test/appid_api_test.cc +++ b/src/network_inspectors/appid/test/appid_api_test.cc @@ -34,6 +34,7 @@ #include "appid_http_session.h" #include "tp_appid_module_api.h" #include "tp_appid_session_api.h" +#include "app_cpu_profile_table.h" #include "appid_mock_definitions.h" #include "appid_mock_http_session.h" @@ -50,6 +51,7 @@ using namespace snort; static SnortProtocolId dummy_http2_protocol_id = 1; char const* APPID_UT_ORG_UNIT = "Google"; +THREAD_LOCAL bool TimeProfilerStats::enabled = false; namespace snort { @@ -202,6 +204,10 @@ void AppIdSession::set_ss_application_ids(AppId client_id, AppId payload_id, } } +bool OdpContext::is_appid_cpu_profiler_enabled() { return false; } + +void AppidCPUProfilingManager::cleanup_appid_cpu_profiler_table() {} + AppIdHttpSession* AppIdSession::get_http_session(uint32_t) const { return nullptr; } Flow* flow = nullptr; diff --git a/src/network_inspectors/appid/test/appid_debug_test.cc b/src/network_inspectors/appid/test/appid_debug_test.cc index 4542ffb13..e1abc0a14 100644 --- a/src/network_inspectors/appid/test/appid_debug_test.cc +++ b/src/network_inspectors/appid/test/appid_debug_test.cc @@ -34,6 +34,7 @@ #include #include +THREAD_LOCAL bool TimeProfilerStats::enabled = false; // Mocks @@ -69,6 +70,7 @@ public: AppIdConfig::~AppIdConfig() = default; OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } +AppidCPUProfilingManager::AppidCPUProfilingManager() {} AppIdConfig stub_config; AppIdContext stub_ctxt(stub_config); diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index ac8731d00..d9c1dd3df 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -43,6 +43,7 @@ #include uint32_t ThirdPartyAppIdContext::next_version = 0; +THREAD_LOCAL bool TimeProfilerStats::enabled = false; namespace snort { @@ -184,6 +185,9 @@ AppId OdpContext::get_port_service_id(IpProtocol, uint16_t) return APP_ID_NONE; } +bool OdpContext::is_appid_cpu_profiler_enabled() { return false; } +bool OdpContext::is_appid_cpu_profiler_running() { return false; } + AppId OdpContext::get_protocol_service_id(IpProtocol) { return APP_ID_NONE; diff --git a/src/network_inspectors/appid/test/appid_http_event_test.cc b/src/network_inspectors/appid/test/appid_http_event_test.cc index 412333ee9..eb2d98e2f 100644 --- a/src/network_inspectors/appid/test/appid_http_event_test.cc +++ b/src/network_inspectors/appid/test/appid_http_event_test.cc @@ -43,6 +43,7 @@ THREAD_LOCAL AppIdDebug* appidDebug = nullptr; ThirdPartyAppIdContext* AppIdContext::tp_appid_ctxt = nullptr; THREAD_LOCAL bool ThirdPartyAppIdContext::tp_reload_in_progress = false; +THREAD_LOCAL bool TimeProfilerStats::enabled = false; bool DiscoveryFilter::is_app_monitored(const snort::Packet*, uint8_t*){return true;} void AppIdDebug::activate(const Flow*, const AppIdSession*, bool) { active = true; } void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { } @@ -84,6 +85,9 @@ class FakeHttpMsgHeader }; FakeHttpMsgHeader* fake_msg_header = nullptr; +bool OdpContext::is_appid_cpu_profiler_enabled() { return false; } +bool OdpContext::is_appid_cpu_profiler_running() { return false; } + AppIdSession* AppIdSession::allocate_session(const Packet*, IpProtocol, AppidSessionDirection, AppIdInspector&, OdpContext&) { diff --git a/src/network_inspectors/appid/test/appid_http_session_test.cc b/src/network_inspectors/appid/test/appid_http_session_test.cc index 3fc88b655..81adda7ad 100644 --- a/src/network_inspectors/appid/test/appid_http_session_test.cc +++ b/src/network_inspectors/appid/test/appid_http_session_test.cc @@ -46,6 +46,7 @@ #include using namespace snort; +THREAD_LOCAL bool TimeProfilerStats::enabled = false; namespace snort { @@ -174,6 +175,7 @@ void Profiler::reset_stats(snort::ProfilerType) { } void Profiler::show_stats() { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } +AppidCPUProfilingManager::AppidCPUProfilingManager() { } AppIdConfig::~AppIdConfig() = default; diff --git a/src/network_inspectors/appid/test/appid_mock_session.h b/src/network_inspectors/appid/test/appid_mock_session.h index 2b3d38b18..d89e099b4 100644 --- a/src/network_inspectors/appid/test/appid_mock_session.h +++ b/src/network_inspectors/appid/test/appid_mock_session.h @@ -210,5 +210,6 @@ bool AppIdSession::is_tp_appid_available() const return true; } -#endif +AppidCPUProfilingManager::AppidCPUProfilingManager() { } +#endif diff --git a/src/network_inspectors/appid/test/service_state_test.cc b/src/network_inspectors/appid/test/service_state_test.cc index f0d52560c..7ff85e657 100644 --- a/src/network_inspectors/appid/test/service_state_test.cc +++ b/src/network_inspectors/appid/test/service_state_test.cc @@ -28,6 +28,8 @@ #include +THREAD_LOCAL bool TimeProfilerStats::enabled = false; + namespace snort { Packet::Packet(bool) @@ -102,6 +104,7 @@ void ServiceAppDescriptor::set_port_service_id(AppId){} void ClientAppDescriptor::update_user(AppId, const char*, AppidChangeBits&){} AppIdConfig::~AppIdConfig() = default; OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } +AppidCPUProfilingManager::AppidCPUProfilingManager() {} AppIdConfig stub_config; AppIdContext stub_ctxt(stub_config); OdpContext stub_odp_ctxt(stub_config, nullptr); diff --git a/src/network_inspectors/appid/test/tp_lib_handler_test.cc b/src/network_inspectors/appid/test/tp_lib_handler_test.cc index 7abc3d8b1..1dc854b7e 100644 --- a/src/network_inspectors/appid/test/tp_lib_handler_test.cc +++ b/src/network_inspectors/appid/test/tp_lib_handler_test.cc @@ -65,6 +65,7 @@ SslPatternMatchers::~SslPatternMatchers() = default; AlpnPatternMatchers::~AlpnPatternMatchers() = default; CipPatternMatchers::~CipPatternMatchers() = default; AppIdConfig::~AppIdConfig() = default; +AppidCPUProfilingManager::AppidCPUProfilingManager() = default; OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } void ServiceDiscovery::initialize(AppIdInspector&) { } void ServiceDiscovery::reload() { } diff --git a/src/network_inspectors/appid/tp_appid_module_api.cc b/src/network_inspectors/appid/tp_appid_module_api.cc index bf318d606..6574ece61 100644 --- a/src/network_inspectors/appid/tp_appid_module_api.cc +++ b/src/network_inspectors/appid/tp_appid_module_api.cc @@ -1,4 +1,24 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// tp_appid_module_api.cc author Lukasz Czarnik + #ifdef HAVE_CONFIG_H #include "config.h" #endif diff --git a/src/network_inspectors/appid/tp_appid_utils.cc b/src/network_inspectors/appid/tp_appid_utils.cc index d755308a5..cb2c6264b 100644 --- a/src/network_inspectors/appid/tp_appid_utils.cc +++ b/src/network_inspectors/appid/tp_appid_utils.cc @@ -491,6 +491,16 @@ static void set_tp_reinspect(AppIdSession& asd, const Packet* p, AppidSessionDir asd.tp_reinspect_by_initiator = true; asd.set_session_flags(APPID_SESSION_APP_REINSPECT); appid_log(p, TRACE_DEBUG_LEVEL, "3rd party allow reinspect http\n"); + + // If on reinspection, payload is found, a new record would be inserted for that + // payload, only for the time and packets processed from this point onwards + if (asd.get_odp_ctxt().is_appid_cpu_profiler_running() and asd.get_payload_id() > APP_ID_NONE) + { + asd.stats.prev_payload_processing_time = asd.stats.processing_time; + asd.stats.prev_payload_processing_packets = asd.stats.cpu_profiler_pkt_count; + asd.get_odp_ctxt().get_appid_cpu_profiler_mgr().check_appid_cpu_profiler_table_entry(&asd, asd.get_payload_id()); + } + asd.init_tpPackets = 0; asd.resp_tpPackets = 0; asd.clear_http_data(); diff --git a/src/profiler/profiler.cc b/src/profiler/profiler.cc index 7792f876c..56e0d0cfd 100644 --- a/src/profiler/profiler.cc +++ b/src/profiler/profiler.cc @@ -37,6 +37,7 @@ #include "profiler_nodes.h" #include "rule_profiler.h" #include "time_profiler.h" +#include #ifdef UNIT_TEST #include "catch/snort_catch.h" @@ -131,6 +132,7 @@ void Profiler::reset_stats(snort::ProfilerType type) } s_profiler_nodes.reset_nodes(type); + appid_api.reset_appid_cpu_profiler_stats(); } void Profiler::prepare_stats() -- 2.47.3