From 58ac5ea597fb29ef2bf76038012fe340d03171e8 Mon Sep 17 00:00:00 2001 From: "Umang Sharma (umasharm)" Date: Thu, 6 Jun 2024 13:04:13 +0000 Subject: [PATCH] Pull request #4336: appid: appid cpu profiler table rows limit and totals Merge in SNORT/snort3 from ~UMASHARM/snort3:appid_profiler_limit_rows to master Squashed commit of the following: commit cd9740e5236d8ab167df5693fd03650a5822d6d0 Author: Umang Sharma Date: Thu May 30 08:45:47 2024 -0400 appid: display rows limit of table and totals --- src/network_inspectors/appid/appid_config.cc | 2 +- .../appid/appid_cpu_profile_table.cc | 57 ++++++++++++++----- .../appid/appid_cpu_profile_table.h | 11 +++- src/network_inspectors/appid/appid_module.cc | 12 +++- 4 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc index ebcb1fd3b..a6a6c90b8 100644 --- a/src/network_inspectors/appid/appid_config.cc +++ b/src/network_inspectors/appid/appid_config.cc @@ -112,7 +112,7 @@ void AppIdContext::pterm() { odp_ctxt->get_app_info_mgr().cleanup_appid_info_table(); if (odp_ctxt->is_appid_cpu_profiler_enabled()) - odp_ctxt->get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(*odp_ctxt, true); + odp_ctxt->get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(*odp_ctxt, APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS, true); odp_ctxt->get_appid_cpu_profiler_mgr().cleanup_appid_cpu_profiler_table(); delete odp_ctxt; diff --git a/src/network_inspectors/appid/appid_cpu_profile_table.cc b/src/network_inspectors/appid/appid_cpu_profile_table.cc index c0b4ac6f4..44d25f660 100644 --- a/src/network_inspectors/appid/appid_cpu_profile_table.cc +++ b/src/network_inspectors/appid/appid_cpu_profile_table.cc @@ -27,15 +27,16 @@ #include #include #include +#include #include "appid_session.h" #include "appid_cpu_profile_table.h" using namespace snort; -static const char* table_header = "AppId Performance Statistics (all)\n===========================================================================================================================================================\n"; -static const char* columns = " AppId App Name Usecs Pkts AvgUsecs/Pkt Sessions AvgUsecs/Sess MaxPkts/Sess MaxUsecs/Sess\n"; -static const char* partition = "-----------------------------------------------------------------------------------------------------------------------------------------------------------\n"; +#define TABLE_HEADER(num_rows) "AppId Performance Statistics (top %d appids)\n===================================================================================================================================================\n", num_rows +static const char* columns = " AppId App Name Usecs Pkts AvgUsecs/Pkt Sessions AvgUsecs/Sess MaxPkts/Sess MaxUsecs/Sess %%/Total\n"; +static const char* partition = "---------------------------------------------------------------------------------------------------------------------------------------------------\n"; static std::string FormatWithCommas(uint64_t value) { @@ -52,7 +53,7 @@ static std::string FormatWithCommas(uint64_t value) // Comparator for priority queue based on avg_processing_time/session struct CompareByAvgProcessingTime { bool operator()(const std::pair& a, const std::pair& b) const { - if (a.second.per_appid_sessions == 0 or b.second.per_appid_sessions == 0) + if (!a.second.per_appid_sessions or !b.second.per_appid_sessions) return false; return a.second.processing_time/a.second.per_appid_sessions < b.second.processing_time/b.second.per_appid_sessions; @@ -70,15 +71,15 @@ AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_ if (bucket != appid_cpu_profiling_table.end()) { - appid_log(nullptr, TRACE_INFO_LEVEL, table_header); + appid_log(nullptr, TRACE_INFO_LEVEL, TABLE_HEADER(1)); appid_log(nullptr, TRACE_INFO_LEVEL, columns); appid_log(nullptr, TRACE_INFO_LEVEL, partition); - appid_log(nullptr, TRACE_INFO_LEVEL, " %5d %-25.25s %14.14s %9.9s %17.17s %12.12s %14.14s %15.14s %17.16s\n", + appid_log(nullptr, TRACE_INFO_LEVEL, " %5d %-15.15s %14.14s %10.10s %15.14s %11.11s %16.14s %15.14s %15.14s %10.2f\n", appid, bucket->second.app_name.c_str(), FormatWithCommas(bucket->second.processing_time).c_str(), FormatWithCommas(bucket->second.processed_packets).c_str(), FormatWithCommas(bucket->second.processing_time/bucket->second.processed_packets).c_str(), FormatWithCommas(bucket->second.per_appid_sessions).c_str(), FormatWithCommas(bucket->second.processing_time/bucket->second.per_appid_sessions).c_str(), FormatWithCommas(bucket->second.max_processed_pkts_per_session).c_str(), - FormatWithCommas(bucket->second.max_processing_time_per_session).c_str()); + FormatWithCommas(bucket->second.max_processing_time_per_session).c_str(), (static_cast(bucket->second.processing_time)/total_processing_time)*100); } else { @@ -87,7 +88,7 @@ AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_ return DISPLAY_SUCCESS; } -AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_table(OdpContext& odp_ctxt, bool override_running_flag) +AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_table(OdpContext& odp_ctxt, uint32_t display_rows_limit, bool override_running_flag) { if (odp_ctxt.is_appid_cpu_profiler_running() and !override_running_flag) return DISPLAY_ERROR_APPID_PROFILER_RUNNING; @@ -99,23 +100,36 @@ AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_ for (const auto& entry : appid_cpu_profiling_table) sorted_appid_cpu_profiler_table.push(entry); - appid_log(nullptr, TRACE_INFO_LEVEL, table_header); + display_rows_limit = static_cast(std::min({static_cast(display_rows_limit), sorted_appid_cpu_profiler_table.size(), static_cast(APPID_CPU_PROFILER_MAX_DISPLAY_ROWS)})); + appid_log(nullptr, TRACE_INFO_LEVEL, TABLE_HEADER(display_rows_limit)); appid_log(nullptr, TRACE_INFO_LEVEL, columns); appid_log(nullptr, TRACE_INFO_LEVEL, partition); - while (!sorted_appid_cpu_profiler_table.empty()) + uint32_t rows_displayed = 0; + + while (!sorted_appid_cpu_profiler_table.empty() and rows_displayed < display_rows_limit) { auto entry = sorted_appid_cpu_profiler_table.top(); sorted_appid_cpu_profiler_table.pop(); if (!entry.second.processed_packets or !entry.second.per_appid_sessions) continue; - appid_log(nullptr, TRACE_INFO_LEVEL, " %5d %-25.25s %14.14s %9.9s %17.17s %12.12s %14.14s %15.14s %17.16s\n", + appid_log(nullptr, TRACE_INFO_LEVEL, " %5d %-15.15s %14.14s %10.10s %15.14s %11.11s %16.14s %15.14s %15.14s %10.2f\n", entry.first, entry.second.app_name.c_str(), FormatWithCommas(entry.second.processing_time).c_str(), FormatWithCommas(entry.second.processed_packets).c_str(), FormatWithCommas(entry.second.processing_time/entry.second.processed_packets).c_str(), FormatWithCommas(entry.second.per_appid_sessions).c_str(), - FormatWithCommas(entry.second.processing_time/entry.second.per_appid_sessions).c_str(), FormatWithCommas(entry.second.max_processed_pkts_per_session).c_str(), - FormatWithCommas(entry.second.max_processing_time_per_session).c_str()); + FormatWithCommas(entry.second.processing_time/entry.second.per_appid_sessions).c_str(), FormatWithCommas(entry.second.max_processed_pkts_per_session).c_str(), + FormatWithCommas(entry.second.max_processing_time_per_session).c_str(), (static_cast(entry.second.processing_time)/total_processing_time)*100); + + rows_displayed += 1; } + + appid_log(nullptr, TRACE_INFO_LEVEL, partition); + + appid_log(nullptr, TRACE_INFO_LEVEL, "Totals(all_sessions) : %15.15s %10.10s %15.14s %10.10s %15.15s %15.14s %16.15s %9d\n", + FormatWithCommas(total_processing_time).c_str(), FormatWithCommas(total_processed_packets).c_str(), FormatWithCommas(total_processing_time/total_processed_packets).c_str(), + FormatWithCommas(total_per_appid_sessions).c_str(), FormatWithCommas(total_processing_time/total_per_appid_sessions).c_str(), + FormatWithCommas(max_processed_pkts_per_session).c_str(), FormatWithCommas(max_processing_time_per_session).c_str(), 100); + return DISPLAY_SUCCESS; } @@ -123,6 +137,22 @@ void AppidCPUProfilingManager::cleanup_appid_cpu_profiler_table() { std::lock_guard lock(appid_cpu_profiler_mutex); appid_cpu_profiling_table.clear(); + total_processing_time = 0; + total_processed_packets = 0; + total_per_appid_sessions = 0; + max_processing_time_per_session = 0; + max_processed_pkts_per_session = 0; +} + +void AppidCPUProfilingManager::update_totals(const AppidCPUProfilerStats& stats) +{ + total_processing_time += stats.processing_time; + total_processed_packets += stats.processed_packets; + total_per_appid_sessions += stats.per_appid_sessions; + if (stats.max_processed_pkts_per_session >= max_processed_pkts_per_session) + max_processed_pkts_per_session = stats.max_processed_pkts_per_session; + if (stats.max_processing_time_per_session >= max_processing_time_per_session) + max_processing_time_per_session = stats.max_processing_time_per_session; } void AppidCPUProfilingManager::insert_appid_cpu_profiler_record(AppId appId, const AppidCPUProfilerStats& stats) @@ -145,6 +175,7 @@ void AppidCPUProfilingManager::insert_appid_cpu_profiler_record(AppId appId, con if (stats.processing_time > it->second.max_processing_time_per_session) it->second.max_processing_time_per_session = stats.processing_time; } + update_totals(stats); } void AppidCPUProfilingManager::check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId payload_id) diff --git a/src/network_inspectors/appid/appid_cpu_profile_table.h b/src/network_inspectors/appid/appid_cpu_profile_table.h index 5a3f94d11..32a6d2764 100644 --- a/src/network_inspectors/appid/appid_cpu_profile_table.h +++ b/src/network_inspectors/appid/appid_cpu_profile_table.h @@ -34,6 +34,9 @@ class AppIdSession; class OdpContext; +#define APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS 100 +#define APPID_CPU_PROFILER_MAX_DISPLAY_ROWS 2000 + enum AppidCpuTableDisplayStatus { DISPLAY_SUCCESS = 0, DISPLAY_ERROR_TABLE_EMPTY, @@ -59,6 +62,11 @@ private: using AppidCPUProfilingTable = std::unordered_map; AppidCPUProfilingTable appid_cpu_profiling_table; std::mutex appid_cpu_profiler_mutex; + uint64_t total_processing_time = 0; + uint64_t total_processed_packets = 0; + uint32_t total_per_appid_sessions = 0; + uint64_t max_processing_time_per_session = 0; + uint64_t max_processed_pkts_per_session = 0; public: AppidCPUProfilingManager() = default; @@ -67,8 +75,9 @@ public: void insert_appid_cpu_profiler_record(AppId appId, const AppidCPUProfilerStats& stats); void check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId service_id, AppId client_id, AppId payload_id, AppId misc_id); void check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId payload_id); + void update_totals(const AppidCPUProfilerStats& stats); - AppidCpuTableDisplayStatus display_appid_cpu_profiler_table(OdpContext&, bool override_running_flag = false); + AppidCpuTableDisplayStatus display_appid_cpu_profiler_table(OdpContext&, uint32_t display_rows_limit = APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS, bool override_running_flag = false); AppidCpuTableDisplayStatus display_appid_cpu_profiler_table(AppId, OdpContext&); void cleanup_appid_cpu_profiler_table(); diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 952548485..344c4492d 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -396,6 +396,8 @@ static void clear_dynamic_host_cache_services() static int show_cpu_profiler_stats(lua_State* L) { int appid = luaL_optint(L, 1, 0); + int display_rows_limit = luaL_optint(L, 2, APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS); + ControlConn* ctrlcon = ControlConn::query_from_lua(L); AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME); if (!inspector) @@ -411,7 +413,12 @@ static int show_cpu_profiler_stats(lua_State* L) AppidCpuTableDisplayStatus displayed = DISPLAY_SUCCESS; ctrlcon->respond("== showing appid cpu profiler table\n"); if (!appid) - displayed = odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(odp_ctxt); + { + if (display_rows_limit > APPID_CPU_PROFILER_MAX_DISPLAY_ROWS) + ctrlcon->respond("given number of rows exceeds maximum limit of %d, limiting to %d\n", + APPID_CPU_PROFILER_MAX_DISPLAY_ROWS, APPID_CPU_PROFILER_MAX_DISPLAY_ROWS); + displayed = odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(odp_ctxt, display_rows_limit); + } else displayed = odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(appid, odp_ctxt); @@ -532,7 +539,8 @@ static const Parameter enable_debug_params[] = static const Parameter appid_cpu_params[] = { - { "appid", Parameter::PT_INT, nullptr, nullptr, "show appid cpu profiling stats" }, + { "appid", Parameter::PT_INT, nullptr, "0", "show appid cpu profiling stats" }, + { "display_rows_limit", Parameter::PT_INT, "1:2000", "100", "num of rows to be displayed" }, { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; -- 2.47.3