Pull request #4336: appid: appid cpu profiler table rows limit and totals

Merge in SNORT/snort3 from ~UMASHARM/snort3:appid_profiler_limit_rows to master

Squashed commit of the following:

commit cd9740e5236d8ab167df5693fd03650a5822d6d0
Author: Umang Sharma <umasharm@cisco.com>
Date:   Thu May 30 08:45:47 2024 -0400

    appid: display rows limit of table and totals

diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc
index ebcb1fd3bc1abb769f9013df61bde5681d5e3ba9..a6a6c90b85256cd199ad1b48460169ef308afceb 100644 (file)
--- a/src/network_inspectors/appid/appid_config.cc
+++ b/src/network_inspectors/appid/appid_config.cc
@@ -112,7 +112,7 @@ void AppIdContext::pterm()
     {
         odp_ctxt->get_app_info_mgr().cleanup_appid_info_table();
         if (odp_ctxt->is_appid_cpu_profiler_enabled())
-            odp_ctxt->get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(*odp_ctxt, true);
+            odp_ctxt->get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(*odp_ctxt, APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS, true);
 
         odp_ctxt->get_appid_cpu_profiler_mgr().cleanup_appid_cpu_profiler_table();
         delete odp_ctxt;

diff --git a/src/network_inspectors/appid/appid_cpu_profile_table.cc b/src/network_inspectors/appid/appid_cpu_profile_table.cc
index c0b4ac6f49a69d4ac4cabdcdbd7c05b0d93c06e2..44d25f66051b7a59e1f9e99b3f64f7f8e67c77ff 100644 (file)
--- a/src/network_inspectors/appid/appid_cpu_profile_table.cc
+++ b/src/network_inspectors/appid/appid_cpu_profile_table.cc
@@ -27,15 +27,16 @@
 #include <iomanip>
 #include <sstream>
 #include <queue>
+#include <algorithm>
 
 #include "appid_session.h"
 #include "appid_cpu_profile_table.h"
 
 using namespace snort;
 
-static const char* table_header = "AppId Performance Statistics (all)\n===========================================================================================================================================================\n";
-static const char* columns = " AppId   App Name                             Usecs        Pkts        AvgUsecs/Pkt       Sessions     AvgUsecs/Sess       MaxPkts/Sess       MaxUsecs/Sess\n";
-static const char* partition = "-----------------------------------------------------------------------------------------------------------------------------------------------------------\n";
+#define TABLE_HEADER(num_rows) "AppId Performance Statistics (top %d appids)\n===================================================================================================================================================\n", num_rows
+static const char* columns = " AppId   App Name                   Usecs       Pkts     AvgUsecs/Pkt     Sessions     AvgUsecs/Sess     MaxPkts/Sess     MaxUsecs/Sess     %%/Total\n";
+static const char* partition = "---------------------------------------------------------------------------------------------------------------------------------------------------\n";
 
 static std::string FormatWithCommas(uint64_t value)
 {
@@ -52,7 +53,7 @@ static std::string FormatWithCommas(uint64_t value)
 // Comparator for priority queue based on avg_processing_time/session
 struct CompareByAvgProcessingTime {
     bool operator()(const std::pair<AppId, AppidCPUProfilerStats>& a, const std::pair<AppId, AppidCPUProfilerStats>& b) const {
-        if (a.second.per_appid_sessions == 0 or b.second.per_appid_sessions == 0)
+        if (!a.second.per_appid_sessions or !b.second.per_appid_sessions)
             return false;
 
         return a.second.processing_time/a.second.per_appid_sessions < b.second.processing_time/b.second.per_appid_sessions;
@@ -70,15 +71,15 @@ AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_
 
     if (bucket != appid_cpu_profiling_table.end())
     {
-        appid_log(nullptr, TRACE_INFO_LEVEL, table_header);
+        appid_log(nullptr, TRACE_INFO_LEVEL, TABLE_HEADER(1));
         appid_log(nullptr, TRACE_INFO_LEVEL, columns);
         appid_log(nullptr, TRACE_INFO_LEVEL, partition);
 
-        appid_log(nullptr, TRACE_INFO_LEVEL, " %5d   %-25.25s   %14.14s   %9.9s   %17.17s   %12.12s    %14.14s    %15.14s   %17.16s\n",
+        appid_log(nullptr, TRACE_INFO_LEVEL, " %5d   %-15.15s   %14.14s %10.10s  %15.14s  %11.11s  %16.14s  %15.14s   %15.14s  %10.2f\n",
                 appid, bucket->second.app_name.c_str(), FormatWithCommas(bucket->second.processing_time).c_str(), FormatWithCommas(bucket->second.processed_packets).c_str(), 
                 FormatWithCommas(bucket->second.processing_time/bucket->second.processed_packets).c_str(), FormatWithCommas(bucket->second.per_appid_sessions).c_str(), 
                 FormatWithCommas(bucket->second.processing_time/bucket->second.per_appid_sessions).c_str(), FormatWithCommas(bucket->second.max_processed_pkts_per_session).c_str(), 
-                FormatWithCommas(bucket->second.max_processing_time_per_session).c_str());
+                FormatWithCommas(bucket->second.max_processing_time_per_session).c_str(), (static_cast<float>(bucket->second.processing_time)/total_processing_time)*100);
     }
     else
     {
@@ -87,7 +88,7 @@ AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_
     return DISPLAY_SUCCESS;
 }
 
-AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_table(OdpContext& odp_ctxt, bool override_running_flag)
+AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_table(OdpContext& odp_ctxt, uint32_t display_rows_limit, bool override_running_flag)
 {
     if (odp_ctxt.is_appid_cpu_profiler_running() and !override_running_flag)
         return DISPLAY_ERROR_APPID_PROFILER_RUNNING;
@@ -99,23 +100,36 @@ AppidCpuTableDisplayStatus AppidCPUProfilingManager::display_appid_cpu_profiler_
     for (const auto& entry : appid_cpu_profiling_table) 
         sorted_appid_cpu_profiler_table.push(entry);
 
-    appid_log(nullptr, TRACE_INFO_LEVEL, table_header);
+    display_rows_limit = static_cast<uint32_t>(std::min({static_cast<size_t>(display_rows_limit), sorted_appid_cpu_profiler_table.size(), static_cast<size_t>(APPID_CPU_PROFILER_MAX_DISPLAY_ROWS)}));
+    appid_log(nullptr, TRACE_INFO_LEVEL, TABLE_HEADER(display_rows_limit));
     appid_log(nullptr, TRACE_INFO_LEVEL, columns);
     appid_log(nullptr, TRACE_INFO_LEVEL, partition);
     
-    while (!sorted_appid_cpu_profiler_table.empty()) 
+    uint32_t rows_displayed = 0;
+
+    while (!sorted_appid_cpu_profiler_table.empty() and rows_displayed < display_rows_limit)
     {
         auto entry = sorted_appid_cpu_profiler_table.top();
         sorted_appid_cpu_profiler_table.pop();
         if (!entry.second.processed_packets or !entry.second.per_appid_sessions)
             continue;
             
-        appid_log(nullptr, TRACE_INFO_LEVEL, " %5d   %-25.25s   %14.14s   %9.9s   %17.17s   %12.12s    %14.14s    %15.14s   %17.16s\n",
+        appid_log(nullptr, TRACE_INFO_LEVEL, " %5d   %-15.15s   %14.14s %10.10s  %15.14s  %11.11s  %16.14s  %15.14s   %15.14s  %10.2f\n",
                 entry.first, entry.second.app_name.c_str(), FormatWithCommas(entry.second.processing_time).c_str(), FormatWithCommas(entry.second.processed_packets).c_str(), 
                 FormatWithCommas(entry.second.processing_time/entry.second.processed_packets).c_str(), FormatWithCommas(entry.second.per_appid_sessions).c_str(), 
-                FormatWithCommas(entry.second.processing_time/entry.second.per_appid_sessions).c_str(), FormatWithCommas(entry.second.max_processed_pkts_per_session).c_str(), 
-                FormatWithCommas(entry.second.max_processing_time_per_session).c_str());
+                FormatWithCommas(entry.second.processing_time/entry.second.per_appid_sessions).c_str(), FormatWithCommas(entry.second.max_processed_pkts_per_session).c_str(),
+                FormatWithCommas(entry.second.max_processing_time_per_session).c_str(), (static_cast<float>(entry.second.processing_time)/total_processing_time)*100);
+
+        rows_displayed += 1;
     } 
+
+    appid_log(nullptr, TRACE_INFO_LEVEL, partition);
+
+    appid_log(nullptr, TRACE_INFO_LEVEL, "Totals(all_sessions)    : %15.15s %10.10s  %15.14s   %10.10s   %15.15s  %15.14s  %16.15s   %9d\n",
+            FormatWithCommas(total_processing_time).c_str(), FormatWithCommas(total_processed_packets).c_str(),  FormatWithCommas(total_processing_time/total_processed_packets).c_str(),
+            FormatWithCommas(total_per_appid_sessions).c_str(), FormatWithCommas(total_processing_time/total_per_appid_sessions).c_str(),
+            FormatWithCommas(max_processed_pkts_per_session).c_str(), FormatWithCommas(max_processing_time_per_session).c_str(), 100);
+
     return DISPLAY_SUCCESS;
 }
 
@@ -123,6 +137,22 @@ void AppidCPUProfilingManager::cleanup_appid_cpu_profiler_table()
 { 
     std::lock_guard<std::mutex> lock(appid_cpu_profiler_mutex);
     appid_cpu_profiling_table.clear();
+    total_processing_time = 0;
+    total_processed_packets = 0;
+    total_per_appid_sessions = 0;
+    max_processing_time_per_session = 0;
+    max_processed_pkts_per_session = 0;
+}
+
+void AppidCPUProfilingManager::update_totals(const AppidCPUProfilerStats& stats)
+{
+    total_processing_time += stats.processing_time;
+    total_processed_packets += stats.processed_packets;
+    total_per_appid_sessions += stats.per_appid_sessions;
+    if (stats.max_processed_pkts_per_session >= max_processed_pkts_per_session)
+        max_processed_pkts_per_session = stats.max_processed_pkts_per_session;
+    if (stats.max_processing_time_per_session >= max_processing_time_per_session)
+        max_processing_time_per_session = stats.max_processing_time_per_session;
 }
 
 void AppidCPUProfilingManager::insert_appid_cpu_profiler_record(AppId appId, const AppidCPUProfilerStats& stats)
@@ -145,6 +175,7 @@ void AppidCPUProfilingManager::insert_appid_cpu_profiler_record(AppId appId, con
         if (stats.processing_time > it->second.max_processing_time_per_session)
             it->second.max_processing_time_per_session = stats.processing_time;
     }
+    update_totals(stats);
 }
 
 void AppidCPUProfilingManager::check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId payload_id)

diff --git a/src/network_inspectors/appid/appid_cpu_profile_table.h b/src/network_inspectors/appid/appid_cpu_profile_table.h
index 5a3f94d116b1806b409da195d904e7511e8e89e1..32a6d2764746b1e7dbb5fea7bd88f0a021fef959 100644 (file)
--- a/src/network_inspectors/appid/appid_cpu_profile_table.h
+++ b/src/network_inspectors/appid/appid_cpu_profile_table.h
@@ -34,6 +34,9 @@
 class AppIdSession;
 class OdpContext;
 
+#define APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS 100
+#define APPID_CPU_PROFILER_MAX_DISPLAY_ROWS 2000
+
 enum AppidCpuTableDisplayStatus {
     DISPLAY_SUCCESS = 0,
     DISPLAY_ERROR_TABLE_EMPTY,
@@ -59,6 +62,11 @@ private:
     using AppidCPUProfilingTable = std::unordered_map<AppId, AppidCPUProfilerStats>;
     AppidCPUProfilingTable appid_cpu_profiling_table;
     std::mutex appid_cpu_profiler_mutex;
+    uint64_t total_processing_time = 0;
+    uint64_t total_processed_packets = 0;
+    uint32_t total_per_appid_sessions = 0;
+    uint64_t max_processing_time_per_session = 0;
+    uint64_t max_processed_pkts_per_session = 0;
         
 public:
     AppidCPUProfilingManager() = default;
@@ -67,8 +75,9 @@ public:
     void insert_appid_cpu_profiler_record(AppId appId, const AppidCPUProfilerStats& stats);
     void check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId service_id, AppId client_id, AppId payload_id, AppId misc_id);
     void check_appid_cpu_profiler_table_entry(const AppIdSession* asd, AppId payload_id);
+    void update_totals(const AppidCPUProfilerStats& stats);
 
-    AppidCpuTableDisplayStatus display_appid_cpu_profiler_table(OdpContext&, bool override_running_flag = false);
+    AppidCpuTableDisplayStatus display_appid_cpu_profiler_table(OdpContext&, uint32_t display_rows_limit = APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS, bool override_running_flag = false);
     AppidCpuTableDisplayStatus display_appid_cpu_profiler_table(AppId, OdpContext&);
     
     void cleanup_appid_cpu_profiler_table();

diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc
index 95254848528eb8b763520aaca7eea652bcdd1558..344c4492d2f486c68123ebc817ea2fdb27e23ce8 100644 (file)
--- a/src/network_inspectors/appid/appid_module.cc
+++ b/src/network_inspectors/appid/appid_module.cc
@@ -396,6 +396,8 @@ static void clear_dynamic_host_cache_services()
 static int show_cpu_profiler_stats(lua_State* L)
 {
     int appid = luaL_optint(L, 1, 0);
+    int display_rows_limit = luaL_optint(L, 2, APPID_CPU_PROFILER_DEFAULT_DISPLAY_ROWS);
+
     ControlConn* ctrlcon = ControlConn::query_from_lua(L);
     AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME);
     if (!inspector)
@@ -411,7 +413,12 @@ static int show_cpu_profiler_stats(lua_State* L)
         AppidCpuTableDisplayStatus displayed = DISPLAY_SUCCESS;
         ctrlcon->respond("== showing appid cpu profiler table\n");
         if (!appid)
-            displayed = odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(odp_ctxt);
+        {
+            if (display_rows_limit > APPID_CPU_PROFILER_MAX_DISPLAY_ROWS)
+                ctrlcon->respond("given number of rows exceeds maximum limit of %d, limiting to %d\n",
+                                                   APPID_CPU_PROFILER_MAX_DISPLAY_ROWS, APPID_CPU_PROFILER_MAX_DISPLAY_ROWS);
+            displayed = odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(odp_ctxt, display_rows_limit);
+        }
         else
             displayed = odp_ctxt.get_appid_cpu_profiler_mgr().display_appid_cpu_profiler_table(appid, odp_ctxt);
 
@@ -532,7 +539,8 @@ static const Parameter enable_debug_params[] =
 
 static const Parameter appid_cpu_params[] =
 {
-    { "appid", Parameter::PT_INT, nullptr, nullptr, "show appid cpu profiling stats" },
+    { "appid", Parameter::PT_INT, nullptr, "0", "show appid cpu profiling stats" },
+    { "display_rows_limit", Parameter::PT_INT, "1:2000", "100", "num of rows to be displayed" },
 
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };