From: Tom Peters (thopeter) <thopeter@cisco.com>
Date: Tue, 10 Apr 2018 17:22:37 +0000 (-0400)
Subject: Merge pull request #1183 in SNORT/snort3 from appid_pegs to master
X-Git-Tag: 3.0.0-245~46
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=3df191e7ae3c1c7a2c1d0ae1c866616ae5aaa18c;p=thirdparty%2Fsnort3.git

Merge pull request #1183 in SNORT/snort3 from appid_pegs to master

Squashed commit of the following:

commit 24afa26ce1178c47b2cd2f77c326d1a24babe00b
Author: mdagon <mdagon@cisco.com>
Date:   Thu Apr 5 13:49:49 2018 -0400

    appid: peg count / dynamic peg count update.

    Split peg counts into the ones known at compile time and dynamic ones.
    Update stats , module manager and module to support dumping dynamic stats.
---

diff --git a/src/file_api/file_module.cc b/src/file_api/file_module.cc
index e3d3d9a12..ef53357af 100644
--- a/src/file_api/file_module.cc
+++ b/src/file_api/file_module.cc
@@ -412,3 +412,8 @@ void FileIdModule::load_config(FileConfig*& dst)
         fc = nullptr;
     }
 }
+
+void FileIdModule::show_dynamic_stats()
+{
+    file_stats_print();
+}
diff --git a/src/file_api/file_module.h b/src/file_api/file_module.h
index d4d6de284..ae860930d 100644
--- a/src/file_api/file_module.h
+++ b/src/file_api/file_module.h
@@ -52,6 +52,8 @@ public:
     Usage get_usage() const override
     { return GLOBAL; }
 
+    void show_dynamic_stats() override;
+
 private:
     FileMagicRule rule;
     FileMagicData magic;
diff --git a/src/framework/module.h b/src/framework/module.h
index c20311c58..1e65f86e5 100644
--- a/src/framework/module.h
+++ b/src/framework/module.h
@@ -166,6 +166,7 @@ public:
     virtual void show_interval_stats(IndexVec&, FILE*);
     virtual void show_stats();
     virtual void reset_stats();
+    virtual void show_dynamic_stats() {}
 
     // Wrappers to check that lists are not tables
     bool verified_begin(const char*, int, SnortConfig*);
diff --git a/src/managers/module_manager.cc b/src/managers/module_manager.cc
index 5ee151fbf..36221fe67 100644
--- a/src/managers/module_manager.cc
+++ b/src/managers/module_manager.cc
@@ -1417,14 +1417,17 @@ void ModuleManager::dump_msg_map(const char* pfx)
         cout << "no match" << endl;
 }
 
-void ModuleManager::dump_stats(SnortConfig*, const char* skip)
+void ModuleManager::dump_stats(SnortConfig*, const char* skip, bool dynamic)
 {
     for ( auto p : s_modules )
     {
         if ( !skip || !strstr(skip, p->mod->get_name()) )
         {
             std::lock_guard<std::mutex> lock(stats_mutex);
-            p->mod->show_stats();
+            if (dynamic)
+                p->mod->show_dynamic_stats();
+            else
+                p->mod->show_stats();
         }
     }
 }
diff --git a/src/managers/module_manager.h b/src/managers/module_manager.h
index 4ed969c2b..5309ed6a8 100644
--- a/src/managers/module_manager.h
+++ b/src/managers/module_manager.h
@@ -75,7 +75,8 @@ public:
     static void reset_errors();
     static unsigned get_errors();
 
-    static void dump_stats(snort::SnortConfig*, const char* skip = nullptr);
+    static void dump_stats(snort::SnortConfig*, const char* skip = nullptr, bool dynamic = false);
+ 
     static void accumulate(snort::SnortConfig*);
     static void reset_stats(snort::SnortConfig*);
 
diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc
index fc5c678e8..b020cab0e 100644
--- a/src/network_inspectors/appid/app_info_table.cc
+++ b/src/network_inspectors/appid/app_info_table.cc
@@ -477,7 +477,6 @@ void AppInfoManager::init_appid_info_table(AppIdModuleConfig* mod_config,
 {
     if ( !mod_config->app_detector_dir )
     {
-        AppIdPegCounts::set_detectors_configured();
         return;  // no lua detectors, no rule support, already warned
     }
 
@@ -573,7 +572,6 @@ void AppInfoManager::init_appid_info_table(AppIdModuleConfig* mod_config,
         }
         fclose(tableFile);
 
-        AppIdPegCounts::add_unknown_app_peg();
         snprintf(filepath, sizeof(filepath), "%s/odp/%s", mod_config->app_detector_dir,
             APP_CONFIG_FILE);
         load_appid_config (mod_config, filepath);
@@ -581,7 +579,5 @@ void AppInfoManager::init_appid_info_table(AppIdModuleConfig* mod_config,
             USR_CONFIG_FILE);
         load_appid_config (mod_config, filepath);
     }
-
-    AppIdPegCounts::set_detectors_configured();
 }
 
diff --git a/src/network_inspectors/appid/appid_app_descriptor.h b/src/network_inspectors/appid/appid_app_descriptor.h
index 3c2b4f38a..5d911d031 100644
--- a/src/network_inspectors/appid/appid_app_descriptor.h
+++ b/src/network_inspectors/appid/appid_app_descriptor.h
@@ -30,6 +30,7 @@
 
 #include <string>
 
+#include "appid_module.h"
 #include "appid_peg_counts.h"
 
 class ApplicationDescriptor
@@ -67,7 +68,7 @@ public:
             if ( app_id > APP_ID_NONE )
                 update_stats(app_id);
             else if ( app_id == APP_ID_UNKNOWN )
-                AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::APPID_UNKNOWN);
+                appid_stats.appid_unknown++;
         }
     }
 
diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index b578d25ea..6c9392679 100644
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -329,7 +329,7 @@ static bool is_packet_ignored(AppIdSession* asd, Packet* p, int& direction)
         if ( !p->is_rebuilt() )
         {
             // For HTTP/2, only examine packets that have been rebuilt as HTTP/1 packets.
-            AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::IGNORED_PACKETS);
+            appid_stats.ignored_packets++;
             return true;
         }
     }
@@ -349,7 +349,7 @@ static bool is_packet_ignored(AppIdSession* asd, Packet* p, int& direction)
                         hsession->get_field_offset(REQ_COOKIE_FID),
                         hsession->get_field_end_offset(REQ_COOKIE_FID));
             }
-        AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::IGNORED_PACKETS);
+        appid_stats.ignored_packets++;
         return true;
         }
     }
@@ -606,7 +606,7 @@ void AppIdDiscovery::do_application_discovery(Packet* p, AppIdInspector& inspect
     AppIdSession* asd = (AppIdSession*)p->flow->get_flow_data(AppIdSession::inspector_id);
     if ( !set_network_attributes(asd, p, protocol, direction) )
     {
-        AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::IGNORED_PACKETS);
+        appid_stats.ignored_packets++;
         return;
     }
 
@@ -684,7 +684,7 @@ void AppIdDiscovery::do_application_discovery(Packet* p, AppIdInspector& inspect
 
     // FIXIT-L - from this point on we always have a valid ptr to an AppIdSession and a Packet
     //           refactor to pass these as refs and delete any checks for null
-    AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::PROCESSED_PACKETS);
+    appid_stats.processed_packets++;
     asd->session_packet_count++;
 
     if (direction == APP_ID_FROM_INITIATOR)
diff --git a/src/network_inspectors/appid/appid_inspector.cc b/src/network_inspectors/appid/appid_inspector.cc
index b73306a78..7b18b9618 100644
--- a/src/network_inspectors/appid/appid_inspector.cc
+++ b/src/network_inspectors/appid/appid_inspector.cc
@@ -183,7 +183,7 @@ void AppIdInspector::eval(Packet* p)
 {
     Profile profile(appidPerfStats);
 
-    AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::PACKETS);
+    appid_stats.packets++;
     if (p->flow)
     {
         AppIdDiscovery::do_application_discovery(p, *this);
@@ -192,7 +192,7 @@ void AppIdInspector::eval(Packet* p)
             add_appid_to_packet_trace(*p->flow);
     }
     else
-        AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::IGNORED_PACKETS);
+        appid_stats.ignored_packets++;
 }
 
 //-------------------------------------------------------------------------
diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc
index 1011090f9..38e8ef361 100644
--- a/src/network_inspectors/appid/appid_module.cc
+++ b/src/network_inspectors/appid/appid_module.cc
@@ -47,6 +47,7 @@ Trace TRACE_NAME(appid_module);
 //-------------------------------------------------------------------------
 
 THREAD_LOCAL ProfileStats appidPerfStats;
+THREAD_LOCAL AppIdStats appid_stats;
 
 static const Parameter s_params[] =
 {
@@ -183,6 +184,16 @@ static const RuleMap appid_rules[] =
 };
 #endif
 
+static const PegInfo appid_pegs[] =
+{
+    { CountType::SUM, "packets", "count of packets received" },
+    { CountType::SUM, "processed_packets", "count of packets processed" },
+    { CountType::SUM, "ignored_packets", "count of packets ignored" },
+    { CountType::SUM, "total_sessions", "count of sessions created" },
+    { CountType::SUM, "appid_unknown", "count of sessions where appid could not be determined" },
+    { CountType::END, nullptr, nullptr},
+};
+
 AppIdModule::AppIdModule() :
     Module(MOD_NAME, MOD_HELP, s_params, false, &TRACE_NAME(appid_module))
 {
@@ -269,11 +280,21 @@ const Command* AppIdModule::get_commands() const
 
 const PegInfo* AppIdModule::get_pegs() const
 {
-    return AppIdPegCounts::get_peg_info();
+    return appid_pegs;
 }
 
 PegCount* AppIdModule::get_counts() const
 {
-    return AppIdPegCounts::get_peg_counts();
+    return (PegCount*)&appid_stats;
+}
+
+void AppIdModule::sum_stats(bool accumulate_now_stats)
+{
+    AppIdPegCounts::sum_stats();
+    Module::sum_stats(accumulate_now_stats);
 }
 
+void AppIdModule::show_dynamic_stats()
+{
+    AppIdPegCounts::print();
+}
diff --git a/src/network_inspectors/appid/appid_module.h b/src/network_inspectors/appid/appid_module.h
index dad5114bf..c47b03449 100644
--- a/src/network_inspectors/appid/appid_module.h
+++ b/src/network_inspectors/appid/appid_module.h
@@ -35,6 +35,17 @@ extern Trace TRACE_NAME(appid_module);
 #define MOD_NAME "appid"
 #define MOD_HELP "application and service identification"
 
+struct AppIdStats
+{
+    PegCount packets;
+    PegCount processed_packets;
+    PegCount ignored_packets;
+    PegCount total_sessions;
+    PegCount appid_unknown;  
+};
+
+extern THREAD_LOCAL AppIdStats appid_stats;
+
 class AppIdModule : public snort::Module
 {
 public:
@@ -54,6 +65,8 @@ public:
 
     Usage get_usage() const override
     { return CONTEXT; }
+    void sum_stats(bool) override;
+    void show_dynamic_stats() override;
 
 private:
     AppIdModuleConfig* config;
diff --git a/src/network_inspectors/appid/appid_peg_counts.cc b/src/network_inspectors/appid/appid_peg_counts.cc
index cadeef48b..e78a0d0e8 100644
--- a/src/network_inspectors/appid/appid_peg_counts.cc
+++ b/src/network_inspectors/appid/appid_peg_counts.cc
@@ -24,30 +24,24 @@
 #endif
 
 #include "appid_peg_counts.h"
-#include "app_info_table.h"
 
 #include <algorithm>
 #include <string>
 
-bool AppIdPegCounts::detectors_configured = false;
-bool AppIdPegCounts::dynamic_counts_imported = false;
-uint32_t AppIdPegCounts::unknown_app_idx = 0;
-std::map<AppId, uint32_t> AppIdPegCounts::appid_detector_pegs_idx;
-std::vector<PegInfo> AppIdPegCounts::appid_detectors_peg_info;
-std::vector<PegInfo> AppIdPegCounts::appid_pegs =
-{
-    { CountType::SUM, "packets", "count of packets received" },
-    { CountType::SUM, "processed_packets", "count of packets processed" },
-    { CountType::SUM, "ignored_packets", "count of packets ignored" },
-    { CountType::SUM, "total_sessions", "count of sessions created" },
-    { CountType::SUM, "appid_unknown", "count of sessions where appid could not be determined" },
-};
+#include "utils/stats.h"
 
-THREAD_LOCAL std::vector<PegCount>* AppIdPegCounts::appid_peg_counts;
+std::map<AppId, uint32_t> AppIdPegCounts::appid_detector_pegs_idx;
+std::vector<std::string> AppIdPegCounts::appid_detectors_info;
+THREAD_LOCAL std::vector<AppIdPegCounts::AppIdDynamicPeg>* AppIdPegCounts::appid_peg_counts;
+AppIdPegCounts::AppIdDynamicPeg AppIdPegCounts::appid_dynamic_sum[SF_APPID_MAX + 1];
+uint32_t AppIdPegCounts::unknown_app_idx;
 
 void AppIdPegCounts::init_pegs()
 {
-    appid_peg_counts = new std::vector<PegCount>(appid_detectors_peg_info.size() + NUM_APPID_GLOBAL_PEGS, 0);
+    AppIdPegCounts::AppIdDynamicPeg zeroed_peg = AppIdPegCounts::AppIdDynamicPeg();
+    appid_peg_counts = new std::vector<AppIdPegCounts::AppIdDynamicPeg>(
+        appid_detectors_info.size() + 1, zeroed_peg);
+    AppIdPegCounts::unknown_app_idx = appid_detectors_info.size();
 }
 
 void AppIdPegCounts::cleanup_pegs()
@@ -55,122 +49,112 @@ void AppIdPegCounts::cleanup_pegs()
     delete appid_peg_counts;
 }
 
-void AppIdPegCounts::init_detector_peg_info(const std::string& app_name, const std::string& name_suffix,
-    const std::string& help_suffix)
+void AppIdPegCounts::cleanup_peg_info()
 {
-    std::string name = app_name + name_suffix;
-    std::string help = "count of ";
-    help += app_name + help_suffix;
-    appid_detectors_peg_info.push_back({CountType::SUM, snort_strdup(name.c_str()), snort_strdup(help.c_str())});
+    appid_detectors_info.clear();
+    appid_detector_pegs_idx.clear();
 }
 
-void AppIdPegCounts::add_app_peg_info(std::string app_name, AppId app_id )
+void AppIdPegCounts::add_app_peg_info(std::string app_name, AppId app_id)
 {
     std::replace(app_name.begin(), app_name.end(), ' ', '_');
 
-    appid_detector_pegs_idx[app_id] = appid_detectors_peg_info.size() + NUM_APPID_GLOBAL_PEGS;
-    init_detector_peg_info(app_name, "_flows", " services detected");
-    init_detector_peg_info(app_name, "_clients", " clients detected");
-    init_detector_peg_info(app_name, "_users", " users detected");
-    init_detector_peg_info(app_name, "_payloads", " payloads detected");
-    init_detector_peg_info(app_name, "_misc", " misc detected");
-    init_detector_peg_info(app_name, "_incompatible", " incompatible");
-    init_detector_peg_info(app_name, "_failed", " failed");
+    appid_detector_pegs_idx[app_id] = appid_detectors_info.size();
+    appid_detectors_info.push_back({ app_name });
+}
+
+void AppIdPegCounts::sum_stats()
+{
+    if (!appid_peg_counts)
+        return;
+
+    const unsigned peg_num = appid_peg_counts->size() - 1;
+    const AppIdDynamicPeg* ptr = (AppIdDynamicPeg*)appid_peg_counts->data();
+
+    for ( unsigned i = 0; i < peg_num; ++i )
+    {
+        for (unsigned j = 0; j < DetectorPegs::NUM_APPID_DETECTOR_PEGS; ++j)
+            appid_dynamic_sum[i].stats[j] += ptr[i].stats[j];
+    }
+
+    // unknown_app stats
+    for (unsigned j = 0; j < DetectorPegs::NUM_APPID_DETECTOR_PEGS; ++j)
+        appid_dynamic_sum[SF_APPID_MAX].stats[j] += ptr[peg_num].stats[j];
+}
+
+void AppIdPegCounts::inc_service_count(AppId id)
+{
+    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::SERVICE_DETECTS]++;
+}
+
+void AppIdPegCounts::inc_client_count(AppId id)
+{
+    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::CLIENT_DETECTS]++;
+}
+
+void AppIdPegCounts::inc_user_count(AppId id)
+{
+    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::USER_DETECTS]++;
 }
 
-void AppIdPegCounts::add_unknown_app_peg()
+void AppIdPegCounts::inc_payload_count(AppId id)
 {
-    std::string app_name = "unknown_app";
-
-    AppIdPegCounts::unknown_app_idx = appid_detectors_peg_info.size() + NUM_APPID_GLOBAL_PEGS;
-    init_detector_peg_info(app_name, "_flows", " services detected");
-    init_detector_peg_info(app_name, "_clients", " clients detected");
-    init_detector_peg_info(app_name, "_users", " users detected");
-    init_detector_peg_info(app_name, "_payloads", " payloads detected");
-    init_detector_peg_info(app_name, "_misc", " misc detected");
+    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::PAYLOAD_DETECTS]++;
 }
 
-PegCount* AppIdPegCounts::get_peg_counts()
+void AppIdPegCounts::inc_misc_count(AppId id)
 {
-    if ( AppIdPegCounts::detectors_configured )
-        return appid_peg_counts->data();
+    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::MISC_DETECTS]++;
+}
+
+uint32_t AppIdPegCounts::get_stats_index(AppId id)
+{
+    std::map<AppId, uint32_t>::iterator stats_idx_it = appid_detector_pegs_idx.find(id);
+    if ( stats_idx_it != appid_detector_pegs_idx.end() )
+        return stats_idx_it->second;
     else
-        return nullptr;
+        return AppIdPegCounts::unknown_app_idx;
 }
 
-PegInfo* AppIdPegCounts::get_peg_info()
+void AppIdPegCounts::print()
 {
-    if ( AppIdPegCounts::detectors_configured )
+    bool print = false;
+    unsigned app_num = AppIdPegCounts::appid_detectors_info.size();
+
+    for (unsigned i = 0; i < app_num; i++)
     {
-        if ( !AppIdPegCounts::dynamic_counts_imported )
+        AppIdDynamicPeg* pegs = &appid_dynamic_sum[i];
+        if (!pegs->all_zeros())
         {
-            appid_pegs.insert( appid_pegs.end(), appid_detectors_peg_info.begin(), appid_detectors_peg_info.end());
-            // add the sentinel entry at the end
-            appid_pegs.push_back({ CountType::END, nullptr, nullptr });
-            AppIdPegCounts::dynamic_counts_imported = true;
+            print = true;
+            break;
         }
-        return appid_pegs.data();
     }
-    else
-        return nullptr;
-}
 
-void AppIdPegCounts::cleanup_peg_info()
-{
-    for ( auto& app_info : appid_detectors_peg_info )
+    AppIdDynamicPeg* unknown_pegs = &appid_dynamic_sum[SF_APPID_MAX];
+    if (!print && unknown_pegs->all_zeros())
+        return;
+
+    LogLabel("Appid dynamic stats:");
+
+    for (unsigned i = 0; i < app_num; i++)
     {
-        snort_free((void*)app_info.name);
-        snort_free((void*)app_info.help);
+        AppIdDynamicPeg* pegs = &appid_dynamic_sum[i];
+        if (pegs->all_zeros())
+            continue;
+
+        std::string app_name = AppIdPegCounts::appid_detectors_info[i];
+        LogMessage("%s: ", app_name.c_str());
+        pegs->print();
     }
-    appid_detectors_peg_info.clear();
-}
 
-void AppIdPegCounts::inc_disco_peg(enum DiscoveryPegs stat)
- {
-     (*appid_peg_counts)[stat]++;
- }
-
- PegCount AppIdPegCounts::get_disco_peg(enum DiscoveryPegs stat)
- {
-     return (*appid_peg_counts)[stat];
- }
-
- void AppIdPegCounts::inc_service_count(AppId id)
- {
-     (*appid_peg_counts)[get_stats_index(id) + DetectorPegs::SERVICE_DETECTS]++;
- }
-
- void AppIdPegCounts::inc_client_count(AppId id)
- {
-     (*appid_peg_counts)[get_stats_index(id) + DetectorPegs::CLIENT_DETECTS]++;
- }
-
- void AppIdPegCounts::inc_user_count(AppId id)
- {
-     (*appid_peg_counts)[get_stats_index(id) + DetectorPegs::USER_DETECTS]++;
- }
-
- void AppIdPegCounts::inc_payload_count(AppId id)
- {
-     (*appid_peg_counts)[get_stats_index(id)+ DetectorPegs::PAYLOAD_DETECTS]++;
- }
-
- void AppIdPegCounts::inc_misc_count(AppId id)
- {
-     (*appid_peg_counts)[get_stats_index(id) + DetectorPegs::MISC_DETECTS]++;
- }
-
- void AppIdPegCounts::set_detectors_configured()
- {
-     detectors_configured = true;
- }
-
- uint32_t AppIdPegCounts::get_stats_index(AppId id)
- {
-     std::map<AppId, uint32_t>::iterator stats_idx_it = appid_detector_pegs_idx.find(id);
-     if ( stats_idx_it != appid_detector_pegs_idx.end() )
-         return stats_idx_it->second;
-     else
-         return AppIdPegCounts::unknown_app_idx;
- }
+    // Print unknown app stats
+    if (!unknown_pegs->all_zeros())
+    {
+        LogMessage("unknown_app: flows: %" PRIu64 ", clients: %" PRIu64 ", users: %" PRIu64 ", payloads %"
+            PRIu64 ", misc: %" PRIu64 "\n",
+            unknown_pegs->stats[0], unknown_pegs->stats[1], unknown_pegs->stats[2],
+            unknown_pegs->stats[3], unknown_pegs->stats[4]);
+    }
+}
 
diff --git a/src/network_inspectors/appid/appid_peg_counts.h b/src/network_inspectors/appid/appid_peg_counts.h
index e6b33a683..c8c1c0691 100644
--- a/src/network_inspectors/appid/appid_peg_counts.h
+++ b/src/network_inspectors/appid/appid_peg_counts.h
@@ -21,38 +21,30 @@
 #ifndef APPID_PEG_COUNTS_H
 #define APPID_PEG_COUNTS_H
 
-// The AppIdPegCounts class provides an API to manage the peg counts maintained by AppId.
-// AppId defines peg counts that are known at compile time as well as a set of counts for
-// each application that it can detect.  This list of applications is not known until the
+// The AppIdPegCounts class provides an API to manage the dynamic peg counts maintained by AppId.
+// AppId defines peg counts that are known at compile time in appid_module.h. The counts here are
+// for each application that it can detect.  This list of applications is not known until the
 // appMapping.data configuration file is loaded so methods are provided to dynamically
-// initialize the PegInfo and PegCount array when that file is loaded.
-// Functions for incrementing the peg counts are also provided.  The AppId can be a very large
-// number so using it as the array index is not practical, therefore when the dynamic pegs are
-// added we also initialize a std::map that is used to translate the AppId to its array index.
+// initialize the PegCount array when that file is loaded.
+// Functions for incrementing the peg counts are also provided.
+// The AppId can be a very large number so using it as the array index is not practical.
+// Packet threads are using dynamic pegs, and std::map that is used to translate the AppId to its
+// array index.
+// Only the main thread is using a static array.
 
 #include <map>
 #include <vector>
 
 #include "application_ids.h"
+#include "app_info_table.h"
 #include "framework/counts.h"
+#include "log/messages.h"
 #include "main/thread.h"
 #include "utils/util.h"
 
-class AppInfoTableEntry;
-
 class AppIdPegCounts
 {
 public:
-    enum DiscoveryPegs
-    {
-        PACKETS = 0,
-        PROCESSED_PACKETS,
-        IGNORED_PACKETS,
-        TOTAL_SESSIONS,
-        APPID_UNKNOWN,
-        NUM_APPID_GLOBAL_PEGS
-    };
-
     enum DetectorPegs
     {
         SERVICE_DETECTS = 0,
@@ -65,45 +57,57 @@ public:
         NUM_APPID_DETECTOR_PEGS
     };
 
+    class AppIdDynamicPeg
+    {
+    public:
+        PegCount stats[DetectorPegs::NUM_APPID_DETECTOR_PEGS] = { 0 };
+
+        bool all_zeros()
+        {
+            PegCount zeroed_peg[DetectorPegs::NUM_APPID_DETECTOR_PEGS] = { 0 };
+            return !memcmp(stats, &zeroed_peg, sizeof(stats));
+        }
+
+        void print()
+        {
+            LogMessage("flows: %" PRIu64 ", clients: %" PRIu64 ", users: %" PRIu64 ", payloads %" PRIu64
+                ", misc: %" PRIu64 ", incompatible: %" PRIu64 ", failed: %" PRIu64 "\n",
+                stats[0], stats[1], stats[2], stats[3], stats[4], stats[5], stats[6]);
+        }
+    };
+
     static void add_app_peg_info(std::string app_name, AppId);
-    static void add_unknown_app_peg();
-    static PegCount* get_peg_counts();
-    static PegInfo* get_peg_info();
     static void init_pegs();
     static void cleanup_pegs();
     static void cleanup_peg_info();
-    static void inc_disco_peg(enum DiscoveryPegs stat);
-    static PegCount get_disco_peg(enum DiscoveryPegs stat);
     static void inc_service_count(AppId id);
     static void inc_client_count(AppId id);
     static void inc_user_count(AppId id);
     static void inc_payload_count(AppId id);
     static void inc_misc_count(AppId id);
-    static void set_detectors_configured();
 
     static void inc_incompatible_count(AppId id)
     {
-        if ( appid_detector_pegs_idx[id] )
-            (*appid_peg_counts)[appid_detector_pegs_idx[id] + DetectorPegs::INCOMPATIBLE]++;
+        if ( appid_detector_pegs_idx[id] != unknown_app_idx)
+            (*appid_peg_counts)[appid_detector_pegs_idx[id]].stats[DetectorPegs::INCOMPATIBLE]++;
     }
 
     static void inc_failed_count(AppId id)
     {
-        if ( appid_detector_pegs_idx[id] )
-            (*appid_peg_counts)[appid_detector_pegs_idx[id] + DetectorPegs::FAILED]++;
+        if ( appid_detector_pegs_idx[id] != unknown_app_idx)
+            (*appid_peg_counts)[appid_detector_pegs_idx[id]].stats[DetectorPegs::FAILED]++;
     }
 
-private:
-     static bool detectors_configured;
-     static bool dynamic_counts_imported;
-     static uint32_t unknown_app_idx;
-     static std::map<AppId, uint32_t> appid_detector_pegs_idx;
-     static std::vector<PegInfo> appid_detectors_peg_info;
-     static std::vector<PegInfo> appid_pegs;
-     static THREAD_LOCAL std::vector<PegCount>* appid_peg_counts;
+    static void sum_stats();
+    static void print();
 
-     static void init_detector_peg_info(const std::string& app_name, const std::string& name_suffix,
-         const std::string& help_suffix);
-     static uint32_t get_stats_index(AppId id);
+private:
+    static uint32_t unknown_app_idx;
+    static std::map<AppId, uint32_t> appid_detector_pegs_idx;
+    static std::vector<std::string> appid_detectors_info;
+    static AppIdDynamicPeg appid_dynamic_sum[SF_APPID_MAX+1];
+    static THREAD_LOCAL std::vector<AppIdDynamicPeg>* appid_peg_counts;
+    static uint32_t get_stats_index(AppId id);
 };
 #endif
+
diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc
index 0bcb0b74f..b7a025003 100644
--- a/src/network_inspectors/appid/appid_session.cc
+++ b/src/network_inspectors/appid/appid_session.cc
@@ -105,7 +105,7 @@ AppIdSession::AppIdSession(IpProtocol proto, const SfIp* ip, uint16_t port,
     length_sequence.sequence_cnt = 0;
     memset(length_sequence.sequence, '\0', sizeof(length_sequence.sequence));
 
-    AppIdPegCounts::inc_disco_peg(AppIdPegCounts::DiscoveryPegs::TOTAL_SESSIONS);
+    appid_stats.total_sessions++;
 }
 
 AppIdSession::~AppIdSession()
diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
index f2dd7609c..4db9ba5cc 100644
--- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
+++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
@@ -103,15 +103,14 @@ AppIdHttpSession::AppIdHttpSession(AppIdSession& session)
 AppIdHttpSession::~AppIdHttpSession() = default;
 
 // Stubs for AppIdPegCounts
-void AppIdPegCounts::inc_disco_peg(enum DiscoveryPegs) {}
 void AppIdPegCounts::inc_service_count(AppId) {}
 void AppIdPegCounts::inc_client_count(AppId) {}
 void AppIdPegCounts::inc_user_count(AppId) {}
 void AppIdPegCounts::inc_payload_count(AppId) {}
-PegCount AppIdPegCounts::get_disco_peg(enum DiscoveryPegs)
-{
-    return 0;
-}
+
+THREAD_LOCAL AppIdStats appid_stats;
+void AppIdModule::sum_stats(bool) {}
+void AppIdModule::show_dynamic_stats() {}
 
 namespace snort
 {
diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index 4a5ca4898..a7ea071c3 100644
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -760,8 +760,7 @@ static int detector_get_packet_dst_port(lua_State* L)
 static int detector_get_packet_count(lua_State* L)
 {
     lua_checkstack (L, 1);
-    lua_pushnumber(L,
-        AppIdPegCounts::get_disco_peg(AppIdPegCounts::DiscoveryPegs::PROCESSED_PACKETS));
+    lua_pushnumber(L, appid_stats.processed_packets);
     return 1;
 }
 
diff --git a/src/network_inspectors/appid/test/appid_mock_definitions.h b/src/network_inspectors/appid/test/appid_mock_definitions.h
index e58e6b131..7d070c0eb 100644
--- a/src/network_inspectors/appid/test/appid_mock_definitions.h
+++ b/src/network_inspectors/appid/test/appid_mock_definitions.h
@@ -57,6 +57,8 @@ void WarningMessage(const char*,...) { }
 void LogMessage(const char*,...) { }
 void ParseWarning(WarningGroup, const char*, ...) { }
 
+void LogLabel(const char*, FILE*) {}
+
 int ServiceDiscovery::add_ftp_service_state(AppIdSession&)
 {
     return 0;
@@ -97,9 +99,6 @@ int ServiceDiscovery::fail_service(AppIdSession&, const Packet*, int, ServiceDet
 
 void mock_init_appid_pegs()
 {
-    AppIdPegCounts::set_detectors_configured();
-    AppIdPegCounts::add_unknown_app_peg();
-    AppIdPegCounts::get_peg_info();
     AppIdPegCounts::init_pegs();
 }
 
@@ -108,5 +107,8 @@ void mock_cleanup_appid_pegs()
     AppIdPegCounts::cleanup_pegs();
     AppIdPegCounts::cleanup_peg_info();
 }
+
+THREAD_LOCAL AppIdStats appid_stats;
+
 #endif
 
diff --git a/src/network_inspectors/appid/test/appid_mock_inspector.h b/src/network_inspectors/appid/test/appid_mock_inspector.h
index 6c765545d..eab1054ae 100644
--- a/src/network_inspectors/appid/test/appid_mock_inspector.h
+++ b/src/network_inspectors/appid/test/appid_mock_inspector.h
@@ -32,15 +32,28 @@ Inspector::~Inspector() = default;
 bool Inspector::likes(Packet*) { return true; }
 bool Inspector::get_buf(const char*, Packet*, InspectionBuffer&) { return true; }
 class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; }
-}
 
-class AppIdModule
-{
-public:
-    AppIdModule() = default;
-    ~AppIdModule() = default;
+Module::Module(char const*, char const*) {}
+bool Module::set(const char*, Value&, SnortConfig*) { return true; }
+void Module::sum_stats(bool) {}
+void Module::show_interval_stats(std::vector<unsigned int, std::allocator<unsigned int> >&, _IO_FILE*) {}
+void Module::show_stats() {}
+void Module::reset_stats() {}
+PegCount Module::get_global_count(char const*) const { return 0; }
 
-};
+}
+
+AppIdModule::AppIdModule(): snort::Module("appid_mock", "appid_mock_help") {}
+AppIdModule::~AppIdModule() {}
+void AppIdModule::sum_stats(bool) {}
+void AppIdModule::show_dynamic_stats() {}
+bool AppIdModule::begin(char const*, int, snort::SnortConfig*) { return true; }
+bool AppIdModule::end(char const*, int, snort::SnortConfig*) { return true; }
+bool AppIdModule::set(char const*, snort::Value&, snort::SnortConfig*) { return true; }
+const snort::Command* AppIdModule::get_commands() const { return nullptr; }
+const PegInfo* AppIdModule::get_pegs() const { return nullptr; }
+PegCount* AppIdModule::get_counts() const { return nullptr; }
+snort::ProfileStats* AppIdModule::get_profile() const { return nullptr; }
 
 class AppIdInspector : public snort::Inspector
 {
diff --git a/src/utils/stats.cc b/src/utils/stats.cc
index 5ba9199d2..b89a02f72 100644
--- a/src/utils/stats.cc
+++ b/src/utils/stats.cc
@@ -212,9 +212,8 @@ void DropStats()
 
     LogLabel("Module Statistics");
     const char* exclude = "daq snort";
-    ModuleManager::dump_stats(SnortConfig::get_conf(), exclude);
-
-    file_stats_print();
+    ModuleManager::dump_stats(SnortConfig::get_conf(), exclude, false);
+    ModuleManager::dump_stats(SnortConfig::get_conf(), exclude, true);
 
     LogLabel("Summary Statistics");
     show_stats((PegCount*)&proc_stats, proc_names, array_size(proc_names)-1, "process");