Pull request #4103: flow_cache: added new protocol base counters

Merge in SNORT/snort3 from ~RSHAFIQ/snort3:proto_prune_stats to master

Squashed commit of the following:

commit e098d60d6b616a4dcda7bd9561ff932429fd9360
Author: rshafiq <rshafiq@cisco.com>
Date:   Fri Oct 20 16:57:29 2023 -0400

    flow_cache: added new protocol base counters

diff --git a/src/flow/flow_cache.cc b/src/flow/flow_cache.cc
index d218fcb99b8bf1373354f0562346bb1784f44cc0..39d241e8878200b58a326eed1eb8f86091938351 100644 (file)
--- a/src/flow/flow_cache.cc
+++ b/src/flow/flow_cache.cc
@@ -51,7 +51,8 @@ static const unsigned OFFLOADED_FLOWS_TOO = 2;
 static const unsigned ALL_FLOWS = 3;
 static const unsigned WDT_MASK = 7; // kick watchdog once for every 8 flows deleted
 
-const uint8_t MAX_PROTOCOLS = (uint8_t)to_utype(PktType::MAX) - 1; //removing PktType::NONE from count
+constexpr uint8_t MAX_PROTOCOLS = (uint8_t)to_utype(PktType::MAX) - 1; //removing PktType::NONE from count
+constexpr uint64_t max_skip_protos = (1ULL << MAX_PROTOCOLS) - 1;
 
 //-------------------------------------------------------------------------
 // FlowCache stuff
@@ -204,7 +205,7 @@ bool FlowCache::release(Flow* flow, PruneReason reason, bool do_cleanup)
     }
 
     flow->reset(do_cleanup);
-    prune_stats.update(reason);
+    prune_stats.update(reason, flow->key->pkt_type);
     remove(flow);
     return true;
 }
@@ -212,7 +213,7 @@ bool FlowCache::release(Flow* flow, PruneReason reason, bool do_cleanup)
 void FlowCache::retire(Flow* flow)
 {
     flow->reset(true);
-    prune_stats.update(PruneReason::NONE);
+    prune_stats.update(PruneReason::NONE, flow->key->pkt_type);
     remove(flow);
 }
 
@@ -225,8 +226,6 @@ unsigned FlowCache::prune_idle(uint32_t thetime, const Flow* save_me)
 
     assert(MAX_PROTOCOLS < 8 * sizeof(skip_protos));
 
-    const uint64_t max_skip_protos = (1ULL << MAX_PROTOCOLS) - 1;
-
     {
         PacketTracerSuspend pt_susp;
         while ( pruned <= cleanup_flows and 
@@ -238,13 +237,15 @@ unsigned FlowCache::prune_idle(uint32_t thetime, const Flow* save_me)
                 if( pruned > cleanup_flows )
                     break;
 
-                if ( skip_protos & (1ULL << proto_idx) )
+                const uint64_t proto_mask = 1ULL << proto_idx;
+
+                if ( skip_protos & proto_mask )
                     continue;
 
                 auto flow = static_cast<Flow*>(hash_table->lru_first(proto_idx));
                 if ( !flow )
                 {
-                    skip_protos |= (1ULL << proto_idx);
+                    skip_protos |= proto_mask;
                     continue;
                 }
                 
@@ -252,7 +253,7 @@ unsigned FlowCache::prune_idle(uint32_t thetime, const Flow* save_me)
                     or flow->is_suspended()
                     or flow->last_data_seen + config.pruning_timeout >= thetime )
                 {
-                    skip_protos |= (1ULL << proto_idx);
+                    skip_protos |= proto_mask;
                     continue;
                 }
 
@@ -323,7 +324,6 @@ unsigned FlowCache::prune_excess(const Flow* save_me)
 
     assert(MAX_PROTOCOLS < 8 * sizeof(skip_protos));
 
-    const uint64_t max_skip_protos = (1ULL << MAX_PROTOCOLS) - 1;
 
     {
         PacketTracerSuspend pt_susp;
@@ -341,14 +341,16 @@ unsigned FlowCache::prune_excess(const Flow* save_me)
                 num_nodes = hash_table->get_num_nodes();
                 if ( num_nodes <= max_cap or num_nodes <= blocks )
                     break;
-                
-                if ( skip_protos & (1ULL << proto_idx) ) 
+
+                const uint64_t proto_mask = 1ULL << proto_idx;
+
+                if ( skip_protos & proto_mask ) 
                     continue;
 
                 auto flow = static_cast<Flow*>(hash_table->lru_first(proto_idx));
                 if ( !flow )
                 {
-                    skip_protos |= (1ULL << proto_idx);
+                    skip_protos |= proto_mask;
                     continue;
                 }
 
@@ -415,14 +417,14 @@ unsigned FlowCache::prune_multiple(PruneReason reason, bool do_cleanup)
 
     assert(MAX_PROTOCOLS < 8 * sizeof(skip_protos));
 
-    const uint64_t max_skip_protos = (1ULL << MAX_PROTOCOLS) - 1;
-
+    
     while ( pruned < config.prune_flows )
     {
-        if ( (skip_protos & (1ULL << proto)) or !prune_one(reason, do_cleanup, proto) )
+        const uint64_t proto_mask = 1ULL << proto;
+        if ( (skip_protos & proto_mask) or !prune_one(reason, do_cleanup, proto) )
         {
 
-            skip_protos |= (1ULL << proto);
+            skip_protos |= proto_mask;
             if ( skip_protos == max_skip_protos )
                 break;
         }
@@ -448,7 +450,6 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime)
 
     assert(MAX_PROTOCOLS < 8 * sizeof(skip_protos));
 
-    const uint64_t max_skip_protos = (1ULL << MAX_PROTOCOLS) - 1;
     {
         PacketTracerSuspend pt_susp;
 
@@ -458,8 +459,10 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime)
             {
                 if( retired >= num_flows )
                     break;
+                
+                const uint64_t proto_mask = 1ULL << proto_idx;
 
-                if ( skip_protos & (1ULL << proto_idx) ) 
+                if ( skip_protos & proto_mask ) 
                     continue;
 
                 auto flow = static_cast<Flow*>(hash_table->lru_current(proto_idx));
@@ -467,7 +470,7 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime)
                     flow = static_cast<Flow*>(hash_table->lru_first(proto_idx));
                 if ( !flow )
                 {
-                    skip_protos |= (1ULL << proto_idx);
+                    skip_protos |= proto_mask;
                     continue;
                 }
 
@@ -475,13 +478,13 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime)
                 {
                     if ( flow->expire_time > static_cast<uint64_t>(thetime) )
                     {
-                        skip_protos |= (1ULL << proto_idx);
+                        skip_protos |= proto_mask;
                         continue;
                     }
                 }
                 else if ( flow->last_data_seen + flow->idle_timeout > thetime )
                 {
-                    skip_protos |= (1ULL << proto_idx);
+                    skip_protos |= proto_mask;
                     continue;
                 }
 
@@ -508,7 +511,6 @@ unsigned FlowCache::delete_active_flows(unsigned mode, unsigned num_to_delete, u
 
     assert(MAX_PROTOCOLS < 8 * sizeof(skip_protos));
 
-    const uint64_t max_skip_protos = (1ULL << MAX_PROTOCOLS) - 1;
 
     while ( num_to_delete and skip_protos != max_skip_protos and
             undeletable < hash_table->get_num_nodes() )
@@ -518,13 +520,15 @@ unsigned FlowCache::delete_active_flows(unsigned mode, unsigned num_to_delete, u
             if( num_to_delete == 0)
                 break;
             
-            if ( skip_protos & (1ULL << proto_idx) )
+            const uint64_t proto_mask = 1ULL << proto_idx;
+
+            if ( skip_protos & proto_mask )
                 continue;
             
             auto flow = static_cast<Flow*>(hash_table->lru_first(proto_idx));
             if ( !flow )
             {
-                skip_protos |= (1ULL << proto_idx);
+                skip_protos |= proto_mask;
                 continue;
             }
 

diff --git a/src/flow/flow_cache.h b/src/flow/flow_cache.h
index c7db2371d3bbe070b149018b974c955a89b2189e..c01df4f3e7b75ba02a9b73891ea4a75e0d217ced 100644 (file)
--- a/src/flow/flow_cache.h
+++ b/src/flow/flow_cache.h
@@ -75,6 +75,9 @@ public:
     PegCount get_prunes(PruneReason reason) const
     { return prune_stats.get(reason); }
 
+    PegCount get_proto_prune_count(PruneReason reason, PktType type) const
+    { return prune_stats.get_proto_prune_count(reason,type); }
+
     PegCount get_total_deletes() const
     { return delete_stats.get_total(); }
 

diff --git a/src/flow/flow_control.cc b/src/flow/flow_control.cc
index 56c5be9e8241f884e3e39b589c47bf1f341c22d0..0c2b65e4129cd28cf4dd2c64be381e35fb9b5108 100644 (file)
--- a/src/flow/flow_control.cc
+++ b/src/flow/flow_control.cc
@@ -67,6 +67,9 @@ PegCount FlowControl::get_total_prunes() const
 PegCount FlowControl::get_prunes(PruneReason reason) const
 { return cache->get_prunes(reason); }
 
+PegCount FlowControl::get_proto_prune_count(PruneReason reason, PktType type) const
+{ return cache->get_proto_prune_count(reason,type); }
+
 PegCount FlowControl::get_total_deletes() const
 { return cache->get_total_deletes(); }
 

diff --git a/src/flow/flow_control.h b/src/flow/flow_control.h
index f40836611d8aacc90a4c08d831f4dae85524df47..c4a61a60e372138c6ae3b2163be80550b35eed51 100644 (file)
--- a/src/flow/flow_control.h
+++ b/src/flow/flow_control.h
@@ -91,6 +91,7 @@ public:
 
     PegCount get_total_prunes() const;
     PegCount get_prunes(PruneReason) const;
+    PegCount get_proto_prune_count(PruneReason, PktType) const;
     PegCount get_total_deletes() const;
     PegCount get_deletes(FlowDeleteState state) const;
     void clear_counts();

diff --git a/src/flow/prune_stats.h b/src/flow/prune_stats.h
index 650f6397a79bf79c61349c255f34ae9718da212e..afa3081c848ec4b4051fe1eda3844795f7111c07 100644 (file)
--- a/src/flow/prune_stats.h
+++ b/src/flow/prune_stats.h
@@ -39,11 +39,36 @@ enum class PruneReason : uint8_t
     MAX
 };
 
+struct ProtoPruneStats
+{
+    using proto_t = std::underlying_type_t<PktType>;
+    PegCount proto_counts[static_cast<proto_t>(PktType::MAX)] { };
+
+    PegCount get_total() const
+    {
+        PegCount total = 0;
+        for ( proto_t i = 0; i < static_cast<proto_t>(PktType::MAX); ++i )
+            total += proto_counts[i];
+
+        return total;
+    }
+
+    PegCount& get(PktType type)
+    { return proto_counts[static_cast<proto_t>(type)]; }
+
+    const PegCount& get(PktType type) const
+    { return proto_counts[static_cast<proto_t>(type)]; }
+
+    void update(PktType type)
+    { ++get(type); }
+};
+
 struct PruneStats
 {
     using reason_t = std::underlying_type<PruneReason>::type;
 
     PegCount prunes[static_cast<reason_t>(PruneReason::MAX)] { };
+    ProtoPruneStats protoPruneStats[static_cast<reason_t>(PruneReason::MAX)] { };
 
     PegCount get_total() const
     {
@@ -60,8 +85,26 @@ struct PruneStats
     const PegCount& get(PruneReason reason) const
     { return prunes[static_cast<reason_t>(reason)]; }
 
-    void update(PruneReason reason)
-    { ++get(reason); }
+    void update(PruneReason reason, PktType type = PktType::NONE)
+    { 
+        ++get(reason); 
+        protoPruneStats[static_cast<reason_t>(reason)].update(type);
+    }
+
+    PegCount& get_proto_prune_count(PruneReason reason, PktType type)
+    { return protoPruneStats[static_cast<reason_t>(reason)].get(type); }
+
+    const PegCount& get_proto_prune_count(PruneReason reason, PktType type) const
+    { return protoPruneStats[static_cast<reason_t>(reason)].get(type); }
+
+    PegCount get_proto_prune_count(PktType type) const
+    {
+        PegCount total = 0;
+        for ( reason_t i = 0; i < static_cast<reason_t>(PruneReason::NONE); ++i )
+            total += protoPruneStats[i].get(type);
+
+        return total;
+    }
 };
 
 enum class FlowDeleteState : uint8_t

diff --git a/src/flow/test/flow_cache_test.cc b/src/flow/test/flow_cache_test.cc
index 358b01e646f12f464025ea4ea2e6146608d73286..fe63c026a2c0ecb92e8b33a286d0257037760754 100644 (file)
--- a/src/flow/test/flow_cache_test.cc
+++ b/src/flow/test/flow_cache_test.cc
@@ -304,6 +304,50 @@ TEST(flow_prune, prune_proto)
     delete cache;
 }
 
+TEST(flow_prune, prune_counts)
+{
+    PruneStats stats;
+
+    // Simulate a few prunes for different reasons and protocol types
+    stats.update(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP);
+    stats.update(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::TCP);
+    stats.update(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::UDP);
+    stats.update(PruneReason::MEMCAP, PktType::ICMP);
+    stats.update(PruneReason::MEMCAP, PktType::USER);
+
+    // Check the total prunes
+    CHECK_EQUAL(5, stats.get_total());
+
+    // Check individual protocol prunes
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::TCP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::UDP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PruneReason::MEMCAP, PktType::ICMP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PruneReason::MEMCAP, PktType::USER));
+
+    // Check prunes for a specific protocol across all reasons
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PktType::IP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PktType::TCP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PktType::UDP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PktType::ICMP));
+    CHECK_EQUAL(1, stats.get_proto_prune_count(PktType::USER));
+
+    // Reset the counts
+    stats = PruneStats();
+
+    // Ensure that the counts have been reset
+    CHECK_EQUAL(0, stats.get_total());
+    CHECK_EQUAL(0, stats.get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP));
+    CHECK_EQUAL(0, stats.get_proto_prune_count(PruneReason::MEMCAP, PktType::TCP));
+
+     // Update the same protocol and reason multiple times
+    stats.update(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP);
+    stats.update(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP);
+    stats.update(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP);
+
+    CHECK_EQUAL(3, stats.get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP));
+}
+
 int main(int argc, char** argv)
 {
     return CommandLineTestRunner::RunAllTests(argc, argv);

diff --git a/src/stream/base/stream_base.cc b/src/stream/base/stream_base.cc
index ba7c975de9d467f73f69b85981ed95514450a24a..1d0ff065e5883ab8c2b89f2d6c514d4adbbd2237 100644 (file)
--- a/src/stream/base/stream_base.cc
+++ b/src/stream/base/stream_base.cc
@@ -82,6 +82,20 @@ const PegInfo base_pegs[] =
     { CountType::SUM, "reload_allowed_deletes", "number of allowed flows deleted by config reloads" },
     { CountType::SUM, "reload_blocked_deletes", "number of blocked flows deleted by config reloads" },
     { CountType::SUM, "reload_offloaded_deletes", "number of offloaded flows deleted by config reloads" },
+    { CountType::SUM, "ip_timeout_prunes", "number of IP flows pruned due to timeout" },
+    { CountType::SUM, "tcp_timeout_prunes", "number of TCP flows pruned due to timeout" },
+    { CountType::SUM, "udp_timeout_prunes", "number of UDP flows pruned due to timeout" },
+    { CountType::SUM, "icmp_timeout_prunes", "number of ICMP flows pruned due to timeout" },
+    { CountType::SUM, "user_timeout_prunes", "number of USER flows pruned due to timeout" },
+    { CountType::SUM, "file_timeout_prunes", "number of FILE flows pruned due to timeout" },
+    { CountType::SUM, "pdu_timeout_prunes", "number of PDU flows pruned due to timeout" },
+    { CountType::SUM, "ip_memcap_prunes", "number of IP flows pruned due to memcap" },
+    { CountType::SUM, "tcp_memcap_prunes", "number of TCP flows pruned due to memcap" },
+    { CountType::SUM, "udp_memcap_prunes", "number of UDP flows pruned due to memcap" },
+    { CountType::SUM, "icmp_memcap_prunes", "number of ICMP flows pruned due to memcap" },
+    { CountType::SUM, "user_memcap_prunes", "number of USER flows pruned due to memcap" },
+    { CountType::SUM, "file_memcap_prunes", "number of FILE flows pruned due to memcap" },
+    { CountType::SUM, "pdu_memcap_prunes", "number of PDU flows pruned due to memcap" },
 
     // Keep the NOW stats at the bottom as it requires special sum_stats logic
     { CountType::NOW, "current_flows", "current number of flows in cache" },
@@ -111,6 +125,20 @@ void base_prep()
     stream_base_stats.reload_allowed_flow_deletes = flow_con->get_deletes(FlowDeleteState::ALLOWED);
     stream_base_stats.reload_offloaded_flow_deletes= flow_con->get_deletes(FlowDeleteState::OFFLOADED);
     stream_base_stats.reload_blocked_flow_deletes= flow_con->get_deletes(FlowDeleteState::BLOCKED);
+    stream_base_stats.ip_timeout_prunes = flow_con->get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::IP);
+    stream_base_stats.tcp_timeout_prunes = flow_con->get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::TCP);
+    stream_base_stats.udp_timeout_prunes = flow_con->get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::UDP);
+    stream_base_stats.icmp_timeout_prunes = flow_con->get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::ICMP);
+    stream_base_stats.user_timeout_prunes = flow_con->get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::USER);
+    stream_base_stats.file_timeout_prunes = flow_con->get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::FILE);
+    stream_base_stats.pdu_timeout_prunes = flow_con->get_proto_prune_count(PruneReason::IDLE_PROTOCOL_TIMEOUT, PktType::PDU);
+    stream_base_stats.ip_memcap_prunes = flow_con->get_proto_prune_count(PruneReason::MEMCAP, PktType::IP);
+    stream_base_stats.tcp_memcap_prunes = flow_con->get_proto_prune_count(PruneReason::MEMCAP, PktType::TCP);
+    stream_base_stats.udp_memcap_prunes = flow_con->get_proto_prune_count(PruneReason::MEMCAP, PktType::UDP);
+    stream_base_stats.icmp_memcap_prunes = flow_con->get_proto_prune_count(PruneReason::MEMCAP, PktType::ICMP);
+    stream_base_stats.user_memcap_prunes = flow_con->get_proto_prune_count(PruneReason::MEMCAP, PktType::USER);
+    stream_base_stats.file_memcap_prunes = flow_con->get_proto_prune_count(PruneReason::MEMCAP, PktType::FILE);
+    stream_base_stats.pdu_memcap_prunes = flow_con->get_proto_prune_count(PruneReason::MEMCAP, PktType::PDU);
 
     stream_base_stats.current_flows = flow_con->get_num_flows();
     stream_base_stats.uni_flows = flow_con->get_uni_flows();

diff --git a/src/stream/base/stream_module.h b/src/stream/base/stream_module.h
index f21718cf65de35ff7405697a2ef95aea2fc65001..a800e564ab422b3a9103926d722227412ad80853 100644 (file)
--- a/src/stream/base/stream_module.h
+++ b/src/stream/base/stream_module.h
@@ -75,6 +75,20 @@ struct BaseStats
      PegCount reload_allowed_flow_deletes;
      PegCount reload_blocked_flow_deletes;
      PegCount reload_offloaded_flow_deletes;
+     PegCount ip_timeout_prunes;
+     PegCount tcp_timeout_prunes;
+     PegCount udp_timeout_prunes;
+     PegCount icmp_timeout_prunes;
+     PegCount user_timeout_prunes;
+     PegCount file_timeout_prunes;
+     PegCount pdu_timeout_prunes;  
+     PegCount ip_memcap_prunes;
+     PegCount tcp_memcap_prunes;
+     PegCount udp_memcap_prunes;
+     PegCount icmp_memcap_prunes;
+     PegCount user_memcap_prunes;
+     PegCount file_memcap_prunes;
+     PegCount pdu_memcap_prunes;
 
      // Keep the NOW stats at the bottom as it requires special sum_stats logic
      PegCount current_flows;