Merge pull request #1745 in SNORT/snort3 from ~BBANTWAL/snort3:expect_cache_fix to...

Squashed commit of the following:

commit d7228380a4b95305ea45e59b14087af1b9b95a6e
Author: Bhagya Tholpady <bbantwal@cisco.com>
Date:   Thu Sep 12 14:16:13 2019 -0400

    flow: check if control packet has a valid daq instance before setting up daq expected flow and add pegcounts for expected flows

diff --git a/src/flow/expect_cache.cc b/src/flow/expect_cache.cc
index 7299374a1e7eb3a41dcaaa47195be541fb44dc9a..1bc043e64bf8104b47ab7c8e9de7d5084f8cd335 100644 (file)
--- a/src/flow/expect_cache.cc
+++ b/src/flow/expect_cache.cc
@@ -401,7 +401,8 @@ int ExpectCache::add_flow(const Packet *ctrlPkt,
         node->count = 0;
         last = nullptr;
         /* Only add TCP and UDP expected flows for now via the DAQ module. */
-        if (ip_proto == IpProtocol::TCP || ip_proto == IpProtocol::UDP)
+        if ((ip_proto == IpProtocol::TCP || ip_proto == IpProtocol::UDP) 
+                && ctrlPkt->daq_instance)
             ctrlPkt->daq_instance->add_expected(ctrlPkt, cliIP, cliPort, srvIP, srvPort,
                     ip_proto, 1000, 0);
     }

diff --git a/src/flow/flow_control.h b/src/flow/flow_control.h
index e6d954e57af242d24eed624bd47e3c03d722d2d7..825733ff9ada726eaec9db16b3d728e003a0d1f9 100644 (file)
--- a/src/flow/flow_control.h
+++ b/src/flow/flow_control.h
@@ -83,6 +83,9 @@ public:
         const snort::SfIp *dstIP, uint16_t dstPort,
         SnortProtocolId snort_protocol_id, snort::FlowData*);
 
+    class ExpectCache* get_exp_cache()
+    { return exp_cache; }
+
     PegCount get_flows()
     { return num_flows; }
 

diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc
index 88c694b1645df6689fd8fc20a7d356938fd1e054..32acc5c7ac35b7e7b155bd570d870aa09e8f34c5 100644 (file)
--- a/src/main/analyzer.cc
+++ b/src/main/analyzer.cc
@@ -55,6 +55,7 @@
 #include "packet_io/sfdaq.h"
 #include "packet_io/sfdaq_config.h"
 #include "packet_io/sfdaq_instance.h"
+#include "packet_io/sfdaq_module.h"
 #include "packet_tracer/packet_tracer.h"
 #include "profiler/profiler.h"
 #include "pub_sub/finalize_packet_event.h"
@@ -176,7 +177,7 @@ static bool process_packet(Packet* p)
 {
     assert(p->pkth && p->pkt);
 
-    aux_counts.rx_bytes += p->pktlen;
+    daq_stats.rx_bytes += p->pktlen;
 
     PacketTracer::activate(*p);
 
@@ -210,7 +211,7 @@ static DAQ_Verdict distill_verdict(Packet* p)
             verdict = DAQ_VERDICT_PASS;
         else if ( act->get_tunnel_bypass() )
         {
-            aux_counts.internal_blacklist++;
+            daq_stats.internal_blacklist++;
             verdict = DAQ_VERDICT_BLOCK;
         }
         else if ( SnortConfig::inline_mode() || act->packet_force_dropped() )
@@ -250,7 +251,7 @@ static DAQ_Verdict distill_verdict(Packet* p)
         else
         {
             verdict = DAQ_VERDICT_PASS;
-            aux_counts.internal_whitelist++;
+            daq_stats.internal_whitelist++;
         }
     }
     else if ( p->ptrs.decode_flags & DECODE_PKT_TRUST )
@@ -277,7 +278,7 @@ void Analyzer::post_process_daq_pkt_msg(Packet* p)
     if (p->active->packet_retry_requested())
     {
         retry_queue->put(p->daq_msg);
-        aux_counts.retries_queued++;
+        daq_stats.retries_queued++;
     }
     else if (!p->active->is_packet_held())
         verdict = distill_verdict(p);
@@ -368,7 +369,7 @@ void Analyzer::process_daq_msg(DAQ_Msg_h msg, bool retry)
         default:
             {
                 OtherMessageEvent event(msg, verdict);
-                aux_counts.other_messages++;
+                daq_stats.other_messages++;
                 // the verdict can be updated by event handler
                 DataBus::publish(OTHER_MESSAGE_EVENT, event);
             }
@@ -390,7 +391,7 @@ void Analyzer::process_retry_queue()
         while ((msg = retry_queue->get(&now)) != nullptr)
         {
             process_daq_msg(msg, true);
-            aux_counts.retries_processed++;
+            daq_stats.retries_processed++;
         }
     }
 }
@@ -476,7 +477,7 @@ const char* Analyzer::get_state_string()
 void Analyzer::idle()
 {
     // FIXIT-L this whole thing could be pub-sub
-    aux_counts.idle++;
+    daq_stats.idle++;
 
     // This should only be called if the DAQ timeout elapsed, so increment the packet time
     // by the DAQ timeout.
@@ -564,7 +565,7 @@ void Analyzer::term()
     DAQ_Msg_h msg;
     while ((msg = retry_queue->get()) != nullptr)
     {
-        aux_counts.retries_discarded++;
+        daq_stats.retries_discarded++;
         Profile profile(daqPerfStats);
         daq_instance->finalize_message(msg, DAQ_VERDICT_BLOCK);
     }
@@ -761,7 +762,7 @@ DAQ_RecvStatus Analyzer::process_messages()
         if (skip_cnt > 0)
         {
             Profile profile(daqPerfStats);
-            aux_counts.skipped++;
+            daq_stats.skipped++;
             skip_cnt--;
             daq_instance->finalize_message(msg, DAQ_VERDICT_PASS);
             continue;

diff --git a/src/packet_io/active.cc b/src/packet_io/active.cc
index f52d9835b37ed4bbe886c2290a34a150055b8f8e..8c5359927e3b55fff879f1a694d5b2d31fbea7b0 100644 (file)
--- a/src/packet_io/active.cc
+++ b/src/packet_io/active.cc
@@ -35,6 +35,7 @@
 
 #include "sfdaq.h"
 #include "sfdaq_instance.h"
+#include "sfdaq_module.h"
 
 using namespace snort;
 
@@ -431,7 +432,7 @@ bool Active::retry_packet(const Packet* p)
     {
         // Fall back on dropping the packet and relying on the host to retransmit
         active_action = ACT_DROP;
-        aux_counts.retries_dropped++;
+        daq_stats.retries_dropped++;
         return false;
     }
 

diff --git a/src/packet_io/sfdaq_instance.cc b/src/packet_io/sfdaq_instance.cc
index 49870169cb95120859c8a1f1538c5ba6d11b982d..9aa78f0ab61bd6a96c835d734240a32d340c9d00 100644 (file)
--- a/src/packet_io/sfdaq_instance.cc
+++ b/src/packet_io/sfdaq_instance.cc
@@ -32,6 +32,7 @@
 #include "protocols/vlan.h"
 
 #include "sfdaq_config.h"
+#include "sfdaq_module.h"
 
 using namespace snort;
 
@@ -275,17 +276,20 @@ const DAQ_Stats_t* SFDAQInstance::get_stats()
 {
     if (instance)
     {
-        int rval = daq_instance_get_stats(instance, &daq_stats);
+        int rval = daq_instance_get_stats(instance, &daq_instance_stats);
         if (rval != DAQ_SUCCESS)
             LogMessage("Couldn't query DAQ stats: %s (%d)\n", daq_instance_get_error(instance), rval);
 
         // Some DAQ modules don't provide hardware numbers, so we default HW RX to the SW equivalent
         // (this means outstanding packets = 0)
-        if (daq_stats.hw_packets_received == 0)
-            daq_stats.hw_packets_received = daq_stats.packets_received + daq_stats.packets_filtered;
+        if (daq_instance_stats.hw_packets_received == 0)
+        {
+            daq_instance_stats.hw_packets_received = daq_instance_stats.packets_received +
+                daq_instance_stats.packets_filtered;
+        }
     }
 
-    return &daq_stats;
+    return &daq_instance_stats;
 }
 
 int SFDAQInstance::ioctl(DAQ_IoctlCmd cmd, void *arg, size_t arglen)
@@ -375,5 +379,7 @@ int SFDAQInstance::add_expected(const Packet* ctrlPkt, const SfIp* cliIP, uint16
     d_cef.data = nullptr;
     d_cef.length = 0;
 
+    daq_stats.expected_flows++;
+
     return daq_instance_ioctl(instance, DIOCTL_CREATE_EXPECTED_FLOW, &d_cef, sizeof(d_cef));
 }

diff --git a/src/packet_io/sfdaq_instance.h b/src/packet_io/sfdaq_instance.h
index b8cc809788345611e63b139674ac30bacb552291..9d1968a62ee60aaf15b30036b63c05aeda9c5bb9 100644 (file)
--- a/src/packet_io/sfdaq_instance.h
+++ b/src/packet_io/sfdaq_instance.h
@@ -95,7 +95,7 @@ private:
     uint32_t pool_size = 0;
     uint32_t pool_available = 0;
     int dlt = -1;
-    DAQ_Stats_t daq_stats = { };
+    DAQ_Stats_t daq_instance_stats = { };
     uint8_t daq_tunnel_mask = 0;
 };
 }

diff --git a/src/packet_io/sfdaq_module.cc b/src/packet_io/sfdaq_module.cc
index ccb25eb4748bfc5c6542ad7714482b13e0fe9c87..09675ee7c34bde79e6435d125951d6ce85262776 100644 (file)
--- a/src/packet_io/sfdaq_module.cc
+++ b/src/packet_io/sfdaq_module.cc
@@ -167,32 +167,6 @@ bool SFDAQModule::end(const char* fqn, int idx, SnortConfig* sc)
     return true;
 }
 
-/*
- * Module Counters
- */
-
-struct DAQStats
-{
-    PegCount pcaps;
-    PegCount received;
-    PegCount analyzed;
-    PegCount dropped;
-    PegCount filtered;
-    PegCount outstanding;
-    PegCount injected;
-    PegCount verdicts[MAX_DAQ_VERDICT];
-    PegCount internal_blacklist;
-    PegCount internal_whitelist;
-    PegCount skipped;
-    PegCount idle;
-    PegCount rx_bytes;
-    PegCount retries_queued;
-    PegCount retries_dropped;
-    PegCount retries_processed;
-    PegCount retries_discarded;
-    PegCount other_messages;
-};
-
 const PegInfo daq_names[] =
 {
     { CountType::MAX, "pcaps", "total files and interfaces processed" },
@@ -218,6 +192,7 @@ const PegInfo daq_names[] =
     { CountType::SUM, "skipped", "packets skipped at startup" },
     { CountType::SUM, "idle", "attempts to acquire from DAQ without available packets" },
     { CountType::SUM, "rx_bytes", "total bytes received" },
+    { CountType::SUM, "expected_flows", "expected flows created in DAQ" },
     { CountType::SUM, "retries_queued", "messages queued for retry" },
     { CountType::SUM, "retries_dropped", "messages dropped when overrunning the retry queue" },
     { CountType::SUM, "retries_processed", "messages processed from the retry queue" },
@@ -226,7 +201,7 @@ const PegInfo daq_names[] =
     { CountType::END, nullptr, nullptr }
 };
 
-static THREAD_LOCAL DAQStats stats;
+THREAD_LOCAL DAQStats daq_stats;
 
 const PegInfo* SFDAQModule::get_pegs() const
 {
@@ -235,7 +210,7 @@ const PegInfo* SFDAQModule::get_pegs() const
 
 PegCount* SFDAQModule::get_counts() const
 {
-    return (PegCount*) &stats;
+    return (PegCount*) &daq_stats;
 }
 
 static DAQ_Stats_t operator-(const DAQ_Stats_t& left, const DAQ_Stats_t& right)
@@ -277,29 +252,16 @@ void SFDAQModule::prep_counts()
                         new_sfdaq_stats.packets_filtered -
                         new_sfdaq_stats.packets_received;
 
-    stats.pcaps = Trough::get_file_count();
-    stats.received = sfdaq_stats_delta.hw_packets_received;
-    stats.analyzed = sfdaq_stats_delta.packets_received;
-    stats.dropped = sfdaq_stats_delta.hw_packets_dropped;
-    stats.filtered =  sfdaq_stats_delta.packets_filtered;
-    stats.outstanding =  pkts_out;
-    stats.injected =  sfdaq_stats_delta.packets_injected;
+    daq_stats.pcaps = Trough::get_file_count();
+    daq_stats.received = sfdaq_stats_delta.hw_packets_received;
+    daq_stats.analyzed = sfdaq_stats_delta.packets_received;
+    daq_stats.dropped = sfdaq_stats_delta.hw_packets_dropped;
+    daq_stats.filtered =  sfdaq_stats_delta.packets_filtered;
+    daq_stats.outstanding =  pkts_out;
+    daq_stats.injected =  sfdaq_stats_delta.packets_injected;
 
     for ( unsigned i = 0; i < MAX_DAQ_VERDICT; i++ )
-        stats.verdicts[i] = sfdaq_stats_delta.verdicts[i];
-
-    stats.internal_blacklist = aux_counts.internal_blacklist;
-    stats.internal_whitelist = aux_counts.internal_whitelist;
-    stats.skipped = aux_counts.skipped;
-    stats.idle = aux_counts.idle;
-    stats.rx_bytes = aux_counts.rx_bytes;
-    stats.retries_queued = aux_counts.retries_queued;
-    stats.retries_dropped = aux_counts.retries_dropped;
-    stats.retries_processed = aux_counts.retries_processed;
-    stats.retries_discarded = aux_counts.retries_discarded;
-    stats.other_messages = aux_counts.other_messages;
-
-    memset(&aux_counts, 0, sizeof(AuxCount));
+        daq_stats.verdicts[i] = sfdaq_stats_delta.verdicts[i];
 
     sfdaq_stats = new_sfdaq_stats;
     for ( unsigned i = 0; i < MAX_DAQ_VERDICT; i++ )

diff --git a/src/packet_io/sfdaq_module.h b/src/packet_io/sfdaq_module.h
index b855a8b0f992820583fe1ee746f3ce07ea16d013..0c9ae1a8cfa1a82a7c15508499c138256c3c73f5 100644 (file)
--- a/src/packet_io/sfdaq_module.h
+++ b/src/packet_io/sfdaq_module.h
@@ -54,4 +54,29 @@ private:
     SFDAQModuleConfig* module_config;
 };
 
+struct DAQStats
+{
+    PegCount pcaps;
+    PegCount received;
+    PegCount analyzed;
+    PegCount dropped;
+    PegCount filtered;
+    PegCount outstanding;
+    PegCount injected;
+    PegCount verdicts[MAX_DAQ_VERDICT];
+    PegCount internal_blacklist;
+    PegCount internal_whitelist;
+    PegCount skipped;
+    PegCount idle;
+    PegCount rx_bytes;
+    PegCount expected_flows;
+    PegCount retries_queued;
+    PegCount retries_dropped;
+    PegCount retries_processed;
+    PegCount retries_discarded;
+    PegCount other_messages;
+};
+
+extern THREAD_LOCAL DAQStats daq_stats;
+
 #endif

diff --git a/src/stream/base/stream_base.cc b/src/stream/base/stream_base.cc
index 70f989e549711a6aa2bc1031fb36bd308423cb34..dc61eb48e5a2ca7380f34f02a327b2fbf3d60340 100644 (file)
--- a/src/stream/base/stream_base.cc
+++ b/src/stream/base/stream_base.cc
@@ -22,6 +22,7 @@
 
 #include <functional>
 
+#include "flow/expect_cache.h"
 #include "flow/flow_control.h"
 #include "flow/prune_stats.h"
 #include "framework/data_bus.h"
@@ -60,6 +61,10 @@ const PegInfo base_pegs[] =
     { CountType::SUM, "preemptive_prunes", "sessions pruned during preemptive pruning" },
     { CountType::SUM, "memcap_prunes", "sessions pruned due to memcap" },
     { CountType::SUM, "ha_prunes", "sessions pruned by high availability sync" },
+    { CountType::SUM, "expected_flows", "total expected flows created within snort" },
+    { CountType::SUM, "expected_realized", "number of expected flows realized" },
+    { CountType::SUM, "expected_pruned", "number of expected flows pruned" },
+    { CountType::SUM, "expected_overflows", "number of expected cache overflows" },
     { CountType::END, nullptr, nullptr }
 };
 
@@ -78,6 +83,16 @@ void base_sum()
     stream_base_stats.memcap_prunes = flow_con->get_prunes(PruneReason::MEMCAP);
     stream_base_stats.ha_prunes = flow_con->get_prunes(PruneReason::HA);
 
+    ExpectCache* exp_cache = flow_con->get_exp_cache();
+
+    if ( exp_cache )
+    {
+        stream_base_stats.expected_flows = exp_cache->get_expects();
+        stream_base_stats.expected_realized = exp_cache->get_realized();
+        stream_base_stats.expected_pruned = exp_cache->get_prunes();
+        stream_base_stats.expected_overflows = exp_cache->get_overflows();
+    }
+
     sum_stats((PegCount*)&g_stats, (PegCount*)&stream_base_stats,
         array_size(base_pegs)-1);
     base_reset();
@@ -247,15 +262,6 @@ void StreamBase::eval(Packet* p)
     };
 }
 
-#if 0
-// FIXIT-L add method to get exp cache?
-LogMessage("            Expected Flows\n");
-LogMessage("                  Expected: %lu\n", exp_cache->get_expects());
-LogMessage("                  Realized: %lu\n", exp_cache->get_realized());
-LogMessage("                    Pruned: %lu\n", exp_cache->get_prunes());
-LogMessage("                 Overflows: %lu\n", exp_cache->get_overflows());
-#endif
-
 //-------------------------------------------------------------------------
 // api stuff
 //-------------------------------------------------------------------------

diff --git a/src/stream/base/stream_module.h b/src/stream/base/stream_module.h
index 2305a07bd629af61bdcd85afdc08cb1841cc6253..a959168294c04809ea68c04d3f8a13215c0f99cf 100644 (file)
--- a/src/stream/base/stream_module.h
+++ b/src/stream/base/stream_module.h
@@ -49,6 +49,10 @@ struct BaseStats
      PegCount preemptive_prunes;
      PegCount memcap_prunes;
      PegCount ha_prunes;
+     PegCount expected_flows;
+     PegCount expected_realized;
+     PegCount expected_pruned;
+     PegCount expected_overflows;
 };
 
 extern const PegInfo base_pegs[];

diff --git a/src/utils/stats.cc b/src/utils/stats.cc
index 5501bfaade50b14462a5b48f7cea41274bd02167..8b6d5ed0460eea688eb01541538080b8ae06c4df 100644 (file)
--- a/src/utils/stats.cc
+++ b/src/utils/stats.cc
@@ -43,7 +43,6 @@
 #define STATS_SEPARATOR \
     "--------------------------------------------------"
 
-THREAD_LOCAL AuxCount aux_counts;
 ProcessCount proc_stats;
 
 namespace snort

diff --git a/src/utils/stats.h b/src/utils/stats.h
index 3f69240da07f25f76298025fc079e38d4b446a0d..e15d91a84a72851c22377cfac6279bf467cd9c68 100644 (file)
--- a/src/utils/stats.h
+++ b/src/utils/stats.h
@@ -78,22 +78,7 @@ struct ProcessCount
     PegCount attribute_table_hosts;
 };
 
-struct AuxCount
-{
-    PegCount internal_blacklist;
-    PegCount internal_whitelist;
-    PegCount idle;
-    PegCount rx_bytes;
-    PegCount skipped;
-    PegCount retries_queued;
-    PegCount retries_dropped;
-    PegCount retries_processed;
-    PegCount retries_discarded;
-    PegCount other_messages;
-};
-
 extern ProcessCount proc_stats;
-extern THREAD_LOCAL AuxCount aux_counts;
 
 extern const PegInfo daq_names[];
 extern const PegInfo pc_names[];