From: Michael Altizer (mialtize) Date: Mon, 23 Sep 2019 19:02:10 +0000 (-0400) Subject: Merge pull request #1745 in SNORT/snort3 from ~BBANTWAL/snort3:expect_cache_fix to... X-Git-Tag: 3.0.0-262~32 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=30894e7cb9f45640ebaadbd61df5ac17eac5669c;p=thirdparty%2Fsnort3.git Merge pull request #1745 in SNORT/snort3 from ~BBANTWAL/snort3:expect_cache_fix to master Squashed commit of the following: commit d7228380a4b95305ea45e59b14087af1b9b95a6e Author: Bhagya Tholpady Date: Thu Sep 12 14:16:13 2019 -0400 flow: check if control packet has a valid daq instance before setting up daq expected flow and add pegcounts for expected flows --- diff --git a/src/flow/expect_cache.cc b/src/flow/expect_cache.cc index 7299374a1..1bc043e64 100644 --- a/src/flow/expect_cache.cc +++ b/src/flow/expect_cache.cc @@ -401,7 +401,8 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, node->count = 0; last = nullptr; /* Only add TCP and UDP expected flows for now via the DAQ module. */ - if (ip_proto == IpProtocol::TCP || ip_proto == IpProtocol::UDP) + if ((ip_proto == IpProtocol::TCP || ip_proto == IpProtocol::UDP) + && ctrlPkt->daq_instance) ctrlPkt->daq_instance->add_expected(ctrlPkt, cliIP, cliPort, srvIP, srvPort, ip_proto, 1000, 0); } diff --git a/src/flow/flow_control.h b/src/flow/flow_control.h index e6d954e57..825733ff9 100644 --- a/src/flow/flow_control.h +++ b/src/flow/flow_control.h @@ -83,6 +83,9 @@ public: const snort::SfIp *dstIP, uint16_t dstPort, SnortProtocolId snort_protocol_id, snort::FlowData*); + class ExpectCache* get_exp_cache() + { return exp_cache; } + PegCount get_flows() { return num_flows; } diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc index 88c694b16..32acc5c7a 100644 --- a/src/main/analyzer.cc +++ b/src/main/analyzer.cc @@ -55,6 +55,7 @@ #include "packet_io/sfdaq.h" #include "packet_io/sfdaq_config.h" #include "packet_io/sfdaq_instance.h" +#include "packet_io/sfdaq_module.h" #include "packet_tracer/packet_tracer.h" #include "profiler/profiler.h" #include "pub_sub/finalize_packet_event.h" @@ -176,7 +177,7 @@ static bool process_packet(Packet* p) { assert(p->pkth && p->pkt); - aux_counts.rx_bytes += p->pktlen; + daq_stats.rx_bytes += p->pktlen; PacketTracer::activate(*p); @@ -210,7 +211,7 @@ static DAQ_Verdict distill_verdict(Packet* p) verdict = DAQ_VERDICT_PASS; else if ( act->get_tunnel_bypass() ) { - aux_counts.internal_blacklist++; + daq_stats.internal_blacklist++; verdict = DAQ_VERDICT_BLOCK; } else if ( SnortConfig::inline_mode() || act->packet_force_dropped() ) @@ -250,7 +251,7 @@ static DAQ_Verdict distill_verdict(Packet* p) else { verdict = DAQ_VERDICT_PASS; - aux_counts.internal_whitelist++; + daq_stats.internal_whitelist++; } } else if ( p->ptrs.decode_flags & DECODE_PKT_TRUST ) @@ -277,7 +278,7 @@ void Analyzer::post_process_daq_pkt_msg(Packet* p) if (p->active->packet_retry_requested()) { retry_queue->put(p->daq_msg); - aux_counts.retries_queued++; + daq_stats.retries_queued++; } else if (!p->active->is_packet_held()) verdict = distill_verdict(p); @@ -368,7 +369,7 @@ void Analyzer::process_daq_msg(DAQ_Msg_h msg, bool retry) default: { OtherMessageEvent event(msg, verdict); - aux_counts.other_messages++; + daq_stats.other_messages++; // the verdict can be updated by event handler DataBus::publish(OTHER_MESSAGE_EVENT, event); } @@ -390,7 +391,7 @@ void Analyzer::process_retry_queue() while ((msg = retry_queue->get(&now)) != nullptr) { process_daq_msg(msg, true); - aux_counts.retries_processed++; + daq_stats.retries_processed++; } } } @@ -476,7 +477,7 @@ const char* Analyzer::get_state_string() void Analyzer::idle() { // FIXIT-L this whole thing could be pub-sub - aux_counts.idle++; + daq_stats.idle++; // This should only be called if the DAQ timeout elapsed, so increment the packet time // by the DAQ timeout. @@ -564,7 +565,7 @@ void Analyzer::term() DAQ_Msg_h msg; while ((msg = retry_queue->get()) != nullptr) { - aux_counts.retries_discarded++; + daq_stats.retries_discarded++; Profile profile(daqPerfStats); daq_instance->finalize_message(msg, DAQ_VERDICT_BLOCK); } @@ -761,7 +762,7 @@ DAQ_RecvStatus Analyzer::process_messages() if (skip_cnt > 0) { Profile profile(daqPerfStats); - aux_counts.skipped++; + daq_stats.skipped++; skip_cnt--; daq_instance->finalize_message(msg, DAQ_VERDICT_PASS); continue; diff --git a/src/packet_io/active.cc b/src/packet_io/active.cc index f52d9835b..8c5359927 100644 --- a/src/packet_io/active.cc +++ b/src/packet_io/active.cc @@ -35,6 +35,7 @@ #include "sfdaq.h" #include "sfdaq_instance.h" +#include "sfdaq_module.h" using namespace snort; @@ -431,7 +432,7 @@ bool Active::retry_packet(const Packet* p) { // Fall back on dropping the packet and relying on the host to retransmit active_action = ACT_DROP; - aux_counts.retries_dropped++; + daq_stats.retries_dropped++; return false; } diff --git a/src/packet_io/sfdaq_instance.cc b/src/packet_io/sfdaq_instance.cc index 49870169c..9aa78f0ab 100644 --- a/src/packet_io/sfdaq_instance.cc +++ b/src/packet_io/sfdaq_instance.cc @@ -32,6 +32,7 @@ #include "protocols/vlan.h" #include "sfdaq_config.h" +#include "sfdaq_module.h" using namespace snort; @@ -275,17 +276,20 @@ const DAQ_Stats_t* SFDAQInstance::get_stats() { if (instance) { - int rval = daq_instance_get_stats(instance, &daq_stats); + int rval = daq_instance_get_stats(instance, &daq_instance_stats); if (rval != DAQ_SUCCESS) LogMessage("Couldn't query DAQ stats: %s (%d)\n", daq_instance_get_error(instance), rval); // Some DAQ modules don't provide hardware numbers, so we default HW RX to the SW equivalent // (this means outstanding packets = 0) - if (daq_stats.hw_packets_received == 0) - daq_stats.hw_packets_received = daq_stats.packets_received + daq_stats.packets_filtered; + if (daq_instance_stats.hw_packets_received == 0) + { + daq_instance_stats.hw_packets_received = daq_instance_stats.packets_received + + daq_instance_stats.packets_filtered; + } } - return &daq_stats; + return &daq_instance_stats; } int SFDAQInstance::ioctl(DAQ_IoctlCmd cmd, void *arg, size_t arglen) @@ -375,5 +379,7 @@ int SFDAQInstance::add_expected(const Packet* ctrlPkt, const SfIp* cliIP, uint16 d_cef.data = nullptr; d_cef.length = 0; + daq_stats.expected_flows++; + return daq_instance_ioctl(instance, DIOCTL_CREATE_EXPECTED_FLOW, &d_cef, sizeof(d_cef)); } diff --git a/src/packet_io/sfdaq_instance.h b/src/packet_io/sfdaq_instance.h index b8cc80978..9d1968a62 100644 --- a/src/packet_io/sfdaq_instance.h +++ b/src/packet_io/sfdaq_instance.h @@ -95,7 +95,7 @@ private: uint32_t pool_size = 0; uint32_t pool_available = 0; int dlt = -1; - DAQ_Stats_t daq_stats = { }; + DAQ_Stats_t daq_instance_stats = { }; uint8_t daq_tunnel_mask = 0; }; } diff --git a/src/packet_io/sfdaq_module.cc b/src/packet_io/sfdaq_module.cc index ccb25eb47..09675ee7c 100644 --- a/src/packet_io/sfdaq_module.cc +++ b/src/packet_io/sfdaq_module.cc @@ -167,32 +167,6 @@ bool SFDAQModule::end(const char* fqn, int idx, SnortConfig* sc) return true; } -/* - * Module Counters - */ - -struct DAQStats -{ - PegCount pcaps; - PegCount received; - PegCount analyzed; - PegCount dropped; - PegCount filtered; - PegCount outstanding; - PegCount injected; - PegCount verdicts[MAX_DAQ_VERDICT]; - PegCount internal_blacklist; - PegCount internal_whitelist; - PegCount skipped; - PegCount idle; - PegCount rx_bytes; - PegCount retries_queued; - PegCount retries_dropped; - PegCount retries_processed; - PegCount retries_discarded; - PegCount other_messages; -}; - const PegInfo daq_names[] = { { CountType::MAX, "pcaps", "total files and interfaces processed" }, @@ -218,6 +192,7 @@ const PegInfo daq_names[] = { CountType::SUM, "skipped", "packets skipped at startup" }, { CountType::SUM, "idle", "attempts to acquire from DAQ without available packets" }, { CountType::SUM, "rx_bytes", "total bytes received" }, + { CountType::SUM, "expected_flows", "expected flows created in DAQ" }, { CountType::SUM, "retries_queued", "messages queued for retry" }, { CountType::SUM, "retries_dropped", "messages dropped when overrunning the retry queue" }, { CountType::SUM, "retries_processed", "messages processed from the retry queue" }, @@ -226,7 +201,7 @@ const PegInfo daq_names[] = { CountType::END, nullptr, nullptr } }; -static THREAD_LOCAL DAQStats stats; +THREAD_LOCAL DAQStats daq_stats; const PegInfo* SFDAQModule::get_pegs() const { @@ -235,7 +210,7 @@ const PegInfo* SFDAQModule::get_pegs() const PegCount* SFDAQModule::get_counts() const { - return (PegCount*) &stats; + return (PegCount*) &daq_stats; } static DAQ_Stats_t operator-(const DAQ_Stats_t& left, const DAQ_Stats_t& right) @@ -277,29 +252,16 @@ void SFDAQModule::prep_counts() new_sfdaq_stats.packets_filtered - new_sfdaq_stats.packets_received; - stats.pcaps = Trough::get_file_count(); - stats.received = sfdaq_stats_delta.hw_packets_received; - stats.analyzed = sfdaq_stats_delta.packets_received; - stats.dropped = sfdaq_stats_delta.hw_packets_dropped; - stats.filtered = sfdaq_stats_delta.packets_filtered; - stats.outstanding = pkts_out; - stats.injected = sfdaq_stats_delta.packets_injected; + daq_stats.pcaps = Trough::get_file_count(); + daq_stats.received = sfdaq_stats_delta.hw_packets_received; + daq_stats.analyzed = sfdaq_stats_delta.packets_received; + daq_stats.dropped = sfdaq_stats_delta.hw_packets_dropped; + daq_stats.filtered = sfdaq_stats_delta.packets_filtered; + daq_stats.outstanding = pkts_out; + daq_stats.injected = sfdaq_stats_delta.packets_injected; for ( unsigned i = 0; i < MAX_DAQ_VERDICT; i++ ) - stats.verdicts[i] = sfdaq_stats_delta.verdicts[i]; - - stats.internal_blacklist = aux_counts.internal_blacklist; - stats.internal_whitelist = aux_counts.internal_whitelist; - stats.skipped = aux_counts.skipped; - stats.idle = aux_counts.idle; - stats.rx_bytes = aux_counts.rx_bytes; - stats.retries_queued = aux_counts.retries_queued; - stats.retries_dropped = aux_counts.retries_dropped; - stats.retries_processed = aux_counts.retries_processed; - stats.retries_discarded = aux_counts.retries_discarded; - stats.other_messages = aux_counts.other_messages; - - memset(&aux_counts, 0, sizeof(AuxCount)); + daq_stats.verdicts[i] = sfdaq_stats_delta.verdicts[i]; sfdaq_stats = new_sfdaq_stats; for ( unsigned i = 0; i < MAX_DAQ_VERDICT; i++ ) diff --git a/src/packet_io/sfdaq_module.h b/src/packet_io/sfdaq_module.h index b855a8b0f..0c9ae1a8c 100644 --- a/src/packet_io/sfdaq_module.h +++ b/src/packet_io/sfdaq_module.h @@ -54,4 +54,29 @@ private: SFDAQModuleConfig* module_config; }; +struct DAQStats +{ + PegCount pcaps; + PegCount received; + PegCount analyzed; + PegCount dropped; + PegCount filtered; + PegCount outstanding; + PegCount injected; + PegCount verdicts[MAX_DAQ_VERDICT]; + PegCount internal_blacklist; + PegCount internal_whitelist; + PegCount skipped; + PegCount idle; + PegCount rx_bytes; + PegCount expected_flows; + PegCount retries_queued; + PegCount retries_dropped; + PegCount retries_processed; + PegCount retries_discarded; + PegCount other_messages; +}; + +extern THREAD_LOCAL DAQStats daq_stats; + #endif diff --git a/src/stream/base/stream_base.cc b/src/stream/base/stream_base.cc index 70f989e54..dc61eb48e 100644 --- a/src/stream/base/stream_base.cc +++ b/src/stream/base/stream_base.cc @@ -22,6 +22,7 @@ #include +#include "flow/expect_cache.h" #include "flow/flow_control.h" #include "flow/prune_stats.h" #include "framework/data_bus.h" @@ -60,6 +61,10 @@ const PegInfo base_pegs[] = { CountType::SUM, "preemptive_prunes", "sessions pruned during preemptive pruning" }, { CountType::SUM, "memcap_prunes", "sessions pruned due to memcap" }, { CountType::SUM, "ha_prunes", "sessions pruned by high availability sync" }, + { CountType::SUM, "expected_flows", "total expected flows created within snort" }, + { CountType::SUM, "expected_realized", "number of expected flows realized" }, + { CountType::SUM, "expected_pruned", "number of expected flows pruned" }, + { CountType::SUM, "expected_overflows", "number of expected cache overflows" }, { CountType::END, nullptr, nullptr } }; @@ -78,6 +83,16 @@ void base_sum() stream_base_stats.memcap_prunes = flow_con->get_prunes(PruneReason::MEMCAP); stream_base_stats.ha_prunes = flow_con->get_prunes(PruneReason::HA); + ExpectCache* exp_cache = flow_con->get_exp_cache(); + + if ( exp_cache ) + { + stream_base_stats.expected_flows = exp_cache->get_expects(); + stream_base_stats.expected_realized = exp_cache->get_realized(); + stream_base_stats.expected_pruned = exp_cache->get_prunes(); + stream_base_stats.expected_overflows = exp_cache->get_overflows(); + } + sum_stats((PegCount*)&g_stats, (PegCount*)&stream_base_stats, array_size(base_pegs)-1); base_reset(); @@ -247,15 +262,6 @@ void StreamBase::eval(Packet* p) }; } -#if 0 -// FIXIT-L add method to get exp cache? -LogMessage(" Expected Flows\n"); -LogMessage(" Expected: %lu\n", exp_cache->get_expects()); -LogMessage(" Realized: %lu\n", exp_cache->get_realized()); -LogMessage(" Pruned: %lu\n", exp_cache->get_prunes()); -LogMessage(" Overflows: %lu\n", exp_cache->get_overflows()); -#endif - //------------------------------------------------------------------------- // api stuff //------------------------------------------------------------------------- diff --git a/src/stream/base/stream_module.h b/src/stream/base/stream_module.h index 2305a07bd..a95916829 100644 --- a/src/stream/base/stream_module.h +++ b/src/stream/base/stream_module.h @@ -49,6 +49,10 @@ struct BaseStats PegCount preemptive_prunes; PegCount memcap_prunes; PegCount ha_prunes; + PegCount expected_flows; + PegCount expected_realized; + PegCount expected_pruned; + PegCount expected_overflows; }; extern const PegInfo base_pegs[]; diff --git a/src/utils/stats.cc b/src/utils/stats.cc index 5501bfaad..8b6d5ed04 100644 --- a/src/utils/stats.cc +++ b/src/utils/stats.cc @@ -43,7 +43,6 @@ #define STATS_SEPARATOR \ "--------------------------------------------------" -THREAD_LOCAL AuxCount aux_counts; ProcessCount proc_stats; namespace snort diff --git a/src/utils/stats.h b/src/utils/stats.h index 3f69240da..e15d91a84 100644 --- a/src/utils/stats.h +++ b/src/utils/stats.h @@ -78,22 +78,7 @@ struct ProcessCount PegCount attribute_table_hosts; }; -struct AuxCount -{ - PegCount internal_blacklist; - PegCount internal_whitelist; - PegCount idle; - PegCount rx_bytes; - PegCount skipped; - PegCount retries_queued; - PegCount retries_dropped; - PegCount retries_processed; - PegCount retries_discarded; - PegCount other_messages; -}; - extern ProcessCount proc_stats; -extern THREAD_LOCAL AuxCount aux_counts; extern const PegInfo daq_names[]; extern const PegInfo pc_names[];