From: Steve Chew (stechew) Date: Wed, 31 Mar 2021 12:40:16 +0000 (+0000) Subject: Merge pull request #2781 in SNORT/snort3 from ~STECHEW/snort3:ftps_tls_alert to master X-Git-Tag: 3.1.4.0~31 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1dbd0e430cdf77026ebda21e89d7b931bf427510;p=thirdparty%2Fsnort3.git Merge pull request #2781 in SNORT/snort3 from ~STECHEW/snort3:ftps_tls_alert to master Squashed commit of the following: commit 41c0f9f0404feb00411a381fddc5a4d8b5fe8d2a Author: Steve Chew Date: Wed Mar 24 18:40:40 2021 -0400 main: Log holding verdict only if packet was actually held. commit f85ee407474f867c021381d2c5dad01676c100a2 Author: Steve Chew Date: Fri Mar 19 09:30:21 2021 -0400 dce_rpc: Fixed prototype mismatch. Smb2Tid doesn't need to be inline. commit b1c00248536485223c00d2cd66df1fa236d18673 Author: Steve Chew Date: Mon Mar 8 13:56:29 2021 -0500 main: Update memcap for detained packets. commit 9850db7b66e50454048a7744497c057628a07429 Author: Steve Chew Date: Tue Mar 9 18:06:56 2021 -0500 packet_io: If packet has no daq_instance, use thread-local daq_instance. commit a0479bf54f4d882e2bd19c4044c3776330be787a Author: Steve Chew Date: Mon Mar 8 13:49:00 2021 -0500 stream: Add held packet to retry queue when requested. commit 2e8c00a0ca58c338491bc3a38ed039cc1baba01a Author: Steve Chew Date: Mon Mar 8 02:22:02 2021 -0500 stream: Add partial_flush. Flush one side of flow immediately. --- diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc index eb329a63b..5891e2149 100644 --- a/src/main/analyzer.cc +++ b/src/main/analyzer.cc @@ -51,6 +51,7 @@ #include "managers/ips_manager.h" #include "managers/event_manager.h" #include "managers/module_manager.h" +#include "memory/memory_cap.h" #include "packet_io/active.h" #include "packet_io/sfdaq.h" #include "packet_io/sfdaq_config.h" @@ -285,27 +286,43 @@ static DAQ_Verdict distill_verdict(Packet* p) return verdict; } +void Analyzer::add_to_retry_queue(DAQ_Msg_h daq_msg) +{ + // Temporarily increase memcap until message is finalized in case + // DAQ makes a copy of the data buffer. + memory::MemoryCap::update_allocations(daq_msg_get_data_len(daq_msg)); + retry_queue->put(daq_msg); +} + /* * Private message processing methods */ void Analyzer::post_process_daq_pkt_msg(Packet* p) { + bool msg_was_held = false; + Active::execute(p); DAQ_Verdict verdict = MAX_DAQ_VERDICT; if (p->active->packet_retry_requested()) { - retry_queue->put(p->daq_msg); + add_to_retry_queue(p->daq_msg); daq_stats.retries_queued++; } - else if (p->active->is_packet_held() and Stream::set_packet_action_to_hold(p)) + else { - if (p->flow->flags.trigger_detained_packet_event) - DataBus::publish(DETAINED_PACKET_EVENT, p); + msg_was_held = (p->active->is_packet_held() and Stream::set_packet_action_to_hold(p)); + if (msg_was_held) + { + if (p->flow->flags.trigger_detained_packet_event) + { + DataBus::publish(DETAINED_PACKET_EVENT, p); + } + } + else + verdict = distill_verdict(p); } - else - verdict = distill_verdict(p); if (PacketTracer::is_active()) { @@ -315,7 +332,7 @@ void Analyzer::post_process_daq_pkt_msg(Packet* p) if (p->active->packet_retry_requested()) PacketTracer::log("Verdict: Queuing for Retry\n"); - else if (p->active->is_packet_held()) + else if (msg_was_held) PacketTracer::log("Verdict: Holding for Detection\n"); else PacketTracer::log("Verdict: %s\n", SFDAQ::verdict_to_string(verdict)); @@ -426,6 +443,9 @@ void Analyzer::process_retry_queue() { process_daq_msg(msg, true); daq_stats.retries_processed++; + + // Decrease memcap now that msg has been finalized. + memory::MemoryCap::update_deallocations(daq_msg_get_data_len(msg)); } } } diff --git a/src/main/analyzer.h b/src/main/analyzer.h index c244c16e5..18e00e3e0 100644 --- a/src/main/analyzer.h +++ b/src/main/analyzer.h @@ -96,6 +96,7 @@ public: bool process_rebuilt_packet(snort::Packet*, const DAQ_PktHdr_t*, const uint8_t* pkt, uint32_t pktlen); bool inspect_rebuilt(snort::Packet*); void finalize_daq_message(DAQ_Msg_h, DAQ_Verdict); + void add_to_retry_queue(DAQ_Msg_h); // Functions called by analyzer commands void start(); diff --git a/src/main/test/distill_verdict_test.cc b/src/main/test/distill_verdict_test.cc index 42765c8cb..cf57a8796 100644 --- a/src/main/test/distill_verdict_test.cc +++ b/src/main/test/distill_verdict_test.cc @@ -26,7 +26,9 @@ #include "stubs.h" #include "main/analyzer.h" +#include "memory/memory_cap.h" #include "packet_io/sfdaq_instance.h" +#include "packet_io/sfdaq.h" #include #include @@ -45,8 +47,13 @@ void DeferredTrust::set_deferred_trust(unsigned, bool on) deferred_trust = on ? TRUST_DEFER_ON : TRUST_DEFER_OFF; } void Flow::trust() { } + +SFDAQInstance* SFDAQ::get_local_instance() { return nullptr; } } +void memory::MemoryCap::update_allocations(size_t) { } +void memory::MemoryCap::update_deallocations(size_t) { } + using namespace snort; //-------------------------------------------------------------------------- diff --git a/src/packet_io/active.cc b/src/packet_io/active.cc index 15b51a575..b93d12116 100644 --- a/src/packet_io/active.cc +++ b/src/packet_io/active.cc @@ -606,7 +606,8 @@ bool Active::retry_packet(const Packet* p) return false; // FIXIT-L semi-arbitrary heuristic for preventing retry queue saturation - reevaluate later - if (!p->daq_instance || p->daq_instance->get_pool_available() < p->daq_instance->get_batch_size()) + SFDAQInstance* daq_instance = p->daq_instance ? p->daq_instance : SFDAQ::get_local_instance(); + if (!daq_instance || daq_instance->get_pool_available() < daq_instance->get_batch_size()) { // Fall back on dropping the packet and relying on the host to retransmit active_action = ACT_DROP; diff --git a/src/service_inspectors/dce_rpc/dce_smb2_session.cc b/src/service_inspectors/dce_rpc/dce_smb2_session.cc index 180f60e56..bff04ba47 100644 --- a/src/service_inspectors/dce_rpc/dce_smb2_session.cc +++ b/src/service_inspectors/dce_rpc/dce_smb2_session.cc @@ -24,7 +24,7 @@ #include "dce_smb2_session.h" -inline uint32_t Smb2Tid(const Smb2Hdr* hdr) +uint32_t Smb2Tid(const Smb2Hdr* hdr) { return snort::alignedNtohl(&(((const Smb2SyncHdr*)hdr)->tree_id)); } diff --git a/src/stream/stream.cc b/src/stream/stream.cc index 332c99154..9bf827f89 100644 --- a/src/stream/stream.cc +++ b/src/stream/stream.cc @@ -807,6 +807,17 @@ void Stream::set_no_ack_mode(Flow* flow, bool on_off) tcp_session->set_no_ack(on_off); } +void Stream::partial_flush(Flow* flow, bool to_server) +{ + if ( flow->pkt_type == PktType::TCP ) + { + if ( to_server ) + ((TcpStreamSession*)flow->session)->server.perform_partial_flush(); + else + ((TcpStreamSession*)flow->session)->client.perform_partial_flush(); + } +} + #ifdef UNIT_TEST #include "catch/snort_catch.h" diff --git a/src/stream/stream.h b/src/stream/stream.h index 2fb9a61a1..372852e5d 100644 --- a/src/stream/stream.h +++ b/src/stream/stream.h @@ -241,6 +241,7 @@ public: static bool set_packet_action_to_hold(Packet*); static void set_no_ack_mode(Flow*, bool); + static void partial_flush(Flow*, bool to_server); private: static void set_ip_protocol(Flow*); diff --git a/src/stream/tcp/tcp_module.cc b/src/stream/tcp/tcp_module.cc index cc4714cbe..663686efa 100644 --- a/src/stream/tcp/tcp_module.cc +++ b/src/stream/tcp/tcp_module.cc @@ -103,6 +103,7 @@ const PegInfo tcp_pegs[] = { CountType::SUM, "held_packets_passed", "number of held packets passed" }, { CountType::SUM, "held_packet_timeouts", "number of held packets that timed out" }, { CountType::SUM, "held_packet_purges", "number of held packets that were purged without flushing" }, + { CountType::SUM, "held_packet_retries", "number of held packets that were added to the retry queue" }, { CountType::NOW, "cur_packets_held", "number of packets currently held" }, { CountType::MAX, "max_packets_held", "maximum number of packets held simultaneously" }, { CountType::SUM, "partial_flushes", "number of partial flushes initiated" }, diff --git a/src/stream/tcp/tcp_module.h b/src/stream/tcp/tcp_module.h index 61653daef..2f6e84596 100644 --- a/src/stream/tcp/tcp_module.h +++ b/src/stream/tcp/tcp_module.h @@ -103,6 +103,7 @@ struct TcpStats PegCount held_packets_passed; PegCount held_packet_timeouts; PegCount held_packet_purges; + PegCount held_packet_retries; PegCount current_packets_held; PegCount max_packets_held; PegCount partial_flushes; diff --git a/src/stream/tcp/tcp_stream_tracker.cc b/src/stream/tcp/tcp_stream_tracker.cc index 85e1040ac..de21ed73d 100644 --- a/src/stream/tcp/tcp_stream_tracker.cc +++ b/src/stream/tcp/tcp_stream_tracker.cc @@ -25,9 +25,12 @@ #include "tcp_stream_tracker.h" +#include + #include "log/messages.h" #include "main/analyzer.h" #include "main/snort.h" +#include "memory/memory_cap.h" #include "packet_io/active.h" #include "profiler/profiler_defs.h" #include "protocols/eth.h" @@ -627,6 +630,10 @@ bool TcpStreamTracker::set_held_packet(Packet* p) if ( held_packet != null_iterator ) return false; + // Temporarily increase memcap until message is finalized in case + // DAQ makes a copy of the data buffer. + memory::MemoryCap::update_allocations(daq_msg_get_data_len(p->daq_msg)); + held_packet = hpq->append(p->daq_msg, p->ptrs.tcph->seq(), *this); tcpStats.total_packets_held++; @@ -671,18 +678,31 @@ void TcpStreamTracker::finalize_held_packet(Packet* cp) { if ( held_packet != null_iterator ) { + DAQ_Msg_h msg = held_packet->get_daq_msg(); + uint32_t msglen = daq_msg_get_data_len(msg); + if ( cp->active->packet_was_dropped() ) { DAQ_Verdict verdict = held_packet->has_expired() ? DAQ_VERDICT_BLACKLIST : DAQ_VERDICT_BLOCK; - Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), verdict); + Analyzer::get_local_analyzer()->finalize_daq_message(msg, verdict); tcpStats.held_packets_dropped++; } else { - Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), DAQ_VERDICT_PASS); - tcpStats.held_packets_passed++; + if ( cp->active->packet_retry_requested() ) + { + tcpStats.held_packet_retries++; + Analyzer::get_local_analyzer()->add_to_retry_queue(msg); + } + else + { + Analyzer::get_local_analyzer()->finalize_daq_message(msg, DAQ_VERDICT_PASS); + tcpStats.held_packets_passed++; + } } + memory::MemoryCap::update_deallocations(msglen); + hpq->erase(held_packet); held_packet = null_iterator; tcpStats.current_packets_held--; @@ -696,19 +716,24 @@ void TcpStreamTracker::finalize_held_packet(Flow* flow) { if ( held_packet != null_iterator ) { + DAQ_Msg_h msg = held_packet->get_daq_msg(); + uint32_t msglen = daq_msg_get_data_len(msg); + if ( (flow->session_state & STREAM_STATE_BLOCK_PENDING) || (flow->ssn_state.session_flags & SSNFLAG_BLOCK) ) { DAQ_Verdict verdict = held_packet->has_expired() ? DAQ_VERDICT_BLACKLIST : DAQ_VERDICT_BLOCK; - Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), verdict); + Analyzer::get_local_analyzer()->finalize_daq_message(msg, verdict); tcpStats.held_packets_dropped++; } else { - Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), DAQ_VERDICT_PASS); + Analyzer::get_local_analyzer()->finalize_daq_message(msg, DAQ_VERDICT_PASS); tcpStats.held_packets_passed++; } + memory::MemoryCap::update_deallocations(msglen); + hpq->erase(held_packet); held_packet = null_iterator; tcpStats.current_packets_held--;