Merge pull request #2781 in SNORT/snort3 from ~STECHEW/snort3:ftps_tls_alert to master

Squashed commit of the following:

commit 41c0f9f0404feb00411a381fddc5a4d8b5fe8d2a
Author: Steve Chew <stechew@cisco.com>
Date:   Wed Mar 24 18:40:40 2021 -0400

    main: Log holding verdict only if packet was actually held.

commit f85ee407474f867c021381d2c5dad01676c100a2
Author: Steve Chew <stechew@cisco.com>
Date:   Fri Mar 19 09:30:21 2021 -0400

    dce_rpc: Fixed prototype mismatch. Smb2Tid doesn't need to be inline.

commit b1c00248536485223c00d2cd66df1fa236d18673
Author: Steve Chew <stechew@cisco.com>
Date:   Mon Mar 8 13:56:29 2021 -0500

    main: Update memcap for detained packets.

commit 9850db7b66e50454048a7744497c057628a07429
Author: Steve Chew <stechew@cisco.com>
Date:   Tue Mar 9 18:06:56 2021 -0500

    packet_io: If packet has no daq_instance, use thread-local daq_instance.

commit a0479bf54f4d882e2bd19c4044c3776330be787a
Author: Steve Chew <stechew@cisco.com>
Date:   Mon Mar 8 13:49:00 2021 -0500

    stream: Add held packet to retry queue when requested.

commit 2e8c00a0ca58c338491bc3a38ed039cc1baba01a
Author: Steve Chew <stechew@cisco.com>
Date:   Mon Mar 8 02:22:02 2021 -0500

    stream: Add partial_flush. Flush one side of flow immediately.

diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc
index eb329a63bd2ef347ad38134e9045db69c50d2535..5891e2149f74860e85e66d0092576dabc5b839a8 100644 (file)
--- a/src/main/analyzer.cc
+++ b/src/main/analyzer.cc
@@ -51,6 +51,7 @@
 #include "managers/ips_manager.h"
 #include "managers/event_manager.h"
 #include "managers/module_manager.h"
+#include "memory/memory_cap.h"
 #include "packet_io/active.h"
 #include "packet_io/sfdaq.h"
 #include "packet_io/sfdaq_config.h"
@@ -285,27 +286,43 @@ static DAQ_Verdict distill_verdict(Packet* p)
     return verdict;
 }
 
+void Analyzer::add_to_retry_queue(DAQ_Msg_h daq_msg)
+{
+    // Temporarily increase memcap until message is finalized in case
+    // DAQ makes a copy of the data buffer.
+    memory::MemoryCap::update_allocations(daq_msg_get_data_len(daq_msg));
+    retry_queue->put(daq_msg);
+}
+
 /*
  * Private message processing methods
  */
 void Analyzer::post_process_daq_pkt_msg(Packet* p)
 {
+    bool msg_was_held = false;
+
     Active::execute(p);
 
     DAQ_Verdict verdict = MAX_DAQ_VERDICT;
 
     if (p->active->packet_retry_requested())
     {
-        retry_queue->put(p->daq_msg);
+        add_to_retry_queue(p->daq_msg);
         daq_stats.retries_queued++;
     }
-    else if (p->active->is_packet_held() and Stream::set_packet_action_to_hold(p))
+    else
     {
-        if (p->flow->flags.trigger_detained_packet_event)
-            DataBus::publish(DETAINED_PACKET_EVENT, p);
+        msg_was_held = (p->active->is_packet_held() and Stream::set_packet_action_to_hold(p));
+        if (msg_was_held)
+        {
+            if (p->flow->flags.trigger_detained_packet_event)
+            {
+                DataBus::publish(DETAINED_PACKET_EVENT, p);
+            }
+        }
+        else
+            verdict = distill_verdict(p);
     }
-    else
-        verdict = distill_verdict(p);
 
     if (PacketTracer::is_active())
     {
@@ -315,7 +332,7 @@ void Analyzer::post_process_daq_pkt_msg(Packet* p)
 
         if (p->active->packet_retry_requested())
             PacketTracer::log("Verdict: Queuing for Retry\n");
-        else if (p->active->is_packet_held())
+        else if (msg_was_held)
             PacketTracer::log("Verdict: Holding for Detection\n");
         else
             PacketTracer::log("Verdict: %s\n", SFDAQ::verdict_to_string(verdict));
@@ -426,6 +443,9 @@ void Analyzer::process_retry_queue()
         {
             process_daq_msg(msg, true);
             daq_stats.retries_processed++;
+
+            // Decrease memcap now that msg has been finalized.
+            memory::MemoryCap::update_deallocations(daq_msg_get_data_len(msg));
         }
     }
 }

diff --git a/src/main/analyzer.h b/src/main/analyzer.h
index c244c16e51f23c03c808d63168192a462700b435..18e00e3e085b786469b7f95ce555e4d7161066fc 100644 (file)
--- a/src/main/analyzer.h
+++ b/src/main/analyzer.h
@@ -96,6 +96,7 @@ public:
     bool process_rebuilt_packet(snort::Packet*, const DAQ_PktHdr_t*, const uint8_t* pkt, uint32_t pktlen);
     bool inspect_rebuilt(snort::Packet*);
     void finalize_daq_message(DAQ_Msg_h, DAQ_Verdict);
+    void add_to_retry_queue(DAQ_Msg_h);
 
     // Functions called by analyzer commands
     void start();

diff --git a/src/main/test/distill_verdict_test.cc b/src/main/test/distill_verdict_test.cc
index 42765c8cb5209d1f2e7ca26164e731094814db2e..cf57a8796976217e406a16ffb545a2ca4990f28b 100644 (file)
--- a/src/main/test/distill_verdict_test.cc
+++ b/src/main/test/distill_verdict_test.cc
@@ -26,7 +26,9 @@
 #include "stubs.h"
 
 #include "main/analyzer.h"
+#include "memory/memory_cap.h"
 #include "packet_io/sfdaq_instance.h"
+#include "packet_io/sfdaq.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
 #include <CppUTest/TestHarness.h>
@@ -45,8 +47,13 @@ void DeferredTrust::set_deferred_trust(unsigned, bool on)
     deferred_trust = on ? TRUST_DEFER_ON : TRUST_DEFER_OFF;
 }
 void Flow::trust() { }
+
+SFDAQInstance* SFDAQ::get_local_instance() { return nullptr; }
 }
 
+void memory::MemoryCap::update_allocations(size_t) { }
+void memory::MemoryCap::update_deallocations(size_t) { }
+
 using namespace snort;
 
 //--------------------------------------------------------------------------

diff --git a/src/packet_io/active.cc b/src/packet_io/active.cc
index 15b51a575d013b014419420a92e26fc92e73f4f6..b93d12116a76649948f8f9c2614f6dda222539b5 100644 (file)
--- a/src/packet_io/active.cc
+++ b/src/packet_io/active.cc
@@ -606,7 +606,8 @@ bool Active::retry_packet(const Packet* p)
         return false;
 
     // FIXIT-L semi-arbitrary heuristic for preventing retry queue saturation - reevaluate later
-    if (!p->daq_instance || p->daq_instance->get_pool_available() < p->daq_instance->get_batch_size())
+    SFDAQInstance* daq_instance = p->daq_instance ? p->daq_instance : SFDAQ::get_local_instance();
+    if (!daq_instance || daq_instance->get_pool_available() < daq_instance->get_batch_size())
     {
         // Fall back on dropping the packet and relying on the host to retransmit
         active_action = ACT_DROP;

diff --git a/src/service_inspectors/dce_rpc/dce_smb2_session.cc b/src/service_inspectors/dce_rpc/dce_smb2_session.cc
index 180f60e56894b51294cbaf72fb9f06e2f89838c5..bff04ba4777ff75d24a1402c87ddd424f20e7959 100644 (file)
--- a/src/service_inspectors/dce_rpc/dce_smb2_session.cc
+++ b/src/service_inspectors/dce_rpc/dce_smb2_session.cc
@@ -24,7 +24,7 @@
 
 #include "dce_smb2_session.h"
 
-inline uint32_t Smb2Tid(const Smb2Hdr* hdr)
+uint32_t Smb2Tid(const Smb2Hdr* hdr)
 {
     return snort::alignedNtohl(&(((const Smb2SyncHdr*)hdr)->tree_id));
 }

diff --git a/src/stream/stream.cc b/src/stream/stream.cc
index 332c9915462d82fa1189370ecb0b29ff7902a9f5..9bf827f897558f6f026096e3d7e829cffa775674 100644 (file)
--- a/src/stream/stream.cc
+++ b/src/stream/stream.cc
@@ -807,6 +807,17 @@ void Stream::set_no_ack_mode(Flow* flow, bool on_off)
     tcp_session->set_no_ack(on_off);
 }
 
+void Stream::partial_flush(Flow* flow, bool to_server)
+{
+    if ( flow->pkt_type == PktType::TCP )
+    {
+        if ( to_server )
+            ((TcpStreamSession*)flow->session)->server.perform_partial_flush();
+        else
+            ((TcpStreamSession*)flow->session)->client.perform_partial_flush();
+    }
+}
+
 #ifdef UNIT_TEST
 
 #include "catch/snort_catch.h"

diff --git a/src/stream/stream.h b/src/stream/stream.h
index 2fb9a61a1131ce98ca0f210adbdebc63e0c9e691..372852e5d60f90d2f6f2a646dc74c658e2bf674a 100644 (file)
--- a/src/stream/stream.h
+++ b/src/stream/stream.h
@@ -241,6 +241,7 @@ public:
 
     static bool set_packet_action_to_hold(Packet*);
     static void set_no_ack_mode(Flow*, bool);
+    static void partial_flush(Flow*, bool to_server);
 
 private:
     static void set_ip_protocol(Flow*);

diff --git a/src/stream/tcp/tcp_module.cc b/src/stream/tcp/tcp_module.cc
index cc4714cbe43974caaee85296eb6d7c9c5c58a815..663686efa9a3cd21bee697a9b6d97d69114aca72 100644 (file)
--- a/src/stream/tcp/tcp_module.cc
+++ b/src/stream/tcp/tcp_module.cc
@@ -103,6 +103,7 @@ const PegInfo tcp_pegs[] =
     { CountType::SUM, "held_packets_passed", "number of held packets passed" },
     { CountType::SUM, "held_packet_timeouts", "number of held packets that timed out" },
     { CountType::SUM, "held_packet_purges", "number of held packets that were purged without flushing" },
+    { CountType::SUM, "held_packet_retries", "number of held packets that were added to the retry queue" },
     { CountType::NOW, "cur_packets_held", "number of packets currently held" },
     { CountType::MAX, "max_packets_held", "maximum number of packets held simultaneously" },
     { CountType::SUM, "partial_flushes", "number of partial flushes initiated" },

diff --git a/src/stream/tcp/tcp_module.h b/src/stream/tcp/tcp_module.h
index 61653daef4e2b9a095013b9354c0e48092170ad6..2f6e8459616b5217517b661217de8cd31255c19e 100644 (file)
--- a/src/stream/tcp/tcp_module.h
+++ b/src/stream/tcp/tcp_module.h
@@ -103,6 +103,7 @@ struct TcpStats
     PegCount held_packets_passed;
     PegCount held_packet_timeouts;
     PegCount held_packet_purges;
+    PegCount held_packet_retries;
     PegCount current_packets_held;
     PegCount max_packets_held;
     PegCount partial_flushes;

diff --git a/src/stream/tcp/tcp_stream_tracker.cc b/src/stream/tcp/tcp_stream_tracker.cc
index 85e1040ac53221308e55fcf5c8afc8c4c359053c..de21ed73de2390af720aa480b219b88dd8fa5de9 100644 (file)
--- a/src/stream/tcp/tcp_stream_tracker.cc
+++ b/src/stream/tcp/tcp_stream_tracker.cc
@@ -25,9 +25,12 @@
 
 #include "tcp_stream_tracker.h"
 
+#include <daq.h>
+
 #include "log/messages.h"
 #include "main/analyzer.h"
 #include "main/snort.h"
+#include "memory/memory_cap.h"
 #include "packet_io/active.h"
 #include "profiler/profiler_defs.h"
 #include "protocols/eth.h"
@@ -627,6 +630,10 @@ bool TcpStreamTracker::set_held_packet(Packet* p)
     if ( held_packet != null_iterator )
         return false;
 
+    // Temporarily increase memcap until message is finalized in case
+    // DAQ makes a copy of the data buffer.
+    memory::MemoryCap::update_allocations(daq_msg_get_data_len(p->daq_msg));
+
     held_packet = hpq->append(p->daq_msg, p->ptrs.tcph->seq(), *this);
 
     tcpStats.total_packets_held++;
@@ -671,18 +678,31 @@ void TcpStreamTracker::finalize_held_packet(Packet* cp)
 {
     if ( held_packet != null_iterator )
     {
+        DAQ_Msg_h msg = held_packet->get_daq_msg();
+        uint32_t msglen = daq_msg_get_data_len(msg);
+
         if ( cp->active->packet_was_dropped() )
         {
             DAQ_Verdict verdict = held_packet->has_expired() ? DAQ_VERDICT_BLACKLIST : DAQ_VERDICT_BLOCK;
-            Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), verdict);
+            Analyzer::get_local_analyzer()->finalize_daq_message(msg, verdict);
             tcpStats.held_packets_dropped++;
         }
         else
         {
-            Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), DAQ_VERDICT_PASS);
-            tcpStats.held_packets_passed++;
+            if ( cp->active->packet_retry_requested() )
+            {
+                tcpStats.held_packet_retries++;
+                Analyzer::get_local_analyzer()->add_to_retry_queue(msg);
+            }
+            else
+            {
+                Analyzer::get_local_analyzer()->finalize_daq_message(msg, DAQ_VERDICT_PASS);
+                tcpStats.held_packets_passed++;
+            }
         }
 
+        memory::MemoryCap::update_deallocations(msglen);
+
         hpq->erase(held_packet);
         held_packet = null_iterator;
         tcpStats.current_packets_held--;
@@ -696,19 +716,24 @@ void TcpStreamTracker::finalize_held_packet(Flow* flow)
 {
     if ( held_packet != null_iterator )
     {
+        DAQ_Msg_h msg = held_packet->get_daq_msg();
+        uint32_t msglen = daq_msg_get_data_len(msg);
+
         if ( (flow->session_state & STREAM_STATE_BLOCK_PENDING) ||
              (flow->ssn_state.session_flags & SSNFLAG_BLOCK) )
         {
             DAQ_Verdict verdict = held_packet->has_expired() ? DAQ_VERDICT_BLACKLIST : DAQ_VERDICT_BLOCK;
-            Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), verdict);
+            Analyzer::get_local_analyzer()->finalize_daq_message(msg, verdict);
             tcpStats.held_packets_dropped++;
         }
         else
         {
-            Analyzer::get_local_analyzer()->finalize_daq_message(held_packet->get_daq_msg(), DAQ_VERDICT_PASS);
+            Analyzer::get_local_analyzer()->finalize_daq_message(msg, DAQ_VERDICT_PASS);
             tcpStats.held_packets_passed++;
         }
 
+        memory::MemoryCap::update_deallocations(msglen);
+
         hpq->erase(held_packet);
         held_packet = null_iterator;
         tcpStats.current_packets_held--;