Merge pull request #616 in SNORT/snort3 from crc_flush to master

Squashed commit of the following:

commit f7e5fd34e3bee33e92ce677181408374f019f27e
Author: Russ Combs <rucombs@cisco.com>
Date:   Tue Sep 6 21:41:54 2016 -0400

    fully cut over to use stream splitter reassembly buffer and size / max
    squelch bogus gap counts

diff --git a/src/stream/libtcp/tcp_stream_tracker.h b/src/stream/libtcp/tcp_stream_tracker.h
index 8e033af3ea7a1eba0b0a1418e9b83a1cb6aec213..90f879e3081c99871c0ecec90685b364d840c225 100644 (file)
--- a/src/stream/libtcp/tcp_stream_tracker.h
+++ b/src/stream/libtcp/tcp_stream_tracker.h
@@ -240,6 +240,10 @@ public:
         this->fin_final_seq = fin_final_seq;
     }
 
+    // FIXIT-M fin_final_seq can be zero so need to use current state
+    // or other flag to know when it is actually set
+    bool fin_set() { return fin_final_seq != 0; }
+
     uint32_t get_ts_last_packet() const
     {
         return ts_last_packet;

diff --git a/src/stream/stream_splitter.cc b/src/stream/stream_splitter.cc
index 584693a709e47c37e7c7644668c4c1a3dcc15a31..eb8240a1eb7fa73ebc14449a7c681d4ab18f9f60 100644 (file)
--- a/src/stream/stream_splitter.cc
+++ b/src/stream/stream_splitter.cc
@@ -25,7 +25,7 @@
 #include "flush_bucket.h"
 #include "protocols/packet.h"
 
-static THREAD_LOCAL uint8_t pdu_buf[65536];
+static THREAD_LOCAL uint8_t pdu_buf[StreamSplitter::max_buf];
 static THREAD_LOCAL StreamBuffer str_buf;
 
 unsigned StreamSplitter::max_pdu = 16384;

diff --git a/src/stream/stream_splitter.h b/src/stream/stream_splitter.h
index 2be13751231e39449f4af7951603c34983520c74..e45a57095c7b1eb3082f4e3927278bbfddf8943c 100644 (file)
--- a/src/stream/stream_splitter.h
+++ b/src/stream/stream_splitter.h
@@ -84,6 +84,9 @@ public:
     // paf_max; the HI splitter should pull from there
     static void set_max(unsigned);
 
+    // FIXIT-L max_pdu should suffice
+    static const unsigned max_buf = 65536;
+
     virtual void reset() { }
     virtual void update() { }
 

diff --git a/src/stream/tcp/tcp_reassembler.cc b/src/stream/tcp/tcp_reassembler.cc
index c3581023d8824e18f2ce819bfa2e6651aae9015a..e1ce0c7704fc53a0815d21f7d0b4fadc80b13c11 100644 (file)
--- a/src/stream/tcp/tcp_reassembler.cc
+++ b/src/stream/tcp/tcp_reassembler.cc
@@ -457,14 +457,12 @@ void TcpReassembler::show_rebuilt_packet(Packet* pkt)
     }
 }
 
-uint32_t TcpReassembler::get_flush_data_len(TcpSegmentNode* tsn, uint32_t to_seq,
-    uint32_t flushBufSize)
+uint32_t TcpReassembler::get_flush_data_len(TcpSegmentNode* tsn, uint32_t to_seq, unsigned max)
 {
     unsigned int flushSize = tsn->payload_size;
 
-    // copy only till flush buffer gets full
-    if ( flushSize > flushBufSize )
-        flushSize = flushBufSize;
+    if ( flushSize > max )
+        flushSize = max;
 
     // copy only to flush point
     if ( paf_active(&tracker->paf_state) && SEQ_GT(tsn->seq + flushSize, to_seq) )
@@ -474,10 +472,9 @@ uint32_t TcpReassembler::get_flush_data_len(TcpSegmentNode* tsn, uint32_t to_seq
 }
 
 // flush the client seglist up to the most recently acked segment
-int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq, uint8_t* flushbuf,
-    const uint8_t* flushbuf_end)
+int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq)
 {
-    uint16_t bytes_flushed = 0;
+    uint32_t bytes_flushed = 0;
     uint32_t segs = 0;
     uint32_t flags = PKT_PDU_HEAD;
     DEBUG_WRAP(uint32_t bytes_queued = seg_bytes_logical; );
@@ -489,8 +486,7 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq, uint8_t* flus
     while ( SEQ_LT(seglist.next->seq, toSeq) )
     {
         TcpSegmentNode* tsn = seglist.next, * sr = nullptr;
-        unsigned flushbuf_size = flushbuf_end - flushbuf;
-        unsigned bytes_to_copy = get_flush_data_len(tsn, toSeq, flushbuf_size);
+        unsigned bytes_to_copy = get_flush_data_len(tsn, toSeq, tracker->splitter->max(p->flow));
         unsigned bytes_copied = 0;
         assert(bytes_to_copy);
 
@@ -500,25 +496,20 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq, uint8_t* flus
             || SEQ_EQ(tsn->seq +  bytes_to_copy, toSeq) )
             flags |= PKT_PDU_TAIL;
 
-        const StreamBuffer* sb = tracker->splitter->reassemble(p->flow, total, bytes_flushed,
-            tsn->payload,
-            bytes_to_copy, flags, bytes_copied);
+        const StreamBuffer* sb = tracker->splitter->reassemble(
+            p->flow, total, bytes_flushed, tsn->payload, bytes_to_copy, flags, bytes_copied);
+
         flags = 0;
+
         if ( sb )
         {
             s5_pkt->data = sb->data;
             s5_pkt->dsize = sb->length;
             assert(sb->length <= s5_pkt->max_dsize);
 
-            // FIXIT-M flushbuf should be eliminated from this function
-            // since we are actually using the stream splitter buffer
-            flushbuf = ( uint8_t* )s5_pkt->data;
-            // ensure we stop here
             bytes_to_copy = bytes_copied;
         }
         assert(bytes_to_copy == bytes_copied);
-
-        flushbuf += bytes_to_copy;
         bytes_flushed += bytes_to_copy;
 
         if ( bytes_to_copy < tsn->payload_size
@@ -533,9 +524,6 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq, uint8_t* flus
         flush_count++;
         segs++;
 
-        if ( flushbuf >= flushbuf_end )
-            break;
-
         if ( SEQ_EQ(tsn->seq + bytes_to_copy, toSeq) )
             break;
 
@@ -550,13 +538,19 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq, uint8_t* flus
             if ( tsn->next )
                 seglist.next = tsn->next;
 
-            tracker->set_tf_flags(TF_MISSING_PKT);
+            // FIXIT-L this is suboptimal - better to exclude fin from toSeq
+            if ( !tracker->fin_set() or SEQ_LT(toSeq, tracker->fin_final_seq) )
+                tracker->set_tf_flags(TF_MISSING_PKT);
+
             break;
         }
         seglist.next = tsn->next;
 
         if ( sb || !seglist.next )
             break;
+
+        if ( bytes_flushed + seglist.next->payload_size >= StreamSplitter::max_buf )
+            break;
     }
 
     DEBUG_WRAP(bytes_queued -= bytes_flushed; );
@@ -634,10 +628,6 @@ int TcpReassembler::_flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags)
 
     prep_s5_pkt(session->flow, p, pkt_flags);
 
-    // if not specified, set bytes to flush to what was acked
-    if (!bytes && SEQ_GT(tracker->r_win_base, seglist_base_seq))
-        bytes = tracker->r_win_base - seglist_base_seq;
-
     // FIXIT-L this should not be necessary here
     seglist_base_seq = seglist.next->seq;
     stop_seq = seglist_base_seq + bytes;
@@ -676,9 +666,8 @@ int TcpReassembler::_flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags)
 
         /* setup the pseudopacket payload */
         s5_pkt->dsize = 0;
-        s5_pkt->data = s5_pkt->pkt;
-        const uint8_t* s5_pkt_end = s5_pkt->data + s5_pkt->max_dsize;
-        flushed_bytes = flush_data_segments(p, stop_seq, (uint8_t*)s5_pkt->data, s5_pkt_end);
+        s5_pkt->data = nullptr;
+        flushed_bytes = flush_data_segments(p, stop_seq);
 
         if ( flushed_bytes == 0 )
             break; /* No more data... bail */
@@ -783,28 +772,26 @@ int TcpReassembler::flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags)
     return _flush_to_seq(bytes, p, pkt_flags);
 }
 
-// FIXIT-H the seq number math in the following 2 funcs does not handle
-// wrapping get the footprint for the current seglist, the difference
+// get the footprint for the current seglist, the difference
 // between our base sequence and the last ack'd sequence we received
 
 uint32_t TcpReassembler::get_q_footprint()
 {
-    int32_t fp;
-
-    if ( tracker == nullptr )
-        return 0;
-
-    fp = tracker->r_win_base - seglist_base_seq;
-    if ( fp <= 0 )
+    if ( !tracker )
         return 0;
 
     seglist.next = seglist.head;
+    uint32_t fp = tracker->r_win_base - seglist_base_seq;
+
+    // FIXIT-M ideally would exclude fin here
+
     return fp;
 }
 
 // FIXIT-P get_q_sequenced() performance could possibly be
 // boosted by tracking sequenced bytes as seglist is updated
 // to avoid the while loop, etc. below.
+
 uint32_t TcpReassembler::get_q_sequenced()
 {
     int32_t len;

diff --git a/src/stream/tcp/tcp_reassembler.h b/src/stream/tcp/tcp_reassembler.h
index 9e23578ffe06d42bd5d7ab2d4ebeae7afc483099..bcb5b9d5de0e2e24af995bc162fcd136d88cd66f 100644 (file)
--- a/src/stream/tcp/tcp_reassembler.h
+++ b/src/stream/tcp/tcp_reassembler.h
@@ -146,9 +146,8 @@ protected:
     bool is_segment_fasttrack(TcpSegmentNode* tail, TcpSegmentDescriptor&);
     int purge_alerts(uint32_t /*flush_seq*/,  Flow* flow);
     void show_rebuilt_packet(Packet* pkt);
-    uint32_t get_flush_data_len(TcpSegmentNode* ss, uint32_t to_seq, uint32_t flushBufSize);
-    int flush_data_segments(Packet* p, uint32_t toSeq,  uint8_t* flushbuf,
-            const  uint8_t* flushbuf_end);
+    uint32_t get_flush_data_len(TcpSegmentNode* ss, uint32_t to_seq, unsigned max);
+    int flush_data_segments(Packet* p, uint32_t toSeq);
     void prep_s5_pkt(Flow* flow, Packet* p, uint32_t pkt_flags);
     int _flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags);
     int flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags);

diff --git a/src/stream/tcp/tcp_tracker.cc b/src/stream/tcp/tcp_tracker.cc
index 8334709aa9784c27d6a19d348a5eddba74be8355..66d1b4aa92d063091ce9f663004e70439c0b89cf 100644 (file)
--- a/src/stream/tcp/tcp_tracker.cc
+++ b/src/stream/tcp/tcp_tracker.cc
@@ -490,7 +490,7 @@ bool TcpTracker::update_on_fin_recv(TcpSegmentDescriptor& tsd)
     r_nxt_ack++;
 
     // set final seq # any packet rx'ed with seq > is bad
-    if ( fin_final_seq == 0 )
+    if ( !fin_set() )
         fin_final_seq = tsd.get_end_seq() + 1;
 
     return true;

diff --git a/src/stream/tcp/tcp_tracker.h b/src/stream/tcp/tcp_tracker.h
index 79bd6cd8fef3351df8e8b64f63d44a75d795b286..81b4ba790e135475b5e5a7a099b50cc7abf5866a 100644 (file)
--- a/src/stream/tcp/tcp_tracker.h
+++ b/src/stream/tcp/tcp_tracker.h
@@ -87,7 +87,6 @@ public:
     void flush_data_on_fin_recv(TcpSegmentDescriptor& tsd) override;
 
     void init_toolbox() override;
-
 };
 
 #endif