fix paf-type flushing of single segments

thanks to joaosoares11@hotmail.com for reporting the issue

also fixes double counting of reassembled buffers
minor refactor of flush loop for clarity

diff --git a/src/stream/paf.cc b/src/stream/paf.cc
index d49911dbcbacbab999c23c2addfb9fa827253dca..1d9e4b1d67cf8afd13a92d78e0cf68c002361913 100644 (file)
--- a/src/stream/paf.cc
+++ b/src/stream/paf.cc
@@ -144,6 +144,7 @@ static bool paf_callback (
     StreamSplitter* ss, PAF_State* ps, Flow* ssn,
     const uint8_t* data, uint32_t len, uint32_t flags)
 {
+    ps->fpt = 0;
     ps->paf = ss->scan(ssn, data, len, flags, &ps->fpt);
 
     if ( ps->paf == StreamSplitter::ABORT )

diff --git a/src/stream/tcp/tcp_reassembler.cc b/src/stream/tcp/tcp_reassembler.cc
index 1874bf5ee24186692ffe5ba799cfaff269984d11..5029f51c632348ae5089add3bf2e7f53c43254f3 100644 (file)
--- a/src/stream/tcp/tcp_reassembler.cc
+++ b/src/stream/tcp/tcp_reassembler.cc
@@ -456,7 +456,7 @@ uint32_t TcpReassembler::get_flush_data_len(TcpSegmentNode* tsn, uint32_t to_seq
 }
 
 // flush the client seglist up to the most recently acked segment
-int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq)
+int TcpReassembler::flush_data_segments(Packet* p, uint32_t total)
 {
     uint32_t bytes_flushed = 0;
     uint32_t segs = 0;
@@ -466,18 +466,19 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq)
     assert(seglist.next);
     Profile profile(s5TcpBuildPacketPerfStats);
 
-    uint32_t total = toSeq - seglist.next->seq;
-    while ( SEQ_LT(seglist.next->seq, toSeq) )
+    uint32_t to_seq = seglist.next->seq + total;
+
+    while ( SEQ_LT(seglist.next->seq, to_seq) )
     {
         TcpSegmentNode* tsn = seglist.next, * sr = nullptr;
-        unsigned bytes_to_copy = get_flush_data_len(tsn, toSeq, tracker->splitter->max(p->flow));
+        unsigned bytes_to_copy = get_flush_data_len(tsn, to_seq, tracker->splitter->max(p->flow));
         unsigned bytes_copied = 0;
         assert(bytes_to_copy);
 
         DebugFormat(DEBUG_STREAM_STATE, "Flushing %u bytes from %X\n", bytes_to_copy, tsn->seq);
 
         if ( !tsn->next || ( bytes_to_copy < tsn->payload_size )
-            || SEQ_EQ(tsn->seq +  bytes_to_copy, toSeq) )
+            || SEQ_EQ(tsn->seq +  bytes_to_copy, to_seq) )
             flags |= PKT_PDU_TAIL;
 
         const StreamBuffer* sb = tracker->splitter->reassemble(
@@ -508,27 +509,25 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t toSeq)
         flush_count++;
         segs++;
 
-        if ( SEQ_EQ(tsn->seq + bytes_to_copy, toSeq) )
+        seglist.next = tsn->next;
+
+        if ( SEQ_EQ(tsn->seq + bytes_to_copy, to_seq) )
             break;
 
         /* Check for a gap/missing packet */
         // FIXIT-L PAF should account for missing data and resume
         // scanning at the start of next PDU instead of aborting.
-        // FIXIT-L FIN may be in toSeq causing bogus gap counts.
+        // FIXIT-L FIN may be in to_seq causing bogus gap counts.
         if (((tsn->next && (tsn->seq + tsn->payload_size != tsn->next->seq))
-            || (!tsn->next && (tsn->seq + tsn->payload_size < toSeq)))
+            || (!tsn->next && (tsn->seq + tsn->payload_size < to_seq)))
             && !(tracker->get_tf_flags() & TF_FIRST_PKT_MISSING))
         {
-            if ( tsn->next )
-                seglist.next = tsn->next;
-
-            // FIXIT-L this is suboptimal - better to exclude fin from toSeq
-            if ( !tracker->fin_set() or SEQ_LEQ(toSeq, tracker->fin_final_seq) )
+            // FIXIT-L this is suboptimal - better to exclude fin from to_seq
+            if ( !tracker->fin_set() or SEQ_LEQ(to_seq, tracker->fin_final_seq) )
                 tracker->set_tf_flags(TF_MISSING_PKT);
 
             break;
         }
-        seglist.next = tsn->next;
 
         if ( sb || !seglist.next )
             break;
@@ -600,37 +599,29 @@ int TcpReassembler::_flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags)
 {
     Profile profile(s5TcpFlushPerfStats);
 
-    uint32_t stop_seq;
-    uint32_t footprint;
-    uint32_t bytes_processed = 0;
-    int32_t flushed_bytes;
+    DAQ_PktHdr_t pkth;
     EncodeFlags enc_flags = 0;
 
-    DAQ_PktHdr_t pkth;
     session->GetPacketHeaderFoo(&pkth, pkt_flags);
     PacketManager::format_tcp(enc_flags, p, s5_pkt, PSEUDO_PKT_TCP, &pkth, pkth.opaque);
-
     prep_s5_pkt(session->flow, p, pkt_flags);
 
-    // FIXIT-L this should not be necessary here
-    seglist_base_seq = seglist.next->seq;
-    stop_seq = seglist_base_seq + bytes;
+    DebugFormat(DEBUG_STREAM_STATE, "Attempting to flush %u bytes\n", bytes);
+
+    uint32_t bytes_processed = 0;
+    uint32_t stop_seq = seglist.next->seq + bytes;
 
     do
     {
-        footprint = stop_seq - seglist_base_seq;
+        seglist_base_seq = seglist.next->seq;
+        uint32_t footprint = stop_seq - seglist_base_seq;
 
-        if (footprint == 0)
+        if ( footprint == 0 )
             return bytes_processed;
 
-        if (footprint > s5_pkt->max_dsize )
-        {
+        if ( footprint > s5_pkt->max_dsize )
             /* this is as much as we can pack into a stream buffer */
             footprint = s5_pkt->max_dsize;
-            stop_seq = seglist_base_seq + footprint;
-        }
-
-        DebugFormat(DEBUG_STREAM_STATE, "Attempting to flush %u bytes\n", footprint);
 
         ((DAQ_PktHdr_t*)s5_pkt->pkth)->ts.tv_sec = seglist.next->tv.tv_sec;
         ((DAQ_PktHdr_t*)s5_pkt->pkth)->ts.tv_usec = seglist.next->tv.tv_usec;
@@ -638,7 +629,8 @@ int TcpReassembler::_flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags)
         /* setup the pseudopacket payload */
         s5_pkt->dsize = 0;
         s5_pkt->data = nullptr;
-        flushed_bytes = flush_data_segments(p, stop_seq);
+
+        int32_t flushed_bytes = flush_data_segments(p, footprint);
 
         if ( flushed_bytes == 0 )
             break; /* No more data... bail */
@@ -666,8 +658,7 @@ int TcpReassembler::_flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags)
         }
         else
         {
-            tcpStats.rebuilt_buffers++;
-            tcpStats.rebuilt_buffers++;
+            tcpStats.rebuilt_buffers++; // FIXIT-L this is not accurate
         }
 
         DebugFormat(DEBUG_STREAM_STATE, "setting seglist_base_seq to 0x%X\n", seglist_base_seq);
@@ -711,7 +702,8 @@ int TcpReassembler::flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags)
         return 0;
     }
 
-    if ( !flush_data_ready( ) && !( tracker->get_tf_flags() & TF_FORCE_FLUSH ) )
+    if ( !flush_data_ready() and !(tracker->get_tf_flags() & TF_FORCE_FLUSH) and
+        (!tracker->splitter or !tracker->splitter->is_paf()) )
     {
         DebugMessage(DEBUG_STREAM_STATE, "only 1 packet in seglist no need to flush\n");
         return 0;

diff --git a/src/stream/tcp/tcp_reassembler.h b/src/stream/tcp/tcp_reassembler.h
index 53c5b9bb2818121728587193ad7e9f805624decf..24c49857028e260b1f90c24260ad124058311658 100644 (file)
--- a/src/stream/tcp/tcp_reassembler.h
+++ b/src/stream/tcp/tcp_reassembler.h
@@ -130,7 +130,7 @@ protected:
     int add_reassembly_segment(TcpSegmentDescriptor&, int16_t len, uint32_t slide, uint32_t trunc,
         uint32_t seq, TcpSegmentNode* left) override;
     int dup_reassembly_segment(TcpSegmentNode* left, TcpSegmentNode** retSeg) override;
-    int delete_reassembly_segment(TcpSegmentNode* seg) override;
+    int delete_reassembly_segment(TcpSegmentNode*) override;
 
     virtual void insert_segment_in_empty_seglist(TcpSegmentDescriptor&);
     virtual int insert_segment_in_seglist(TcpSegmentDescriptor&);
@@ -139,23 +139,23 @@ protected:
     virtual uint32_t get_pending_segment_count(unsigned max);
 
     bool flush_data_ready();
-    int trim_delete_reassembly_segment(TcpSegmentNode* seg, uint32_t flush_seq);
-    void queue_reassembly_segment(TcpSegmentNode* prev, TcpSegmentNode* ss);
+    int trim_delete_reassembly_segment(TcpSegmentNode*, uint32_t flush_seq);
+    void queue_reassembly_segment(TcpSegmentNode* prev, TcpSegmentNode*);
     void init_overlap_editor(TcpSegmentDescriptor&);
     bool is_segment_fasttrack(TcpSegmentNode* tail, TcpSegmentDescriptor&);
-    int purge_alerts(uint32_t /*flush_seq*/,  Flow* flow);
-    void show_rebuilt_packet(Packet* pkt);
-    uint32_t get_flush_data_len(TcpSegmentNode* ss, uint32_t to_seq, unsigned max);
-    int flush_data_segments(Packet* p, uint32_t toSeq);
-    void prep_s5_pkt(Flow* flow, Packet* p, uint32_t pkt_flags);
-    int _flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags);
-    int flush_to_seq(uint32_t bytes, Packet* p, uint32_t pkt_flags);
+    int purge_alerts(uint32_t /*flush_seq*/,  Flow*);
+    void show_rebuilt_packet(Packet*);
+    uint32_t get_flush_data_len(TcpSegmentNode*, uint32_t to_seq, unsigned max);
+    int flush_data_segments(Packet*, uint32_t total);
+    void prep_s5_pkt(Flow*, Packet*, uint32_t pkt_flags);
+    int _flush_to_seq(uint32_t bytes, Packet*, uint32_t pkt_flags);
+    int flush_to_seq(uint32_t bytes, Packet*, uint32_t pkt_flags);
     uint32_t get_q_footprint();
     uint32_t get_q_sequenced();
-    void final_flush(Packet* p, PegCount& peg, uint32_t dir);
-    uint32_t get_reverse_packet_dir(const Packet* p);
-    uint32_t get_forward_packet_dir(const Packet* p);
-    int32_t flush_pdu_ips(uint32_t* flags);
+    void final_flush(Packet*, PegCount&, uint32_t dir);
+    uint32_t get_reverse_packet_dir(const Packet*);
+    uint32_t get_forward_packet_dir(const Packet*);
+    int32_t flush_pdu_ips(uint32_t*);
     void fallback();
     int32_t flush_pdu_ackd(uint32_t* flags);
     int purge_to_seq(uint32_t flush_seq);