stream: improve FIN checking

After recent next_seq changes, the FIN checks could be too strict
leading to stalling sessions in IPS mode.

This patch requires a FIN to be >= last ack and <= next_win to be
accepted.

(cherry picked from commit 39a6f411e9748e76211b76b1f45b6c1863012972)

diff --git a/src/stream-tcp.c b/src/stream-tcp.c
index f4c7ce8420236799619173ec72264230915257b2..c63e4ec2a6b74d08bfe680a3b4053828bf6797c9 100644 (file)
--- a/src/stream-tcp.c
+++ b/src/stream-tcp.c
@@ -2985,9 +2985,13 @@ static int StreamTcpHandleFin(ThreadVars *tv, StreamTcpThread *stt,
             return -1;
         }
 
-        if (SEQ_LT(TCP_GET_SEQ(p), ssn->client.next_seq) ||
-            SEQ_GT(TCP_GET_SEQ(p), (ssn->client.last_ack + ssn->client.window)))
-        {
+        const uint32_t pkt_re = TCP_GET_SEQ(p) + p->payload_len;
+        SCLogDebug("ssn %p: -> SEQ %u, re %u. last_ack %u next_win %u", ssn, TCP_GET_SEQ(p), pkt_re,
+                ssn->client.last_ack, ssn->client.next_win);
+        if (SEQ_GEQ(TCP_GET_SEQ(p), ssn->client.last_ack) &&
+                SEQ_LEQ(pkt_re, ssn->client.next_win)) {
+            // within expectations
+        } else {
             SCLogDebug("ssn %p: -> SEQ mismatch, packet SEQ %" PRIu32 " != "
                     "%" PRIu32 " from stream", ssn, TCP_GET_SEQ(p),
                     ssn->client.next_seq);
@@ -3038,9 +3042,13 @@ static int StreamTcpHandleFin(ThreadVars *tv, StreamTcpThread *stt,
             return -1;
         }
 
-        if (SEQ_LT(TCP_GET_SEQ(p), ssn->server.next_seq) ||
-                SEQ_GT(TCP_GET_SEQ(p), (ssn->server.last_ack + ssn->server.window)))
-        {
+        const uint32_t pkt_re = TCP_GET_SEQ(p) + p->payload_len;
+        SCLogDebug("ssn %p: -> SEQ %u, re %u. last_ack %u next_win %u", ssn, TCP_GET_SEQ(p), pkt_re,
+                ssn->server.last_ack, ssn->server.next_win);
+        if (SEQ_GEQ(TCP_GET_SEQ(p), ssn->server.last_ack) &&
+                SEQ_LEQ(pkt_re, ssn->server.next_win)) {
+            // within expectations
+        } else {
             SCLogDebug("ssn %p: -> SEQ mismatch, packet SEQ %" PRIu32 " != "
                     "%" PRIu32 " from stream (last_ack %u win %u = %u)", ssn, TCP_GET_SEQ(p),
                     ssn->server.next_seq, ssn->server.last_ack, ssn->server.window, (ssn->server.last_ack + ssn->server.window));