stream: add TCP urgent handling options

TCP urgent handling is a complex topic due to conflicting RFCs and
implementations.

Until now the URG flag and urgent pointer values were simply ignored,
leading to an effective "inline" processing of urgent data. Many
implementations however, do not default to this behavior.

Many actual implementations use the urgent mechanism to send 1 byte of
data out of band to the application.

Complicating the matter is that the way the urgent logic is handled is
generally configurable both of the OS and the app level. So from the
network it is impossible to know with confidence what the settings are.

This patch adds the following policies:

`stream.reassembly.urgent.policy`:

- drop: drop URG packets before they affect the stream engine

- inline: ignore the urgent pointer and process all data inline

- oob (out of band): treat the last byte as out of band

- gap: skip the last byte, but do no adjust sequence offsets, leading to
       gaps in the data

For the `oob` option, tracking of a sequence number offset is required,
as the OOB data does "consume" sequence number space. This is limited to
64k. For this reason, there is a second policy:

`stream.reassembly.urgent.oob-limit-policy`:

- drop: drop URG packets before they affect the stream engine

- inline: ignore the urgent pointer and process all data inline

- gap: skip the last byte, but do no adjust sequence offsets, leading to
       gaps in the data

Bug: #7411.
(cherry picked from commit 6882bcb3e51bd3cf509fb6569cc30f48d7bb53d7)

diff --git a/etc/schema.json b/etc/schema.json
index ede9f6653fb4ff6497a32ff1d0d1fa0645158d49..debdd274da86d0c2ef5e20d63a539f381b2ff33a 100644 (file)
--- a/etc/schema.json
+++ b/etc/schema.json
@@ -4157,6 +4157,11 @@
                                 "stream_reassembly": {
                                     "type": "integer"
                                 },
+                                "stream_urgent": {
+                                    "description":
+                                            "Number of packets dropped due to TCP urgent flag",
+                                    "type": "integer"
+                                },
                                 "nfq_error": {
                                     "type": "integer"
                                 },
@@ -5316,6 +5321,10 @@
                         "urg": {
                             "description": "Number of TCP packets with the urgent flag set",
                             "type": "integer"
+                        },
+                        "urgent_oob_data": {
+                            "description": "Number of OOB bytes tracked in TCP urgent handling",
+                            "type": "integer"
                         }
                     },
                     "additionalProperties": false

diff --git a/rules/stream-events.rules b/rules/stream-events.rules
index e589c81b134d78958e3b8698ed9a2ab73cc583a5..46d34a3d684cbe59aed329feeee2574dec84b04e 100644 (file)
--- a/rules/stream-events.rules
+++ b/rules/stream-events.rules
@@ -109,5 +109,7 @@ alert tcp any any -> any any (msg:"SURICATA STREAM FIN SYN reuse"; stream-event:
 # Depth setting reached for a stream. Very common in normal traffic, so disable by default.
 #alert tcp any any -> any any (msg:"SURICATA STREAM reassembly depth reached"; stream-event:reassembly_depth_reached; classtype:protocol-command-decode; sid:2210062; rev:1;)
 
-# next sid 2210066
+alert tcp any any -> any any (msg:"SURICATA STREAM urgent OOB limit reached";  stream-event:reassembly_urgent_oob_limit_reached; classtype:protocol-command-decode; sid:2210066; rev:1;)
+
+# next sid 2210067
 

diff --git a/src/app-layer.c b/src/app-layer.c
index 794e8e84d315d32de131f215603788e3025c6f50..3c416eef75d021512c650d149f6aa68b5d2495ad 100644 (file)
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -684,6 +684,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
     /* If a gap notification, relay the notification on to the
      * app-layer if known. */
     if (flags & STREAM_GAP) {
+        SCLogDebug("GAP of size %u", data_len);
         if (alproto == ALPROTO_UNKNOWN) {
             StreamTcpSetStreamFlagAppProtoDetectionCompleted(*stream);
             SCLogDebug("ALPROTO_UNKNOWN flow %p, due to GAP in stream start", f);

diff --git a/src/decode-events.c b/src/decode-events.c
index 83f8b22efd93f5fe1000fbcf752b613766747ddf..7ff2d946d1d1549fe6d2e4a949b958fa30af2492 100644 (file)
--- a/src/decode-events.c
+++ b/src/decode-events.c
@@ -872,6 +872,10 @@ const struct DecodeEvents_ DEvents[] = {
             "stream.reassembly_insert_invalid",
             STREAM_REASSEMBLY_INSERT_INVALID,
     },
+    {
+            "stream.reassembly_urgent_oob_limit_reached",
+            STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED,
+    },
 
     { NULL, 0 },
 };

diff --git a/src/decode-events.h b/src/decode-events.h
index 5547866fa59bef5c6442df62da2e1f830d83485c..76169bccbfd6887fa0c9c9bcf4f5523a48cb47f8 100644 (file)
--- a/src/decode-events.h
+++ b/src/decode-events.h
@@ -296,6 +296,7 @@ enum {
     STREAM_REASSEMBLY_INSERT_MEMCAP,
     STREAM_REASSEMBLY_INSERT_LIMIT,
     STREAM_REASSEMBLY_INSERT_INVALID,
+    STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED,
 
     /* should always be last! */
     DECODE_EVENT_MAX,

diff --git a/src/decode.c b/src/decode.c
index 9de0ee7d2aaeb5caff71b8a84b9d7d4a757c0cef..ff06929cca32afdcdce02a3c5f6b90ee1e55818b 100644 (file)
--- a/src/decode.c
+++ b/src/decode.c
@@ -804,6 +804,8 @@ const char *PacketDropReasonToString(enum PacketDropReason r)
             return "stream memcap";
         case PKT_DROP_REASON_STREAM_MIDSTREAM:
             return "stream midstream";
+        case PKT_DROP_REASON_STREAM_URG:
+            return "stream urgent";
         case PKT_DROP_REASON_STREAM_REASSEMBLY:
             return "stream reassembly";
         case PKT_DROP_REASON_APPLAYER_ERROR:
@@ -844,6 +846,8 @@ static const char *PacketDropReasonToJsonString(enum PacketDropReason r)
             return "ips.drop_reason.stream_memcap";
         case PKT_DROP_REASON_STREAM_MIDSTREAM:
             return "ips.drop_reason.stream_midstream";
+        case PKT_DROP_REASON_STREAM_URG:
+            return "ips.drop_reason.stream_urgent";
         case PKT_DROP_REASON_STREAM_REASSEMBLY:
             return "ips.drop_reason.stream_reassembly";
         case PKT_DROP_REASON_APPLAYER_ERROR:

diff --git a/src/decode.h b/src/decode.h
index d604d7f69b46f3d1d33b6128d93b0b11bf4b0630..b8c475c2785fdc4dcae667987e736bcbc50af715 100644 (file)
--- a/src/decode.h
+++ b/src/decode.h
@@ -404,6 +404,7 @@ enum PacketDropReason {
     PKT_DROP_REASON_STREAM_MEMCAP,
     PKT_DROP_REASON_STREAM_MIDSTREAM,
     PKT_DROP_REASON_STREAM_REASSEMBLY,
+    PKT_DROP_REASON_STREAM_URG,
     PKT_DROP_REASON_NFQ_ERROR,    /**< no nfq verdict, must be error */
     PKT_DROP_REASON_INNER_PACKET, /**< drop issued by inner (tunnel) packet */
     PKT_DROP_REASON_MAX,

diff --git a/src/stream-tcp-list.c b/src/stream-tcp-list.c
index 91d0d25616ede8f36fc368e5363d60eaa39d6f6d..ff308d179acfae4284a63304688ee845245d8450 100644 (file)
--- a/src/stream-tcp-list.c
+++ b/src/stream-tcp-list.c
@@ -635,8 +635,7 @@ static void StreamTcpSegmentAddPacketData(
  *  In case of error, this function returns the segment to the pool
  */
 int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
-        TcpStream *stream, TcpSegment *seg, Packet *p,
-        uint8_t *pkt_data, uint16_t pkt_datalen)
+        TcpStream *stream, TcpSegment *seg, Packet *p, uint8_t *pkt_data, uint16_t pkt_datalen)
 {
     SCEnter();
 

diff --git a/src/stream-tcp-private.h b/src/stream-tcp-private.h
index 619398deb7c92f38d3e3777df07453f14398c8a4..d39bb91a88da21dc964364195be0fdd312b4dec3 100644 (file)
--- a/src/stream-tcp-private.h
+++ b/src/stream-tcp-private.h
@@ -288,6 +288,8 @@ typedef struct TcpSession_ {
     int8_t data_first_seen_dir;
     /** track all the tcp flags we've seen */
     uint8_t tcp_packet_flags;
+    uint16_t urg_offset_ts;            /**< SEQ offset from accepted OOB urg bytes */
+    uint16_t urg_offset_tc;            /**< SEQ offset from accepted OOB urg bytes */
     /* coccinelle: TcpSession:flags:STREAMTCP_FLAG */
     uint32_t flags;
     uint32_t reassembly_depth; /**< reassembly depth for the stream */

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index 3d4952bb5abd241c0cc06c500ae208fa1a9c449a..31e338838824b937c6877cdb9719146b6342308a 100644 (file)
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -28,10 +28,12 @@
 
 #include "suricata-common.h"
 #include "suricata.h"
+#include "packet.h"
 #include "detect.h"
 #include "flow.h"
 #include "threads.h"
 #include "conf.h"
+#include "action-globals.h"
 
 #include "flow-util.h"
 
@@ -764,9 +766,64 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
         SCReturnInt(0);
     }
 
+    uint16_t *urg_offset;
+    if (PKT_IS_TOSERVER(p)) {
+        urg_offset = &ssn->urg_offset_ts;
+    } else {
+        urg_offset = &ssn->urg_offset_tc;
+    }
+
+    /* segment sequence number, offset by previously accepted
+     * URG OOB data. */
+    uint32_t seg_seq = TCP_GET_RAW_SEQ(p->tcph) - (*urg_offset);
+    uint8_t urg_data = 0;
+
+    /* if stream_config.urgent_policy == TCP_STREAM_URGENT_DROP, we won't get here */
+    if (p->tcph->th_flags & TH_URG) {
+        const uint16_t urg_ptr = SCNtohs(p->tcph->th_urp);
+        if (urg_ptr > 0 && urg_ptr <= p->payload_len &&
+                (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB ||
+                        stream_config.urgent_policy == TCP_STREAM_URGENT_GAP)) {
+            /* track up to 64k out of band URG bytes. Fall back to inline
+             * when that budget is exceeded. */
+            if ((*urg_offset) < UINT16_MAX) {
+                if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB)
+                    (*urg_offset)++;
+
+                if ((*urg_offset) == UINT16_MAX) {
+                    StreamTcpSetEvent(p, STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED);
+                }
+            } else {
+                /* OOB limit DROP is handled here */
+                if (stream_config.urgent_oob_limit_policy == TCP_STREAM_URGENT_DROP) {
+                    PacketDrop(p, ACTION_DROP, PKT_DROP_REASON_STREAM_URG);
+                    SCReturnInt(0);
+                }
+            }
+            urg_data = 1; /* only treat last 1 byte as out of band. */
+            if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB) {
+                StatsIncr(tv, ra_ctx->counter_tcp_urgent_oob);
+            }
+
+            /* depending on hitting the OOB limit, update urg_data or not */
+            if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB &&
+                    (*urg_offset) == UINT16_MAX &&
+                    stream_config.urgent_oob_limit_policy == TCP_STREAM_URGENT_INLINE) {
+                urg_data = 0;
+            } else {
+                if (urg_ptr == 1 && p->payload_len == 1) {
+                    SCLogDebug("no non-URG data");
+                    SCReturnInt(0);
+                }
+            }
+        }
+    }
+
+    const uint16_t payload_len = p->payload_len - urg_data;
+
     /* If we have reached the defined depth for either of the stream, then stop
        reassembling the TCP session */
-    uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
+    uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, seg_seq, payload_len);
     SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
 
     if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) {
@@ -780,9 +837,9 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
         SCReturnInt(0);
     }
 
-    DEBUG_VALIDATE_BUG_ON(size > p->payload_len);
-    if (size > p->payload_len)
-        size = p->payload_len;
+    DEBUG_VALIDATE_BUG_ON(size > payload_len);
+    if (size > payload_len)
+        size = payload_len;
 
     TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
     if (seg == NULL) {
@@ -794,7 +851,8 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
 
     DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
     TCP_SEG_LEN(seg) = (uint16_t)size;
-    seg->seq = TCP_GET_SEQ(p);
+    /* set SEQUENCE number, adjusted to any URG pointer offset */
+    seg->seq = seg_seq;
 
     /* HACK: for TFO SYN packets the seq for data starts at + 1 */
     if (TCP_HAS_TFO(p) && p->payload_len && (p->tcph->th_flags & TH_SYN))
@@ -808,8 +866,7 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
                 APPLAYER_PROTO_DETECTION_SKIPPED);
     }
 
-    int r = StreamTcpReassembleInsertSegment(
-            tv, ra_ctx, stream, seg, p, p->payload, p->payload_len);
+    int r = StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, p->payload, payload_len);
     if (r < 0) {
         if (r == -SC_ENOMEM) {
             ssn->flags |= STREAMTCP_FLAG_LOSSY_BE_LIBERAL;

diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h
index c883e6c0f5ef759ca57d5d5bd0d5c5e6de02078b..aa61f2bb13b1268c282d9fd312e24cf907b01d17 100644 (file)
--- a/src/stream-tcp-reassemble.h
+++ b/src/stream-tcp-reassemble.h
@@ -80,6 +80,9 @@ typedef struct TcpReassemblyThreadCtx_ {
 
     uint16_t counter_tcp_reass_data_normal_fail;
     uint16_t counter_tcp_reass_data_overlap_fail;
+
+    /** count OOB bytes */
+    uint16_t counter_tcp_urgent_oob;
 } TcpReassemblyThreadCtx;
 
 #define OS_POLICY_DEFAULT   OS_POLICY_BSD

diff --git a/src/stream-tcp.c b/src/stream-tcp.c
index a99029ab8481baa89935deebcba9f21deaf28ab1..6aff1537c151202d7b478692348d7812c292f33a 100644 (file)
--- a/src/stream-tcp.c
+++ b/src/stream-tcp.c
@@ -336,6 +336,17 @@ static inline bool StreamTcpInlineDropInvalid(void)
             && (stream_config.flags & STREAMTCP_INIT_FLAG_DROP_INVALID));
 }
 
+/** \internal
+ *  \brief See if stream engine is dropping URG packets in inline mode
+ *  \retval false no
+ *  \retval true yes
+ */
+static inline bool StreamTcpInlineDropUrg(void)
+{
+    return ((stream_config.flags & STREAMTCP_INIT_FLAG_INLINE) &&
+            stream_config.urgent_policy == TCP_STREAM_URGENT_DROP);
+}
+
 /* hack: stream random range code expects random values in range of 0-RAND_MAX,
  * but we can get both <0 and >RAND_MAX values from RandomGet
  */
@@ -350,6 +361,22 @@ static int RandomGetWrap(void)
     return r % RAND_MAX;
 }
 
+static const char *UrgentPolicyToString(enum TcpStreamUrgentHandling pol)
+{
+    switch (pol) {
+        case TCP_STREAM_URGENT_OOB:
+            return "oob";
+        case TCP_STREAM_URGENT_INLINE:
+            return "inline";
+        case TCP_STREAM_URGENT_DROP:
+            return "drop";
+        case TCP_STREAM_URGENT_GAP:
+            return "gap";
+    }
+    return NULL;
+}
+
+
 /** \brief          To initialize the stream global configuration data
  *
  *  \param  quiet   It tells the mode of operation, if it is true nothing will
@@ -499,6 +526,46 @@ void StreamTcpInitConfig(bool quiet)
         stream_config.flags |= STREAMTCP_INIT_FLAG_DROP_INVALID;
     }
 
+    const char *temp_urgpol = NULL;
+    if (ConfGet("stream.reassembly.urgent.policy", &temp_urgpol) == 1 && temp_urgpol != NULL) {
+        if (strcmp(temp_urgpol, "inline") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_INLINE;
+        } else if (strcmp(temp_urgpol, "drop") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_DROP;
+        } else if (strcmp(temp_urgpol, "oob") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_OOB;
+        } else if (strcmp(temp_urgpol, "gap") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_GAP;
+        } else {
+            FatalError("stream.reassembly.urgent.policy: invalid value '%s'", temp_urgpol);
+        }
+    } else {
+        stream_config.urgent_policy = TCP_STREAM_URGENT_DEFAULT;
+    }
+    if (!quiet) {
+        SCLogConfig("stream.reassembly.urgent.policy\": %s", UrgentPolicyToString(stream_config.urgent_policy));
+    }
+    if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB) {
+        const char *temp_urgoobpol = NULL;
+        if (ConfGet("stream.reassembly.urgent.oob-limit-policy", &temp_urgoobpol) == 1 &&
+                temp_urgoobpol != NULL) {
+            if (strcmp(temp_urgoobpol, "inline") == 0) {
+                stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_INLINE;
+            } else if (strcmp(temp_urgoobpol, "drop") == 0) {
+                stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_DROP;
+            } else if (strcmp(temp_urgoobpol, "gap") == 0) {
+                stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_GAP;
+            } else {
+                FatalError("stream.reassembly.urgent.oob-limit-policy: invalid value '%s'", temp_urgoobpol);
+            }
+        } else {
+            stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_DEFAULT;
+        }
+        if (!quiet) {
+            SCLogConfig("stream.reassembly.urgent.oob-limit-policy\": %s", UrgentPolicyToString(stream_config.urgent_oob_limit_policy));
+        }
+    }
+
     if ((ConfGetInt("stream.max-syn-queued", &value)) == 1) {
         if (value >= 0 && value <= 255) {
             stream_config.max_syn_queued = (uint8_t)value;
@@ -5351,6 +5418,12 @@ int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt,
         StreamTcpSetEvent(p, STREAM_PKT_BROKEN_ACK);
     }
 
+    if ((p->tcph->th_flags & TH_URG) && StreamTcpInlineDropUrg()) {
+        PacketDrop(p, ACTION_DROP, PKT_DROP_REASON_STREAM_URG);
+        SCLogDebug("dropping urgent packet");
+        SCReturnInt(0);
+    }
+
     /* If we are on IPS mode, and got a drop action triggered from
      * the IP only module, or from a reassembled msg and/or from an
      * applayer detection, then drop the rest of the packets of the
@@ -5793,6 +5866,7 @@ TmEcode StreamTcpThreadInit(ThreadVars *tv, void *initdata, void **data)
 
     stt->ra_ctx->counter_tcp_reass_data_normal_fail = StatsRegisterCounter("tcp.insert_data_normal_fail", tv);
     stt->ra_ctx->counter_tcp_reass_data_overlap_fail = StatsRegisterCounter("tcp.insert_data_overlap_fail", tv);
+    stt->ra_ctx->counter_tcp_urgent_oob = StatsRegisterCounter("tcp.urgent_oob_data", tv);
 
     SCLogDebug("StreamTcp thread specific ctx online at %p, reassembly ctx %p",
                 stt, stt->ra_ctx);

diff --git a/src/stream-tcp.h b/src/stream-tcp.h
index 598edc6bd11589ee2f7799042f5b497a34df98c1..cc227cacfbb7fa613267836e3d0887ceb8c71df6 100644 (file)
--- a/src/stream-tcp.h
+++ b/src/stream-tcp.h
@@ -38,6 +38,16 @@
 #define STREAMTCP_INIT_FLAG_DROP_INVALID           BIT_U8(1)
 #define STREAMTCP_INIT_FLAG_BYPASS                 BIT_U8(2)
 #define STREAMTCP_INIT_FLAG_INLINE                 BIT_U8(3)
+/** flag to drop packets with URG flag set */
+#define STREAMTCP_INIT_FLAG_DROP_URG BIT_U8(4)
+
+enum TcpStreamUrgentHandling {
+    TCP_STREAM_URGENT_INLINE, /**< treat as inline data */
+#define TCP_STREAM_URGENT_DEFAULT TCP_STREAM_URGENT_INLINE
+    TCP_STREAM_URGENT_DROP, /**< drop TCP packet with URG flag */
+    TCP_STREAM_URGENT_OOB,  /**< treat 1 byte of URG data as OOB */
+    TCP_STREAM_URGENT_GAP,  /**< treat 1 byte of URG data as GAP */
+};
 
 /*global flow data*/
 typedef struct TcpStreamCnf_ {
@@ -69,6 +79,8 @@ typedef struct TcpStreamCnf_ {
     enum ExceptionPolicy ssn_memcap_policy;
     enum ExceptionPolicy reassembly_memcap_policy;
     enum ExceptionPolicy midstream_policy;
+    enum TcpStreamUrgentHandling urgent_policy;
+    enum TcpStreamUrgentHandling urgent_oob_limit_policy;
 
     /* default to "LINUX" timestamp behavior if true*/
     bool liberal_timestamps;

diff --git a/suricata.yaml.in b/suricata.yaml.in
index 504bfd0b66b85a96708fdd2e839d63022e2e6887..05aa170d9276cc1293d321d2bfa8f068cb8e240a 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -1592,6 +1592,9 @@ stream:
   #midstream-policy: ignore
   inline: auto                  # auto will use inline mode in IPS mode, yes or no set it statically
   reassembly:
+    urgent:
+      policy: oob              # drop, inline, oob (1 byte, see RFC 6093, 3.1), gap
+      oob-limit-policy: drop
     memcap: 256mb
     #memcap-policy: ignore
     depth: 1mb                  # reassemble 1mb into a stream