stream: improve 'no app layer' handling

When the session/flow was flagged as 'no applayer inspect', which
could happen as a result various reasons, packets would still be
considered by the app layer reassembly.

When ACK'd, they would be removed again. Depending also on the raw
reassembly.

In very long sessions however, this meganism could fail leading to
virtually endlessly growing segment lists.

This patch makes sure that segments that come in on a 'no app layer'
session are tagged properly or even not added at all.

Use a new ssn flag instead of flow flag for no app tracking.

diff --git a/src/stream-tcp-private.h b/src/stream-tcp-private.h
index 50c6421a872b9bc7a5ca5cb9dc894acc93ff2a65..51ced8882824d68f54be2452a56a235bf8fada4b 100644 (file)
--- a/src/stream-tcp-private.h
+++ b/src/stream-tcp-private.h
@@ -142,6 +142,8 @@ enum
  *  normal packet we assume 3whs to be completed. Only used for SYN/ACK resend
  *  event. */
 #define STREAMTCP_FLAG_3WHS_CONFIRMED               0x1000
+/** App Layer tracking/reassembly is disabled */
+#define STREAMTCP_FLAG_APP_LAYER_DISABLED           0x2000
 
 /*
  * Per STREAM flags
@@ -237,5 +239,8 @@ typedef struct TcpSession_ {
     ((stream)->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED)
 #define StreamTcpResetStreamFlagAppProtoDetectionCompleted(stream) \
     ((stream)->flags &= ~STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED);
+#define StreamTcpDisableAppLayerReassembly(ssn) do { \
+        ((ssn)->flags |= STREAMTCP_FLAG_APP_LAYER_DISABLED); \
+    } while (0);
 
 #endif /* __STREAM_TCP_PRIVATE_H__ */

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index ae14fd7c7863d3ae24f594336d0e1e9c6a23775b..c319c468f7688fd09764935766c69cbe692a7fd1 100644 (file)
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -1870,6 +1870,12 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
         }
     }
 
+    if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) &&
+        (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)) {
+        SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
+        SCReturnInt(0);
+    }
+
     /* If we have reached the defined depth for either of the stream, then stop
        reassembling the TCP session */
     uint32_t size = StreamTcpReassembleCheckDepth(stream, TCP_GET_SEQ(p), p->payload_len);
@@ -1907,6 +1913,9 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
     seg->payload_len = size;
     seg->seq = TCP_GET_SEQ(p);
 
+    if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)
+        seg->flags |= SEGMENTTCP_FLAG_APPLAYER_PROCESSED;
+
     /* if raw reassembly is disabled for new segments, flag each
      * segment as complete for raw before insert */
     if (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED) {
@@ -2049,8 +2058,9 @@ static int StreamTcpReassembleRawCheckLimit(TcpSession *ssn, TcpStream *stream,
  *  \retval 1 app layer is done with this segment
  *  \retval 0 not done yet
  */
-#define StreamTcpAppLayerSegmentProcessed(stream, segment) \
-    (( ( (stream)->flags & STREAMTCP_STREAM_FLAG_GAP ) || \
+#define StreamTcpAppLayerSegmentProcessed(ssn, stream, segment) \
+    (( ( (ssn)->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) || \
+       ( (stream)->flags & STREAMTCP_STREAM_FLAG_GAP ) || \
        ( (segment)->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED ) ? 1 :0 ))
 
 /** \internal
@@ -2082,10 +2092,8 @@ static inline int StreamTcpReturnSegmentCheck(const Flow *f, TcpSession *ssn, Tc
     }
 
     /* check app layer conditions */
-    if (!(f->flags & FLOW_NO_APPLAYER_INSPECTION)) {
-        if (!(StreamTcpAppLayerSegmentProcessed(stream, seg))) {
-            SCReturnInt(0);
-        }
+    if (!(StreamTcpAppLayerSegmentProcessed(ssn, stream, seg))) {
+        SCReturnInt(0);
     }
 
     /* check raw reassembly conditions */
@@ -2188,7 +2196,7 @@ static int StreamTcpReassembleInlineRaw (TcpReassemblyThreadCtx *ra_ctx,
                         seg->payload_len);
 
             /* only remove if app layer reassembly is ready too */
-            if (StreamTcpAppLayerSegmentProcessed(stream, seg)) {
+            if (StreamTcpAppLayerSegmentProcessed(ssn, stream, seg)) {
                 TcpSegment *next_seg = seg->next;
                 StreamTcpRemoveSegmentFromStream(stream, seg);
                 StreamTcpSegmentReturntoPool(seg);
@@ -2207,7 +2215,7 @@ static int StreamTcpReassembleInlineRaw (TcpReassemblyThreadCtx *ra_ctx,
          * If the stream is in GAP state the app layer flag won't be set */
         if (StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream) &&
                 (seg->flags & SEGMENTTCP_FLAG_RAW_PROCESSED) &&
-                StreamTcpAppLayerSegmentProcessed(stream, seg))
+                StreamTcpAppLayerSegmentProcessed(ssn, stream, seg))
         {
             SCLogDebug("segment(%p) of length %"PRIu16" has been processed,"
                     " so return it to pool", seg, seg->payload_len);
@@ -2421,7 +2429,7 @@ static int StreamTcpReassembleInlineRaw (TcpReassemblyThreadCtx *ra_ctx,
         SCLogDebug("seg %p seq %"PRIu32", len %"PRIu16", sum %"PRIu32, seg, seg->seq, seg->payload_len, seg->seq+seg->payload_len);
 
         /* only remove if app layer reassembly is ready too */
-        if (StreamTcpAppLayerSegmentProcessed(stream, seg)) {
+        if (StreamTcpAppLayerSegmentProcessed(ssn, stream, seg)) {
             TcpSegment *next_seg = seg->next;
             StreamTcpRemoveSegmentFromStream(stream, seg);
             StreamTcpSegmentReturntoPool(seg);
@@ -2909,7 +2917,7 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
      * next_seq, we know we are missing data that has been ack'd. That
      * won't get retransmitted, so it's a data gap.
      */
-    if (!(p->flow->flags & FLOW_NO_APPLAYER_INSPECTION)) {
+    if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) {
         if (SEQ_GT(seg->seq, next_seq) && SEQ_LT(seg->seq, stream->last_ack)) {
             /* send gap signal */
             AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
@@ -2942,11 +2950,6 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                 seg, seg->seq, seg->payload_len,
                 (uint32_t)(seg->seq + seg->payload_len));
 
-        if (p->flow->flags & FLOW_NO_APPLAYER_INSPECTION) {
-            SCLogDebug("FLOW_NO_APPLAYER_INSPECTION set, breaking out");
-            break;
-        }
-
         if (StreamTcpReturnSegmentCheck(p->flow, ssn, stream, seg) == 1) {
             SCLogDebug("removing segment");
             TcpSegment *next_seg = seg->next;
@@ -2954,7 +2957,7 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
             StreamTcpSegmentReturntoPool(seg);
             seg = next_seg;
             continue;
-        } else if (StreamTcpAppLayerSegmentProcessed(stream, seg)) {
+        } else if (StreamTcpAppLayerSegmentProcessed(ssn, stream, seg)) {
             TcpSegment *next_seg = seg->next;
             seg = next_seg;
             continue;