stream: introduce min inspect depth logic

Some rules need to inspect both raw stream data and higher level
buffers together. When this higher level buffer is a streaming
buffer itself, the risk of mismatch exists.

This patch allows an app-layer parser to set a 'min inspect depth'.
The value is used by the stream engine to keep at least this
depth worth of data, so that the detection engine can request
all of it for inspection.

For rules that have the SIG_FLAG_FLUSH flag set, data is inspected
not from offset raw_progress, but from raw_progress minus
min_inspect_depth.

At this time this is only used for sigs that have their fast_pattern
in a HTTP body and have raw stream match as well.

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index eff10cb070ef0e2d91bee35e4e8b904507933ce6..28eaea3d6562f3133c5097377e00334e7b133851 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -695,6 +695,10 @@ static int Setup(Flow *f, HtpState *hstate)
 
     htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
 
+    StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER,
+            htp_cfg_rec->request.inspect_min_size);
+    StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOCLIENT,
+            htp_cfg_rec->response.inspect_min_size);
     return 0;
 error:
     return -1;

diff --git a/src/detect-engine-payload.c b/src/detect-engine-payload.c
index 2393a0b1c7b261ec9b20a860466c32da5736d079..fbdf3da5b39959cdac79c823b16097aaec3292dd 100644 (file)
--- a/src/detect-engine-payload.c
+++ b/src/detect-engine-payload.c
@@ -82,7 +82,8 @@ static void PrefilterPktStream(DetectEngineThreadCtx *det_ctx,
         struct StreamMpmData stream_mpm_data = { det_ctx, mpm_ctx };
         StreamReassembleRaw(p->flow->protoctx, p,
                 StreamMpmFunc, &stream_mpm_data,
-                &det_ctx->raw_stream_progress);
+                &det_ctx->raw_stream_progress,
+                false /* mpm doesn't use min inspect depth */);
         SCLogDebug("POST det_ctx->raw_stream_progress %"PRIu64,
                 det_ctx->raw_stream_progress);
     } else {
@@ -268,7 +269,7 @@ int DetectEngineInspectStreamPayload(DetectEngineCtx *de_ctx,
     struct StreamContentInspectData inspect_data = { de_ctx, det_ctx, s, f };
     int r = StreamReassembleRaw(f->protoctx, p,
             StreamContentInspectFunc, &inspect_data,
-            &unused);
+            &unused, ((s->flags & SIG_FLAG_FLUSH) != 0));
     return r;
 }
 
@@ -338,7 +339,7 @@ int DetectEngineInspectStream(ThreadVars *tv,
     struct StreamContentInspectEngineData inspect_data = { de_ctx, det_ctx, s, smd, f };
     int match = StreamReassembleRaw(f->protoctx, p,
             StreamContentInspectEngineFunc, &inspect_data,
-            &unused);
+            &unused, ((s->flags & SIG_FLAG_FLUSH) != 0));
 
     bool is_last = false;
     if (flags & STREAM_TOSERVER) {

diff --git a/src/stream-tcp-list.c b/src/stream-tcp-list.c
index cbaa7e73deff0e20e27d2cc0d0ba37682380ff3d..31401d023674597bccfa142b5bec53a99cbabfa7 100644 (file)
--- a/src/stream-tcp-list.c
+++ b/src/stream-tcp-list.c
@@ -698,6 +698,19 @@ static inline uint64_t GetLeftEdge(TcpSession *ssn, TcpStream *stream)
                 raw_progress -= (uint64_t)chunk_size;
             }
         }
+
+        /* apply min inspect depth: if it is set we need to keep data
+         * before the raw progress. */
+        if (use_app && stream->min_inspect_depth) {
+            if (raw_progress < stream->min_inspect_depth)
+                raw_progress = 0;
+            else
+                raw_progress -= stream->min_inspect_depth;
+
+            SCLogDebug("stream->min_inspect_depth %u, raw_progress %"PRIu64,
+                    stream->min_inspect_depth, raw_progress);
+        }
+
         if (use_app) {
             left_edge = MIN(STREAM_APP_PROGRESS(stream), raw_progress);
             SCLogDebug("left_edge %"PRIu64", using both app:%"PRIu64", raw:%"PRIu64,

diff --git a/src/stream-tcp-private.h b/src/stream-tcp-private.h
index 25d010b690613cfb9b598d1e314b2501340104de..0a338e7deda1557e3af202cda65e36674d676de8 100644 (file)
--- a/src/stream-tcp-private.h
+++ b/src/stream-tcp-private.h
@@ -115,6 +115,9 @@ typedef struct TcpStream_ {
     uint32_t raw_progress_rel;      /**< raw reassembly progress relative to STREAM_BASE_OFFSET */
     uint32_t log_progress_rel;      /**< streaming logger progress relative to STREAM_BASE_OFFSET */
 
+    uint32_t min_inspect_depth;     /**< min inspect size set by the app layer, to make sure enough data
+                                     *   remains available for inspection together with app layer buffers */
+
     StreamingBuffer sb;
     struct TCPSEG seg_tree;         /**< red black tree of TCP segments. Data is stored in TcpStream::sb */
     uint32_t segs_right_edge;

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index 6a689d65eec021f6007707ca35053f7dead6eacf..cdbef821d65b3c4c7f3a91975654b967acb0a951 100644 (file)
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -1526,11 +1526,17 @@ static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
  *  \param[in] progress_in progress to work from
  *  \param[out] progress_out absolute progress value of the data this
  *                           call handled.
+ *  \param eof we're wrapping up so inspect all data we have, incl unACKd
+ *  \param respect_inspect_depth use Stream::min_inspect_depth if set
+ *
+ *  `respect_inspect_depth` is used to avoid useless inspection of too
+ *  much data.
  */
 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
                         StreamReassembleRawFunc Callback, void *cb_data,
                         const uint64_t progress_in,
-                        uint64_t *progress_out, bool eof)
+                        uint64_t *progress_out, bool eof,
+                        bool respect_inspect_depth)
 {
     SCEnter();
     int r = 0;
@@ -1539,6 +1545,27 @@ static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
     uint64_t progress = progress_in;
     uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
 
+    /* if the app layer triggered a flush, and we're supposed to
+     * use a minimal inspect depth, we actually take the app progress
+     * as that is the right edge of the data. Then we take the window
+     * of 'min_inspect_depth' before that. */
+    if (respect_inspect_depth &&
+        (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW)
+        && stream->min_inspect_depth)
+    {
+        progress = STREAM_APP_PROGRESS(stream);
+        if (stream->min_inspect_depth >= progress) {
+            progress = 0;
+        } else {
+            progress -= stream->min_inspect_depth;
+        }
+        SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
+
+        SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
+    }
+
+    SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
+
     /* get window of data that is acked */
     if (STREAM_LASTACK_GT_BASESEQ(stream)) {
         SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
@@ -1626,7 +1653,7 @@ end:
 
 int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
                         StreamReassembleRawFunc Callback, void *cb_data,
-                        uint64_t *progress_out)
+                        uint64_t *progress_out, bool respect_inspect_depth)
 {
     /* handle inline seperately as the logic is very different */
     if (StreamTcpInlineMode() == TRUE) {
@@ -1649,7 +1676,7 @@ int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
 
     return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
             STREAM_RAW_PROGRESS(stream), progress_out,
-            (p->flags & PKT_PSEUDO_STREAM_END));
+            (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
 }
 
 int StreamReassembleLog(TcpSession *ssn, TcpStream *stream,
@@ -1661,7 +1688,7 @@ int StreamReassembleLog(TcpSession *ssn, TcpStream *stream,
         return 0;
 
     return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
-            progress_in, progress_out, eof);
+            progress_in, progress_out, eof, false);
 }
 
 /** \internal
@@ -1819,6 +1846,23 @@ void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
     }
 }
 
+void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
+{
+#ifdef DEBUG
+    BUG_ON(ssn == NULL);
+#endif
+
+    if (ssn != NULL) {
+        if (direction == STREAM_TOSERVER) {
+            ssn->client.min_inspect_depth = depth;
+            SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
+        } else {
+            ssn->server.min_inspect_depth = depth;
+            SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
+        }
+    }
+}
+
 #ifdef UNITTESTS
 /** unit tests and it's support functions below */
 

diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h
index 28f1c438e31f611a0a4b481e90727aff4c483fc3..886318f2bd99a44941027cb91e03e87ec4ae8ba3 100644 (file)
--- a/src/stream-tcp-reassemble.h
+++ b/src/stream-tcp-reassemble.h
@@ -128,6 +128,7 @@ int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *);
 #endif
 
 bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p);
+void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth);
 
 static inline bool STREAM_LASTACK_GT_BASESEQ(const TcpStream *stream)
 {

diff --git a/src/stream-tcp.h b/src/stream-tcp.h
index 4c06d93b38067dd91b8a3a624a46c0772f758893..7aae41f584e6fed8577dde0a4414215c1fd8679e 100644 (file)
--- a/src/stream-tcp.h
+++ b/src/stream-tcp.h
@@ -132,7 +132,8 @@ int StreamReassembleLog(TcpSession *ssn, TcpStream *stream,
         uint64_t progress_in,
         uint64_t *progress_out, bool eof);
 int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
-        StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out);
+        StreamReassembleRawFunc Callback, void *cb_data,
+        uint64_t *progress_out, bool respect_inspect_depth);
 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress);
 
 void StreamTcpDetectLogFlush(ThreadVars *tv, StreamTcpThread *stt, Flow *f, Packet *p, PacketQueue *pq);

diff --git a/src/tests/stream-tcp-reassemble.c b/src/tests/stream-tcp-reassemble.c
index 89faa9d6d897541487e61d522a3cdcb66524b8cc..d7bf3d9e8b15c48912aa293bcaa87640653260a7 100644 (file)
--- a/src/tests/stream-tcp-reassemble.c
+++ b/src/tests/stream-tcp-reassemble.c
@@ -54,7 +54,7 @@ static int TestReassembleRawValidate(TcpSession *ssn, Packet *p,
 {
     struct TestReassembleRawCallbackData cb = { data, data_len };
     uint64_t progress = 0;
-    int r = StreamReassembleRaw(ssn, p, TestReassembleRawCallback, &cb, &progress);
+    int r = StreamReassembleRaw(ssn, p, TestReassembleRawCallback, &cb, &progress, false);
     if (r == 1) {
         StreamReassembleRawUpdateProgress(ssn, p, progress);
     }