detect/http: flush bodies when inspecting stream

The HTTP bodies (http_client_body and http_server_body/file_data) use
settings to control how much data we have before doing first inspection:

    request-body-minimal-inspect-size
    response-body-minimal-inspect-size

These settings default to 32k as quite some existing rules need this.

At the same time, the 'raw stream' inspection uses its own limits. By
default it inspects the data in blocks of about 2.5k. This could lead
to a situation where rules would not match.

For example, with 2 rules like this:

    content:"abc"; content:"data="; http_client_body; depth:5; sid:1;
    content:"xyz"; sid:2;

Sid 1 would only be inspected when the POST body reached the 32k limit
or when it was complete. Observed case shows the POST body to be 18k.
Sid 2 is inspected as soon as the 2.5k limit is reached, and then again
for each 2.5k increment. This moves the raw stream tracker forward.

So by the time sid 1 is inspected, some 18/19k into the stream, the
raw stream tracker is actually already moved forward for approximately
17.5k, this leads to the stream match of sid 1 possibly not matching.
Since the body match is at the start of the buffer, it makes sense
that the body and stream are inspected together.

The body inspection uses a tracker 'body_inspected', that keeps track
of how far into the body both MPM and per signature inspection has
moved.

This patch updates the logic in 2 ways:

1. it triggers earlier HTTP body inspection, which is matched to the
   stream inspection. When the detection engine finds it has stream
   data available for inspection, it passes the new 'STREAM_FLUSH'
   flag to the HTTP body inspection code. Which will then do an
   early inspection, even if still before the min inspect size.

2. to still somewhat adhere to the min inspect size, the body
   tracker is not updated until the min inspect size is reached.
   This will lead to some re-evaluation of the same body data.

If raw stream reassembly is disabled, this 'STREAM_FLUSH' flag is
never set, and the old behavior is used.

Bug #2522.

diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index 97a6977e7119eb2e8fa580c93e44045a349abbdc..9c95ca40f94c3f2d3575601507d137bc53e74aa4 100644 (file)
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -107,7 +107,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
-                                               uint8_t flags,
+                                               const uint8_t flags,
                                                uint32_t *buffer_len,
                                                uint32_t *stream_start_offset)
 {
@@ -163,7 +163,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
         goto end;
     }
 
-    if (!htp_state->cfg->http_body_inline) {
+    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->request.body_limit == 0 ||
@@ -205,10 +205,13 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
             offset);
     det_ctx->hcbd[index].offset = offset;
 
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
-     * the window sizes when freeing data */
-    htud->request_body.body_inspected = htud->request_body.content_len_so_far;
-
+    if (htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size) {
+        SCLogDebug("not updating tracker as we're still below inspect_min_size");
+    } else {
+        /* move inspected tracker to end of the data. HtpBodyPrune will consider
+         * the window sizes when freeing data */
+        htud->request_body.body_inspected = htud->request_body.content_len_so_far;
+    }
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
     *stream_start_offset = det_ctx->hcbd[index].offset;

diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 1b71eb9a8e69b68570978f0fc9fb3ef00d9cc8f7..6cfeef3cb2f724d95263b3bae573f0df1f2d0a2d 100644 (file)
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -118,7 +118,7 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
               flags & STREAM_EOF ? "true" : "false",
                (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false");
 
-    if (!htp_state->cfg->http_body_inline) {
+    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->response.body_limit == 0 ||
@@ -178,10 +178,14 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
         }
     }
 
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
-     * the window sizes when freeing data */
-    body->body_inspected = body->content_len_so_far;
-    SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
+    if (body->content_len_so_far < htp_state->cfg->response.inspect_min_size) {
+        SCLogDebug("not updating tracker as we're still below inspect_min_size");
+    } else {
+        /* move inspected tracker to end of the data. HtpBodyPrune will consider
+         * the window sizes when freeing data */
+        body->body_inspected = body->content_len_so_far;
+        SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
+    }
 
     SCReturnPtr(buffer, "InspectionBuffer");
 }

diff --git a/src/detect.c b/src/detect.c
index 1dea649a131da68c69be580e1cec9fb44ce1b1dc..bb16ad75013d6b1b3e4e2ed0a04877b37ba14ffb 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -984,6 +984,7 @@ static DetectRunScratchpad DetectRunSetup(
             if (p->proto == IPPROTO_TCP && pflow->protoctx &&
                     StreamReassembleRawHasDataReady(pflow->protoctx, p)) {
                 p->flags |= PKT_DETECT_HAS_STREAMDATA;
+                flow_flags |= STREAM_FLUSH;
             }
             SCLogDebug("alproto %u", alproto);
         } else {

diff --git a/src/stream.h b/src/stream.h
index 68cb2dcefcdd8e2163aafad0bb922e72a1cfa700..ab5ccdaf18df5fb2ef2a8e94d9bff541e4c8c039 100644 (file)
--- a/src/stream.h
+++ b/src/stream.h
@@ -33,6 +33,7 @@
 #define STREAM_GAP              0x10    /**< data gap encountered */
 #define STREAM_DEPTH            0x20    /**< depth reached */
 #define STREAM_MIDSTREAM        0x40
+#define STREAM_FLUSH            0x80
 
 typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t);
 int StreamSegmentForEach(const Packet *p, uint8_t flag,