detect/http: flush bodies when inspecting stream

The HTTP bodies (http_client_body and http_server_body/file_data) use
settings to control how much data we have before doing first inspection:

    request-body-minimal-inspect-size
    response-body-minimal-inspect-size

These settings default to 32k as quite some existing rules need this.

At the same time, the 'raw stream' inspection uses its own limits. By
default it inspects the data in blocks of about 2.5k. This could lead
to a situation where rules would not match.

For example, with 2 rules like this:

    content:"abc"; content:"data="; http_client_body; depth:5; sid:1;
    content:"xyz"; sid:2;

Sid 1 would only be inspected when the POST body reached the 32k limit
or when it was complete. Observed case shows the POST body to be 18k.
Sid 2 is inspected as soon as the 2.5k limit is reached, and then again
for each 2.5k increment. This moves the raw stream tracker forward.

So by the time sid 1 is inspected, some 18/19k into the stream, the
raw stream tracker is actually already moved forward for approximately
17.5k, this leads to the stream match of sid 1 possibly not matching.
Since the body match is at the start of the buffer, it makes sense
that the body and stream are inspected together.

The body inspection uses a tracker 'body_inspected', that keeps track
of how far into the body both MPM and per signature inspection has
moved.

This patch updates the logic in 2 ways:

1. it triggers earlier HTTP body inspection, which is matched to the
   stream inspection. When the detection engine finds it has stream
   data available for inspection, it passes the new 'STREAM_FLUSH'
   flag to the HTTP body inspection code. Which will then do an
   early inspection, even if still before the min inspect size.

2. to still somewhat adhere to the min inspect size, the body
   tracker is not updated until the min inspect size is reached.
   This will lead to some re-evaluation of the same body data.

If raw stream reassembly is disabled, this 'STREAM_FLUSH' flag is
never set, and the old behavior is used.

Bug #2522.

diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index 6dce727d66148564f7a8555cb87274ec0e71dab8..568cd1cff24667aaf1c85b64642755cae20e02ce 100644 (file)
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -106,7 +106,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
-                                               uint8_t flags,
+                                               const uint8_t flags,
                                                uint32_t *buffer_len,
                                                uint32_t *stream_start_offset)
 {
@@ -162,7 +162,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
         goto end;
     }
 
-    if (!htp_state->cfg->http_body_inline) {
+    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->request.body_limit == 0 ||
@@ -204,10 +204,13 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
             offset);
     det_ctx->hcbd[index].offset = offset;
 
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
-     * the window sizes when freeing data */
-    htud->request_body.body_inspected = htud->request_body.content_len_so_far;
-
+    if (htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size) {
+        SCLogDebug("not updating tracker as we're still below inspect_min_size");
+    } else {
+        /* move inspected tracker to end of the data. HtpBodyPrune will consider
+         * the window sizes when freeing data */
+        htud->request_body.body_inspected = htud->request_body.content_len_so_far;
+    }
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
     *stream_start_offset = det_ctx->hcbd[index].offset;

diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 2355dd5befa7f9d9e4787f93c12cadf18743063f..3e7cd84a3f8042f35cd446b1a14b265f550fe1e5 100644 (file)
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -167,7 +167,7 @@ static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
               flags & STREAM_EOF ? "true" : "false",
                (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false");
 
-    if (!htp_state->cfg->http_body_inline) {
+    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->response.body_limit == 0 ||
@@ -209,11 +209,14 @@ static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
             offset);
     det_ctx->hsbd[index].offset = offset;
 
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
-     * the window sizes when freeing data */
-    htud->response_body.body_inspected = htud->response_body.content_len_so_far;
-    SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected);
-
+    if (htud->response_body.content_len_so_far < htp_state->cfg->response.inspect_min_size) {
+        SCLogDebug("not updating tracker as we're still below inspect_min_size");
+    } else {
+        /* move inspected tracker to end of the data. HtpBodyPrune will consider
+         * the window sizes when freeing data */
+        htud->response_body.body_inspected = htud->response_body.content_len_so_far;
+        SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected);
+    }
     buffer = det_ctx->hsbd[index].buffer;
     *buffer_len = det_ctx->hsbd[index].buffer_len;
     *stream_start_offset = det_ctx->hsbd[index].offset;

diff --git a/src/detect.c b/src/detect.c
index efc4ba0c48dadc06f551a6f31d6596b6f4d7c334..a5bea255bce56998aa4c28b9b45fec007ad38f2e 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -1016,6 +1016,7 @@ void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineT
                 if (p->proto == IPPROTO_TCP && pflow->protoctx &&
                     StreamReassembleRawHasDataReady(pflow->protoctx, p)) {
                     p->flags |= PKT_DETECT_HAS_STREAMDATA;
+                    flow_flags |= STREAM_FLUSH;
                 }
                 SCLogDebug("alstate %s, alproto %u", has_state ? "true" : "false", alproto);
             } else {

diff --git a/src/stream.h b/src/stream.h
index d421c99a70ff2dc5ba5e1d8cb619ccb237de2d08..46590ce846333bea34b35c6bdb94e7008ed218ca 100644 (file)
--- a/src/stream.h
+++ b/src/stream.h
@@ -32,6 +32,7 @@
 #define STREAM_TOCLIENT         0x08
 #define STREAM_GAP              0x10    /**< data gap encountered */
 #define STREAM_DEPTH            0x20    /**< depth reached */
+#define STREAM_FLUSH            0x80
 
 typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t);
 int StreamSegmentForEach(const Packet *p, uint8_t flag,