http: improve body inspection

Enforce inspect window also in IDS mode. Try always to get at least
'inspect win' worth of data. In case there is more new data, take
some of the old data as well to make sure there is always some overlap.

This unifies IDS and IPS modes, the only difference left is the start
of inspection. IDS waits until min_size is available, IPS starts right
away.

diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index deaaa8a14a103e97ec48c52e14ca564d210fe549..fddfaa56d06c8562e11f59b34c10ec067f9329e6 100644 (file)
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -157,29 +157,57 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
         goto end;
     }
 
-    /* inspect the body if the transfer is complete or we have hit
-     * our body size limit */
-    if ((htp_state->cfg->request.body_limit == 0 ||
-         htud->request_body.content_len_so_far < htp_state->cfg->request.body_limit) &&
-        htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size &&
-        !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_REQUEST_BODY) &&
-        !(flags & STREAM_EOF)) {
-        SCLogDebug("we still haven't seen the entire request body.  "
-                   "Let's defer body inspection till we see the "
-                   "entire body.");
-        goto end;
+    if (!htp_state->cfg->http_body_inline) {
+        /* inspect the body if the transfer is complete or we have hit
+        * our body size limit */
+        if ((htp_state->cfg->request.body_limit == 0 ||
+             htud->request_body.content_len_so_far < htp_state->cfg->request.body_limit) &&
+            htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size &&
+            !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_REQUEST_BODY) &&
+            !(flags & STREAM_EOF)) {
+            SCLogDebug("we still haven't seen the entire request body.  "
+                       "Let's defer body inspection till we see the "
+                       "entire body.");
+            goto end;
+        }
+    }
+
+    /* get the inspect buffer
+     *
+     * make sure that we have at least the configured inspect_win size.
+     * If we have more, take at least 1/4 of the inspect win size before
+     * the new data.
+     */
+    uint64_t offset = 0;
+    if (htud->request_body.body_inspected > htp_state->cfg->request.inspect_min_size) {
+        BUG_ON(htud->request_body.content_len_so_far < htud->request_body.body_inspected);
+        uint64_t inspect_win = htud->request_body.content_len_so_far - htud->request_body.body_inspected;
+        SCLogDebug("inspect_win %u", (uint)inspect_win);
+        if (inspect_win < htp_state->cfg->request.inspect_window) {
+            uint64_t inspect_short = htp_state->cfg->request.inspect_window - inspect_win;
+            if (htud->request_body.body_inspected < inspect_short)
+                offset = 0;
+            else
+                offset = htud->request_body.body_inspected - inspect_short;
+        } else {
+            offset = htud->request_body.body_inspected - (htp_state->cfg->request.inspect_window / 4);
+        }
     }
 
-    StreamingBufferGetData(htud->request_body.sb,
+    StreamingBufferGetDataAtOffset(htud->request_body.sb,
             &det_ctx->hcbd[index].buffer, &det_ctx->hcbd[index].buffer_len,
-            &det_ctx->hcbd[index].offset);
+            offset);
+    det_ctx->hcbd[index].offset = offset;
 
-    /* update inspected tracker */
-    htud->request_body.body_inspected = htud->request_body.last->sbseg.stream_offset + htud->request_body.last->sbseg.segment_len;
+    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+     * the window sizes when freeing data */
+    htud->request_body.body_inspected = htud->request_body.content_len_so_far;
 
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
     *stream_start_offset = det_ctx->hcbd[index].offset;
+
+    SCLogDebug("buffer_len %u (%u)", *buffer_len, (uint)htud->request_body.content_len_so_far);
  end:
     return buffer;
 }

diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 8ef2c17a5aa71a94d1d0c98686c92833b837e39c..bf980bd9bb65574a52edbd39abffefca4997fa17 100644 (file)
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -178,23 +178,41 @@ static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
         }
     }
 
-    StreamingBufferGetData(htud->response_body.sb,
+    /* get the inspect buffer
+     *
+     * make sure that we have at least the configured inspect_win size.
+     * If we have more, take at least 1/4 of the inspect win size before
+     * the new data.
+     */
+    uint64_t offset = 0;
+    if (htud->response_body.body_inspected > htp_state->cfg->response.inspect_min_size) {
+        BUG_ON(htud->response_body.content_len_so_far < htud->response_body.body_inspected);
+        uint64_t inspect_win = htud->response_body.content_len_so_far - htud->response_body.body_inspected;
+        SCLogDebug("inspect_win %u", (uint)inspect_win);
+        if (inspect_win < htp_state->cfg->response.inspect_window) {
+            uint64_t inspect_short = htp_state->cfg->response.inspect_window - inspect_win;
+            if (htud->response_body.body_inspected < inspect_short)
+                offset = 0;
+            else
+                offset = htud->response_body.body_inspected - inspect_short;
+        } else {
+            offset = htud->response_body.body_inspected - (htp_state->cfg->response.inspect_window / 4);
+        }
+    }
+
+    StreamingBufferGetDataAtOffset(htud->response_body.sb,
             &det_ctx->hsbd[index].buffer, &det_ctx->hsbd[index].buffer_len,
-            &det_ctx->hsbd[index].offset);
+            offset);
+    det_ctx->hsbd[index].offset = offset;
 
-    /* update inspected tracker */
-    if (!htp_state->cfg->http_body_inline) {
-        htud->response_body.body_inspected = htud->response_body.last->sbseg.stream_offset +
-                                             htud->response_body.last->sbseg.segment_len;
-    } else {
-        htud->response_body.body_inspected = htud->response_body.content_len_so_far;
-    }
+    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+     * the window sizes when freeing data */
+    htud->response_body.body_inspected = htud->response_body.content_len_so_far;
     SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected);
 
     buffer = det_ctx->hsbd[index].buffer;
     *buffer_len = det_ctx->hsbd[index].buffer_len;
     *stream_start_offset = det_ctx->hsbd[index].offset;
-
  end:
     return buffer;
 }
@@ -4111,8 +4129,8 @@ libhtp:\n\
   default-config:\n\
 \n\
     http-body-inline: yes\n\
-    response-body-minimal-inspect-size: 6\n\
-    response-body-inspect-window: 3\n\
+    response-body-minimal-inspect-size: 9\n\
+    response-body-inspect-window: 12\n\
 ";
 
     struct TestSteps steps[] = {
@@ -4153,8 +4171,8 @@ libhtp:\n\
   default-config:\n\
 \n\
     http-body-inline: yes\n\
-    response-body-minimal-inspect-size: 6\n\
-    response-body-inspect-window: 3\n\
+    response-body-minimal-inspect-size: 9\n\
+    response-body-inspect-window: 12\n\
 ";
 
     struct TestSteps steps[] = {
@@ -4189,8 +4207,8 @@ libhtp:\n\
   default-config:\n\
 \n\
     http-body-inline: yes\n\
-    response-body-minimal-inspect-size: 6\n\
-    response-body-inspect-window: 3\n\
+    response-body-minimal-inspect-size: 9\n\
+    response-body-inspect-window: 12\n\
 ";
 
     struct TestSteps steps[] = {
@@ -4225,8 +4243,8 @@ libhtp:\n\
   default-config:\n\
 \n\
     http-body-inline: yes\n\
-    response-body-minimal-inspect-size: 6\n\
-    response-body-inspect-window: 3\n\
+    response-body-minimal-inspect-size: 9\n\
+    response-body-inspect-window: 12\n\
 ";
 
     struct TestSteps steps[] = {
@@ -4265,8 +4283,8 @@ libhtp:\n\
   default-config:\n\
 \n\
     http-body-inline: yes\n\
-    response-body-minimal-inspect-size: 6\n\
-    response-body-inspect-window: 3\n\
+    response-body-minimal-inspect-size: 8\n\
+    response-body-inspect-window: 4\n\
 ";
 
     struct TestSteps steps[] = {
@@ -4305,8 +4323,8 @@ libhtp:\n\
   default-config:\n\
 \n\
     http-body-inline: yes\n\
-    response-body-minimal-inspect-size: 6\n\
-    response-body-inspect-window: 3\n\
+    response-body-minimal-inspect-size: 8\n\
+    response-body-inspect-window: 4\n\
 ";
 
     struct TestSteps steps[] = {