From: Victor Julien Date: Wed, 8 Aug 2018 12:06:43 +0000 (+0200) Subject: detect/http: flush bodies when inspecting stream X-Git-Tag: suricata-4.0.6~47 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4ca131d9bc05c7f182b58dd402a168d2fc846639;p=thirdparty%2Fsuricata.git detect/http: flush bodies when inspecting stream The HTTP bodies (http_client_body and http_server_body/file_data) use settings to control how much data we have before doing first inspection: request-body-minimal-inspect-size response-body-minimal-inspect-size These settings default to 32k as quite some existing rules need this. At the same time, the 'raw stream' inspection uses its own limits. By default it inspects the data in blocks of about 2.5k. This could lead to a situation where rules would not match. For example, with 2 rules like this: content:"abc"; content:"data="; http_client_body; depth:5; sid:1; content:"xyz"; sid:2; Sid 1 would only be inspected when the POST body reached the 32k limit or when it was complete. Observed case shows the POST body to be 18k. Sid 2 is inspected as soon as the 2.5k limit is reached, and then again for each 2.5k increment. This moves the raw stream tracker forward. So by the time sid 1 is inspected, some 18/19k into the stream, the raw stream tracker is actually already moved forward for approximately 17.5k, this leads to the stream match of sid 1 possibly not matching. Since the body match is at the start of the buffer, it makes sense that the body and stream are inspected together. The body inspection uses a tracker 'body_inspected', that keeps track of how far into the body both MPM and per signature inspection has moved. This patch updates the logic in 2 ways: 1. it triggers earlier HTTP body inspection, which is matched to the stream inspection. When the detection engine finds it has stream data available for inspection, it passes the new 'STREAM_FLUSH' flag to the HTTP body inspection code. Which will then do an early inspection, even if still before the min inspect size. 2. to still somewhat adhere to the min inspect size, the body tracker is not updated until the min inspect size is reached. This will lead to some re-evaluation of the same body data. If raw stream reassembly is disabled, this 'STREAM_FLUSH' flag is never set, and the old behavior is used. Bug #2522. --- diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c index 6dce727d66..568cd1cff2 100644 --- a/src/detect-engine-hcbd.c +++ b/src/detect-engine-hcbd.c @@ -106,7 +106,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Flow *f, HtpState *htp_state, - uint8_t flags, + const uint8_t flags, uint32_t *buffer_len, uint32_t *stream_start_offset) { @@ -162,7 +162,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i goto end; } - if (!htp_state->cfg->http_body_inline) { + if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) { /* inspect the body if the transfer is complete or we have hit * our body size limit */ if ((htp_state->cfg->request.body_limit == 0 || @@ -204,10 +204,13 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i offset); det_ctx->hcbd[index].offset = offset; - /* move inspected tracker to end of the data. HtpBodyPrune will consider - * the window sizes when freeing data */ - htud->request_body.body_inspected = htud->request_body.content_len_so_far; - + if (htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size) { + SCLogDebug("not updating tracker as we're still below inspect_min_size"); + } else { + /* move inspected tracker to end of the data. HtpBodyPrune will consider + * the window sizes when freeing data */ + htud->request_body.body_inspected = htud->request_body.content_len_so_far; + } buffer = det_ctx->hcbd[index].buffer; *buffer_len = det_ctx->hcbd[index].buffer_len; *stream_start_offset = det_ctx->hcbd[index].offset; diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c index 2355dd5bef..3e7cd84a3f 100644 --- a/src/detect-engine-hsbd.c +++ b/src/detect-engine-hsbd.c @@ -167,7 +167,7 @@ static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i flags & STREAM_EOF ? "true" : "false", (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false"); - if (!htp_state->cfg->http_body_inline) { + if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) { /* inspect the body if the transfer is complete or we have hit * our body size limit */ if ((htp_state->cfg->response.body_limit == 0 || @@ -209,11 +209,14 @@ static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i offset); det_ctx->hsbd[index].offset = offset; - /* move inspected tracker to end of the data. HtpBodyPrune will consider - * the window sizes when freeing data */ - htud->response_body.body_inspected = htud->response_body.content_len_so_far; - SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected); - + if (htud->response_body.content_len_so_far < htp_state->cfg->response.inspect_min_size) { + SCLogDebug("not updating tracker as we're still below inspect_min_size"); + } else { + /* move inspected tracker to end of the data. HtpBodyPrune will consider + * the window sizes when freeing data */ + htud->response_body.body_inspected = htud->response_body.content_len_so_far; + SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected); + } buffer = det_ctx->hsbd[index].buffer; *buffer_len = det_ctx->hsbd[index].buffer_len; *stream_start_offset = det_ctx->hsbd[index].offset; diff --git a/src/detect.c b/src/detect.c index efc4ba0c48..a5bea255bc 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1016,6 +1016,7 @@ void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineT if (p->proto == IPPROTO_TCP && pflow->protoctx && StreamReassembleRawHasDataReady(pflow->protoctx, p)) { p->flags |= PKT_DETECT_HAS_STREAMDATA; + flow_flags |= STREAM_FLUSH; } SCLogDebug("alstate %s, alproto %u", has_state ? "true" : "false", alproto); } else { diff --git a/src/stream.h b/src/stream.h index d421c99a70..46590ce846 100644 --- a/src/stream.h +++ b/src/stream.h @@ -32,6 +32,7 @@ #define STREAM_TOCLIENT 0x08 #define STREAM_GAP 0x10 /**< data gap encountered */ #define STREAM_DEPTH 0x20 /**< depth reached */ +#define STREAM_FLUSH 0x80 typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t); int StreamSegmentForEach(const Packet *p, uint8_t flag,