From: Victor Julien Date: Thu, 13 Dec 2018 10:29:15 +0000 (+0100) Subject: detect/http-client-body: convert to inspect api v2 X-Git-Tag: suricata-5.0.0-beta1~245 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=477e46da7871c68f5d3dce9aa5d13c123e9e3668;p=thirdparty%2Fsuricata.git detect/http-client-body: convert to inspect api v2 --- diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c index 045399f84d..8c0eed2c1d 100644 --- a/src/detect-engine-hcbd.c +++ b/src/detect-engine-hcbd.c @@ -65,257 +65,6 @@ #include "util-validate.h" -#define BUFFER_STEP 50 - -static inline int HCBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size) -{ - if (size >= (USHRT_MAX - BUFFER_STEP)) - return -1; - - void *ptmp; - if (size > det_ctx->hcbd_buffers_size) { - uint16_t grow_by = size - det_ctx->hcbd_buffers_size; - grow_by = MAX(grow_by, BUFFER_STEP); - - ptmp = SCRealloc(det_ctx->hcbd, - (det_ctx->hcbd_buffers_size + grow_by) * sizeof(HttpReassembledBody)); - if (ptmp == NULL) { - SCFree(det_ctx->hcbd); - det_ctx->hcbd = NULL; - det_ctx->hcbd_buffers_size = 0; - det_ctx->hcbd_buffers_list_len = 0; - return -1; - } - det_ctx->hcbd = ptmp; - - memset(det_ctx->hcbd + det_ctx->hcbd_buffers_size, 0, grow_by * sizeof(HttpReassembledBody)); - det_ctx->hcbd_buffers_size += grow_by; - - uint16_t i; - for (i = det_ctx->hcbd_buffers_list_len; i < det_ctx->hcbd_buffers_size; i++) { - det_ctx->hcbd[i].buffer_len = 0; - det_ctx->hcbd[i].offset = 0; - } - } - - return 0; -} - -/** - */ -static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, - DetectEngineCtx *de_ctx, - DetectEngineThreadCtx *det_ctx, - Flow *f, HtpState *htp_state, - const uint8_t flags, - uint32_t *buffer_len, - uint32_t *stream_start_offset) -{ - int index = 0; - const uint8_t *buffer = NULL; - *buffer_len = 0; - *stream_start_offset = 0; - - if (det_ctx->hcbd_buffers_list_len == 0) { - /* get the inspect id to use as a 'base id' */ - uint64_t base_inspect_id = AppLayerParserGetTransactionInspectId(f->alparser, flags); - BUG_ON(base_inspect_id > tx_id); - /* see how many space we need for the current tx_id */ - uint64_t txs = (tx_id - base_inspect_id) + 1; - if (HCBDCreateSpace(det_ctx, txs) < 0) - goto end; - - index = (tx_id - base_inspect_id); - det_ctx->hcbd_start_tx_id = base_inspect_id; - det_ctx->hcbd_buffers_list_len = txs; - } else { - if ((tx_id - det_ctx->hcbd_start_tx_id) < det_ctx->hcbd_buffers_list_len) { - if (det_ctx->hcbd[(tx_id - det_ctx->hcbd_start_tx_id)].buffer_len != 0) { - *buffer_len = det_ctx->hcbd[(tx_id - det_ctx->hcbd_start_tx_id)].buffer_len; - *stream_start_offset = det_ctx->hcbd[(tx_id - det_ctx->hcbd_start_tx_id)].offset; - return det_ctx->hcbd[(tx_id - det_ctx->hcbd_start_tx_id)].buffer; - } - } else { - uint64_t txs = (tx_id - det_ctx->hcbd_start_tx_id) + 1; - if (HCBDCreateSpace(det_ctx, txs) < 0) - goto end; /* let's consider it as stage not done for now */ - - det_ctx->hcbd_buffers_list_len = txs; - } - index = (tx_id - det_ctx->hcbd_start_tx_id); - } - - HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); - if (htud == NULL) { - SCLogDebug("no htud"); - goto end; - } - - /* no new data */ - if (htud->request_body.body_inspected == htud->request_body.content_len_so_far) { - SCLogDebug("no new data"); - goto end; - } - - HtpBodyChunk *cur = htud->request_body.first; - if (cur == NULL) { - SCLogDebug("No http chunks to inspect for this transacation"); - goto end; - } - - if (!htp_state->cfg->http_body_inline) { - /* inspect the body if the transfer is complete or we have hit - * our body size limit */ - if ((htp_state->cfg->request.body_limit == 0 || - htud->request_body.content_len_so_far < htp_state->cfg->request.body_limit) && - htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size && - !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_REQUEST_BODY) && - !(flags & STREAM_EOF)) { - SCLogDebug("we still haven't seen the entire request body. " - "Let's defer body inspection till we see the " - "entire body."); - goto end; - } - } - - /* get the inspect buffer - * - * make sure that we have at least the configured inspect_win size. - * If we have more, take at least 1/4 of the inspect win size before - * the new data. - */ - uint64_t offset = 0; - if (htud->request_body.body_inspected > htp_state->cfg->request.inspect_min_size) { - BUG_ON(htud->request_body.content_len_so_far < htud->request_body.body_inspected); - uint64_t inspect_win = htud->request_body.content_len_so_far - htud->request_body.body_inspected; - SCLogDebug("inspect_win %"PRIu64, inspect_win); - if (inspect_win < htp_state->cfg->request.inspect_window) { - uint64_t inspect_short = htp_state->cfg->request.inspect_window - inspect_win; - if (htud->request_body.body_inspected < inspect_short) - offset = 0; - else - offset = htud->request_body.body_inspected - inspect_short; - } else { - offset = htud->request_body.body_inspected - (htp_state->cfg->request.inspect_window / 4); - } - } - - StreamingBufferGetDataAtOffset(htud->request_body.sb, - &det_ctx->hcbd[index].buffer, &det_ctx->hcbd[index].buffer_len, - offset); - det_ctx->hcbd[index].offset = offset; - - /* move inspected tracker to end of the data. HtpBodyPrune will consider - * the window sizes when freeing data */ - htud->request_body.body_inspected = htud->request_body.content_len_so_far; - - buffer = det_ctx->hcbd[index].buffer; - *buffer_len = det_ctx->hcbd[index].buffer_len; - *stream_start_offset = det_ctx->hcbd[index].offset; - - SCLogDebug("buffer_len %u (%"PRIu64")", *buffer_len, htud->request_body.content_len_so_far); - end: - return buffer; -} - -/** \brief HTTP Client Body Mpm prefilter callback - * - * \param det_ctx detection engine thread ctx - * \param p packet to inspect - * \param f flow to inspect - * \param txv tx to inspect - * \param pectx inspection context - */ -static void PrefilterTxHttpRequestBody(DetectEngineThreadCtx *det_ctx, - const void *pectx, - Packet *p, Flow *f, void *txv, - const uint64_t idx, const uint8_t flags) -{ - SCEnter(); - - const MpmCtx *mpm_ctx = (MpmCtx *)pectx; - htp_tx_t *tx = (htp_tx_t *)txv; - HtpState *htp_state = f->alstate; - uint32_t buffer_len = 0; - uint32_t stream_start_offset = 0; - const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx, - NULL, det_ctx, - f, htp_state, - flags, - &buffer_len, - &stream_start_offset); - - if (buffer_len >= mpm_ctx->minlen) { - (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx, - &det_ctx->mtcu, &det_ctx->pmq, buffer, buffer_len); - } -} - -int PrefilterTxHttpRequestBodyRegister(DetectEngineCtx *de_ctx, - SigGroupHead *sgh, MpmCtx *mpm_ctx) -{ - SCEnter(); - - return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHttpRequestBody, - ALPROTO_HTTP, HTP_REQUEST_BODY, - mpm_ctx, NULL, "http_client_body"); -} - -int DetectEngineInspectHttpClientBody(ThreadVars *tv, - DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, - const Signature *s, const SigMatchData *smd, - Flow *f, uint8_t flags, void *alstate, void *tx, uint64_t tx_id) -{ - HtpState *htp_state = (HtpState *)alstate; - uint32_t buffer_len = 0; - uint32_t stream_start_offset = 0; - const bool eof = (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_REQUEST_BODY); - const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id, - de_ctx, det_ctx, - f, htp_state, - flags, - &buffer_len, - &stream_start_offset); - if (buffer_len == 0) - goto end; - - uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0; - ci_flags |= (stream_start_offset == 0 ? DETECT_CI_FLAGS_START : 0); - - det_ctx->buffer_offset = 0; - det_ctx->discontinue_matching = 0; - det_ctx->inspection_recursion_counter = 0; - int r = DetectEngineContentInspection(de_ctx, det_ctx, s, smd, - f, - (uint8_t *)buffer, - buffer_len, - stream_start_offset, ci_flags, - DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL); - if (r == 1) - return DETECT_ENGINE_INSPECT_SIG_MATCH; - - - end: - if (eof) - return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH; - else - return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; -} - -void DetectEngineCleanHCBDBuffers(DetectEngineThreadCtx *det_ctx) -{ - if (det_ctx->hcbd_buffers_list_len > 0) { - for (int i = 0; i < det_ctx->hcbd_buffers_list_len; i++) { - det_ctx->hcbd[i].buffer_len = 0; - det_ctx->hcbd[i].offset = 0; - } - } - det_ctx->hcbd_buffers_list_len = 0; - det_ctx->hcbd_start_tx_id = 0; - - return; -} - /***********************************Unittests**********************************/ #ifdef UNITTESTS diff --git a/src/detect-engine.c b/src/detect-engine.c index d2178fb90a..0f75399056 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2544,12 +2544,6 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) if (det_ctx->bj_values != NULL) SCFree(det_ctx->bj_values); - /* HSCB */ - if (det_ctx->hcbd != NULL) { - SCLogDebug("det_ctx hcbd %u", det_ctx->hcbd_buffers_size); - SCFree(det_ctx->hcbd); - } - /* Decoded base64 data. */ if (det_ctx->base64_decoded != NULL) { SCFree(det_ctx->base64_decoded); diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c index f7e795c850..5ac0004ddf 100644 --- a/src/detect-http-client-body.c +++ b/src/detect-http-client-body.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2018 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -39,6 +39,7 @@ #include "detect-engine.h" #include "detect-engine-mpm.h" #include "detect-engine-state.h" +#include "detect-engine-prefilter.h" #include "detect-content.h" #include "detect-pcre.h" @@ -64,6 +65,12 @@ static void DetectHttpClientBodySetupCallback(const DetectEngineCtx *de_ctx, Signature *s); static int g_http_client_body_buffer_id = 0; +static InspectionBuffer *HttpClientBodyGetDataCallback( + DetectEngineThreadCtx *det_ctx, + const DetectEngineTransforms *transforms, + Flow *f, const uint8_t flow_flags, + void *txv, const int list_id); + /** * \brief Registers the keyword handlers for the "http_client_body" keyword. */ @@ -76,12 +83,14 @@ void DetectHttpClientBodyRegister(void) sigmatch_table[DETECT_AL_HTTP_CLIENT_BODY].RegisterTests = DetectHttpClientBodyRegisterTests; sigmatch_table[DETECT_AL_HTTP_CLIENT_BODY].flags |= SIGMATCH_NOOPT ; - DetectAppLayerMpmRegister("http_client_body", SIG_FLAG_TOSERVER, 2, - PrefilterTxHttpRequestBodyRegister); + DetectAppLayerInspectEngineRegister2("http_client_body", ALPROTO_HTTP, + SIG_FLAG_TOSERVER, HTP_REQUEST_BODY, + DetectEngineInspectBufferGeneric, + HttpClientBodyGetDataCallback); - DetectAppLayerInspectEngineRegister("http_client_body", - ALPROTO_HTTP, SIG_FLAG_TOSERVER, HTP_REQUEST_BODY, - DetectEngineInspectHttpClientBody); + DetectAppLayerMpmRegister2("http_client_body", SIG_FLAG_TOSERVER, 2, + PrefilterGenericMpmRegister, HttpClientBodyGetDataCallback, + ALPROTO_HTTP, HTP_REQUEST_BODY); DetectBufferTypeSetDescriptionByName("http_client_body", "http request body"); @@ -123,6 +132,109 @@ int DetectHttpClientBodySetup(DetectEngineCtx *de_ctx, Signature *s, const char ALPROTO_HTTP); } +static inline HtpBody *GetRequestBody(htp_tx_t *tx) +{ + HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); + if (htud == NULL) { + SCLogDebug("no htud"); + return NULL; + } + + return &htud->request_body; +} + +static InspectionBuffer *HttpClientBodyGetDataCallback(DetectEngineThreadCtx *det_ctx, + const DetectEngineTransforms *transforms, + Flow *f, const uint8_t flow_flags, + void *txv, const int list_id) +{ + SCEnter(); + + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); + if (buffer->inspect != NULL) + return buffer; + + htp_tx_t *tx = txv; + HtpState *htp_state = f->alstate; + const uint8_t flags = flow_flags; + + HtpBody *body = GetRequestBody(tx); + if (body == NULL) { + return NULL; + } + + /* no new data */ + if (body->body_inspected == body->content_len_so_far) { + SCLogDebug("no new data"); + return NULL; + } + + HtpBodyChunk *cur = body->first; + if (cur == NULL) { + SCLogDebug("No http chunks to inspect for this transacation"); + return NULL; + } + + SCLogDebug("request.body_limit %u request_body.content_len_so_far %"PRIu64 + ", request.inspect_min_size %"PRIu32", EOF %s, progress > body? %s", + htp_state->cfg->request.body_limit, body->content_len_so_far, + htp_state->cfg->request.inspect_min_size, + flags & STREAM_EOF ? "true" : "false", + (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_REQUEST_BODY) ? "true" : "false"); + + if (!htp_state->cfg->http_body_inline) { + /* inspect the body if the transfer is complete or we have hit + * our body size limit */ + if ((htp_state->cfg->request.body_limit == 0 || + body->content_len_so_far < htp_state->cfg->request.body_limit) && + body->content_len_so_far < htp_state->cfg->request.inspect_min_size && + !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_REQUEST_BODY) && + !(flags & STREAM_EOF)) { + SCLogDebug("we still haven't seen the entire request body. " + "Let's defer body inspection till we see the " + "entire body."); + return NULL; + } + } + + /* get the inspect buffer + * + * make sure that we have at least the configured inspect_win size. + * If we have more, take at least 1/4 of the inspect win size before + * the new data. + */ + uint64_t offset = 0; + if (body->body_inspected > htp_state->cfg->request.inspect_min_size) { + BUG_ON(body->content_len_so_far < body->body_inspected); + uint64_t inspect_win = body->content_len_so_far - body->body_inspected; + SCLogDebug("inspect_win %"PRIu64, inspect_win); + if (inspect_win < htp_state->cfg->request.inspect_window) { + uint64_t inspect_short = htp_state->cfg->request.inspect_window - inspect_win; + if (body->body_inspected < inspect_short) + offset = 0; + else + offset = body->body_inspected - inspect_short; + } else { + offset = body->body_inspected - (htp_state->cfg->request.inspect_window / 4); + } + } + + const uint8_t *data; + uint32_t data_len; + + StreamingBufferGetDataAtOffset(body->sb, + &data, &data_len, offset); + InspectionBufferSetup(buffer, data, data_len); + buffer->inspect_offset = offset; + + /* move inspected tracker to end of the data. HtpBodyPrune will consider + * the window sizes when freeing data */ + body->body_inspected = body->content_len_so_far; + SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected); + + SCReturnPtr(buffer, "InspectionBuffer"); +} + /************************************Unittests*********************************/ #ifdef UNITTESTS diff --git a/src/detect.c b/src/detect.c index 5270b5649a..c1c2c99e8e 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1044,8 +1044,6 @@ static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx, { StreamReassembleRawUpdateProgress(pflow->protoctx, p, det_ctx->raw_stream_progress); - - DetectEngineCleanHCBDBuffers(det_ctx); } } PACKET_PROFILING_DETECT_END(p, PROF_DETECT_CLEANUP); diff --git a/src/detect.h b/src/detect.h index 6d436373c0..e04b6e748d 100644 --- a/src/detect.h +++ b/src/detect.h @@ -995,11 +995,6 @@ typedef struct DetectEngineThreadCtx_ { /* counter for the filestore array below -- up here for cache reasons. */ uint16_t filestore_cnt; - HttpReassembledBody *hcbd; - uint64_t hcbd_start_tx_id; - uint16_t hcbd_buffers_size; - uint16_t hcbd_buffers_list_len; - /** id for alert counter */ uint16_t counter_alerts; #ifdef PROFILING