From: Victor Julien Date: Fri, 17 Aug 2018 15:53:16 +0000 (+0200) Subject: http: implement min size stream logic X-Git-Tag: suricata-4.0.6~33 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7c4f0ac9014f1a9f3ef087bba37dbcc0cd5666dc;p=thirdparty%2Fsuricata.git http: implement min size stream logic Update HTTP parser to set the min inspect depth per transaction. This allows for signatures to have their fast_pattern in the HTTP body, while still being able to inspect the raw stream reliably with it. The inspect depth is set per transaction as it: - depends on the per personality config for min inspect size - is set to the size of the actual body if it is smaller After the initial inspection is done, it is set to 0 which disables the feature for the rest of the transaction. This removes the rescanning flush logic in commit 7e004f52c60c5e4d7cd8f5ed09491291d18f42d2 and provides an alternative fix for bug #2522. The old approach caused too much rescanning of HTTP body data leading to a performance degradation. Bug #2522 --- diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 24108572b8..de1f2a7479 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1816,6 +1816,32 @@ end: /* set the new chunk flag */ hstate->flags |= HTP_FLAG_NEW_BODY_SET; + if (hstate->conn != NULL) { + SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")", + tx_ud->request_body.content_len_so_far, + hstate->cfg->request.inspect_min_size, + (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp); + + /* if we reach the inspect_min_size we'll trigger inspection, + * so make sure that raw stream is also inspected. Set the + * data to be used to the ammount of raw bytes we've seen to + * get here. */ + if (tx_ud->request_body.body_inspected == 0 && + tx_ud->request_body.content_len_so_far >= hstate->cfg->request.inspect_min_size) { + if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp && + (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX) + { + uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp); + + /* body still in progress, but due to min inspect size we need to inspect now */ + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, x); + AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER); + } + /* after the start of the body, disable the depth logic */ + } else if (tx_ud->request_body.body_inspected > 0) { + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0); + } + } SCReturnInt(HTP_OK); } @@ -1891,6 +1917,31 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d) /* set the new chunk flag */ hstate->flags |= HTP_FLAG_NEW_BODY_SET; + if (hstate->conn != NULL) { + SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")", + tx_ud->response_body.content_len_so_far, + hstate->cfg->response.inspect_min_size, + (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp); + /* if we reach the inspect_min_size we'll trigger inspection, + * so make sure that raw stream is also inspected. Set the + * data to be used to the ammount of raw bytes we've seen to + * get here. */ + if (tx_ud->response_body.body_inspected == 0 && + tx_ud->response_body.content_len_so_far >= hstate->cfg->response.inspect_min_size) { + if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp && + (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX) + { + uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp); + + /* body still in progress, but due to min inspect size we need to inspect now */ + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, x); + AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT); + } + /* after the start of the body, disable the depth logic */ + } else if (tx_ud->response_body.body_inspected > 0) { + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0); + } + } SCReturnInt(HTP_OK); } @@ -1952,6 +2003,41 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx) return HTP_OK; } +/**\internal + * \brief called at start of request + * Set min inspect size. + */ +static int HTPCallbackRequestStart(htp_tx_t *tx) +{ + HtpState *hstate = htp_connp_get_user_data(tx->connp); + if (hstate == NULL) { + SCReturnInt(HTP_ERROR); + } + + if (hstate->cfg) + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, + hstate->cfg->request.inspect_min_size); + SCReturnInt(HTP_OK); +} + +/**\internal + * \brief called at start of response + * Set min inspect size. + */ +static int HTPCallbackResponseStart(htp_tx_t *tx) +{ + HtpState *hstate = htp_connp_get_user_data(tx->connp); + if (hstate == NULL) { + SCReturnInt(HTP_ERROR); + } + + if (hstate->cfg) + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, + hstate->cfg->response.inspect_min_size); + SCReturnInt(HTP_OK); +} + + /** * \brief callback for request to store the recent incoming request in to the recent_in_tx for the given htp state @@ -1984,9 +2070,12 @@ static int HTPCallbackRequest(htp_tx_t *tx) SCLogDebug("closing file that was being stored"); (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER); htud->tsflags &= ~HTP_FILENAME_SET; + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, + (uint32_t)hstate->conn->in_data_counter); } } + hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter; /* request done, do raw reassembly now to inspect state and stream * at the same time. */ AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER); @@ -2045,6 +2134,7 @@ static int HTPCallbackResponse(htp_tx_t *tx) } } + hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter; SCReturnInt(HTP_OK); } @@ -2202,7 +2292,10 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec) htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData); htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData); + htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart); htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest); + + htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart); htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse); htp_config_set_parse_request_cookies(cfg_prec->cfg, 0); diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index 9926f5c756..2cbd2704da 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -247,6 +247,8 @@ typedef struct HtpState_ { uint16_t events; uint16_t htp_messages_offset; /**< offset into conn->messages list */ uint64_t tx_with_detect_state_cnt; + uint64_t last_request_data_stamp; + uint64_t last_response_data_stamp; } HtpState; /** part of the engine needs the request body (e.g. http_client_body keyword) */ diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c index 568cd1cff2..b0e29d802d 100644 --- a/src/detect-engine-hcbd.c +++ b/src/detect-engine-hcbd.c @@ -162,7 +162,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i goto end; } - if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) { + if (!htp_state->cfg->http_body_inline) { /* inspect the body if the transfer is complete or we have hit * our body size limit */ if ((htp_state->cfg->request.body_limit == 0 || @@ -204,13 +204,10 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i offset); det_ctx->hcbd[index].offset = offset; - if (htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size) { - SCLogDebug("not updating tracker as we're still below inspect_min_size"); - } else { - /* move inspected tracker to end of the data. HtpBodyPrune will consider - * the window sizes when freeing data */ - htud->request_body.body_inspected = htud->request_body.content_len_so_far; - } + /* move inspected tracker to end of the data. HtpBodyPrune will consider + * the window sizes when freeing data */ + htud->request_body.body_inspected = htud->request_body.content_len_so_far; + buffer = det_ctx->hcbd[index].buffer; *buffer_len = det_ctx->hcbd[index].buffer_len; *stream_start_offset = det_ctx->hcbd[index].offset; diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c index 3e7cd84a3f..2169e8f840 100644 --- a/src/detect-engine-hsbd.c +++ b/src/detect-engine-hsbd.c @@ -167,7 +167,7 @@ static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i flags & STREAM_EOF ? "true" : "false", (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false"); - if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) { + if (!htp_state->cfg->http_body_inline) { /* inspect the body if the transfer is complete or we have hit * our body size limit */ if ((htp_state->cfg->response.body_limit == 0 || @@ -209,14 +209,10 @@ static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i offset); det_ctx->hsbd[index].offset = offset; - if (htud->response_body.content_len_so_far < htp_state->cfg->response.inspect_min_size) { - SCLogDebug("not updating tracker as we're still below inspect_min_size"); - } else { - /* move inspected tracker to end of the data. HtpBodyPrune will consider - * the window sizes when freeing data */ - htud->response_body.body_inspected = htud->response_body.content_len_so_far; - SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected); - } + /* move inspected tracker to end of the data. HtpBodyPrune will consider + * the window sizes when freeing data */ + htud->response_body.body_inspected = htud->response_body.content_len_so_far; + SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected); buffer = det_ctx->hsbd[index].buffer; *buffer_len = det_ctx->hsbd[index].buffer_len; *stream_start_offset = det_ctx->hsbd[index].offset;