From: Victor Julien Date: Fri, 17 Aug 2018 15:53:16 +0000 (+0200) Subject: http: implement min size stream logic X-Git-Tag: suricata-4.1.0-rc2~60 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=e02b74dee783b4434c0b9885fc8d499155498b66;p=thirdparty%2Fsuricata.git http: implement min size stream logic Update HTTP parser to set the min inspect depth per transaction. This allows for signatures to have their fast_pattern in the HTTP body, while still being able to inspect the raw stream reliably with it. The inspect depth is set per transaction as it: - depends on the per personality config for min inspect size - is set to the size of the actual body if it is smaller After the initial inspection is done, it is set to 0 which disables the feature for the rest of the transaction. This removes the rescanning flush logic in commit 7e004f52c60c5e4d7cd8f5ed09491291d18f42d2 and provides an alternative fix for bug #2522. The old approach caused too much rescanning of HTTP body data leading to a performance degradation. Bug #2522 --- diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 28eaea3d65..648b3d2b54 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1773,6 +1773,32 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d) } end: + if (hstate->conn != NULL) { + SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")", + tx_ud->request_body.content_len_so_far, + hstate->cfg->request.inspect_min_size, + (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp); + + /* if we reach the inspect_min_size we'll trigger inspection, + * so make sure that raw stream is also inspected. Set the + * data to be used to the ammount of raw bytes we've seen to + * get here. */ + if (tx_ud->request_body.body_inspected == 0 && + tx_ud->request_body.content_len_so_far >= hstate->cfg->request.inspect_min_size) { + if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp && + (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX) + { + uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp); + + /* body still in progress, but due to min inspect size we need to inspect now */ + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, x); + AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER); + } + /* after the start of the body, disable the depth logic */ + } else if (tx_ud->request_body.body_inspected > 0) { + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0); + } + } SCReturnInt(HTP_OK); } @@ -1844,6 +1870,31 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d) } } + if (hstate->conn != NULL) { + SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")", + tx_ud->response_body.content_len_so_far, + hstate->cfg->response.inspect_min_size, + (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp); + /* if we reach the inspect_min_size we'll trigger inspection, + * so make sure that raw stream is also inspected. Set the + * data to be used to the ammount of raw bytes we've seen to + * get here. */ + if (tx_ud->response_body.body_inspected == 0 && + tx_ud->response_body.content_len_so_far >= hstate->cfg->response.inspect_min_size) { + if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp && + (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX) + { + uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp); + + /* body still in progress, but due to min inspect size we need to inspect now */ + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, x); + AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT); + } + /* after the start of the body, disable the depth logic */ + } else if (tx_ud->response_body.body_inspected > 0) { + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0); + } + } SCReturnInt(HTP_OK); } @@ -1906,6 +1957,41 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx) return HTP_OK; } +/**\internal + * \brief called at start of request + * Set min inspect size. + */ +static int HTPCallbackRequestStart(htp_tx_t *tx) +{ + HtpState *hstate = htp_connp_get_user_data(tx->connp); + if (hstate == NULL) { + SCReturnInt(HTP_ERROR); + } + + if (hstate->cfg) + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, + hstate->cfg->request.inspect_min_size); + SCReturnInt(HTP_OK); +} + +/**\internal + * \brief called at start of response + * Set min inspect size. + */ +static int HTPCallbackResponseStart(htp_tx_t *tx) +{ + HtpState *hstate = htp_connp_get_user_data(tx->connp); + if (hstate == NULL) { + SCReturnInt(HTP_ERROR); + } + + if (hstate->cfg) + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, + hstate->cfg->response.inspect_min_size); + SCReturnInt(HTP_OK); +} + + /** * \brief callback for request to store the recent incoming request in to the recent_in_tx for the given htp state @@ -1938,9 +2024,12 @@ static int HTPCallbackRequest(htp_tx_t *tx) SCLogDebug("closing file that was being stored"); (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER); htud->tsflags &= ~HTP_FILENAME_SET; + StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, + (uint32_t)hstate->conn->in_data_counter); } } + hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter; /* request done, do raw reassembly now to inspect state and stream * at the same time. */ AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER); @@ -1996,6 +2085,7 @@ static int HTPCallbackResponse(htp_tx_t *tx) } } + hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter; SCReturnInt(HTP_OK); } @@ -2153,7 +2243,10 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec) htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData); htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData); + htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart); htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest); + + htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart); htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse); htp_config_set_parse_request_cookies(cfg_prec->cfg, 0); diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index e0fcce12b7..249c1cf335 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -234,6 +234,8 @@ typedef struct HtpState_ { uint16_t flags; uint16_t events; uint16_t htp_messages_offset; /**< offset into conn->messages list */ + uint64_t last_request_data_stamp; + uint64_t last_response_data_stamp; } HtpState; /** part of the engine needs the request body (e.g. http_client_body keyword) */ diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c index 9c95ca40f9..b7415b62aa 100644 --- a/src/detect-engine-hcbd.c +++ b/src/detect-engine-hcbd.c @@ -163,7 +163,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i goto end; } - if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) { + if (!htp_state->cfg->http_body_inline) { /* inspect the body if the transfer is complete or we have hit * our body size limit */ if ((htp_state->cfg->request.body_limit == 0 || @@ -205,13 +205,10 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i offset); det_ctx->hcbd[index].offset = offset; - if (htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size) { - SCLogDebug("not updating tracker as we're still below inspect_min_size"); - } else { - /* move inspected tracker to end of the data. HtpBodyPrune will consider - * the window sizes when freeing data */ - htud->request_body.body_inspected = htud->request_body.content_len_so_far; - } + /* move inspected tracker to end of the data. HtpBodyPrune will consider + * the window sizes when freeing data */ + htud->request_body.body_inspected = htud->request_body.content_len_so_far; + buffer = det_ctx->hcbd[index].buffer; *buffer_len = det_ctx->hcbd[index].buffer_len; *stream_start_offset = det_ctx->hcbd[index].offset; diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c index 6cfeef3cb2..1b71eb9a8e 100644 --- a/src/detect-engine-hsbd.c +++ b/src/detect-engine-hsbd.c @@ -118,7 +118,7 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx, flags & STREAM_EOF ? "true" : "false", (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false"); - if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) { + if (!htp_state->cfg->http_body_inline) { /* inspect the body if the transfer is complete or we have hit * our body size limit */ if ((htp_state->cfg->response.body_limit == 0 || @@ -178,14 +178,10 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx, } } - if (body->content_len_so_far < htp_state->cfg->response.inspect_min_size) { - SCLogDebug("not updating tracker as we're still below inspect_min_size"); - } else { - /* move inspected tracker to end of the data. HtpBodyPrune will consider - * the window sizes when freeing data */ - body->body_inspected = body->content_len_so_far; - SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected); - } + /* move inspected tracker to end of the data. HtpBodyPrune will consider + * the window sizes when freeing data */ + body->body_inspected = body->content_len_so_far; + SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected); SCReturnPtr(buffer, "InspectionBuffer"); }