From: Victor Julien Date: Wed, 18 Apr 2018 13:25:40 +0000 (+0200) Subject: detect: improve inspect buffer handling X-Git-Tag: suricata-4.1.1~48 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0b3220a0df9689e3cc9633981c0e558f451d23b8;p=thirdparty%2Fsuricata.git detect: improve inspect buffer handling Fix and Optimize cleanup. For the simple single inspect buffer optimize the cleanup by keeping track of the actually used buffers. This avoid looping over unused buffers. Fix the case of cleaning not being done after a tx if the next tx is also inspected in the context of the same packet. Fix cleanup of the multi-inspect buffers. Optimize in 2 ways. First like with single keep track of which multi-inspect buffers have been used. Second, keep a max of the buffers within a multi-inspect buffer. Use this max to limit (nested) looping. --- diff --git a/src/detect-dns-query.c b/src/detect-dns-query.c index 6b74f52d79..e22e19ba13 100644 --- a/src/detect-dns-query.c +++ b/src/detect-dns-query.c @@ -77,41 +77,14 @@ struct DnsQueryGetDataArgs { #endif }; -/** \brief get an InspectionBuffer. Make space if we have to. */ -static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id) -{ - if (id >= fb->size) { - uint32_t old_size = fb->size; - uint32_t new_size = id + 1; - uint32_t grow_by = new_size - old_size; - SCLogDebug("size is %u, need %u, so growing by %u", - old_size, new_size, grow_by); - - void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer)); - if (ptr == NULL) - return NULL; - - InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size; - SCLogDebug("fb->inspection_buffers %p ptr %p to_zero %p", - fb->inspection_buffers, ptr, to_zero); - memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer))); - fb->inspection_buffers = ptr; - fb->size = new_size; - } - - InspectionBuffer *buffer = &fb->inspection_buffers[id]; - SCLogDebug("using file_data buffer %p", buffer); - return buffer; -} - static InspectionBuffer *DnsQueryGetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *f, struct DnsQueryGetDataArgs *cbdata, int list_id, bool first) { SCEnter(); - InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id]; - InspectionBuffer *buffer = GetBuffer(fb, cbdata->local_id); + InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id); + InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, cbdata->local_id); if (buffer == NULL) return NULL; if (!first && buffer->inspect != NULL) diff --git a/src/detect-engine-filedata.c b/src/detect-engine-filedata.c index dfe0bef516..d8d3721042 100644 --- a/src/detect-engine-filedata.c +++ b/src/detect-engine-filedata.c @@ -36,31 +36,6 @@ #include "app-layer-parser.h" -static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id) -{ - if (id >= fb->size) { - uint32_t old_size = fb->size; - uint32_t new_size = id + 1; - uint32_t grow_by = new_size - old_size; - SCLogDebug("size is %u, need %u, so growing by %u", old_size, new_size, grow_by); - - SCLogDebug("fb->inspection_buffers %p", fb->inspection_buffers); - void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer)); - if (ptr == NULL) - return NULL; - - InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size; - SCLogDebug("ptr %p to_zero %p", ptr, to_zero); - memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer))); - fb->inspection_buffers = ptr; - fb->size = new_size; - } - - InspectionBuffer *buffer = &fb->inspection_buffers[id]; - SCLogDebug("using file_data buffer %p", buffer); - return buffer; -} - static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *f, uint8_t flow_flags, File *cur_file, @@ -68,8 +43,8 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx, { SCEnter(); - InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id]; - InspectionBuffer *buffer = GetBuffer(fb, local_file_id); + InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id); + InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, local_file_id); if (buffer == NULL) return NULL; if (!first && buffer->inspect != NULL) diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c index 1b71eb9a8e..9ae7cb817c 100644 --- a/src/detect-engine-hsbd.c +++ b/src/detect-engine-hsbd.c @@ -85,7 +85,7 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx, { SCEnter(); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect != NULL) return buffer; diff --git a/src/detect-engine.c b/src/detect-engine.c index 830a41427d..fd08700798 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -844,6 +844,81 @@ int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s) return 0; } +void InspectionBufferClean(DetectEngineThreadCtx *det_ctx) +{ + /* single buffers */ + for (uint32_t i = 0; i < det_ctx->inspect.to_clear_idx; i++) + { + const uint32_t idx = det_ctx->inspect.to_clear_queue[i]; + InspectionBuffer *buffer = &det_ctx->inspect.buffers[idx]; + buffer->inspect = NULL; + } + det_ctx->inspect.to_clear_idx = 0; + + /* multi buffers */ + for (uint32_t i = 0; i < det_ctx->multi_inspect.to_clear_idx; i++) + { + const uint32_t idx = det_ctx->multi_inspect.to_clear_queue[i]; + InspectionBufferMultipleForList *mbuffer = &det_ctx->multi_inspect.buffers[idx]; + for (uint32_t x = 0; x <= mbuffer->max; x++) { + InspectionBuffer *buffer = &mbuffer->inspection_buffers[x]; + buffer->inspect = NULL; + } + mbuffer->init = 0; + mbuffer->max = 0; + } + det_ctx->multi_inspect.to_clear_idx = 0; +} + +InspectionBuffer *InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id) +{ + InspectionBuffer *buffer = &det_ctx->inspect.buffers[list_id]; + if (buffer->inspect == NULL) { + det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id; + } + return buffer; +} + +/** \brief for a InspectionBufferMultipleForList get a InspectionBuffer + * \param fb the multiple buffer array + * \param local_id the index to get a buffer + * \param buffer the inspect buffer or NULL in case of error */ +InspectionBuffer *InspectionBufferMultipleForListGet(InspectionBufferMultipleForList *fb, uint32_t local_id) +{ + if (local_id >= fb->size) { + uint32_t old_size = fb->size; + uint32_t new_size = local_id + 1; + uint32_t grow_by = new_size - old_size; + SCLogDebug("size is %u, need %u, so growing by %u", old_size, new_size, grow_by); + + SCLogDebug("fb->inspection_buffers %p", fb->inspection_buffers); + void *ptr = SCRealloc(fb->inspection_buffers, (local_id + 1) * sizeof(InspectionBuffer)); + if (ptr == NULL) + return NULL; + + InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size; + SCLogDebug("ptr %p to_zero %p", ptr, to_zero); + memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer))); + fb->inspection_buffers = ptr; + fb->size = new_size; + } + + fb->max = MAX(fb->max, local_id); + InspectionBuffer *buffer = &fb->inspection_buffers[local_id]; + SCLogDebug("using file_data buffer %p", buffer); + return buffer; +} + +InspectionBufferMultipleForList *InspectionBufferGetMulti(DetectEngineThreadCtx *det_ctx, const int list_id) +{ + InspectionBufferMultipleForList *buffer = &det_ctx->multi_inspect.buffers[list_id]; + if (!buffer->init) { + det_ctx->multi_inspect.to_clear_queue[det_ctx->multi_inspect.to_clear_idx++] = list_id; + buffer->init = 1; + } + return buffer; +} + void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size) { memset(buffer, 0, sizeof(*buffer)); @@ -2253,16 +2328,28 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx * det_ctx->base64_decoded_len = 0; } - det_ctx->inspect_buffers_size = de_ctx->buffer_type_id; - det_ctx->inspect_buffers = SCCalloc(det_ctx->inspect_buffers_size, sizeof(InspectionBuffer)); - if (det_ctx->inspect_buffers == NULL) { + det_ctx->inspect.buffers_size = de_ctx->buffer_type_id; + det_ctx->inspect.buffers = SCCalloc(det_ctx->inspect.buffers_size, sizeof(InspectionBuffer)); + if (det_ctx->inspect.buffers == NULL) { return TM_ECODE_FAILED; } - det_ctx->multi_inspect_buffers_size = de_ctx->buffer_type_id; - det_ctx->multi_inspect_buffers = SCCalloc(det_ctx->multi_inspect_buffers_size, sizeof(InspectionBufferMultipleForList)); - if (det_ctx->multi_inspect_buffers == NULL) { + det_ctx->inspect.to_clear_queue = SCCalloc(det_ctx->inspect.buffers_size, sizeof(uint32_t)); + if (det_ctx->inspect.to_clear_queue == NULL) { return TM_ECODE_FAILED; } + det_ctx->inspect.to_clear_idx = 0; + + det_ctx->multi_inspect.buffers_size = de_ctx->buffer_type_id; + det_ctx->multi_inspect.buffers = SCCalloc(det_ctx->multi_inspect.buffers_size, sizeof(InspectionBufferMultipleForList)); + if (det_ctx->multi_inspect.buffers == NULL) { + return TM_ECODE_FAILED; + } + det_ctx->multi_inspect.to_clear_queue = SCCalloc(det_ctx->multi_inspect.buffers_size, sizeof(uint32_t)); + if (det_ctx->multi_inspect.to_clear_queue == NULL) { + return TM_ECODE_FAILED; + } + det_ctx->multi_inspect.to_clear_idx = 0; + DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx); DetectEngineThreadCtxInitGlobalKeywords(det_ctx); @@ -2469,23 +2556,28 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) SCFree(det_ctx->base64_decoded); } - if (det_ctx->inspect_buffers) { - for (uint32_t i = 0; i < det_ctx->inspect_buffers_size; i++) { - InspectionBufferFree(&det_ctx->inspect_buffers[i]); + if (det_ctx->inspect.buffers) { + for (uint32_t i = 0; i < det_ctx->inspect.buffers_size; i++) { + InspectionBufferFree(&det_ctx->inspect.buffers[i]); } - SCFree(det_ctx->inspect_buffers); + SCFree(det_ctx->inspect.buffers); + } + if (det_ctx->inspect.to_clear_queue) { + SCFree(det_ctx->inspect.to_clear_queue); } - if (det_ctx->multi_inspect_buffers) { - for (uint32_t i = 0; i < det_ctx->multi_inspect_buffers_size; i++) { - InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[i]; + if (det_ctx->multi_inspect.buffers) { + for (uint32_t i = 0; i < det_ctx->multi_inspect.buffers_size; i++) { + InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect.buffers[i]; for (uint32_t x = 0; x < fb->size; x++) { InspectionBufferFree(&fb->inspection_buffers[x]); } SCFree(fb->inspection_buffers); } - SCFree(det_ctx->multi_inspect_buffers); + SCFree(det_ctx->multi_inspect.buffers); + } + if (det_ctx->multi_inspect.to_clear_queue) { + SCFree(det_ctx->multi_inspect.to_clear_queue); } - DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx); if (det_ctx->de_ctx != NULL) { diff --git a/src/detect-engine.h b/src/detect-engine.h index cfcc057885..e4435a5413 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -35,6 +35,10 @@ void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size) void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len); void InspectionBufferApplyTransforms(InspectionBuffer *buffer, const DetectEngineTransforms *transforms); +void InspectionBufferClean(DetectEngineThreadCtx *det_ctx); +InspectionBuffer *InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id); +InspectionBuffer *InspectionBufferMultipleForListGet(InspectionBufferMultipleForList *fb, uint32_t local_id); +InspectionBufferMultipleForList *InspectionBufferGetMulti(DetectEngineThreadCtx *det_ctx, const int list_id); int DetectBufferTypeRegister(const char *name); int DetectBufferTypeGetByName(const char *name); diff --git a/src/detect-http-request-line.c b/src/detect-http-request-line.c index af6d096214..455fb4236c 100644 --- a/src/detect-http-request-line.c +++ b/src/detect-http-request-line.c @@ -122,9 +122,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { htp_tx_t *tx = (htp_tx_t *)txv; if (unlikely(tx->request_line == NULL)) { diff --git a/src/detect-krb5-cname.c b/src/detect-krb5-cname.c index df580f07b8..2c1a9a8232 100644 --- a/src/detect-krb5-cname.c +++ b/src/detect-krb5-cname.c @@ -55,33 +55,6 @@ static int DetectKrb5CNameSetup(DetectEngineCtx *de_ctx, Signature *s, const cha return 0; } -/** \brief get an InspectionBuffer. Make space if we have to. */ -static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id) -{ - if (id >= fb->size) { - uint32_t old_size = fb->size; - uint32_t new_size = id + 1; - uint32_t grow_by = new_size - old_size; - SCLogDebug("size is %u, need %u, so growing by %u", - old_size, new_size, grow_by); - - void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer)); - if (ptr == NULL) - return NULL; - - InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size; - SCLogDebug("fb->inspection_buffers %p ptr %p to_zero %p", - fb->inspection_buffers, ptr, to_zero); - memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer))); - fb->inspection_buffers = ptr; - fb->size = new_size; - } - - InspectionBuffer *buffer = &fb->inspection_buffers[id]; - SCLogDebug("using file_data buffer %p", buffer); - return buffer; -} - static InspectionBuffer *GetKrb5CNameData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const struct Krb5PrincipalNameDataArgs *cbdata, @@ -89,8 +62,8 @@ static InspectionBuffer *GetKrb5CNameData(DetectEngineThreadCtx *det_ctx, { SCEnter(); - InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id]; - InspectionBuffer *buffer = GetBuffer(fb, cbdata->local_id); + InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id); + InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, cbdata->local_id); if (buffer == NULL) return NULL; if (!first && buffer->inspect != NULL) diff --git a/src/detect-krb5-sname.c b/src/detect-krb5-sname.c index f9f0f6a3cb..dd01c7d963 100644 --- a/src/detect-krb5-sname.c +++ b/src/detect-krb5-sname.c @@ -55,33 +55,6 @@ static int DetectKrb5SNameSetup(DetectEngineCtx *de_ctx, Signature *s, const cha return 0; } -/** \brief get an InspectionBuffer. Make space if we have to. */ -static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id) -{ - if (id >= fb->size) { - uint32_t old_size = fb->size; - uint32_t new_size = id + 1; - uint32_t grow_by = new_size - old_size; - SCLogDebug("size is %u, need %u, so growing by %u", - old_size, new_size, grow_by); - - void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer)); - if (ptr == NULL) - return NULL; - - InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size; - SCLogDebug("fb->inspection_buffers %p ptr %p to_zero %p", - fb->inspection_buffers, ptr, to_zero); - memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer))); - fb->inspection_buffers = ptr; - fb->size = new_size; - } - - InspectionBuffer *buffer = &fb->inspection_buffers[id]; - SCLogDebug("using file_data buffer %p", buffer); - return buffer; -} - static InspectionBuffer *GetKrb5SNameData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const struct Krb5PrincipalNameDataArgs *cbdata, @@ -89,8 +62,8 @@ static InspectionBuffer *GetKrb5SNameData(DetectEngineThreadCtx *det_ctx, { SCEnter(); - InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id]; - InspectionBuffer *buffer = GetBuffer(fb, cbdata->local_id); + InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id); + InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, cbdata->local_id); if (buffer == NULL) return NULL; if (!first && buffer->inspect != NULL) diff --git a/src/detect-smb-share.c b/src/detect-smb-share.c index 3e899f19ae..95e19f7481 100644 --- a/src/detect-smb-share.c +++ b/src/detect-smb-share.c @@ -56,9 +56,7 @@ static InspectionBuffer *GetNamedPipeData(DetectEngineThreadCtx *det_ctx, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { uint32_t b_len = 0; uint8_t *b = NULL; @@ -119,9 +117,7 @@ static InspectionBuffer *GetShareData(DetectEngineThreadCtx *det_ctx, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { uint32_t b_len = 0; uint8_t *b = NULL; diff --git a/src/detect-template-buffer.c b/src/detect-template-buffer.c index a1ee663178..6264f81b58 100644 --- a/src/detect-template-buffer.c +++ b/src/detect-template-buffer.c @@ -117,14 +117,11 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, Flow *_f, const uint8_t flow_flags, void *txv, const int list_id) { - const uint8_t *data = NULL; - uint32_t data_len = 0; - - BUG_ON(det_ctx->inspect_buffers == NULL); - - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { const TemplateTransaction *tx = (TemplateTransaction *)txv; + const uint8_t *data = NULL; + uint32_t data_len = 0; if (flow_flags & STREAM_TOSERVER) { data = tx->request_buffer; diff --git a/src/detect-tls-cert-fingerprint.c b/src/detect-tls-cert-fingerprint.c index 4f26542d4e..1d5244ed19 100644 --- a/src/detect-tls-cert-fingerprint.c +++ b/src/detect-tls-cert-fingerprint.c @@ -125,9 +125,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { SSLState *ssl_state = (SSLState *)_f->alstate; diff --git a/src/detect-tls-cert-issuer.c b/src/detect-tls-cert-issuer.c index 6edcf1d729..2896549aef 100644 --- a/src/detect-tls-cert-issuer.c +++ b/src/detect-tls-cert-issuer.c @@ -112,9 +112,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { SSLState *ssl_state = (SSLState *)_f->alstate; diff --git a/src/detect-tls-cert-serial.c b/src/detect-tls-cert-serial.c index 3ae70875f7..c58ea9553b 100644 --- a/src/detect-tls-cert-serial.c +++ b/src/detect-tls-cert-serial.c @@ -124,9 +124,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { SSLState *ssl_state = (SSLState *)_f->alstate; diff --git a/src/detect-tls-cert-subject.c b/src/detect-tls-cert-subject.c index b6e0d1e89d..dd9f043abc 100644 --- a/src/detect-tls-cert-subject.c +++ b/src/detect-tls-cert-subject.c @@ -111,9 +111,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { SSLState *ssl_state = (SSLState *)_f->alstate; diff --git a/src/detect-tls-ja3-hash.c b/src/detect-tls-ja3-hash.c index 0a7ec1e485..1e55df2030 100644 --- a/src/detect-tls-ja3-hash.c +++ b/src/detect-tls-ja3-hash.c @@ -130,9 +130,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { SSLState *ssl_state = (SSLState *)_f->alstate; diff --git a/src/detect-tls-ja3-string.c b/src/detect-tls-ja3-string.c index 70aa1c702c..43e18f8832 100644 --- a/src/detect-tls-ja3-string.c +++ b/src/detect-tls-ja3-string.c @@ -120,9 +120,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { SSLState *ssl_state = (SSLState *)_f->alstate; diff --git a/src/detect-tls-sni.c b/src/detect-tls-sni.c index ab7f94c82e..b4c27a2ee8 100644 --- a/src/detect-tls-sni.c +++ b/src/detect-tls-sni.c @@ -110,9 +110,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv, const int list_id) { - BUG_ON(det_ctx->inspect_buffers == NULL); - InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id]; - + InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id); if (buffer->inspect == NULL) { SSLState *ssl_state = (SSLState *)_f->alstate; diff --git a/src/detect.c b/src/detect.c index b83ca708a7..a7b2124d2b 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1038,13 +1038,6 @@ static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx, PacketPatternCleanup(det_ctx); if (pflow != NULL) { - // TODO clean this up - const int nlists = det_ctx->de_ctx->buffer_type_id; - for (int i = 0; i < nlists; i++) { - InspectionBuffer *buffer = &det_ctx->inspect_buffers[i]; - buffer->inspect = NULL; - } - /* update inspected tracker for raw reassembly */ if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL) { StreamReassembleRawUpdateProgress(pflow->protoctx, p, @@ -1600,6 +1593,8 @@ static void DetectRunTx(ThreadVars *tv, flow_flags, new_detect_flags); } next: + InspectionBufferClean(det_ctx); + if (!ires.has_next) break; } diff --git a/src/detect.h b/src/detect.h index 1ebeac49d7..906ba39fe0 100644 --- a/src/detect.h +++ b/src/detect.h @@ -366,7 +366,9 @@ typedef struct InspectionBuffer { typedef struct InspectionBufferMultipleForList { InspectionBuffer *inspection_buffers; - uint32_t size; /**< size in number of elements */ + uint32_t size; /**< size in number of elements */ + uint32_t max:31; /**< max id in use in this run */ + uint32_t init:1; /**< first time used this run. Used for clean logic */ } InspectionBufferMultipleForList; typedef struct DetectEngineTransforms { @@ -1008,14 +1010,22 @@ typedef struct DetectEngineThreadCtx_ { #endif int inspect_list; /**< list we're currently inspecting, DETECT_SM_LIST_* */ - InspectionBuffer *inspect_buffers; - uint32_t inspect_buffers_size; /**< in number of elements */ - uint32_t multi_inspect_buffers_size; /**< in number of elements */ + struct { + InspectionBuffer *buffers; + uint32_t buffers_size; /**< in number of elements */ + uint32_t to_clear_idx; + uint32_t *to_clear_queue; + } inspect; - /* inspection buffers for more complete case. As we can inspect multiple - * buffers in parallel, we need this extra wrapper struct */ - InspectionBufferMultipleForList *multi_inspect_buffers; + struct { + /** inspection buffers for more complex case. As we can inspect multiple + * buffers in parallel, we need this extra wrapper struct */ + InspectionBufferMultipleForList *buffers; + uint32_t buffers_size; /**< in number of elements */ + uint32_t to_clear_idx; + uint32_t *to_clear_queue; + } multi_inspect; /* used to discontinue any more matching */ uint16_t discontinue_matching;