From 0de86211c6179f9a5829f69c495358a9b2f18619 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 30 Oct 2017 17:14:34 +0100 Subject: [PATCH] detect: register dynamic buffers into de_ctx Register buffers that are created during rule parsing. Currently this means an existing buffer with one or more transformations. --- src/detect-byte-extract.c | 9 +++---- src/detect-content.c | 5 ++-- src/detect-engine-analyzer.c | 2 +- src/detect-engine-build.c | 18 ++++++-------- src/detect-engine-mpm.c | 9 +++++-- src/detect-engine.c | 47 +++++++++++++++++++++++++++-------- src/detect-fast-pattern.c | 5 ++-- src/detect-flowbits.c | 4 +-- src/detect-parse.c | 21 ++++++---------- src/detect.c | 2 +- src/detect.h | 5 ++++ src/util-profiling-keywords.c | 12 ++++----- 12 files changed, 81 insertions(+), 58 deletions(-) diff --git a/src/detect-byte-extract.c b/src/detect-byte-extract.c index 26166d315c..8afe2b982a 100644 --- a/src/detect-byte-extract.c +++ b/src/detect-byte-extract.c @@ -641,15 +641,12 @@ static void DetectByteExtractFree(void *ptr) */ SigMatch *DetectByteExtractRetrieveSMVar(const char *arg, const Signature *s) { - DetectByteExtractData *bed = NULL; - int list; - const int nlists = DetectBufferTypeMaxId(); - - for (list = 0; list < nlists; list++) { + const int nlists = s->init_data->smlists_array_size; + for (int list = 0; list < nlists; list++) { SigMatch *sm = s->init_data->smlists[list]; while (sm != NULL) { if (sm->type == DETECT_BYTE_EXTRACT) { - bed = (DetectByteExtractData *)sm->ctx; + const DetectByteExtractData *bed = (const DetectByteExtractData *)sm->ctx; if (strcmp(bed->name, arg) == 0) { return sm; } diff --git a/src/detect-content.c b/src/detect-content.c index 6eb47aeb99..ff8ed27361 100644 --- a/src/detect-content.c +++ b/src/detect-content.c @@ -426,9 +426,8 @@ void DetectContentPropagateLimits(Signature *s) { BUG_ON(s == NULL || s->init_data == NULL); - int nlists = DetectBufferTypeMaxId(); - int list = 0; - for (list = 0; list < nlists; list++) { + uint32_t list = 0; + for (list = 0; list < s->init_data->smlists_array_size; list++) { uint16_t offset = 0; uint16_t offset_plus_pat = 0; uint16_t depth = 0; diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c index aa490119e1..a03c14d695 100644 --- a/src/detect-engine-analyzer.c +++ b/src/detect-engine-analyzer.c @@ -535,7 +535,7 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx, uint32_t warn_no_direction = 0; uint32_t warn_both_direction = 0; - const int nlists = DetectBufferTypeMaxId(); + const int nlists = de_ctx->buffer_type_id; const int filedata_id = DetectBufferTypeGetByName("file_data"); const int httpmethod_id = DetectBufferTypeGetByName("http_method"); const int httpuri_id = DetectBufferTypeGetByName("http_uri"); diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index 01d3e1360f..e68aba7c8d 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -197,7 +197,7 @@ int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s) return 0; /* for now assume that all registered buffer types are incompatible */ - const int nlists = DetectBufferTypeMaxId(); + const int nlists = s->init_data->smlists_array_size; for (int i = 0; i < nlists; i++) { if (s->init_data->smlists[i] == NULL) continue; @@ -266,7 +266,7 @@ static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s) return 0; /* for now assume that all registered buffer types are incompatible */ - const int nlists = DetectBufferTypeMaxId(); + const int nlists = s->init_data->smlists_array_size; for (int i = 0; i < nlists; i++) { if (s->init_data->smlists[i] == NULL) continue; @@ -353,7 +353,7 @@ static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s) } /* for now assume that all registered buffer types are incompatible */ - const int nlists = DetectBufferTypeMaxId(); + const int nlists = s->init_data->smlists_array_size; for (int i = 0; i < nlists; i++) { if (s->init_data->smlists[i] == NULL) continue; @@ -1255,7 +1255,6 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) uint32_t cnt_payload = 0; uint32_t cnt_applayer = 0; uint32_t cnt_deonly = 0; - const int nlists = DetectBufferTypeMaxId(); if (!(de_ctx->flags & DE_QUIET)) { SCLogDebug("building signature grouping structure, stage 1: " @@ -1351,7 +1350,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) int prefilter_list = DETECT_TBLSIZE; /* get the keyword supporting prefilter with the lowest type */ - for (i = 0; i < nlists; i++) { + for (i = 0; i < (int)tmp_s->init_data->smlists_array_size; i++) { SigMatch *sm = tmp_s->init_data->smlists[i]; while (sm != NULL) { if (sigmatch_table[sm->type].SupportsPrefilter != NULL) { @@ -1365,7 +1364,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) /* apply that keyword as prefilter */ if (prefilter_list != DETECT_TBLSIZE) { - for (i = 0; i < nlists; i++) { + for (i = 0; i < (int)tmp_s->init_data->smlists_array_size; i++) { SigMatch *sm = tmp_s->init_data->smlists[i]; while (sm != NULL) { if (sm->type == prefilter_list) { @@ -1382,7 +1381,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) /* run buffer type callbacks if any */ int x; - for (x = 0; x < nlists; x++) { + for (x = 0; x < (int)tmp_s->init_data->smlists_array_size; x++) { if (tmp_s->init_data->smlists[x]) DetectBufferRunSetupCallback(de_ctx, x, tmp_s); } @@ -1809,7 +1808,6 @@ static int SigMatchPrepare(DetectEngineCtx *de_ctx) { SCEnter(); - const int nlists = DetectBufferTypeMaxId(); Signature *s = de_ctx->sig_list; for (; s != NULL; s = s->next) { /* set up inspect engines */ @@ -1823,8 +1821,8 @@ static int SigMatchPrepare(DetectEngineCtx *de_ctx) } /* free lists. Ctx' are xferred to sm_arrays so won't get freed */ - int i; - for (i = 0; i < nlists; i++) { + uint32_t i; + for (i = 0; i < s->init_data->smlists_array_size; i++) { SigMatch *sm = s->init_data->smlists[i]; while (sm != NULL) { SigMatch *nsm = sm->next; diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index ba45cb093d..47b0c1c104 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -642,9 +642,8 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) if (s->init_data->mpm_sm != NULL) return; - const int nlists = DetectBufferTypeMaxId(); - SigMatch *mpm_sm = NULL, *sm = NULL; + const int nlists = s->init_data->smlists_array_size; int nn_sm_list[nlists]; int n_sm_list[nlists]; memset(nn_sm_list, 0, nlists * sizeof(int)); @@ -723,6 +722,9 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) int max_len = 0; int i; for (i = 0; i < count_final_sm_list; i++) { + if (final_sm_list[i] >= (int)s->init_data->smlists_array_size) + continue; + for (sm = s->init_data->smlists[final_sm_list[i]]; sm != NULL; sm = sm->next) { if (sm->type != DETECT_CONTENT) continue; @@ -738,6 +740,9 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) } for (i = 0; i < count_final_sm_list; i++) { + if (final_sm_list[i] >= (int)s->init_data->smlists_array_size) + continue; + for (sm = s->init_data->smlists[final_sm_list[i]]; sm != NULL; sm = sm->next) { if (sm->type != DETECT_CONTENT) continue; diff --git a/src/detect-engine.c b/src/detect-engine.c index cf0db0a168..de3b0f15cc 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -316,7 +316,7 @@ static void AppendStreamInspectEngine(Signature *s, SigMatchData *stream, int di */ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s) { - const int nlists = DetectBufferTypeMaxId(); + const int nlists = s->init_data->smlists_array_size; SigMatchData *ptrs[nlists]; memset(&ptrs, 0, (nlists * sizeof(SigMatchData *))); @@ -342,6 +342,9 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature while (t != NULL) { bool prepend = false; + if (t->sm_list >= nlists) + goto next; + if (ptrs[t->sm_list] == NULL) goto next; @@ -466,12 +469,22 @@ next: */ void DetectEngineAppInspectionEngineSignatureFree(Signature *s) { - const int nlists = DetectBufferTypeMaxId(); + int nlists = 0; + + DetectEngineAppInspectionEngine *ie = s->app_inspect; + while (ie) { + nlists = MAX(ie->sm_list, nlists); + ie = ie->next; + } + if (nlists == 0) + return; + nlists++; + SigMatchData *ptrs[nlists]; memset(&ptrs, 0, (nlists * sizeof(SigMatchData *))); /* free engines and put smd in the array */ - DetectEngineAppInspectionEngine *ie = s->app_inspect; + ie = s->app_inspect; while (ie) { DetectEngineAppInspectionEngine *next = ie->next; BUG_ON(ptrs[ie->sm_list] != NULL && ptrs[ie->sm_list] != ie->smd); @@ -876,11 +889,25 @@ static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx) map->description, map->SetupCallback, map->ValidateCallback); b = HashListTableGetListNext(b); } + + de_ctx->buffer_type_hash = HashListTableInit(256, + DetectBufferTypeHashFunc, + DetectBufferTypeCompareFunc, + DetectBufferTypeFreeFunc); + if (de_ctx->buffer_type_hash == NULL) { + BUG_ON(1); + } + de_ctx->buffer_type_id = g_buffer_type_id; } + static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx) { - if (de_ctx && de_ctx->buffer_type_map) - SCFree(de_ctx->buffer_type_map); + if (de_ctx) { + if (de_ctx->buffer_type_map) + SCFree(de_ctx->buffer_type_map); + if (de_ctx->buffer_type_hash) + HashListTableFree(de_ctx->buffer_type_hash); + } } void DetectBufferTypeCloseRegistration(void) @@ -911,7 +938,7 @@ int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id, t.cnt = transform_cnt; DetectBufferType lookup_map = { (char *)base_map->string, NULL, 0, 0, 0, 0, false, NULL, NULL, t }; - DetectBufferType *res = HashListTableLookup(g_buffer_type_hash, &lookup_map, 0); + DetectBufferType *res = HashListTableLookup(de_ctx->buffer_type_hash, &lookup_map, 0); SCLogDebug("res %p", res); if (res != NULL) { @@ -923,7 +950,7 @@ int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id, return -1; map->string = base_map->string; - map->id = g_buffer_type_id++; + map->id = de_ctx->buffer_type_id++; map->parent_id = base_map->id; map->transforms = t; map->mpm = base_map->mpm; @@ -931,7 +958,7 @@ int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id, map->ValidateCallback = base_map->ValidateCallback; DetectAppLayerMpmRegisterByParentId(map->id, map->parent_id, &map->transforms); - BUG_ON(HashListTableAdd(g_buffer_type_hash, (void *)map, 0) != 0); + BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash, (void *)map, 0) != 0); SCLogDebug("buffer %s registered with id %d, parent %d", map->string, map->id, map->parent_id); if (map->id >= 0 && (uint32_t)map->id >= de_ctx->buffer_type_map_elements) { @@ -2156,12 +2183,12 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx * det_ctx->base64_decoded_len = 0; } - det_ctx->inspect_buffers_size = (uint32_t)DetectBufferTypeMaxId(); + det_ctx->inspect_buffers_size = de_ctx->buffer_type_id; det_ctx->inspect_buffers = SCCalloc(det_ctx->inspect_buffers_size, sizeof(InspectionBuffer)); if (det_ctx->inspect_buffers == NULL) { return TM_ECODE_FAILED; } - det_ctx->multi_inspect_buffers_size = (uint32_t)DetectBufferTypeMaxId(); + det_ctx->multi_inspect_buffers_size = de_ctx->buffer_type_id; det_ctx->multi_inspect_buffers = SCCalloc(det_ctx->multi_inspect_buffers_size, sizeof(InspectionBufferMultipleForList)); if (det_ctx->multi_inspect_buffers == NULL) { return TM_ECODE_FAILED; diff --git a/src/detect-fast-pattern.c b/src/detect-fast-pattern.c index f55f951bc9..7213f295dc 100644 --- a/src/detect-fast-pattern.c +++ b/src/detect-fast-pattern.c @@ -193,7 +193,6 @@ static int DetectFastPatternSetup(DetectEngineCtx *de_ctx, Signature *s, const c int ov[MAX_SUBSTRINGS]; char arg_substr[128] = ""; DetectContentData *cd = NULL; - const int nlists = DetectBufferTypeMaxId(); SigMatch *pm1 = DetectGetLastSMFromMpmLists(de_ctx, s); SigMatch *pm2 = DetectGetLastSMFromLists(s, DETECT_CONTENT, -1); @@ -236,8 +235,8 @@ static int DetectFastPatternSetup(DetectEngineCtx *de_ctx, Signature *s, const c goto error; } else { /*allow only one content to have fast_pattern modifier*/ - int list_id = 0; - for (list_id = 0; list_id < nlists; list_id++) { + uint32_t list_id = 0; + for (list_id = 0; list_id < s->init_data->smlists_array_size; list_id++) { SigMatch *sm = NULL; for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) { if (sm->type == DETECT_CONTENT) { diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 8c65080705..228ac10194 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -332,8 +332,6 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx, void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) { - const int nlists = DetectBufferTypeMaxId(); - const uint32_t max_fb_id = de_ctx->max_fb_id; if (max_fb_id == 0) return; @@ -352,7 +350,7 @@ void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) bool has_state = false; /* see if the signature uses stateful matching */ - for (int x = DETECT_SM_LIST_DYNAMIC_START; x < nlists; x++) { + for (uint32_t x = DETECT_SM_LIST_DYNAMIC_START; x < s->init_data->smlists_array_size; x++) { if (s->init_data->smlists[x] == NULL) continue; has_state = true; diff --git a/src/detect-parse.c b/src/detect-parse.c index b1da674e2f..26db89b8e7 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -362,7 +362,7 @@ SigMatch *DetectGetLastSMFromMpmLists(const DetectEngineCtx *de_ctx, const Signa { SigMatch *sm_last = NULL; SigMatch *sm_new; - int sm_type; + uint32_t sm_type; /* if we have a sticky buffer, use that */ if (s->init_data->list != DETECT_SM_LIST_NOTSET) { @@ -377,8 +377,7 @@ SigMatch *DetectGetLastSMFromMpmLists(const DetectEngineCtx *de_ctx, const Signa } /* otherwise brute force it */ - const int nlists = DetectBufferTypeMaxId(); - for (sm_type = 0; sm_type < nlists; sm_type++) { + for (sm_type = 0; sm_type < s->init_data->smlists_array_size; sm_type++) { if (!DetectBufferTypeSupportsMpmGetById(de_ctx, sm_type)) continue; SigMatch *sm_list = s->init_data->smlists_tail[sm_type]; @@ -404,8 +403,7 @@ SigMatch *DetectGetLastSMFromLists(const Signature *s, ...) SigMatch *sm_new; /* otherwise brute force it */ - const int nlists = DetectBufferTypeMaxId(); - for (int buf_type = 0; buf_type < nlists; buf_type++) { + for (int buf_type = 0; buf_type < (int)s->init_data->smlists_array_size; buf_type++) { if (s->init_data->smlists[buf_type] == NULL) continue; if (s->init_data->list != DETECT_SM_LIST_NOTSET && @@ -505,7 +503,7 @@ SigMatch *DetectGetLastSMByListId(const Signature *s, int list_id, ...) */ SigMatch *DetectGetLastSM(const Signature *s) { - const int nlists = DetectBufferTypeMaxId(); + const int nlists = s->init_data->smlists_array_size; SigMatch *sm_last = NULL; SigMatch *sm_new; int i; @@ -555,7 +553,7 @@ static void SigMatchTransferSigMatchAcrossLists(SigMatch *sm, int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm) { - const int nlists = DetectBufferTypeMaxId(); + const int nlists = s->init_data->smlists_array_size; int list = 0; for (list = 0; list < nlists; list++) { @@ -1291,8 +1289,6 @@ static void SigMatchFreeArrays(Signature *s, int ctxs) void SigFree(Signature *s) { - const int nlists = DetectBufferTypeMaxId(); - if (s == NULL) return; @@ -1304,6 +1300,7 @@ void SigFree(Signature *s) int i; if (s->init_data) { + const int nlists = s->init_data->smlists_array_size; for (i = 0; i < nlists; i++) { SigMatch *sm = s->init_data->smlists[i]; while (sm != NULL) { @@ -1546,7 +1543,7 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s) { uint32_t sig_flags = 0; SigMatch *sm, *pm; - const int nlists = DetectBufferTypeMaxId(); + const int nlists = s->init_data->smlists_array_size; SCEnter(); @@ -1812,9 +1809,7 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, const char *sigstr, SigBuildAddressMatchArray(sig); /* run buffer type callbacks if any */ - const int nlists = DetectBufferTypeMaxId(); - int x; - for (x = 0; x < nlists; x++) { + for (uint32_t x = 0; x < sig->init_data->smlists_array_size; x++) { if (sig->init_data->smlists[x]) DetectBufferRunSetupCallback(de_ctx, x, sig); } diff --git a/src/detect.c b/src/detect.c index 7b03323b0c..a0f5e1a874 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1034,7 +1034,7 @@ static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx, if (pflow != NULL) { // TODO clean this up - const int nlists = DetectBufferTypeMaxId(); + const int nlists = det_ctx->de_ctx->buffer_type_id; for (int i = 0; i < nlists; i++) { InspectionBuffer *buffer = &det_ctx->inspect_buffers[i]; buffer->inspect = NULL; diff --git a/src/detect.h b/src/detect.h index 1e2beb6178..b6548955fd 100644 --- a/src/detect.h +++ b/src/detect.h @@ -853,6 +853,11 @@ typedef struct DetectEngineCtx_ { DetectBufferType **buffer_type_map; uint32_t buffer_type_map_elements; + /* hash table with rule-time buffer registration. Start time registration + * is in detect-engine.c::g_buffer_type_hash */ + HashListTable *buffer_type_hash; + int buffer_type_id; + /** table with mpms and their registration function * \todo we only need this at init, so perhaps this * can move to a DetectEngineCtx 'init' struct */ diff --git a/src/util-profiling-keywords.c b/src/util-profiling-keywords.c index 9f9cbc8c3b..a96555451f 100644 --- a/src/util-profiling-keywords.c +++ b/src/util-profiling-keywords.c @@ -158,7 +158,7 @@ SCProfilingKeywordDump(DetectEngineCtx *de_ctx) if (profiling_keyword_enabled == 0) return; - const int nlists = DetectBufferTypeMaxId(); + const int nlists = de_ctx->buffer_type_id; gettimeofday(&tval, NULL); tms = SCLocalTime(tval.tv_sec, &local_tm); @@ -282,7 +282,7 @@ void SCProfilingKeywordDestroyCtx(DetectEngineCtx *de_ctx) DetroyCtx(de_ctx->profile_keyword_ctx); - const int nlists = DetectBufferTypeMaxId(); + const int nlists = de_ctx->buffer_type_id; int i; for (i = 0; i < nlists; i++) { DetroyCtx(de_ctx->profile_keyword_ctx_per_list[i]); @@ -302,7 +302,7 @@ void SCProfilingKeywordThreadSetup(SCProfileKeywordDetectCtx *ctx, DetectEngineT det_ctx->keyword_perf_data = a; } - const int nlists = DetectBufferTypeMaxId(); + const int nlists = det_ctx->de_ctx->buffer_type_id; det_ctx->keyword_perf_data_per_list = SCCalloc(nlists, sizeof(SCProfileKeywordData *)); BUG_ON(det_ctx->keyword_perf_data_per_list == NULL); @@ -334,7 +334,7 @@ static void SCProfilingKeywordThreadMerge(DetectEngineCtx *de_ctx, DetectEngineT de_ctx->profile_keyword_ctx->data[i].max = det_ctx->keyword_perf_data[i].max; } - const int nlists = DetectBufferTypeMaxId(); + const int nlists = det_ctx->de_ctx->buffer_type_id; int j; for (j = 0; j < nlists; j++) { for (i = 0; i < DETECT_TBLSIZE; i++) { @@ -360,7 +360,7 @@ void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *det_ctx) SCFree(det_ctx->keyword_perf_data); det_ctx->keyword_perf_data = NULL; - const int nlists = DetectBufferTypeMaxId(); + const int nlists = det_ctx->de_ctx->buffer_type_id; int i; for (i = 0; i < nlists; i++) { SCFree(det_ctx->keyword_perf_data_per_list[i]); @@ -380,7 +380,7 @@ SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx) if (profiling_keyword_enabled == 0) return; - const int nlists = DetectBufferTypeMaxId(); + const int nlists = de_ctx->buffer_type_id; de_ctx->profile_keyword_ctx = SCProfilingKeywordInitCtx(); BUG_ON(de_ctx->profile_keyword_ctx == NULL); -- 2.47.2