From: Victor Julien Date: Wed, 2 Apr 2025 07:09:48 +0000 (+0200) Subject: detect/flowbits: implement prefilter support X-Git-Tag: suricata-8.0.0-beta1~83 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f3abee85af9462ae5222f7ba092e822099c440a8;p=thirdparty%2Fsuricata.git detect/flowbits: implement prefilter support Allow for more efficient rules that 'prefilter' on flowbits with 'isset' logic. This prefilter is enabled by default, which means that if no mpm is present or no explicit prefilter is used, the flowbits prefilter will be set up for a rule. flowbits 'isset' prefilter For rules that have a 'flowbits:isset,' statement, a "regular" prefilter facility is created. It means that the rules are removed from the normal match list(s) and added to a prefilter engine that runs prior to the individual rule inspection stage. Implementation: the prefilter is implemented as an RB_TREE of flowbits, with the rule id's they "enable" stored per tree node. The matching logic is walking the list of bits set in the flow and looking each of them up in the RB_TREE, adding the rule ids of each of the matching bits to the list of rule candidates. The 'isset' prefilter has one important corner case, which is that bits can in fact be set during the rule evaluation stage. This is different from all other prefilter engines, that evaluate an immutable state (for the lifetime of the packet inspection). flowbits 'set' post-match prefilter For flowbits 'set' action, special post-match 'prefilter' facilities deal with this corner case. The high level logic is that these track which 'isset' sigs depend on them, and add these dependencies to the candidates list when a 'set' action occurs. This is implemented in a few steps: 1. flowbits 'set' is flagged 2. when 'set' action occurs the flowbit is added to a "post rule match work queue" 3. when the rule evaluation ends, the post-match "prefilter" engine is run on each of the flowbits in the "post rule match work queue" 4. these engines ammend the candidates list with the rule id dependencies for the flowbit 5. the candidates list is sorted to make sure within the execution for that packet the inspection order is maintained Ticket: #2486. --- diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 26bbef19a3..0d4295ceb4 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -42,6 +42,9 @@ #include "detect-engine-mpm.h" #include "detect-engine-state.h" #include "detect-engine-build.h" +#include "detect-engine-prefilter.h" + +#include "tree.h" #include "util-var-name.h" #include "util-unittest.h" @@ -61,6 +64,8 @@ void DetectFlowbitFree (DetectEngineCtx *, void *); #ifdef UNITTESTS void FlowBitsRegisterTests(void); #endif +static bool PrefilterFlowbitIsPrefilterable(const Signature *s); +static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh); void DetectFlowbitsRegister (void) { @@ -76,6 +81,8 @@ void DetectFlowbitsRegister (void) /* this is compatible to ip-only signatures */ sigmatch_table[DETECT_FLOWBITS].flags |= SIGMATCH_IPONLY_COMPAT; + sigmatch_table[DETECT_FLOWBITS].SupportsPrefilter = PrefilterFlowbitIsPrefilterable; + sigmatch_table[DETECT_FLOWBITS].SetupPrefilter = PrefilterSetupFlowbits; DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } @@ -132,11 +139,9 @@ static int FlowbitOrAddData(DetectEngineCtx *de_ctx, DetectFlowbitsData *cd, cha static int DetectFlowbitMatchToggle (Packet *p, const DetectFlowbitsData *fd) { if (p->flow == NULL) - return 0; - - FlowBitToggle(p->flow,fd->idx); + return -1; - return 1; + return FlowBitToggle(p->flow, fd->idx); } static int DetectFlowbitMatchUnset (Packet *p, const DetectFlowbitsData *fd) @@ -152,11 +157,11 @@ static int DetectFlowbitMatchUnset (Packet *p, const DetectFlowbitsData *fd) static int DetectFlowbitMatchSet (Packet *p, const DetectFlowbitsData *fd) { if (p->flow == NULL) - return 0; - - FlowBitSet(p->flow,fd->idx); + return -1; - return 1; + int r = FlowBitSet(p->flow, fd->idx); + SCLogDebug("set %u", fd->idx); + return r; } static int DetectFlowbitMatchIsset (Packet *p, const DetectFlowbitsData *fd) @@ -189,9 +194,8 @@ static int DetectFlowbitMatchIsnotset (Packet *p, const DetectFlowbitsData *fd) } /* - * returns 0: no match + * returns 0: no match (or error) * 1: match - * -1: error */ int DetectFlowbitMatch (DetectEngineThreadCtx *det_ctx, Packet *p, @@ -206,12 +210,25 @@ int DetectFlowbitMatch (DetectEngineThreadCtx *det_ctx, Packet *p, return DetectFlowbitMatchIsset(p,fd); case DETECT_FLOWBITS_CMD_ISNOTSET: return DetectFlowbitMatchIsnotset(p,fd); - case DETECT_FLOWBITS_CMD_SET: - return DetectFlowbitMatchSet(p,fd); + case DETECT_FLOWBITS_CMD_SET: { + int r = DetectFlowbitMatchSet(p, fd); + /* only on a new "set" invoke the prefilter */ + if (r == 1 && fd->post_rule_match_prefilter) { + SCLogDebug("flowbit set, appending to work queue"); + PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); + } + return (r != -1); + } case DETECT_FLOWBITS_CMD_UNSET: return DetectFlowbitMatchUnset(p,fd); - case DETECT_FLOWBITS_CMD_TOGGLE: - return DetectFlowbitMatchToggle(p,fd); + case DETECT_FLOWBITS_CMD_TOGGLE: { + int r = DetectFlowbitMatchToggle(p, fd); + if (r == 1 && fd->post_rule_match_prefilter) { + SCLogDebug("flowbit set (by toggle), appending to work queue"); + PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); + } + return (r != -1); + } default: SCLogError("unknown cmd %" PRIu32 "", fd->cmd); return 0; @@ -388,6 +405,11 @@ void DetectFlowbitFree (DetectEngineCtx *de_ctx, void *ptr) SCFree(fd); } +struct FBAnalyzer { + struct FBAnalyze *array; + uint32_t array_size; +}; + struct FBAnalyze { uint16_t cnts[DETECT_FLOWBITS_CMD_MAX]; uint16_t state_cnts[DETECT_FLOWBITS_CMD_MAX]; @@ -417,20 +439,143 @@ extern bool rule_engine_analysis_set; static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx, struct FBAnalyze *array, uint32_t elements); +static void FBAnalyzerArrayFree(struct FBAnalyze *array, const uint32_t array_size) +{ + if (array) { + for (uint32_t i = 0; i < array_size; i++) { + SCFree(array[i].set_sids); + SCFree(array[i].unset_sids); + SCFree(array[i].isset_sids); + SCFree(array[i].isnotset_sids); + SCFree(array[i].toggle_sids); + } + SCFree(array); + } +} + +static void FBAnalyzerFree(struct FBAnalyzer *fba) +{ + if (fba && fba->array) { + FBAnalyzerArrayFree(fba->array, fba->array_size); + fba->array = NULL; + fba->array_size = 0; + } +} + +#define MAX_SIDS 8 +static bool CheckExpand(const uint32_t sids_idx, uint32_t **sids, uint32_t *sids_size) +{ + if (sids_idx >= *sids_size) { + const uint32_t old_size = *sids_size; + const uint32_t new_size = MAX(2 * old_size, MAX_SIDS); + + void *ptr = SCRealloc(*sids, new_size * sizeof(uint32_t)); + if (ptr == NULL) + return false; + *sids_size = new_size; + *sids = ptr; + } + return true; +} + +static int DetectFlowbitsAnalyzeSignature(const Signature *s, struct FBAnalyzer *fba) +{ + struct FBAnalyze *array = fba->array; + if (array == NULL) + return -1; + + /* see if the signature uses stateful matching TODO is there not a flag? */ + bool has_state = (s->init_data->buffer_index != 0); + + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) + continue; + /* figure out the flowbit action */ + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + // Handle flowbit array in case of ORed flowbits + for (uint8_t k = 0; k < fb->or_list_size; k++) { + struct FBAnalyze *fa = &array[fb->or_list[k]]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size)) + return -1; + fa->isset_sids[fa->isset_sids_idx] = s->num; + fa->isset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { + if (!CheckExpand( + fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size)) + return -1; + fa->isnotset_sids[fa->isnotset_sids_idx] = s->num; + fa->isnotset_sids_idx++; + } + } + if (fb->or_list_size == 0) { + struct FBAnalyze *fa = &array[fb->idx]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size)) + return -1; + fa->isset_sids[fa->isset_sids_idx] = s->num; + fa->isset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { + if (!CheckExpand( + fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size)) + return -1; + fa->isnotset_sids[fa->isnotset_sids_idx] = s->num; + fa->isnotset_sids_idx++; + } + } + } + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) + continue; + /* figure out what flowbit action */ + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + struct FBAnalyze *fa = &array[fb->idx]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { + if (!CheckExpand(fa->set_sids_idx, &fa->set_sids, &fa->set_sids_size)) + return -1; + fa->set_sids[fa->set_sids_idx] = s->num; + fa->set_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) { + if (!CheckExpand(fa->unset_sids_idx, &fa->unset_sids, &fa->unset_sids_size)) + return -1; + fa->unset_sids[fa->unset_sids_idx] = s->num; + fa->unset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { + if (!CheckExpand(fa->toggle_sids_idx, &fa->toggle_sids, &fa->toggle_sids_size)) + return -1; + fa->toggle_sids[fa->toggle_sids_idx] = s->num; + fa->toggle_sids_idx++; + } + } + return 0; +} + int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) { const uint32_t max_fb_id = de_ctx->max_fb_id; if (max_fb_id == 0) return 0; -#define MAX_SIDS 8 - uint32_t array_size = max_fb_id + 1; + struct FBAnalyzer fba = { .array = NULL, .array_size = 0 }; + const uint32_t array_size = max_fb_id + 1; struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); - if (array == NULL) { SCLogError("Unable to allocate flowbit analyze array"); return -1; } + fba.array = array; + fba.array_size = array_size; SCLogDebug("fb analyzer array size: %"PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze))); @@ -439,143 +584,10 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) for (uint32_t i = 0; i < de_ctx->sig_array_len; i++) { const Signature *s = de_ctx->sig_array[i]; - /* see if the signature uses stateful matching */ - bool has_state = (s->init_data->buffer_index != 0); - - for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) { - switch (sm->type) { - case DETECT_FLOWBITS: - { - /* figure out the flowbit action */ - const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; - // Handle flowbit array in case of ORed flowbits - for (uint8_t k = 0; k < fb->or_list_size; k++) { - array[fb->or_list[k]].cnts[fb->cmd]++; - if (has_state) - array[fb->or_list[k]].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { - if (array[fb->or_list[k]].isset_sids_idx >= array[fb->or_list[k]].isset_sids_size) { - uint32_t old_size = array[fb->or_list[k]].isset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->or_list[k]].isset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->or_list[k]].isset_sids_size = new_size; - array[fb->or_list[k]].isset_sids = ptr; - } - - array[fb->or_list[k]].isset_sids[array[fb->or_list[k]].isset_sids_idx] = s->num; - array[fb->or_list[k]].isset_sids_idx++; - } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { - if (array[fb->or_list[k]].isnotset_sids_idx >= array[fb->or_list[k]].isnotset_sids_size) { - uint32_t old_size = array[fb->or_list[k]].isnotset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->or_list[k]].isnotset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->or_list[k]].isnotset_sids_size = new_size; - array[fb->or_list[k]].isnotset_sids = ptr; - } - - array[fb->or_list[k]].isnotset_sids[array[fb->or_list[k]].isnotset_sids_idx] = s->num; - array[fb->or_list[k]].isnotset_sids_idx++; - } - } - if (fb->or_list_size == 0) { - array[fb->idx].cnts[fb->cmd]++; - if (has_state) - array[fb->idx].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { - if (array[fb->idx].isset_sids_idx >= array[fb->idx].isset_sids_size) { - uint32_t old_size = array[fb->idx].isset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].isset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].isset_sids_size = new_size; - array[fb->idx].isset_sids = ptr; - } - - array[fb->idx].isset_sids[array[fb->idx].isset_sids_idx] = s->num; - array[fb->idx].isset_sids_idx++; - } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { - if (array[fb->idx].isnotset_sids_idx >= array[fb->idx].isnotset_sids_size) { - uint32_t old_size = array[fb->idx].isnotset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].isnotset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].isnotset_sids_size = new_size; - array[fb->idx].isnotset_sids = ptr; - } - - array[fb->idx].isnotset_sids[array[fb->idx].isnotset_sids_idx] = s->num; - array[fb->idx].isnotset_sids_idx++; - } - } - } - } - } - for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH] ; sm != NULL; sm = sm->next) { - switch (sm->type) { - case DETECT_FLOWBITS: - { - /* figure out what flowbit action */ - const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; - array[fb->idx].cnts[fb->cmd]++; - if (has_state) - array[fb->idx].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { - if (array[fb->idx].set_sids_idx >= array[fb->idx].set_sids_size) { - uint32_t old_size = array[fb->idx].set_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].set_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].set_sids_size = new_size; - array[fb->idx].set_sids = ptr; - } - - array[fb->idx].set_sids[array[fb->idx].set_sids_idx] = s->num; - array[fb->idx].set_sids_idx++; - } - else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) { - if (array[fb->idx].unset_sids_idx >= array[fb->idx].unset_sids_size) { - uint32_t old_size = array[fb->idx].unset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].unset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].unset_sids_size = new_size; - array[fb->idx].unset_sids = ptr; - } - - array[fb->idx].unset_sids[array[fb->idx].unset_sids_idx] = s->num; - array[fb->idx].unset_sids_idx++; - } - else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { - if (array[fb->idx].toggle_sids_idx >= array[fb->idx].toggle_sids_size) { - uint32_t old_size = array[fb->idx].toggle_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].toggle_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].toggle_sids_size = new_size; - array[fb->idx].toggle_sids = ptr; - } - - array[fb->idx].toggle_sids[array[fb->idx].toggle_sids_idx] = s->num; - array[fb->idx].toggle_sids_idx++; - } - } - } + int r = DetectFlowbitsAnalyzeSignature(s, &fba); + if (r < 0) { + FBAnalyzerFree(&fba); + return -1; } } @@ -640,14 +652,14 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) SCCalloc(sids_array_size, sizeof(uint32_t)); if (s->init_data->rule_state_dependant_sids_array == NULL) { SCLogError("Failed to allocate memory for rule_state_dependant_ids"); - goto end; + goto error; } s->init_data->rule_state_flowbits_ids_size = 1; s->init_data->rule_state_flowbits_ids_array = SCCalloc(s->init_data->rule_state_flowbits_ids_size, sizeof(uint32_t)); if (s->init_data->rule_state_flowbits_ids_array == NULL) { SCLogError("Failed to allocate memory for rule_state_variable_idx"); - goto end; + goto error; } s->init_data->rule_state_dependant_sids_size = sids_array_size; SCLogDebug("alloc'ed array for rule dependency and fbs idx array, sid %u, " @@ -661,7 +673,7 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) new_array_size * sizeof(uint32_t)); if (tmp_ptr == NULL) { SCLogError("Failed to allocate memory for rule_state_variable_idx"); - goto end; + goto error; } s->init_data->rule_state_dependant_sids_array = tmp_ptr; s->init_data->rule_state_dependant_sids_size = new_array_size; @@ -673,7 +685,7 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) s->init_data->rule_state_flowbits_ids_array = tmp_fb_ptr; if (s->init_data->rule_state_flowbits_ids_array == NULL) { SCLogError("Failed to reallocate memory for rule_state_variable_idx"); - goto end; + goto error; } SCLogDebug( "realloc'ed array for flowbits ids, new size is %u", new_fb_array_size); @@ -704,17 +716,96 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) DetectFlowbitsAnalyzeDump(de_ctx, array, array_size); } -end: + FBAnalyzerFree(&fba); + return 0; +error: + FBAnalyzerFree(&fba); + return -1; +} + +// TODO misses IPOnly rules. IPOnly flowbit rules are set only though. +static struct FBAnalyzer DetectFlowbitsAnalyzeForGroup( + const DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + struct FBAnalyzer fba = { .array = NULL, .array_size = 0 }; + + const uint32_t max_fb_id = de_ctx->max_fb_id; + if (max_fb_id == 0) + return fba; + + uint32_t array_size = max_fb_id + 1; + struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); + if (array == NULL) { + SCLogError("Unable to allocate flowbit analyze array"); + return fba; + } + SCLogDebug( + "fb analyzer array size: %" PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze))); + fba.array = array; + fba.array_size = array_size; + + /* fill flowbit array, updating counters per sig */ + for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) { + const Signature *s = sgh->init->match_array[i]; + SCLogDebug("sgh %p: s->id %u", sgh, s->id); + + int r = DetectFlowbitsAnalyzeSignature(s, &fba); + if (r < 0) { + FBAnalyzerFree(&fba); + return fba; + } + } + + /* walk array to see if all bits make sense */ for (uint32_t i = 0; i < array_size; i++) { - SCFree(array[i].set_sids); - SCFree(array[i].unset_sids); - SCFree(array[i].isset_sids); - SCFree(array[i].isnotset_sids); - SCFree(array[i].toggle_sids); + const char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT); + if (varname == NULL) + continue; + + bool to_state = false; + if (array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] && + array[i].state_cnts[DETECT_FLOWBITS_CMD_SET] == 0) { + SCLogDebug("flowbit %s/%u: isset in state, set not in state", varname, i); + } + + /* if signature depends on 'stateful' flowbits, then turn the + * sig into a stateful sig itself */ + if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] > 0 && + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] == 0 && + array[i].state_cnts[DETECT_FLOWBITS_CMD_SET]) { + SCLogDebug("flowbit %s/%u: isset not in state, set in state", varname, i); + to_state = true; + } + + SCLogDebug("ALL flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", + varname, i, array[i].cnts[DETECT_FLOWBITS_CMD_SET], + array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE], array[i].cnts[DETECT_FLOWBITS_CMD_UNSET], + array[i].cnts[DETECT_FLOWBITS_CMD_ISNOTSET], + array[i].cnts[DETECT_FLOWBITS_CMD_ISSET]); + SCLogDebug("STATE flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", + varname, i, array[i].state_cnts[DETECT_FLOWBITS_CMD_SET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_TOGGLE], + array[i].state_cnts[DETECT_FLOWBITS_CMD_UNSET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISNOTSET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET]); + for (uint32_t x = 0; x < array[i].set_sids_idx; x++) { + SCLogDebug("SET flowbit %s/%u: SID %u", varname, i, + de_ctx->sig_array[array[i].set_sids[x]]->id); + } + for (uint32_t x = 0; x < array[i].isset_sids_idx; x++) { + Signature *s = de_ctx->sig_array[array[i].isset_sids[x]]; + SCLogDebug("GET flowbit %s/%u: SID %u", varname, i, s->id); + + if (to_state) { + s->init_data->init_flags |= SIG_FLAG_INIT_STATE_MATCH; + SCLogDebug("made SID %u stateful because it depends on " + "stateful rules that set flowbit %s", + s->id, varname); + } + } } - SCFree(array); - return 0; + return fba; } SCMutex g_flowbits_dump_write_m = SCMUTEX_INITIALIZER; @@ -809,6 +900,470 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx, jb_free(js); } +static bool PrefilterFlowbitIsPrefilterable(const Signature *s) +{ + SCLogDebug("sid:%u: checking", s->id); + + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; + sm = sm->next) { + switch (sm->type) { + case DETECT_FLOWBITS: { + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + SCLogDebug("sid:%u: FLOWBITS ISSET can prefilter", s->id); + return true; + } + break; + } + } + } + SCLogDebug("sid:%u: no flowbit prefilter", s->id); + return false; +} + +/** core flowbit data structure: map a flowbit id to the signatures that need inspecting after it is + * found. Part of a rb-tree. */ +typedef struct PrefilterFlowbit { + uint32_t id; /**< flowbit id */ + uint32_t rule_id_size; /**< size in elements of `rule_id` */ + uint32_t rule_id_cnt; /**< usage in elements of `rule_id` */ + uint32_t *rule_id; /**< array of signature iid that are part of this prefilter */ + RB_ENTRY(PrefilterFlowbit) __attribute__((__packed__)) rb; +} __attribute__((__packed__)) PrefilterFlowbit; + +static int PrefilterFlowbitCompare(const PrefilterFlowbit *a, const PrefilterFlowbit *b) +{ + if (a->id > b->id) + return 1; + else if (a->id < b->id) + return -1; + else + return 0; +} + +/** red-black tree prototype for PFB (Prefilter Flow Bits) */ +RB_HEAD(PFB, PrefilterFlowbit); +RB_PROTOTYPE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); +RB_GENERATE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); + +struct PrefilterEngineFlowbits { + struct PFB fb_tree; +}; + +static void PrefilterFlowbitFree(void *vctx) +{ + struct PrefilterEngineFlowbits *ctx = vctx; + struct PrefilterFlowbit *rec, *safe = NULL; + RB_FOREACH_SAFE (rec, PFB, &ctx->fb_tree, safe) { + PFB_RB_REMOVE(&ctx->fb_tree, rec); + SCFree(rec->rule_id); + SCFree(rec); + } + + SCFree(ctx); +} + +static void PrefilterFlowbitMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx; + SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx); + + if (p->flow == NULL) { + SCReturn; + } + + for (GenericVar *gv = p->flow->flowvar; gv != NULL; gv = gv->next) { + if (gv->type != DETECT_FLOWBITS) + continue; + + PrefilterFlowbit lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.id = gv->idx; + SCLogDebug("flowbit %u", gv->idx); + + PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup); + if (b == NULL) { + SCLogDebug("flowbit %u not in the tree", lookup.id); + } else { + SCLogDebug("flowbit %u found in the tree: %u", lookup.id, b->id); + + PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt); +#ifdef DEBUG + for (uint32_t x = 0; x < b->rule_id_cnt; x++) { + const Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; + SCLogDebug("flowbit %u -> sig %u", gv->idx, s->id); + } +#endif + } + } +} + +static void PrefilterFlowbitPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f) +{ + struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx; + SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx); + + if (p->flow == NULL) { + SCReturn; + } + + for (uint32_t i = 0; i < det_ctx->post_rule_work_queue.len; i++) { + const PostRuleMatchWorkQueueItem *w = &det_ctx->post_rule_work_queue.q[i]; + if (w->sm_type != DETECT_FLOWBITS) + continue; + + PrefilterFlowbit lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.id = w->value; + + PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup); + if (b == NULL) { + SCLogDebug("flowbit %u not in the tree", lookup.id); + } else { + SCLogDebug("flowbit %u found in the tree: %u. Adding %u sids", lookup.id, b->id, + b->rule_id_cnt); + PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt); +#ifdef DEBUG + // SCLogDebug("b %u", b->rule_id_cnt); + for (uint32_t x = 0; x < b->rule_id_cnt; x++) { + Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; + SCLogDebug("flowbit %u -> sig %u (triggered by %u)", w->value, s->id, + det_ctx->de_ctx->sig_array[w->id]->id); + } +#endif + } + } +} + +#define BLOCK_SIZE 8 + +static int AddBitAndSid( + struct PrefilterEngineFlowbits *ctx, const Signature *s, const uint32_t flowbit_id) +{ + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = flowbit_id; + + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + if (add == NULL) + return -1; + + add->id = flowbit_id; + add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t)); + if (add->rule_id == NULL) { + SCFree(add); + return -1; + } + add->rule_id_size = BLOCK_SIZE; + add->rule_id_cnt = 1; + add->rule_id[0] = s->num; + + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + if (res != NULL) { + // duplicate, shouldn't be possible after the FIND above + BUG_ON(1); + return -1; + } + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + if (pfb->rule_id_cnt < pfb->rule_id_size) { + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } else { + uint32_t *ptr = + SCRealloc(pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + if (ptr == NULL) { + // memory stays in the tree + return -1; + } + pfb->rule_id = ptr; + pfb->rule_id_size += BLOCK_SIZE; + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } + } + return 0; +} + +static int AddBitsAndSid(const DetectEngineCtx *de_ctx, struct PrefilterEngineFlowbits *ctx, + const DetectFlowbitsData *fb, const Signature *s) +{ + if (fb->or_list_size == 0) { + if (AddBitAndSid(ctx, s, fb->idx) < 0) { + return -1; + } + } else { + for (uint8_t i = 0; i < fb->or_list_size; i++) { + SCLogDebug("flowbit OR: bit %u", fb->or_list[i]); + if (AddBitAndSid(ctx, s, fb->or_list[i]) < 0) { + return -1; + } + } + } + return 0; +} + +static uint32_t NextMultiple(const uint32_t v, const uint32_t m) +{ + return v + (m - v % m); +} + +/** \internal + * \brief adds sids for 'isset' prefilter flowbits + * \retval int 1 if we added sid(s), 0 if we didn't, -1 on error */ +// TODO skip sids that aren't set by this sgh +// TODO skip sids that doesn't have a isset in the same direction +static int AddIssetSidsForBit(const DetectEngineCtx *de_ctx, const struct FBAnalyzer *fba, + const DetectFlowbitsData *fb, PrefilterFlowbit *add) +{ + int added = 0; + for (uint32_t i = 0; i < fba->array[fb->idx].isset_sids_idx; i++) { + const uint32_t sig_iid = fba->array[fb->idx].isset_sids[i]; + const Signature *s = de_ctx->sig_array[sig_iid]; + SCLogDebug("flowbit: %u => considering sid %u (iid:%u)", fb->idx, s->id, s->num); + + /* Skip sids that aren't prefilter. These would just run all the time. */ + if (s->init_data->prefilter_sm == NULL || + s->init_data->prefilter_sm->type != DETECT_FLOWBITS) { +#ifdef DEBUG + const char *name = s->init_data->prefilter_sm + ? sigmatch_table[s->init_data->prefilter_sm->type].name + : "none"; + SCLogDebug("flowbit: %u => rejected sid %u (iid:%u). No prefilter or prefilter not " + "flowbits (%p, %s, %d)", + fb->idx, s->id, sig_iid, s->init_data->prefilter_sm, name, + s->init_data->prefilter_sm ? s->init_data->prefilter_sm->type : -1); +#endif + continue; + } + + /* only add sids that match our bit */ + const DetectFlowbitsData *fs_fb = + (const DetectFlowbitsData *)s->init_data->prefilter_sm->ctx; + if (fs_fb->idx != fb->idx) { + SCLogDebug( + "flowbit: %u => rejected sid %u (iid:%u). Sig prefilters on different bit %u", + fb->idx, s->id, sig_iid, fs_fb->idx); + continue; + } + + bool dup = false; + for (uint32_t x = 0; x < add->rule_id_cnt; x++) { + if (add->rule_id[x] == sig_iid) { + dup = true; + } + } + + if (!dup) { + if (add->rule_id_cnt < add->rule_id_size) { + add->rule_id[add->rule_id_cnt++] = sig_iid; + } else { + uint32_t *ptr = SCRealloc( + add->rule_id, (add->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + if (ptr == NULL) { + return -1; + } + add->rule_id = ptr; + add->rule_id_size += BLOCK_SIZE; + add->rule_id[add->rule_id_cnt++] = sig_iid; + } + added = 1; + SCLogDebug("flowbit: %u => accepted sid %u (iid:%u)", fb->idx, s->id, sig_iid); + } + } + return added; +} + +/* TODO shouldn't add sids for which Signature::num is < our num. Is this possible after sorting? */ + +/** \brief For set/toggle flowbits, build "set" post-rule-match engine + * + * For set/toggle flowbits, a special post-rule-match engine is constructed + * to update the running match array during rule matching. + */ +static int AddBitSetToggle(const DetectEngineCtx *de_ctx, struct FBAnalyzer *fba, + struct PrefilterEngineFlowbits *ctx, const DetectFlowbitsData *fb, const Signature *s) +{ + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = fb->idx; + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + if (add == NULL) + return -1; + + add->id = fb->idx; + add->rule_id_size = NextMultiple(fba->array[fb->idx].isset_sids_idx, BLOCK_SIZE); + add->rule_id = SCCalloc(1, add->rule_id_size * sizeof(uint32_t)); + if (add->rule_id == NULL) { + SCFree(add); + return -1; + } + + if (AddIssetSidsForBit(de_ctx, fba, fb, add) != 1) { + SCLogDebug("no sids added"); + SCFree(add->rule_id); + SCFree(add); + return 0; + } + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + BUG_ON(res != NULL); // TODO if res != NULL we have a duplicate which should be impossible + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + int r = AddIssetSidsForBit(de_ctx, fba, fb, pfb); + if (r < 0) { + return -1; + } else if (r == 0) { + SCLogDebug("no sids added"); + return 0; + } + } + return 1; +} + +/** \brief build flowbit prefilter state(s) + * + * Build "set" and "isset" states. + * + * For each flowbit "isset" in the sgh, we need to check: + * 1. is it supported + * 2. is prefilter enabled + * 3. does it match in the same dir or only opposing dir + */ +static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + if (sgh == NULL) + return 0; + + SCLogDebug("sgh %p: setting up prefilter", sgh); + struct PrefilterEngineFlowbits *isset_ctx = NULL; + struct PrefilterEngineFlowbits *set_ctx = NULL; + + struct FBAnalyzer fb_analysis = DetectFlowbitsAnalyzeForGroup(de_ctx, sgh); + if (fb_analysis.array == NULL) + goto error; + + for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) { + Signature *s = sgh->init->match_array[i]; + if (s == NULL) + continue; + + SCLogDebug("checking sid %u", s->id); + + /* first build the 'set' state */ + for (SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) { + SCLogDebug("skip non flowbits sm"); + continue; + } + + DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { + SCLogDebug( + "DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS set %u", s->id, fb->idx); + } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { + SCLogDebug("DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS toggle %u", s->id, + fb->idx); + } else { + SCLogDebug("unsupported flowbits setting"); + continue; + } + + if (fb_analysis.array[fb->idx].isnotset_sids_idx || + fb_analysis.array[fb->idx].unset_sids_idx) { + SCLogDebug("flowbit %u not supported: unset in use", fb->idx); + continue; + } + + if (set_ctx == NULL) { + set_ctx = SCCalloc(1, sizeof(*set_ctx)); + if (set_ctx == NULL) + goto error; + } + + SCLogDebug("setting up sets/toggles for sid %u", s->id); + if (AddBitSetToggle(de_ctx, &fb_analysis, set_ctx, fb, s) == 1) { + // flag the set/toggle to trigger the post-rule match logic + SCLogDebug("set up sets/toggles for sid %u", s->id); + fb->post_rule_match_prefilter = true; + } + + // TODO don't add for sigs that don't have isset in this sgh. Reasoning: + // prefilter post match logic only makes sense in the same dir as otherwise + // the regular 'isset' logic can simply run with the regular prefilters + // before the rule loop + } + + /* next, build the 'isset' state */ + if (s->init_data->prefilter_sm == NULL || + s->init_data->prefilter_sm->type != DETECT_FLOWBITS) { + SCLogDebug("no prefilter or prefilter not flowbits"); + continue; + } + + const DetectFlowbitsData *fb = (DetectFlowbitsData *)s->init_data->prefilter_sm->ctx; + if (fb_analysis.array[fb->idx].isnotset_sids_idx || + fb_analysis.array[fb->idx].unset_sids_idx) { + SCLogDebug("flowbit %u not supported: toggle or unset in use", fb->idx); + s->init_data->prefilter_sm = NULL; + s->flags &= ~SIG_FLAG_PREFILTER; + continue; + } + + SCLogDebug("isset: adding sid %u, flowbit %u", s->id, fb->idx); + + if (isset_ctx == NULL) { + isset_ctx = SCCalloc(1, sizeof(*isset_ctx)); + if (isset_ctx == NULL) + goto error; + } + if (AddBitsAndSid(de_ctx, isset_ctx, fb, s) < 0) { + goto error; + } + } + + /* finally, register the states with their engines */ + static const char *g_prefilter_flowbits_isset = "flowbits:isset"; + if (isset_ctx != NULL) { + PrefilterAppendEngine(de_ctx, sgh, PrefilterFlowbitMatch, SIG_MASK_REQUIRE_FLOW, isset_ctx, + PrefilterFlowbitFree, g_prefilter_flowbits_isset); + SCLogDebug("isset: added prefilter engine"); + + if (set_ctx != NULL && !RB_EMPTY(&set_ctx->fb_tree)) { + static const char *g_prefilter_flowbits_set = "flowbits:set"; + PrefilterAppendPostRuleEngine(de_ctx, sgh, PrefilterFlowbitPostRuleMatch, set_ctx, + PrefilterFlowbitFree, g_prefilter_flowbits_set); + SCLogDebug("set/toggle: added prefilter engine"); + } else { + if (set_ctx) { + PrefilterFlowbitFree(set_ctx); + } + SCLogDebug("set/toggle: NO prefilter engine added"); + } + } else if (set_ctx != NULL) { + PrefilterFlowbitFree(set_ctx); + } + FBAnalyzerFree(&fb_analysis); + return 0; + +error: + if (set_ctx) { + PrefilterFlowbitFree(set_ctx); + } + if (isset_ctx) { + PrefilterFlowbitFree(isset_ctx); + } + FBAnalyzerFree(&fb_analysis); + return -1; +} + #ifdef UNITTESTS static int FlowBitsTestParse01(void) diff --git a/src/detect-flowbits.h b/src/detect-flowbits.h index 0a5823ef60..38694752ae 100644 --- a/src/detect-flowbits.h +++ b/src/detect-flowbits.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2025 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -36,6 +36,9 @@ typedef struct DetectFlowbitsData_ { uint32_t idx; uint8_t cmd; uint8_t or_list_size; + /** Flag to trigger post rule match prefilter following a 'set' match. */ + bool post_rule_match_prefilter; /**< set/toggle command should trigger post-rule-match + "prefilter" */ uint32_t *or_list; } DetectFlowbitsData;