From: Victor Julien Date: Wed, 2 Apr 2025 07:06:57 +0000 (+0200) Subject: detect/prefilter: add post-match 'prefilter' engine X-Git-Tag: suricata-8.0.0-beta1~84 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=52c071b14a0ef5db56feab96136cb52c8d75be7e;p=thirdparty%2Fsuricata.git detect/prefilter: add post-match 'prefilter' engine Add support for special post-match engines. This allows a rule to enable other rules when it matches. Implementation is similar to prefilter engines, however prefilter engines run before individual rules while this post-match engine runs after and individual rule match. It will then add the new rules to the existing rule list. --- diff --git a/src/app-layer-events.c b/src/app-layer-events.c index eb32ea3946..9128c5d54b 100644 --- a/src/app-layer-events.c +++ b/src/app-layer-events.c @@ -161,6 +161,10 @@ SCEnumCharMap det_ctx_event_table[] = { "TOO_MANY_BUFFERS", DETECT_EVENT_TOO_MANY_BUFFERS, }, + { + "POST_MATCH_QUEUE_FAILED", + DETECT_EVENT_POST_MATCH_QUEUE_FAILED, + }, { NULL, -1 }, }; diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index ab3b345d27..9a01f155c5 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2016-2021 Open Information Security Foundation +/* Copyright (C) 2016-2025 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -155,6 +155,35 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, } } +/** \brief invoke post-rule match "prefilter" engines + * + * Invoke prefilter engines that depend on a rule match to run. + * e.g. the flowbits:set prefilter that adds sids that depend on + * a flowbit "set" to the match array. + */ +void PrefilterPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f) +{ + SCLogDebug("post-rule-match engines %p", sgh->post_rule_match_engines); + if (sgh->post_rule_match_engines) { + PrefilterEngine *engine = sgh->post_rule_match_engines; + do { + SCLogDebug("running post-rule-match engine"); + PREFILTER_PROFILING_START(det_ctx); + engine->cb.PrefilterPostRule(det_ctx, engine->pectx, p, f); + PREFILTER_PROFILING_END(det_ctx, engine->gid); + + if (engine->is_last) + break; + engine++; + } while (1); + + if (det_ctx->pmq.rule_id_array_cnt > 1) { + QuickSortSigIntId(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt); + } + } +} + void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t flags, const SignatureMask mask) { @@ -353,6 +382,39 @@ int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, return 0; } +int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + void (*PrefilterPostRuleFunc)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f), + void *pectx, void (*FreeFunc)(void *pectx), const char *name) +{ + if (sgh == NULL || PrefilterPostRuleFunc == NULL || pectx == NULL) + return -1; + + PrefilterEngineList *e = SCMallocAligned(sizeof(*e), CLS); + if (e == NULL) + return -1; + memset(e, 0x00, sizeof(*e)); + e->PrefilterPostRule = PrefilterPostRuleFunc; + e->pectx = pectx; + e->Free = FreeFunc; + + if (sgh->init->post_rule_match_engines == NULL) { + sgh->init->post_rule_match_engines = e; + } else { + PrefilterEngineList *t = sgh->init->post_rule_match_engines; + while (t->next != NULL) { + t = t->next; + } + + t->next = e; + e->id = t->id + 1; + } + + e->name = name; + e->gid = PrefilterStoreGetId(de_ctx, e->name, e->Free); + return 0; +} + static void PrefilterFreeEngineList(PrefilterEngineList *e) { if (e->Free && e->pectx) { @@ -407,6 +469,10 @@ void PrefilterCleanupRuleGroup(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) PrefilterFreeEngines(de_ctx, sgh->frame_engines); sgh->frame_engines = NULL; } + if (sgh->post_rule_match_engines) { + PrefilterFreeEngines(de_ctx, sgh->post_rule_match_engines); + sgh->post_rule_match_engines = NULL; + } } static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b) @@ -600,6 +666,30 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) e++; } } + if (sgh->init->post_rule_match_engines != NULL) { + uint32_t cnt = 0; + for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) { + cnt++; + } + sgh->post_rule_match_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS); + if (sgh->post_rule_match_engines == NULL) { + return; + } + memset(sgh->post_rule_match_engines, 0x00, (cnt * sizeof(PrefilterEngine))); + + uint16_t local_id = 0; + PrefilterEngine *e = sgh->post_rule_match_engines; + for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) { + e->local_id = local_id++; + e->cb.PrefilterPostRule = el->PrefilterPostRule; + e->pectx = el->pectx; + el->pectx = NULL; // e now owns the ctx + e->gid = el->gid; + e->is_last = (el->next == NULL); + e++; + } + SCLogDebug("sgh %p max local_id %u", sgh, local_id); + } } /* hash table for assigning a unique id to each engine type. */ @@ -900,3 +990,36 @@ int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, M } return r; } + +#define QUEUE_STEP 16 + +void PostRuleMatchWorkQueueAppend( + DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value) +{ + if (det_ctx->post_rule_work_queue.q == NULL) { + det_ctx->post_rule_work_queue.q = + SCCalloc(1, sizeof(PostRuleMatchWorkQueueItem) * QUEUE_STEP); + if (det_ctx->post_rule_work_queue.q == NULL) { + DetectEngineSetEvent(det_ctx, DETECT_EVENT_POST_MATCH_QUEUE_FAILED); + return; + } + det_ctx->post_rule_work_queue.size = QUEUE_STEP; + } else if (det_ctx->post_rule_work_queue.len == det_ctx->post_rule_work_queue.size) { + void *ptr = SCRealloc( + det_ctx->post_rule_work_queue.q, (det_ctx->post_rule_work_queue.size + QUEUE_STEP) * + sizeof(PostRuleMatchWorkQueueItem)); + if (ptr == NULL) { + DetectEngineSetEvent(det_ctx, DETECT_EVENT_POST_MATCH_QUEUE_FAILED); + return; + } + det_ctx->post_rule_work_queue.q = ptr; + det_ctx->post_rule_work_queue.size += QUEUE_STEP; + } + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].sm_type = type; + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].value = value; +#ifdef DEBUG + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].id = s->num; +#endif + det_ctx->post_rule_work_queue.len++; + SCLogDebug("det_ctx->post_rule_work_queue.len %u", det_ctx->post_rule_work_queue.len); +} diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h index 2434fca536..9f37525bd0 100644 --- a/src/detect-engine-prefilter.h +++ b/src/detect-engine-prefilter.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2016 Open Information Security Foundation +/* Copyright (C) 2016-2025 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -55,6 +55,10 @@ void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p, const u int PrefilterAppendEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterPktFn PrefilterFunc, SignatureMask mask, void *pectx, void (*FreeFunc)(void *pectx), const char *name); + +void PrefilterPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f); + int PrefilterAppendPayloadEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterPktFn PrefilterFunc, void *pectx, void (*FreeFunc)(void *pectx), const char *name); int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, @@ -63,6 +67,10 @@ int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterFrameFn PrefilterFrameFunc, AppProto alproto, uint8_t frame_type, void *pectx, void (*FreeFunc)(void *pectx), const char *name); +int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + void (*PrefilterPostRuleFunc)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f), + void *pectx, void (*FreeFunc)(void *pectx), const char *name); void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, @@ -94,4 +102,7 @@ int PrefilterMultiGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectBufferMpmRegistry *mpm_reg, int list_id); +void PostRuleMatchWorkQueueAppend( + DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value); + #endif diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 49982f028b..29bc3b2401 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -81,6 +81,7 @@ void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid) PrefilterFreeEnginesList(sghid->pkt_engines); PrefilterFreeEnginesList(sghid->payload_engines); PrefilterFreeEnginesList(sghid->frame_engines); + PrefilterFreeEnginesList(sghid->post_rule_match_engines); SCFree(sghid); } diff --git a/src/detect-engine.c b/src/detect-engine.c index 62b53e9a61..5a61890779 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -3670,6 +3670,9 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) AlertQueueFree(det_ctx); + if (det_ctx->post_rule_work_queue.q) + SCFree(det_ctx->post_rule_work_queue.q); + if (det_ctx->byte_values != NULL) SCFree(det_ctx->byte_values); diff --git a/src/detect-prefilter.c b/src/detect-prefilter.c index d3f209d69e..c73363b1eb 100644 --- a/src/detect-prefilter.c +++ b/src/detect-prefilter.c @@ -112,6 +112,8 @@ static int DetectPrefilterSetup (DetectEngineCtx *de_ctx, Signature *s, const ch } s->init_data->prefilter_sm = sm; + SCLogDebug( + "sid %u: prefilter is on \"%s\" (%u)", s->id, sigmatch_table[sm->type].name, sm->type); SCReturnInt(0); } diff --git a/src/detect.c b/src/detect.c index 07b8d98001..3cfe50bc7e 100644 --- a/src/detect.c +++ b/src/detect.c @@ -747,6 +747,15 @@ static bool IsOnlyTxInDirection(Flow *f, uint64_t txid, uint8_t dir) return false; } +static int SortHelper(const void *a, const void *b) +{ + const Signature *sa = *(const Signature **)a; + const Signature *sb = *(const Signature **)b; + if (sa->num == sb->num) + return 0; + return sa->num > sb->num ? 1 : -1; +} + static inline void DetectRulePacketRules( ThreadVars * const tv, DetectEngineCtx * const de_ctx, @@ -794,7 +803,7 @@ static inline void DetectRulePacketRules( } const uint8_t s_proto_flags = s->proto.flags; - SCLogDebug("inspecting signature id %"PRIu32"", s->id); + SCLogDebug("packet %" PRIu64 ": inspecting signature id %" PRIu32 "", p->pcap_cnt, s->id); if (s->app_inspect != NULL) { goto next; // handle sig in DetectRunTx @@ -862,6 +871,61 @@ static inline void DetectRulePacketRules( } } AlertQueueAppend(det_ctx, s, p, txid, alert_flags); + + if (det_ctx->post_rule_work_queue.len > 0) { + /* run post match prefilter engines on work queue */ + PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, pflow); + + if (det_ctx->pmq.rule_id_array_cnt > 0) { + /* undo "prefetch" */ + if (next_s) + match_array--; + /* create temporary rule pointer array starting + * at where we are in the current match array */ + const Signature *replace[de_ctx->sig_array_len]; // TODO heap? + SCLogDebug("sig_array_len %u det_ctx->pmq.rule_id_array_cnt %u", + de_ctx->sig_array_len, det_ctx->pmq.rule_id_array_cnt); + const Signature **r = replace; + for (uint32_t x = 0; x < match_cnt; x++) { + *r++ = match_array[x]; + SCLogDebug("appended %u", match_array[x]->id); + } + /* append the prefilter results, then sort it */ + for (uint32_t x = 0; x < det_ctx->pmq.rule_id_array_cnt; x++) { + SCLogDebug("adding iid %u", det_ctx->pmq.rule_id_array[x]); + Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[x]]; + SCLogDebug("adding id %u", ts->id); + if (ts->app_inspect == NULL) { + *r++ = ts; + match_cnt++; + } + } + if (match_cnt > 1) { + qsort(replace, match_cnt, sizeof(Signature *), SortHelper); + } + /* rewrite match_array to include the new additions, and deduplicate + * while at it. */ + Signature **m = match_array; + Signature *last_sig = NULL; + uint32_t skipped = 0; + for (uint32_t x = 0; x < match_cnt; x++) { + /* de-duplicate */ + if (last_sig == *m) { + skipped++; + continue; + } + last_sig = *m; + *m++ = (Signature *)replace[x]; + } + match_cnt -= skipped; + /* prefetch next */ + next_s = *match_array++; + next_sflags = next_s->flags; + SCLogDebug("%u rules added", det_ctx->pmq.rule_id_array_cnt); + det_ctx->post_rule_work_queue.len = 0; + PMQ_RESET(&det_ctx->pmq); + } + } next: DetectVarProcessList(det_ctx, pflow, p); DetectReplaceFree(det_ctx); @@ -1660,6 +1724,38 @@ static void DetectRunTx(ThreadVars *tv, } DetectVarProcessList(det_ctx, p->flow, p); RULE_PROFILING_END(det_ctx, s, r, p); + + if (det_ctx->post_rule_work_queue.len > 0) { + SCLogDebug("%p/%" PRIu64 " post_rule_work_queue len %u", tx.tx_ptr, tx.tx_id, + det_ctx->post_rule_work_queue.len); + /* run post match prefilter engines on work queue */ + PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, f); + + uint32_t prev_array_idx = array_idx; + for (uint32_t j = 0; j < det_ctx->pmq.rule_id_array_cnt; j++) { + const Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[j]]; + if (ts->app_inspect != NULL) { + const SigIntId id = ts->num; + det_ctx->tx_candidates[array_idx].s = ts; + det_ctx->tx_candidates[array_idx].id = id; + det_ctx->tx_candidates[array_idx].flags = NULL; + det_ctx->tx_candidates[array_idx].stream_reset = 0; + array_idx++; + + SCLogDebug("%p/%" PRIu64 " rule %u (%u) added from 'post match' prefilter", + tx.tx_ptr, tx.tx_id, ts->id, id); + } + } + SCLogDebug("%p/%" PRIu64 " rules added from 'post match' prefilter: %u", tx.tx_ptr, + tx.tx_id, array_idx - prev_array_idx); + if (prev_array_idx != array_idx) { + /* sort, but only part of array we're still going to process */ + qsort(det_ctx->tx_candidates + i, array_idx - i, sizeof(RuleMatchCandidateTx), + DetectRunTxSortHelper); + } + det_ctx->post_rule_work_queue.len = 0; + PMQ_RESET(&det_ctx->pmq); + } } det_ctx->tx_id = 0; diff --git a/src/detect.h b/src/detect.h index 2c9b15edb6..639ba0ea34 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1121,6 +1121,25 @@ typedef struct RuleMatchCandidateTx { const Signature *s; /**< ptr to sig */ } RuleMatchCandidateTx; +/** Stores a single u32 for a rule match of the type `sm_type`. Used by + * flowbits prefilter to register DETECT_FLOWBITS, for post + * match handling. */ +typedef struct PostRuleMatchWorkQueueItem { + int sm_type; /**< sigmatch type e.g. DETECT_FLOWBITS */ + uint32_t value; /**< value to be interpreted by the sm_type + * implementation. E.g. flowbit id. */ +#ifdef DEBUG + SigIntId id; +#endif +} PostRuleMatchWorkQueueItem; + +/** Array of PostRuleMatchWorkQueueItem's. */ +typedef struct PostRuleMatchWorkQueue { + PostRuleMatchWorkQueueItem *q; /**< array pointer */ + uint32_t len; /**< number of array elements in use. */ + uint32_t size; /**< allocation size in number of elements. */ +} PostRuleMatchWorkQueue; + /** * Detection engine thread data. */ @@ -1230,6 +1249,9 @@ typedef struct DetectEngineThreadCtx_ { uint32_t non_pf_store_cnt; MpmThreadCtx mtc; /**< thread ctx for the mpm */ + /* work queue for post-rule matching affecting prefilter */ + PostRuleMatchWorkQueue post_rule_work_queue; + PrefilterRuleStore pmq; /* string to replace */ @@ -1358,6 +1380,7 @@ enum { FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR, DETECT_EVENT_TOO_MANY_BUFFERS, + DETECT_EVENT_POST_MATCH_QUEUE_FAILED, }; #define SIG_GROUP_HEAD_HAVERAWSTREAM BIT_U16(0) @@ -1421,6 +1444,8 @@ typedef struct PrefilterEngineList_ { PrefilterPktFn Prefilter; PrefilterTxFn PrefilterTx; PrefilterFrameFn PrefilterFrame; + void (*PrefilterPostRule)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f); struct PrefilterEngineList_ *next; @@ -1454,6 +1479,8 @@ typedef struct PrefilterEngine_ { PrefilterPktFn Prefilter; PrefilterTxFn PrefilterTx; PrefilterFrameFn PrefilterFrame; + void (*PrefilterPostRule)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f); } cb; /* global id for this prefilter */ @@ -1481,6 +1508,7 @@ typedef struct SigGroupHeadInitData_ { PrefilterEngineList *payload_engines; PrefilterEngineList *tx_engines; PrefilterEngineList *frame_engines; + PrefilterEngineList *post_rule_match_engines; /** number of sigs in this group */ SigIntId sig_cnt; @@ -1511,6 +1539,7 @@ typedef struct SigGroupHead_ { PrefilterEngine *payload_engines; PrefilterEngine *tx_engines; PrefilterEngine *frame_engines; + PrefilterEngine *post_rule_match_engines; /**< engines to run after rules modified a state */ /* ptr to our init data we only use at... init :) */ SigGroupHeadInitData *init;