detect/prefilter: add post-match 'prefilter' engine

Add support for special post-match engines. This allows a rule to enable
other rules when it matches.

Implementation is similar to prefilter engines, however prefilter
engines run before individual rules while this post-match engine runs
after and individual rule match. It will then add the new rules to the
existing rule list.

diff --git a/src/app-layer-events.c b/src/app-layer-events.c
index eb32ea394671d91ce55e7434e790b9f01acbee78..9128c5d54b2f1d38f0021dacbe745468e1188440 100644 (file)
--- a/src/app-layer-events.c
+++ b/src/app-layer-events.c
@@ -161,6 +161,10 @@ SCEnumCharMap det_ctx_event_table[] = {
             "TOO_MANY_BUFFERS",
             DETECT_EVENT_TOO_MANY_BUFFERS,
     },
+    {
+            "POST_MATCH_QUEUE_FAILED",
+            DETECT_EVENT_POST_MATCH_QUEUE_FAILED,
+    },
     { NULL, -1 },
 };
 

diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c
index ab3b345d276507237d2b004605f9623efe988043..9a01f155c5f3d52e670ea27a10bb2d1f08f70607 100644 (file)
--- a/src/detect-engine-prefilter.c
+++ b/src/detect-engine-prefilter.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2016-2021 Open Information Security Foundation
+/* Copyright (C) 2016-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -155,6 +155,35 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
     }
 }
 
+/** \brief invoke post-rule match "prefilter" engines
+ *
+ * Invoke prefilter engines that depend on a rule match to run.
+ * e.g. the flowbits:set prefilter that adds sids that depend on
+ * a flowbit "set" to the match array.
+ */
+void PrefilterPostRuleMatch(
+        DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f)
+{
+    SCLogDebug("post-rule-match engines %p", sgh->post_rule_match_engines);
+    if (sgh->post_rule_match_engines) {
+        PrefilterEngine *engine = sgh->post_rule_match_engines;
+        do {
+            SCLogDebug("running post-rule-match engine");
+            PREFILTER_PROFILING_START(det_ctx);
+            engine->cb.PrefilterPostRule(det_ctx, engine->pectx, p, f);
+            PREFILTER_PROFILING_END(det_ctx, engine->gid);
+
+            if (engine->is_last)
+                break;
+            engine++;
+        } while (1);
+
+        if (det_ctx->pmq.rule_id_array_cnt > 1) {
+            QuickSortSigIntId(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt);
+        }
+    }
+}
+
 void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p,
         const uint8_t flags, const SignatureMask mask)
 {
@@ -353,6 +382,39 @@ int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
     return 0;
 }
 
+int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
+        void (*PrefilterPostRuleFunc)(
+                DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f),
+        void *pectx, void (*FreeFunc)(void *pectx), const char *name)
+{
+    if (sgh == NULL || PrefilterPostRuleFunc == NULL || pectx == NULL)
+        return -1;
+
+    PrefilterEngineList *e = SCMallocAligned(sizeof(*e), CLS);
+    if (e == NULL)
+        return -1;
+    memset(e, 0x00, sizeof(*e));
+    e->PrefilterPostRule = PrefilterPostRuleFunc;
+    e->pectx = pectx;
+    e->Free = FreeFunc;
+
+    if (sgh->init->post_rule_match_engines == NULL) {
+        sgh->init->post_rule_match_engines = e;
+    } else {
+        PrefilterEngineList *t = sgh->init->post_rule_match_engines;
+        while (t->next != NULL) {
+            t = t->next;
+        }
+
+        t->next = e;
+        e->id = t->id + 1;
+    }
+
+    e->name = name;
+    e->gid = PrefilterStoreGetId(de_ctx, e->name, e->Free);
+    return 0;
+}
+
 static void PrefilterFreeEngineList(PrefilterEngineList *e)
 {
     if (e->Free && e->pectx) {
@@ -407,6 +469,10 @@ void PrefilterCleanupRuleGroup(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         PrefilterFreeEngines(de_ctx, sgh->frame_engines);
         sgh->frame_engines = NULL;
     }
+    if (sgh->post_rule_match_engines) {
+        PrefilterFreeEngines(de_ctx, sgh->post_rule_match_engines);
+        sgh->post_rule_match_engines = NULL;
+    }
 }
 
 static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b)
@@ -600,6 +666,30 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
             e++;
         }
     }
+    if (sgh->init->post_rule_match_engines != NULL) {
+        uint32_t cnt = 0;
+        for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) {
+            cnt++;
+        }
+        sgh->post_rule_match_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
+        if (sgh->post_rule_match_engines == NULL) {
+            return;
+        }
+        memset(sgh->post_rule_match_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
+
+        uint16_t local_id = 0;
+        PrefilterEngine *e = sgh->post_rule_match_engines;
+        for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) {
+            e->local_id = local_id++;
+            e->cb.PrefilterPostRule = el->PrefilterPostRule;
+            e->pectx = el->pectx;
+            el->pectx = NULL; // e now owns the ctx
+            e->gid = el->gid;
+            e->is_last = (el->next == NULL);
+            e++;
+        }
+        SCLogDebug("sgh %p max local_id %u", sgh, local_id);
+    }
 }
 
 /* hash table for assigning a unique id to each engine type. */
@@ -900,3 +990,36 @@ int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, M
     }
     return r;
 }
+
+#define QUEUE_STEP 16
+
+void PostRuleMatchWorkQueueAppend(
+        DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value)
+{
+    if (det_ctx->post_rule_work_queue.q == NULL) {
+        det_ctx->post_rule_work_queue.q =
+                SCCalloc(1, sizeof(PostRuleMatchWorkQueueItem) * QUEUE_STEP);
+        if (det_ctx->post_rule_work_queue.q == NULL) {
+            DetectEngineSetEvent(det_ctx, DETECT_EVENT_POST_MATCH_QUEUE_FAILED);
+            return;
+        }
+        det_ctx->post_rule_work_queue.size = QUEUE_STEP;
+    } else if (det_ctx->post_rule_work_queue.len == det_ctx->post_rule_work_queue.size) {
+        void *ptr = SCRealloc(
+                det_ctx->post_rule_work_queue.q, (det_ctx->post_rule_work_queue.size + QUEUE_STEP) *
+                                                         sizeof(PostRuleMatchWorkQueueItem));
+        if (ptr == NULL) {
+            DetectEngineSetEvent(det_ctx, DETECT_EVENT_POST_MATCH_QUEUE_FAILED);
+            return;
+        }
+        det_ctx->post_rule_work_queue.q = ptr;
+        det_ctx->post_rule_work_queue.size += QUEUE_STEP;
+    }
+    det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].sm_type = type;
+    det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].value = value;
+#ifdef DEBUG
+    det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].id = s->num;
+#endif
+    det_ctx->post_rule_work_queue.len++;
+    SCLogDebug("det_ctx->post_rule_work_queue.len %u", det_ctx->post_rule_work_queue.len);
+}

diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h
index 2434fca536ee0be64428dac80cc390c0fddcfc08..9f37525bd0d30ae4e13d987666b98e33494883c1 100644 (file)
--- a/src/detect-engine-prefilter.h
+++ b/src/detect-engine-prefilter.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2016 Open Information Security Foundation
+/* Copyright (C) 2016-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -55,6 +55,10 @@ void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p, const u
 
 int PrefilterAppendEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterPktFn PrefilterFunc,
         SignatureMask mask, void *pectx, void (*FreeFunc)(void *pectx), const char *name);
+
+void PrefilterPostRuleMatch(
+        DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f);
+
 int PrefilterAppendPayloadEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
         PrefilterPktFn PrefilterFunc, void *pectx, void (*FreeFunc)(void *pectx), const char *name);
 int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
@@ -63,6 +67,10 @@ int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
         PrefilterFrameFn PrefilterFrameFunc, AppProto alproto, uint8_t frame_type, void *pectx,
         void (*FreeFunc)(void *pectx), const char *name);
+int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
+        void (*PrefilterPostRuleFunc)(
+                DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f),
+        void *pectx, void (*FreeFunc)(void *pectx), const char *name);
 
 void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
         const SigGroupHead *sgh,
@@ -94,4 +102,7 @@ int PrefilterMultiGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
         const DetectBufferMpmRegistry *mpm_reg, int list_id);
 
+void PostRuleMatchWorkQueueAppend(
+        DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value);
+
 #endif

diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index 49982f028b2871a88af2ab7604c12e53605af97f..29bc3b24011b511be914487d48832104cef52fd1 100644 (file)
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -81,6 +81,7 @@ void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid)
     PrefilterFreeEnginesList(sghid->pkt_engines);
     PrefilterFreeEnginesList(sghid->payload_engines);
     PrefilterFreeEnginesList(sghid->frame_engines);
+    PrefilterFreeEnginesList(sghid->post_rule_match_engines);
 
     SCFree(sghid);
 }

diff --git a/src/detect-engine.c b/src/detect-engine.c
index 62b53e9a618f8e2c843b6c297e8f3a81dc580e8c..5a61890779fcaae13df50fd4f6e5e9e59e0cac53 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -3670,6 +3670,9 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
 
     AlertQueueFree(det_ctx);
 
+    if (det_ctx->post_rule_work_queue.q)
+        SCFree(det_ctx->post_rule_work_queue.q);
+
     if (det_ctx->byte_values != NULL)
         SCFree(det_ctx->byte_values);
 

diff --git a/src/detect-prefilter.c b/src/detect-prefilter.c
index d3f209d69eebec514bc36b5a94d73fed928ed460..c73363b1eb04fe8b2aaac13bcf59b46effe03c28 100644 (file)
--- a/src/detect-prefilter.c
+++ b/src/detect-prefilter.c
@@ -112,6 +112,8 @@ static int DetectPrefilterSetup (DetectEngineCtx *de_ctx, Signature *s, const ch
     }
 
     s->init_data->prefilter_sm = sm;
+    SCLogDebug(
+            "sid %u: prefilter is on \"%s\" (%u)", s->id, sigmatch_table[sm->type].name, sm->type);
 
     SCReturnInt(0);
 }

diff --git a/src/detect.c b/src/detect.c
index 07b8d98001b5a199d247905453c58b10abce5de4..3cfe50bc7ec575876c74015a1ebe4901331a4c2b 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -747,6 +747,15 @@ static bool IsOnlyTxInDirection(Flow *f, uint64_t txid, uint8_t dir)
     return false;
 }
 
+static int SortHelper(const void *a, const void *b)
+{
+    const Signature *sa = *(const Signature **)a;
+    const Signature *sb = *(const Signature **)b;
+    if (sa->num == sb->num)
+        return 0;
+    return sa->num > sb->num ? 1 : -1;
+}
+
 static inline void DetectRulePacketRules(
     ThreadVars * const tv,
     DetectEngineCtx * const de_ctx,
@@ -794,7 +803,7 @@ static inline void DetectRulePacketRules(
         }
         const uint8_t s_proto_flags = s->proto.flags;
 
-        SCLogDebug("inspecting signature id %"PRIu32"", s->id);
+        SCLogDebug("packet %" PRIu64 ": inspecting signature id %" PRIu32 "", p->pcap_cnt, s->id);
 
         if (s->app_inspect != NULL) {
             goto next; // handle sig in DetectRunTx
@@ -862,6 +871,61 @@ static inline void DetectRulePacketRules(
             }
         }
         AlertQueueAppend(det_ctx, s, p, txid, alert_flags);
+
+        if (det_ctx->post_rule_work_queue.len > 0) {
+            /* run post match prefilter engines on work queue */
+            PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, pflow);
+
+            if (det_ctx->pmq.rule_id_array_cnt > 0) {
+                /* undo "prefetch" */
+                if (next_s)
+                    match_array--;
+                /* create temporary rule pointer array starting
+                 * at where we are in the current match array */
+                const Signature *replace[de_ctx->sig_array_len]; // TODO heap?
+                SCLogDebug("sig_array_len %u det_ctx->pmq.rule_id_array_cnt %u",
+                        de_ctx->sig_array_len, det_ctx->pmq.rule_id_array_cnt);
+                const Signature **r = replace;
+                for (uint32_t x = 0; x < match_cnt; x++) {
+                    *r++ = match_array[x];
+                    SCLogDebug("appended %u", match_array[x]->id);
+                }
+                /* append the prefilter results, then sort it */
+                for (uint32_t x = 0; x < det_ctx->pmq.rule_id_array_cnt; x++) {
+                    SCLogDebug("adding iid %u", det_ctx->pmq.rule_id_array[x]);
+                    Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[x]];
+                    SCLogDebug("adding id %u", ts->id);
+                    if (ts->app_inspect == NULL) {
+                        *r++ = ts;
+                        match_cnt++;
+                    }
+                }
+                if (match_cnt > 1) {
+                    qsort(replace, match_cnt, sizeof(Signature *), SortHelper);
+                }
+                /* rewrite match_array to include the new additions, and deduplicate
+                 * while at it. */
+                Signature **m = match_array;
+                Signature *last_sig = NULL;
+                uint32_t skipped = 0;
+                for (uint32_t x = 0; x < match_cnt; x++) {
+                    /* de-duplicate */
+                    if (last_sig == *m) {
+                        skipped++;
+                        continue;
+                    }
+                    last_sig = *m;
+                    *m++ = (Signature *)replace[x];
+                }
+                match_cnt -= skipped;
+                /* prefetch next */
+                next_s = *match_array++;
+                next_sflags = next_s->flags;
+                SCLogDebug("%u rules added", det_ctx->pmq.rule_id_array_cnt);
+                det_ctx->post_rule_work_queue.len = 0;
+                PMQ_RESET(&det_ctx->pmq);
+            }
+        }
 next:
         DetectVarProcessList(det_ctx, pflow, p);
         DetectReplaceFree(det_ctx);
@@ -1660,6 +1724,38 @@ static void DetectRunTx(ThreadVars *tv,
             }
             DetectVarProcessList(det_ctx, p->flow, p);
             RULE_PROFILING_END(det_ctx, s, r, p);
+
+            if (det_ctx->post_rule_work_queue.len > 0) {
+                SCLogDebug("%p/%" PRIu64 " post_rule_work_queue len %u", tx.tx_ptr, tx.tx_id,
+                        det_ctx->post_rule_work_queue.len);
+                /* run post match prefilter engines on work queue */
+                PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, f);
+
+                uint32_t prev_array_idx = array_idx;
+                for (uint32_t j = 0; j < det_ctx->pmq.rule_id_array_cnt; j++) {
+                    const Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[j]];
+                    if (ts->app_inspect != NULL) {
+                        const SigIntId id = ts->num;
+                        det_ctx->tx_candidates[array_idx].s = ts;
+                        det_ctx->tx_candidates[array_idx].id = id;
+                        det_ctx->tx_candidates[array_idx].flags = NULL;
+                        det_ctx->tx_candidates[array_idx].stream_reset = 0;
+                        array_idx++;
+
+                        SCLogDebug("%p/%" PRIu64 " rule %u (%u) added from 'post match' prefilter",
+                                tx.tx_ptr, tx.tx_id, ts->id, id);
+                    }
+                }
+                SCLogDebug("%p/%" PRIu64 " rules added from 'post match' prefilter: %u", tx.tx_ptr,
+                        tx.tx_id, array_idx - prev_array_idx);
+                if (prev_array_idx != array_idx) {
+                    /* sort, but only part of array we're still going to process */
+                    qsort(det_ctx->tx_candidates + i, array_idx - i, sizeof(RuleMatchCandidateTx),
+                            DetectRunTxSortHelper);
+                }
+                det_ctx->post_rule_work_queue.len = 0;
+                PMQ_RESET(&det_ctx->pmq);
+            }
         }
 
         det_ctx->tx_id = 0;

diff --git a/src/detect.h b/src/detect.h
index 2c9b15edb60dba882f7b60bd62ab2b2581d156ef..639ba0ea342c05412142e3af4dcdf963ab4de5f6 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -1121,6 +1121,25 @@ typedef struct RuleMatchCandidateTx {
     const Signature *s;     /**< ptr to sig */
 } RuleMatchCandidateTx;
 
+/** Stores a single u32 for a rule match of the type `sm_type`. Used by
+ *  flowbits prefilter to register DETECT_FLOWBITS,<flowbit id> for post
+ *  match handling. */
+typedef struct PostRuleMatchWorkQueueItem {
+    int sm_type;    /**< sigmatch type e.g. DETECT_FLOWBITS */
+    uint32_t value; /**< value to be interpreted by the sm_type
+                     *   implementation. E.g. flowbit id. */
+#ifdef DEBUG
+    SigIntId id;
+#endif
+} PostRuleMatchWorkQueueItem;
+
+/** Array of PostRuleMatchWorkQueueItem's. */
+typedef struct PostRuleMatchWorkQueue {
+    PostRuleMatchWorkQueueItem *q; /**< array pointer */
+    uint32_t len;                  /**< number of array elements in use. */
+    uint32_t size;                 /**< allocation size in number of elements. */
+} PostRuleMatchWorkQueue;
+
 /**
   * Detection engine thread data.
   */
@@ -1230,6 +1249,9 @@ typedef struct DetectEngineThreadCtx_ {
     uint32_t non_pf_store_cnt;
 
     MpmThreadCtx mtc; /**< thread ctx for the mpm */
+    /* work queue for post-rule matching affecting prefilter */
+    PostRuleMatchWorkQueue post_rule_work_queue;
+
     PrefilterRuleStore pmq;
 
     /* string to replace */
@@ -1358,6 +1380,7 @@ enum {
     FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR,
 
     DETECT_EVENT_TOO_MANY_BUFFERS,
+    DETECT_EVENT_POST_MATCH_QUEUE_FAILED,
 };
 
 #define SIG_GROUP_HEAD_HAVERAWSTREAM BIT_U16(0)
@@ -1421,6 +1444,8 @@ typedef struct PrefilterEngineList_ {
     PrefilterPktFn Prefilter;
     PrefilterTxFn PrefilterTx;
     PrefilterFrameFn PrefilterFrame;
+    void (*PrefilterPostRule)(
+            DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f);
 
     struct PrefilterEngineList_ *next;
 
@@ -1454,6 +1479,8 @@ typedef struct PrefilterEngine_ {
         PrefilterPktFn Prefilter;
         PrefilterTxFn PrefilterTx;
         PrefilterFrameFn PrefilterFrame;
+        void (*PrefilterPostRule)(
+                DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f);
     } cb;
 
     /* global id for this prefilter */
@@ -1481,6 +1508,7 @@ typedef struct SigGroupHeadInitData_ {
     PrefilterEngineList *payload_engines;
     PrefilterEngineList *tx_engines;
     PrefilterEngineList *frame_engines;
+    PrefilterEngineList *post_rule_match_engines;
 
     /** number of sigs in this group */
     SigIntId sig_cnt;
@@ -1511,6 +1539,7 @@ typedef struct SigGroupHead_ {
     PrefilterEngine *payload_engines;
     PrefilterEngine *tx_engines;
     PrefilterEngine *frame_engines;
+    PrefilterEngine *post_rule_match_engines; /**< engines to run after rules modified a state */
 
     /* ptr to our init data we only use at... init :) */
     SigGroupHeadInitData *init;