detect: global registery for keyword thread data

Some keywords need a scratch space where they can do store the results
of expensive operations that remain valid for the time of a packets
journey through the detection engine.

An example is the reconstructed 'http_header' field, that is needed
in MPM, and then for each rule that manually inspects it. Storing this
data in the flow is a waste, and reconstructing multiple times on
demand as well.

This API allows for registering a keyword with an init and free function.

It it mean to be used an initialization time, when the keyword is
registered.

diff --git a/src/detect-engine.c b/src/detect-engine.c
index c19d3f0ef85bb3e561dee19aae30717969540b56..d6a412f21ee09e386eef9babf29341918c371273 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -91,7 +91,8 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForReload(
 
 static int DetectEngineCtxLoadConf(DetectEngineCtx *);
 
-static DetectEngineMasterCtx g_master_de_ctx = { SCMUTEX_INITIALIZER, 0, NULL, NULL, TENANT_SELECTOR_UNKNOWN, NULL,};
+static DetectEngineMasterCtx g_master_de_ctx = { SCMUTEX_INITIALIZER,
+    0, NULL, NULL, TENANT_SELECTOR_UNKNOWN, NULL, NULL, 0};
 
 static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len);
 static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len);
@@ -1387,6 +1388,62 @@ void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
     de_ctx->signum = 0;
 }
 
+static int DetectEngineThreadCtxInitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
+{
+    DetectEngineMasterCtx *master = &g_master_de_ctx;
+    SCMutexLock(&master->lock);
+
+    if (master->keyword_id > 0) {
+        det_ctx->global_keyword_ctxs_array = SCCalloc(master->keyword_id, sizeof(void *));
+        if (det_ctx->global_keyword_ctxs_array == NULL) {
+            SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx");
+            goto error;
+        }
+        det_ctx->global_keyword_ctxs_size = master->keyword_id;
+
+        DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
+        while (item) {
+            det_ctx->global_keyword_ctxs_array[item->id] = item->InitFunc(item->data);
+            if (det_ctx->global_keyword_ctxs_array[item->id] == NULL) {
+                SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx "
+                        "for keyword \"%s\" failed", item->name);
+                goto error;
+            }
+            item = item->next;
+        }
+    }
+    SCMutexUnlock(&master->lock);
+    return TM_ECODE_OK;
+error:
+    SCMutexUnlock(&master->lock);
+    return TM_ECODE_FAILED;
+}
+
+static void DetectEngineThreadCtxDeinitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
+{
+    if (det_ctx->global_keyword_ctxs_array == NULL ||
+        det_ctx->global_keyword_ctxs_size == 0) {
+        return;
+    }
+
+    DetectEngineMasterCtx *master = &g_master_de_ctx;
+    SCMutexLock(&master->lock);
+
+    if (master->keyword_id > 0) {
+        DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
+        while (item) {
+            if (det_ctx->global_keyword_ctxs_array[item->id] != NULL)
+                item->FreeFunc(det_ctx->global_keyword_ctxs_array[item->id]);
+
+            item = item->next;
+        }
+        det_ctx->global_keyword_ctxs_size = 0;
+        SCFree(det_ctx->global_keyword_ctxs_array);
+        det_ctx->global_keyword_ctxs_array = NULL;
+    }
+    SCMutexUnlock(&master->lock);
+}
+
 static int DetectEngineThreadCtxInitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
 {
     if (de_ctx->keyword_id > 0) {
@@ -1605,6 +1662,7 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *
     }
 
     DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
+    DetectEngineThreadCtxInitGlobalKeywords(det_ctx);
 #ifdef PROFILING
     SCProfilingRuleThreadSetup(de_ctx->profile_ctx, det_ctx);
     SCProfilingKeywordThreadSetup(de_ctx->profile_keyword_ctx, det_ctx);
@@ -1823,6 +1881,7 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
         SCFree(det_ctx->base64_decoded);
     }
 
+    DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx);
     if (det_ctx->de_ctx != NULL) {
         DetectEngineThreadCtxDeinitKeywords(det_ctx->de_ctx, det_ctx);
 #ifdef UNITTESTS
@@ -1924,6 +1983,75 @@ void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id)
     return det_ctx->keyword_ctxs_array[id];
 }
 
+
+/** \brief Register Thread keyword context Funcs (Global)
+ *
+ *  IDs stay static over reloads and between tenants
+ *
+ *  \param name keyword name for error printing
+ *  \param InitFunc function ptr
+ *  \param FreeFunc function ptr
+ *
+ *  \retval id for retrieval of ctx at runtime
+ *  \retval -1 on error
+ */
+int DetectRegisterThreadCtxGlobalFuncs(const char *name,
+        void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *))
+{
+    int id;
+    BUG_ON(InitFunc == NULL || FreeFunc == NULL);
+
+    DetectEngineMasterCtx *master = &g_master_de_ctx;
+    SCMutexLock(&master->lock);
+
+    /* if already registered, return existing id */
+    DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
+    while (item != NULL) {
+        if (strcmp(name, item->name) == 0) {
+            id = item->id;
+            SCMutexUnlock(&master->lock);
+            return id;
+        }
+
+        item = item->next;
+    }
+
+    item = SCCalloc(1, sizeof(*item));
+    if (unlikely(item == NULL))
+        return -1;
+
+    item->InitFunc = InitFunc;
+    item->FreeFunc = FreeFunc;
+    item->name = name;
+    item->data = data;
+
+    item->next = master->keyword_list;
+    master->keyword_list = item;
+    item->id = master->keyword_id++;
+
+    id = item->id;
+    SCMutexUnlock(&master->lock);
+    return id;
+}
+
+/** \brief Retrieve thread local keyword ctx by id
+ *
+ *  \param det_ctx detection engine thread ctx to retrieve the ctx from
+ *  \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
+ *            keyword init.
+ *
+ *  \retval ctx or NULL on error
+ */
+void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id)
+{
+    if (id < 0 || id > det_ctx->global_keyword_ctxs_size ||
+        det_ctx->global_keyword_ctxs_array == NULL) {
+        return NULL;
+    }
+
+    return det_ctx->global_keyword_ctxs_array[id];
+}
+
 /** \brief Check if detection is enabled
  *  \retval bool true or false */
 int DetectEngineEnabled(void)

diff --git a/src/detect-engine.h b/src/detect-engine.h
index a29db7a5642ac208f8c392ccc25638d2f709880c..9c1c1614a5c33261e129ba5023e234f44740dd19 100644 (file)
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -53,6 +53,10 @@ DetectEngineCtx *DetectEngineCtxInit(void);
 DetectEngineCtx *DetectEngineCtxInitMinimal(void);
 void DetectEngineCtxFree(DetectEngineCtx *);
 
+int DetectRegisterThreadCtxGlobalFuncs(const char *name,
+        void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *));
+void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id);
+
 TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
 TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
 //inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *);

diff --git a/src/detect.c b/src/detect.c
index cb21cbf9e3b856d1a105f06587005b2419f6063e..0bb8167902f209352f1c75ccbe5e2bc535392252 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -1049,6 +1049,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
 
     SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt);
 
+    det_ctx->ticker++;
     p->alerts.cnt = 0;
     det_ctx->filestore_cnt = 0;
     det_ctx->smsg = NULL;

diff --git a/src/detect.h b/src/detect.h
index 86895a5f3b0b0dcdbdb804f27d877ecf94959ca5..c1321914dc66363fc79b8b9a1405b84ebf0a737c 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -778,6 +778,9 @@ typedef struct DetectEngineThreadCtx_ {
      *        on this beeing the first member */
     uint32_t tenant_id;
 
+    /** ticker that is incremented once per packet. */
+    uint64_t ticker;
+
     /* the thread to which this detection engine thread belongs */
     ThreadVars *tv;
 
@@ -900,10 +903,12 @@ typedef struct DetectEngineThreadCtx_ {
     } filestore[DETECT_FILESTORE_MAX];
 
     DetectEngineCtx *de_ctx;
-    /** store for keyword contexts that need a per thread storage because of
-     *  thread safety issues */
+    /** store for keyword contexts that need a per thread storage. Per de_ctx. */
     void **keyword_ctxs_array;
     int keyword_ctxs_size;
+    /** store for keyword contexts that need a per thread storage. Global. */
+    int global_keyword_ctxs_size;
+    void **global_keyword_ctxs_array;
 
     uint8_t *base64_decoded;
     int base64_decoded_len;
@@ -1146,6 +1151,9 @@ typedef struct DetectEngineMasterCtx_ {
      *  structures. */
     DetectEngineTenantMapping *tenant_mapping_list;
 
+    /** list of keywords that need thread local ctxs */
+    DetectEngineThreadKeywordCtxItem *keyword_list;
+    int keyword_id;
 } DetectEngineMasterCtx;
 
 /** \brief Signature loader statistics */