From: Victor Julien <victor@inliniac.net>
Date: Mon, 30 Oct 2017 08:43:42 +0000 (+0100)
Subject: detect: move buffer type map into detect ctx
X-Git-Tag: suricata-4.1.0-beta1~151
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a499a44f7a0c9bcb8755b4e31cf7d908b7666276;p=thirdparty%2Fsuricata.git

detect: move buffer type map into detect ctx

Move previously global table into detect engine ctx. Now that we
can register buffers at rule loading time we need to take concurrency
into account.

Move DetectBufferType to detect.h and update DetectBufferCtx API calls
to include a detect engine ctx reference.
---

diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c
index e8b3b087b4..aa490119e1 100644
--- a/src/detect-engine-analyzer.c
+++ b/src/detect-engine-analyzer.c
@@ -71,7 +71,7 @@ static void FpPatternStatsAdd(int list, uint16_t patlen)
     f->tot += patlen;
 }
 
-void EngineAnalysisFP(Signature *s, char *line)
+void EngineAnalysisFP(const DetectEngineCtx *de_ctx, const Signature *s, char *line)
 {
     int fast_pattern_set = 0;
     int fast_pattern_only_set = 0;
@@ -113,8 +113,8 @@ void EngineAnalysisFP(Signature *s, char *line)
     if (list_type == DETECT_SM_LIST_PMATCH)
         fprintf(fp_engine_analysis_FD, "content\n");
     else {
-        const char *desc = DetectBufferTypeGetDescriptionById(list_type);
-        const char *name = DetectBufferTypeGetNameById(list_type);
+        const char *desc = DetectBufferTypeGetDescriptionById(de_ctx, list_type);
+        const char *name = DetectBufferTypeGetNameById(de_ctx, list_type);
         if (desc && name) {
             fprintf(fp_engine_analysis_FD, "%s (%s)\n", desc, name);
         }
@@ -394,7 +394,7 @@ int PerCentEncodingMatch (uint8_t *content, uint8_t content_len)
     return ret;
 }
 
-static void EngineAnalysisRulesPrintFP(const Signature *s)
+static void EngineAnalysisRulesPrintFP(const DetectEngineCtx *de_ctx, const Signature *s)
 {
     DetectContentData *fp_cd = NULL;
     SigMatch *mpm_sm = s->init_data->mpm_sm;
@@ -447,8 +447,8 @@ static void EngineAnalysisRulesPrintFP(const Signature *s)
                 payload ? (stream ? "payload and reassembled stream" : "payload") : "reassembled stream");
     }
     else {
-        const char *desc = DetectBufferTypeGetDescriptionById(list_type);
-        const char *name = DetectBufferTypeGetNameById(list_type);
+        const char *desc = DetectBufferTypeGetDescriptionById(de_ctx, list_type);
+        const char *name = DetectBufferTypeGetNameById(de_ctx, list_type);
         if (desc && name) {
             fprintf(rule_engine_analysis_FD, "%s (%s)", desc, name);
         }
@@ -481,7 +481,8 @@ void EngineAnalysisRulesFailure(char *line, char *file, int lineno)
  *
  * \param s Pointer to the signature.
  */
-void EngineAnalysisRules(const Signature *s, const char *line)
+void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
+        const Signature *s, const char *line)
 {
     uint32_t rule_bidirectional = 0;
     uint32_t rule_pcre = 0;
@@ -844,7 +845,7 @@ void EngineAnalysisRules(const Signature *s, const char *line)
             fprintf(rule_engine_analysis_FD, "    Prefilter on: %s.\n",
                     sigmatch_table[s->init_data->prefilter_sm->type].name);
         } else {
-            EngineAnalysisRulesPrintFP(s);
+            EngineAnalysisRulesPrintFP(de_ctx, s);
         }
 
         /* this is where the warnings start */
diff --git a/src/detect-engine-analyzer.h b/src/detect-engine-analyzer.h
index 1a46d0a877..87143e0f48 100644
--- a/src/detect-engine-analyzer.h
+++ b/src/detect-engine-analyzer.h
@@ -35,8 +35,10 @@ void CleanupRuleAnalyzer (void);
 int PerCentEncodingSetup (void);
 int PerCentEncodingMatch (uint8_t *content, uint8_t content_len);
 
-void EngineAnalysisFP(Signature *s, char *line);
-void EngineAnalysisRules(const Signature *s, const char *line);
+void EngineAnalysisFP(const DetectEngineCtx *de_ctx,
+        const Signature *s, char *line);
+void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
+        const Signature *s, const char *line);
 void EngineAnalysisRulesFailure(char *line, char *file, int lineno);
 
 #endif /* __DETECT_ENGINE_ANALYZER_H__ */
diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index 7f1a2a63dc..01d3e1360f 100644
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -201,7 +201,7 @@ int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
     for (int i = 0; i < nlists; i++) {
         if (s->init_data->smlists[i] == NULL)
             continue;
-        if (!(DetectBufferTypeGetNameById(i)))
+        if (!(DetectBufferTypeGetNameById(de_ctx, i)))
             continue;
 
         SCReturnInt(0);
@@ -257,7 +257,7 @@ iponly:
  *  \retval 1 sig is dp only
  *  \retval 0 sig is not dp only
  */
-static int SignatureIsPDOnly(const Signature *s)
+static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s)
 {
     if (s->alproto != ALPROTO_UNKNOWN)
         return 0;
@@ -270,7 +270,7 @@ static int SignatureIsPDOnly(const Signature *s)
     for (int i = 0; i < nlists; i++) {
         if (s->init_data->smlists[i] == NULL)
             continue;
-        if (!(DetectBufferTypeGetNameById(i)))
+        if (!(DetectBufferTypeGetNameById(de_ctx, i)))
             continue;
 
         SCReturnInt(0);
@@ -357,7 +357,7 @@ static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
     for (int i = 0; i < nlists; i++) {
         if (s->init_data->smlists[i] == NULL)
             continue;
-        if (!(DetectBufferTypeGetNameById(i)))
+        if (!(DetectBufferTypeGetNameById(de_ctx, i)))
             continue;
 
         SCReturnInt(0);
@@ -1279,7 +1279,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
         SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]);
 
         /* see if the sig is dp only */
-        if (SignatureIsPDOnly(tmp_s) == 1) {
+        if (SignatureIsPDOnly(de_ctx, tmp_s) == 1) {
             tmp_s->flags |= SIG_FLAG_PDONLY;
             SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", tmp_s->id);
 
@@ -1384,7 +1384,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
         int x;
         for (x = 0; x < nlists; x++) {
             if (tmp_s->init_data->smlists[x])
-                DetectBufferRunSetupCallback(x, tmp_s);
+                DetectBufferRunSetupCallback(de_ctx, x, tmp_s);
         }
 
         de_ctx->sig_cnt++;
@@ -1813,7 +1813,7 @@ static int SigMatchPrepare(DetectEngineCtx *de_ctx)
     Signature *s = de_ctx->sig_list;
     for (; s != NULL; s = s->next) {
         /* set up inspect engines */
-        DetectEngineAppInspectionEngine2Signature(s);
+        DetectEngineAppInspectionEngine2Signature(de_ctx, s);
 
         /* built-ins */
         int type;
diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c
index cb07dc6167..7b9a4a2fa6 100644
--- a/src/detect-engine-loader.c
+++ b/src/detect-engine-loader.c
@@ -169,12 +169,12 @@ static int DetectLoadSigFile(DetectEngineCtx *de_ctx, char *sig_file,
         sig = DetectEngineAppendSig(de_ctx, line);
         if (sig != NULL) {
             if (rule_engine_analysis_set || fp_engine_analysis_set) {
-                RetrieveFPForSig(sig);
+                RetrieveFPForSig(de_ctx, sig);
                 if (fp_engine_analysis_set) {
-                    EngineAnalysisFP(sig, line);
+                    EngineAnalysisFP(de_ctx, sig, line);
                 }
                 if (rule_engine_analysis_set) {
-                    EngineAnalysisRules(sig, line);
+                    EngineAnalysisRules(de_ctx, sig, line);
                 }
             }
             SCLogDebug("signature %"PRIu32" loaded", sig->id);
diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 1e1613abc0..ba45cb093d 100644
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -637,7 +637,7 @@ static void SetMpm(Signature *s, SigMatch *mpm_sm)
     return;
 }
 
-void RetrieveFPForSig(Signature *s)
+void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s)
 {
     if (s->init_data->mpm_sm != NULL)
         return;
@@ -659,7 +659,7 @@ void RetrieveFPForSig(Signature *s)
         if (s->init_data->smlists[list_id] == NULL)
             continue;
 
-        if (!FastPatternSupportEnabledForSigMatchList(list_id))
+        if (!FastPatternSupportEnabledForSigMatchList(de_ctx, list_id))
             continue;
 
         for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) {
@@ -708,6 +708,8 @@ void RetrieveFPForSig(Signature *s)
              tmp != NULL && priority == tmp->priority;
              tmp = tmp->next)
         {
+            if (tmp->list_id >= nlists)
+                continue;
             if (curr_sm_list[tmp->list_id] == 0)
                 continue;
             final_sm_list[count_final_sm_list++] = tmp->list_id;
@@ -1440,7 +1442,7 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
         if (s->flags & SIG_FLAG_PREFILTER)
             continue;
 
-        RetrieveFPForSig(s);
+        RetrieveFPForSig(de_ctx, s);
         if (s->init_data->mpm_sm != NULL) {
             DetectContentData *cd = (DetectContentData *)s->init_data->mpm_sm->ctx;
             struct_total_size += sizeof(DetectFPAndItsId);
diff --git a/src/detect-engine-mpm.h b/src/detect-engine-mpm.h
index 3324403d70..3574db758d 100644
--- a/src/detect-engine-mpm.h
+++ b/src/detect-engine-mpm.h
@@ -60,7 +60,7 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
 int SignatureHasPacketContent(const Signature *);
 int SignatureHasStreamContent(const Signature *);
 
-void RetrieveFPForSig(Signature *s);
+void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s);
 
 int MpmStoreInit(DetectEngineCtx *);
 void MpmStoreFree(DetectEngineCtx *);
diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c
index decc8d0855..bfa92f90ef 100644
--- a/src/detect-engine-register.c
+++ b/src/detect-engine-register.c
@@ -480,7 +480,7 @@ void SigTableSetup(void)
     DetectBypassRegister();
 
     /* close keyword registration */
-    DetectBufferTypeFinalizeRegistration();
+    DetectBufferTypeCloseRegistration();
 }
 
 void SigTableRegisterTests(void)
diff --git a/src/detect-engine.c b/src/detect-engine.c
index e017aacc33..cf0db0a168 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -314,7 +314,7 @@ static void AppendStreamInspectEngine(Signature *s, SigMatchData *stream, int di
  *  \note for the file inspect engine, the id DE_STATE_ID_FILE_INSPECT
  *        is assigned.
  */
-int DetectEngineAppInspectionEngine2Signature(Signature *s)
+int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s)
 {
     const int nlists = DetectBufferTypeMaxId();
     SigMatchData *ptrs[nlists];
@@ -364,7 +364,7 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
             exit(EXIT_FAILURE);
         }
         if (mpm_list == t->sm_list) {
-            SCLogDebug("%s is mpm", DetectBufferTypeGetNameById(t->sm_list));
+            SCLogDebug("%s is mpm", DetectBufferTypeGetNameById(de_ctx, t->sm_list));
             prepend = true;
             head_is_mpm = true;
             new_engine->mpm = true;
@@ -446,7 +446,7 @@ next:
     DetectEngineAppInspectionEngine *iter = s->app_inspect;
     while (iter) {
         SCLogDebug("%u: engine %s id %u progress %d %s", s->id,
-                DetectBufferTypeGetNameById(iter->sm_list), iter->id,
+                DetectBufferTypeGetNameById(de_ctx, iter->sm_list), iter->id,
                 iter->progress,
                 iter->sm_list == mpm_list ? "MPM":"");
         iter = iter->next;
@@ -510,22 +510,6 @@ static int g_buffer_type_reg_closed = 0;
 
 static DetectEngineTransforms no_transforms = { .transforms = { 0 }, .cnt = 0, };
 
-typedef struct DetectBufferType_ {
-    const char *string;
-    const char *description;
-    int id;
-    int parent_id;
-    _Bool mpm;
-    _Bool packet; /**< compat to packet matches */
-    bool supports_transforms;
-    void (*SetupCallback)(Signature *);
-    _Bool (*ValidateCallback)(const Signature *, const char **sigerror);
-    DetectEngineTransforms transforms;
-} DetectBufferType;
-
-static DetectBufferType **g_buffer_type_map = NULL;
-static uint32_t g_buffer_type_map_elements = 0;
-
 int DetectBufferTypeMaxId(void)
 {
     return g_buffer_type_id;
@@ -660,23 +644,23 @@ int DetectBufferTypeGetByName(const char *name)
     return exists->id;
 }
 
-const char *DetectBufferTypeGetNameById(const int id)
+const char *DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
 {
-    BUG_ON(id < 0 || id >= g_buffer_type_id);
-    BUG_ON(g_buffer_type_map == NULL);
+    BUG_ON(id < 0 || (uint32_t)id >= de_ctx->buffer_type_map_elements);
+    BUG_ON(de_ctx->buffer_type_map == NULL);
 
-    if (g_buffer_type_map[id] == NULL)
+    if (de_ctx->buffer_type_map[id] == NULL)
         return NULL;
 
-    return g_buffer_type_map[id]->string;
+    return de_ctx->buffer_type_map[id]->string;
 }
 
-static const DetectBufferType *DetectBufferTypeGetById(const int id)
+static const DetectBufferType *DetectBufferTypeGetById(const DetectEngineCtx *de_ctx, const int id)
 {
-    BUG_ON(id < 0 || id >= g_buffer_type_id);
-    BUG_ON(g_buffer_type_map == NULL);
+    BUG_ON(id < 0 || (uint32_t)id >= de_ctx->buffer_type_map_elements);
+    BUG_ON(de_ctx->buffer_type_map == NULL);
 
-    return g_buffer_type_map[id];
+    return de_ctx->buffer_type_map[id];
 }
 
 void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc)
@@ -688,9 +672,9 @@ void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc)
     exists->description = desc;
 }
 
-const char *DetectBufferTypeGetDescriptionById(const int id)
+const char *DetectBufferTypeGetDescriptionById(const DetectEngineCtx *de_ctx, const int id)
 {
-    const DetectBufferType *exists = DetectBufferTypeGetById(id);
+    const DetectBufferType *exists = DetectBufferTypeGetById(de_ctx, id);
     if (!exists) {
         return NULL;
     }
@@ -706,18 +690,18 @@ const char *DetectBufferTypeGetDescriptionByName(const char *name)
     return exists->description;
 }
 
-_Bool DetectBufferTypeSupportsPacketGetById(const int id)
+bool DetectBufferTypeSupportsPacketGetById(const DetectEngineCtx *de_ctx, const int id)
 {
-    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
     if (map == NULL)
         return FALSE;
     SCLogDebug("map %p id %d packet? %d", map, id, map->packet);
     return map->packet;
 }
 
-_Bool DetectBufferTypeSupportsMpmGetById(const int id)
+bool DetectBufferTypeSupportsMpmGetById(const DetectEngineCtx *de_ctx, const int id)
 {
-    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
     if (map == NULL)
         return FALSE;
     SCLogDebug("map %p id %d mpm? %d", map, id, map->mpm);
@@ -734,9 +718,10 @@ void DetectBufferTypeRegisterSetupCallback(const char *name,
     exists->SetupCallback = SetupCallback;
 }
 
-void DetectBufferRunSetupCallback(const int id, Signature *s)
+void DetectBufferRunSetupCallback(const DetectEngineCtx *de_ctx,
+        const int id, Signature *s)
 {
-    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
     if (map && map->SetupCallback) {
         map->SetupCallback(s);
     }
@@ -752,9 +737,10 @@ void DetectBufferTypeRegisterValidateCallback(const char *name,
     exists->ValidateCallback = ValidateCallback;
 }
 
-_Bool DetectBufferRunValidateCallback(const int id, const Signature *s, const char **sigerror)
+bool DetectBufferRunValidateCallback(const DetectEngineCtx *de_ctx,
+        const int id, const Signature *s, const char **sigerror)
 {
-    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
     if (map && map->ValidateCallback) {
         return map->ValidateCallback(s, sigerror);
     }
@@ -774,14 +760,14 @@ int DetectBufferSetActiveList(Signature *s, const int list)
     return 0;
 }
 
-int DetectBufferGetActiveList(Signature *s)
+int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s)
 {
     BUG_ON(s->init_data == NULL);
 
     if (s->init_data->list && s->init_data->transform_cnt) {
         SCLogDebug("buffer %d has transform(s) registered: %d",
                 s->init_data->list, s->init_data->transforms[0]);
-        int new_list = DetectBufferTypeGetByIdTransforms(s->init_data->list,
+        int new_list = DetectBufferTypeGetByIdTransforms(de_ctx, s->init_data->list,
                 s->init_data->transforms, s->init_data->transform_cnt);
         if (new_list == -1) {
             return -1;
@@ -869,35 +855,45 @@ void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
     }
 }
 
-void DetectBufferTypeFinalizeRegistration(void)
+static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx)
 {
-    BUG_ON(g_buffer_type_hash == NULL);
-
     const int size = g_buffer_type_id;
     BUG_ON(!(size > 0));
 
-    g_buffer_type_map = SCCalloc(size, sizeof(DetectBufferType *));
-    BUG_ON(!g_buffer_type_map);
-    g_buffer_type_map_elements = size;
-    SCLogDebug("g_buffer_type_map %p with %u members", g_buffer_type_map, size);
+    de_ctx->buffer_type_map = SCCalloc(size, sizeof(DetectBufferType *));
+    BUG_ON(!de_ctx->buffer_type_map);
+    de_ctx->buffer_type_map_elements = size;
+    SCLogDebug("de_ctx->buffer_type_map %p with %u members", de_ctx->buffer_type_map, size);
 
     SCLogDebug("DETECT_SM_LIST_DYNAMIC_START %u", DETECT_SM_LIST_DYNAMIC_START);
     HashListTableBucket *b = HashListTableGetListHead(g_buffer_type_hash);
     while (b) {
         DetectBufferType *map = HashListTableGetListData(b);
-        g_buffer_type_map[map->id] = map;
+        de_ctx->buffer_type_map[map->id] = map;
         SCLogDebug("name %s id %d mpm %s packet %s -- %s. "
                 "Callbacks: Setup %p Validate %p", map->string, map->id,
                 map->mpm ? "true" : "false", map->packet ? "true" : "false",
                 map->description, map->SetupCallback, map->ValidateCallback);
         b = HashListTableGetListNext(b);
     }
+}
+static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
+{
+    if (de_ctx && de_ctx->buffer_type_map)
+        SCFree(de_ctx->buffer_type_map);
+}
+
+void DetectBufferTypeCloseRegistration(void)
+{
+    BUG_ON(g_buffer_type_hash == NULL);
+
     g_buffer_type_reg_closed = 1;
 }
 
-int DetectBufferTypeGetByIdTransforms(const int id, int *transforms, int transform_cnt)
+int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id,
+        int *transforms, int transform_cnt)
 {
-    const DetectBufferType *base_map = DetectBufferTypeGetById(id);
+    const DetectBufferType *base_map = DetectBufferTypeGetById(de_ctx, id);
     if (!base_map) {
         return -1;
     }
@@ -938,13 +934,13 @@ int DetectBufferTypeGetByIdTransforms(const int id, int *transforms, int transfo
     BUG_ON(HashListTableAdd(g_buffer_type_hash, (void *)map, 0) != 0);
     SCLogDebug("buffer %s registered with id %d, parent %d", map->string, map->id, map->parent_id);
 
-    if (map->id >= 0 && (uint32_t)map->id >= g_buffer_type_map_elements) {
-        void *ptr = SCRealloc(g_buffer_type_map, (map->id + 1) * sizeof(DetectBufferType *));
+    if (map->id >= 0 && (uint32_t)map->id >= de_ctx->buffer_type_map_elements) {
+        void *ptr = SCRealloc(de_ctx->buffer_type_map, (map->id + 1) * sizeof(DetectBufferType *));
         BUG_ON(ptr == NULL);
-        SCLogDebug("g_buffer_type_map resized to %u (was %u)", (map->id + 1), g_buffer_type_map_elements);
-        g_buffer_type_map = ptr;
-        g_buffer_type_map[map->id] = map;
-        g_buffer_type_map_elements = map->id + 1;
+        SCLogDebug("de_ctx->buffer_type_map resized to %u (was %u)", (map->id + 1), de_ctx->buffer_type_map_elements);
+        de_ctx->buffer_type_map = ptr;
+        de_ctx->buffer_type_map[map->id] = map;
+        de_ctx->buffer_type_map_elements = map->id + 1;
 
         DetectAppLayerInspectEngineCopy(map->parent_id, map->id, &map->transforms);
     }
@@ -1439,6 +1435,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix)
     DetectParseDupSigHashInit(de_ctx);
     DetectAddressMapInit(de_ctx);
     DetectMetadataHashInit(de_ctx);
+    DetectBufferTypeSetupDetectEngine(de_ctx);
 
     /* init iprep... ignore errors for now */
     (void)SRepInit(de_ctx);
@@ -1549,6 +1546,7 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
 
     SCClassConfDeInitContext(de_ctx);
     SCRConfDeInitContext(de_ctx);
+    DetectBufferTypeFreeDetectEngine(de_ctx);
 
     SigGroupCleanup(de_ctx);
 
diff --git a/src/detect-engine.h b/src/detect-engine.h
index 5538136c98..fca7902409 100644
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -38,24 +38,26 @@ void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
 
 int DetectBufferTypeRegister(const char *name);
 int DetectBufferTypeGetByName(const char *name);
-int DetectBufferTypeGetByIdTransforms(const int id, int *transforms, int transform_cnt);
-const char *DetectBufferTypeGetNameById(const int id);
 void DetectBufferTypeSupportsMpm(const char *name);
 void DetectBufferTypeSupportsPacket(const char *name);
 void DetectBufferTypeSupportsTransformations(const char *name);
-_Bool DetectBufferTypeSupportsMpmGetById(const int id);
-_Bool DetectBufferTypeSupportsPacketGetById(const int id);
 int DetectBufferTypeMaxId(void);
-void DetectBufferTypeFinalizeRegistration(void);
+void DetectBufferTypeCloseRegistration(void);
 void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc);
-const char *DetectBufferTypeGetDescriptionById(const int id);
 const char *DetectBufferTypeGetDescriptionByName(const char *name);
 void DetectBufferTypeRegisterSetupCallback(const char *name,
         void (*Callback)(Signature *));
-void DetectBufferRunSetupCallback(const int id, Signature *s);
 void DetectBufferTypeRegisterValidateCallback(const char *name,
         _Bool (*ValidateCallback)(const Signature *, const char **sigerror));
-_Bool DetectBufferRunValidateCallback(const int id, const Signature *s, const char **sigerror);
+
+int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id,
+        int *transforms, int transform_cnt);
+const char *DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id);
+bool DetectBufferTypeSupportsMpmGetById(const DetectEngineCtx *de_ctx, const int id);
+bool DetectBufferTypeSupportsPacketGetById(const DetectEngineCtx *de_ctx, const int id);
+const char *DetectBufferTypeGetDescriptionById(const DetectEngineCtx *de_ctx, const int id);
+void DetectBufferRunSetupCallback(const DetectEngineCtx *de_ctx, const int id, Signature *s);
+bool DetectBufferRunValidateCallback(const DetectEngineCtx *de_ctx, const int id, const Signature *s, const char **sigerror);
 
 /* prototypes */
 DetectEngineCtx *DetectEngineCtxInitWithPrefix(const char *prefix);
@@ -134,7 +136,7 @@ void DetectAppLayerInspectEngineRegister2(const char *name,
         InspectEngineFuncPtr2 Callback2,
         InspectionBufferGetDataPtr GetData);
 
-int DetectEngineAppInspectionEngine2Signature(Signature *s);
+int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s);
 void DetectEngineAppInspectionEngineSignatureFree(Signature *s);
 
 void DetectEngineSetParseMetadata(void);
@@ -142,6 +144,6 @@ void DetectEngineUnsetParseMetadata(void);
 int DetectEngineMustParseMetadata(void);
 
 int DetectBufferSetActiveList(Signature *s, const int list);
-int DetectBufferGetActiveList(Signature *s);
+int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s);
 
 #endif /* __DETECT_ENGINE_H__ */
diff --git a/src/detect-fast-pattern.c b/src/detect-fast-pattern.c
index 1bea3e245b..f55f951bc9 100644
--- a/src/detect-fast-pattern.c
+++ b/src/detect-fast-pattern.c
@@ -58,7 +58,8 @@ SCFPSupportSMList *sm_fp_support_smlist_list = NULL;
  * \retval 1 If supported.
  * \retval 0 If not.
  */
-int FastPatternSupportEnabledForSigMatchList(int list_id)
+int FastPatternSupportEnabledForSigMatchList(const DetectEngineCtx *de_ctx,
+        const int list_id)
 {
     if (sm_fp_support_smlist_list == NULL)
         return 0;
@@ -66,7 +67,7 @@ int FastPatternSupportEnabledForSigMatchList(int list_id)
     if (list_id == DETECT_SM_LIST_PMATCH)
         return 1;
 
-    return DetectBufferTypeSupportsMpmGetById(list_id);
+    return DetectBufferTypeSupportsMpmGetById(de_ctx, list_id);
 
 #if 0
     SCFPSupportSMList *tmp_smlist_fp = sm_fp_support_smlist_list;
@@ -194,7 +195,7 @@ static int DetectFastPatternSetup(DetectEngineCtx *de_ctx, Signature *s, const c
     DetectContentData *cd = NULL;
     const int nlists = DetectBufferTypeMaxId();
 
-    SigMatch *pm1 = DetectGetLastSMFromMpmLists(s);
+    SigMatch *pm1 = DetectGetLastSMFromMpmLists(de_ctx, s);
     SigMatch *pm2 = DetectGetLastSMFromLists(s, DETECT_CONTENT, -1);
     if (pm1 == NULL && pm2 == NULL) {
         SCLogError(SC_ERR_INVALID_SIGNATURE, "fast_pattern found inside "
diff --git a/src/detect-fast-pattern.h b/src/detect-fast-pattern.h
index 30172b8cbf..8617db1826 100644
--- a/src/detect-fast-pattern.h
+++ b/src/detect-fast-pattern.h
@@ -35,7 +35,8 @@ typedef struct SCFPSupportSMList_ {
 extern SCFPSupportSMList *sm_fp_support_smlist_list;
 
 void SupportFastPatternForSigMatchList(int list_id, int priority);
-int FastPatternSupportEnabledForSigMatchList(int list_id);
+int FastPatternSupportEnabledForSigMatchList(const DetectEngineCtx *de_ctx,
+        const int list_id);
 
 void SupportFastPatternForSigMatchTypes(void);
 
diff --git a/src/detect-parse.c b/src/detect-parse.c
index 578454f40b..b1da674e2f 100644
--- a/src/detect-parse.c
+++ b/src/detect-parse.c
@@ -358,7 +358,7 @@ static SigMatch *SigMatchGetLastSMByType(SigMatch *sm, int type)
  *  \note only supports the lists that are registered through
  *        DetectBufferTypeSupportsMpm().
  */
-SigMatch *DetectGetLastSMFromMpmLists(const Signature *s)
+SigMatch *DetectGetLastSMFromMpmLists(const DetectEngineCtx *de_ctx, const Signature *s)
 {
     SigMatch *sm_last = NULL;
     SigMatch *sm_new;
@@ -366,7 +366,7 @@ SigMatch *DetectGetLastSMFromMpmLists(const Signature *s)
 
     /* if we have a sticky buffer, use that */
     if (s->init_data->list != DETECT_SM_LIST_NOTSET) {
-        if (!(DetectBufferTypeSupportsMpmGetById(s->init_data->list))) {
+        if (!(DetectBufferTypeSupportsMpmGetById(de_ctx, s->init_data->list))) {
             return NULL;
         }
 
@@ -379,7 +379,7 @@ SigMatch *DetectGetLastSMFromMpmLists(const Signature *s)
     /* otherwise brute force it */
     const int nlists = DetectBufferTypeMaxId();
     for (sm_type = 0; sm_type < nlists; sm_type++) {
-        if (!DetectBufferTypeSupportsMpmGetById(sm_type))
+        if (!DetectBufferTypeSupportsMpmGetById(de_ctx, sm_type))
             continue;
         SigMatch *sm_list = s->init_data->smlists_tail[sm_type];
         sm_new = SigMatchGetLastSMByType(sm_list, DETECT_CONTENT);
@@ -1559,7 +1559,7 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
     int x;
     for (x = 0; x < nlists; x++) {
         if (s->init_data->smlists[x]) {
-            if (DetectBufferRunValidateCallback(x, s, &de_ctx->sigerror) == FALSE) {
+            if (DetectBufferRunValidateCallback(de_ctx, x, s, &de_ctx->sigerror) == FALSE) {
                 SCReturnInt(0);
             }
         }
@@ -1648,10 +1648,10 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
         for (int i = 0; i < nlists; i++) {
             if (s->init_data->smlists[i] == NULL)
                 continue;
-            if (!(DetectBufferTypeGetNameById(i)))
+            if (!(DetectBufferTypeGetNameById(de_ctx, i)))
                 continue;
 
-            if (!(DetectBufferTypeSupportsPacketGetById(i))) {
+            if (!(DetectBufferTypeSupportsPacketGetById(de_ctx, i))) {
                 SCLogError(SC_ERR_INVALID_SIGNATURE, "Signature combines packet "
                         "specific matches (like dsize, flags, ttl) with stream / "
                         "state matching by matching on app layer proto (like using "
@@ -1816,7 +1816,7 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, const char *sigstr,
     int x;
     for (x = 0; x < nlists; x++) {
         if (sig->init_data->smlists[x])
-            DetectBufferRunSetupCallback(x, sig);
+            DetectBufferRunSetupCallback(de_ctx, x, sig);
     }
 
     /* validate signature, SigValidate will report the error reason */
diff --git a/src/detect-parse.h b/src/detect-parse.h
index 7d9af5e942..721f1957f2 100644
--- a/src/detect-parse.h
+++ b/src/detect-parse.h
@@ -64,7 +64,7 @@ const char *DetectListToHumanString(int list);
 const char *DetectListToString(int list);
 
 SigMatch *DetectGetLastSM(const Signature *);
-SigMatch *DetectGetLastSMFromMpmLists(const Signature *s);
+SigMatch *DetectGetLastSMFromMpmLists(const DetectEngineCtx *de_ctx, const Signature *s);
 SigMatch *DetectGetLastSMFromLists(const Signature *s, ...);
 SigMatch *DetectGetLastSMByListPtr(const Signature *s, SigMatch *sm_list, ...);
 SigMatch *DetectGetLastSMByListId(const Signature *s, int list_id, ...);
diff --git a/src/detect.h b/src/detect.h
index 9db4dd860e..1e2beb6178 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -413,6 +413,19 @@ typedef struct DetectEngineAppInspectionEngine_ {
     struct DetectEngineAppInspectionEngine_ *next;
 } DetectEngineAppInspectionEngine;
 
+typedef struct DetectBufferType_ {
+    const char *string;
+    const char *description;
+    int id;
+    int parent_id;
+    _Bool mpm;
+    _Bool packet; /**< compat to packet matches */
+    bool supports_transforms;
+    void (*SetupCallback)(struct Signature_ *);
+    bool (*ValidateCallback)(const struct Signature_ *, const char **sigerror);
+    DetectEngineTransforms transforms;
+} DetectBufferType;
+
 #ifdef UNITTESTS
 #define sm_lists init_data->smlists
 #define sm_lists_tail init_data->smlists_tail
@@ -837,6 +850,9 @@ typedef struct DetectEngineCtx_ {
     /** table to store metadata keys and values */
     HashTable *metadata_table;
 
+    DetectBufferType **buffer_type_map;
+    uint32_t buffer_type_map_elements;
+
     /** table with mpms and their registration function
      *  \todo we only need this at init, so perhaps this
      *        can move to a DetectEngineCtx 'init' struct */
diff --git a/src/util-profiling-keywords.c b/src/util-profiling-keywords.c
index 80ffc3b7d1..9f9cbc8c3b 100644
--- a/src/util-profiling-keywords.c
+++ b/src/util-profiling-keywords.c
@@ -198,7 +198,7 @@ SCProfilingKeywordDump(DetectEngineCtx *de_ctx)
             if (i < DETECT_SM_LIST_DYNAMIC_START) {
                 name = DetectSigmatchListEnumToString(i);
             } else {
-                name = DetectBufferTypeGetNameById(i);
+                name = DetectBufferTypeGetNameById(de_ctx, i);
             }
 
             DoDump(de_ctx->profile_keyword_ctx_per_list[i], fp, name);