var-names: expose outside of detect engine

Until now variable names, such as flowbit names, were local to a detect
engine. This made sense as they were only ever used in that context.

For the purpose of logging these names, this needs a different approach.
The loggers live outside of the detect engine. Also, in the case of
reloads and multi-tenancy, there are even multiple detect engines, so
it would be even more tricky to access them from the outside.

This patch brings a new approach. A any time, there is a single active
hash table mapping the variable names and their id's. For multiple
tenants the table is shared between tenants.

The table is set up in a 'staging' area, where locking makes sure that
multiple loading threads don't mess things up. Then when the preparing
of a detection engine is ready, but before the detect threads are made
aware of the new detect engine, the active varname hash is swapped with
the staging instance.

For this to work, all the mappings from the 'current' or active mapping
are added to the staging table.

After the threads have reloaded and the new detection engine is active,
the old table can be freed.

For multi tenancy things are similar. The staging area is used for
setting up until the new detection engines / tenants are applied to
the system.

This patch also changes the variable 'id'/'idx' field to uint32_t. Due
to data structure padding and alignment, this should have no practical
drawback while allowing for a lot more vars.

diff --git a/src/detect-engine.c b/src/detect-engine.c
index bbfec11e530b61f638edd62a779083d200309b80..db5a9d4197e4bbed221f99f22972c3cb1c1df027 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -934,7 +934,6 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix)
     SigGroupHeadHashInit(de_ctx);
     MpmStoreInit(de_ctx);
     ThresholdHashInit(de_ctx);
-    VariableNameInitHash(de_ctx);
     DetectParseDupSigHashInit(de_ctx);
     DetectAddressMapInit(de_ctx);
 
@@ -958,6 +957,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix)
     }
 
     de_ctx->version = DetectEngineGetVersion();
+    VarNameStoreSetupStaging(de_ctx->version);
     SCLogDebug("dectx with version %u", de_ctx->version);
     return de_ctx;
 error:
@@ -1033,8 +1033,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     SigCleanSignatures(de_ctx);
     SCFree(de_ctx->app_mpms);
     de_ctx->app_mpms = NULL;
-
-    VariableNameFreeHash(de_ctx);
     if (de_ctx->sig_array)
         SCFree(de_ctx->sig_array);
 
@@ -1067,6 +1065,9 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     DetectPortCleanupList(de_ctx->tcp_whitelist);
     DetectPortCleanupList(de_ctx->udp_whitelist);
 
+    /* freed our var name hash */
+    VarNameStoreFree(de_ctx->version);
+
     SCFree(de_ctx);
     //DetectAddressGroupPrintMemory();
     //DetectSigGroupPrintMemory();
@@ -2490,6 +2491,9 @@ int DetectEngineMultiTenantSetup(void)
         if (DetectLoadersSync() != 0) {
             goto error;
         }
+
+        VarNameStoreActivateStaging();
+
     } else {
         SCLogDebug("multi-detect not enabled (multi tenancy)");
     }

diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c
index 41b249efebd500387e24bd46563bfdf80fdcb978..34d4eaf121279958ffa27d0263ffdae305812c96 100644 (file)
--- a/src/detect-flowbits.c
+++ b/src/detect-flowbits.c
@@ -246,7 +246,7 @@ int DetectFlowbitSetup (DetectEngineCtx *de_ctx, Signature *s, char *rawstr)
     if (unlikely(cd == NULL))
         goto error;
 
-    cd->idx = VariableNameGetIdx(de_ctx, fb_name, VAR_TYPE_FLOW_BIT);
+    cd->idx = VarNameStoreSetupAdd(fb_name, VAR_TYPE_FLOW_BIT);
     cd->cmd = fb_cmd;
 
     SCLogDebug("idx %" PRIu32 ", cmd %s, name %s",
@@ -360,6 +360,7 @@ static int FlowBitsTestSig01(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Noalert\"; flowbits:noalert,wrongusage; content:\"GET \"; sid:1;)");
     FAIL_IF_NOT_NULL(s);
 
+    SigGroupBuild(de_ctx);
     DetectEngineCtxFree(de_ctx);
     PASS;
 }
@@ -399,6 +400,7 @@ static int FlowBitsTestSig02(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"toggle rule need an option\"; flowbits:toggle; content:\"GET \"; sid:5;)");
     FAIL_IF_NOT_NULL(s);
 
+    SigGroupBuild(de_ctx);
     DetectEngineCtxFree(de_ctx);
 
     PASS;
@@ -424,6 +426,7 @@ static int FlowBitsTestSig03(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Unknown cmd\"; flowbits:wrongcmd; content:\"GET \"; sid:1;)");
     FAIL_IF_NOT_NULL(s);
 
+    SigGroupBuild(de_ctx);
     DetectEngineCtxFree(de_ctx);
     PASS;
 }
@@ -449,9 +452,10 @@ static int FlowBitsTestSig04(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"isset option\"; flowbits:isset,fbt; content:\"GET \"; sid:1;)");
     FAIL_IF_NULL(s);
 
-    idx = VariableNameGetIdx(de_ctx, "fbt", VAR_TYPE_FLOW_BIT);
+    idx = VarNameStoreSetupAdd("fbt", VAR_TYPE_FLOW_BIT);
     FAIL_IF(idx != 1);
 
+    SigGroupBuild(de_ctx);
     DetectEngineCtxFree(de_ctx);
     PASS;
 }
@@ -477,6 +481,7 @@ static int FlowBitsTestSig05(void)
     FAIL_IF_NULL(s);
     FAIL_IF((s->flags & SIG_FLAG_NOALERT) != SIG_FLAG_NOALERT);
 
+    SigGroupBuild(de_ctx);
     DetectEngineCtxFree(de_ctx);
     PASS;
 }
@@ -504,7 +509,7 @@ static int FlowBitsTestSig06(void)
     Flow f;
     GenericVar flowvar, *gv = NULL;
     int result = 0;
-    int idx = 0;
+    uint32_t idx = 0;
 
     memset(p, 0, SIZE_OF_PACKET);
     memset(&th_v, 0, sizeof(th_v));
@@ -531,15 +536,14 @@ static int FlowBitsTestSig06(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit set\"; flowbits:set,myflow; sid:10;)");
     FAIL_IF_NULL(s);
 
+    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
 
-    idx = VariableNameGetIdx(de_ctx, "myflow", VAR_TYPE_FLOW_BIT);
-
     gv = p->flow->flowvar;
-
+    FAIL_IF_NULL(gv);
     for ( ; gv != NULL; gv = gv->next) {
         if (gv->type == DETECT_FLOWBITS && gv->idx == idx) {
                 result = 1;
@@ -547,13 +551,9 @@ static int FlowBitsTestSig06(void)
     }
     FAIL_IF_NOT(result);
 
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 
-    if(gv) GenericVarFree(gv);
     FLOW_DESTROY(&f);
 
     SCFree(p);
@@ -583,7 +583,7 @@ static int FlowBitsTestSig07(void)
     Flow f;
     GenericVar flowvar, *gv = NULL;
     int result = 0;
-    int idx = 0;
+    uint32_t idx = 0;
 
     memset(p, 0, SIZE_OF_PACKET);
     memset(&th_v, 0, sizeof(th_v));
@@ -611,14 +611,14 @@ static int FlowBitsTestSig07(void)
     s = s->next = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit unset\"; flowbits:unset,myflow2; sid:11;)");
     FAIL_IF_NULL(s);
 
+    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
 
-    idx = VariableNameGetIdx(de_ctx, "myflow", VAR_TYPE_FLOW_BIT);
-
     gv = p->flow->flowvar;
+    FAIL_IF_NULL(gv);
 
     for ( ; gv != NULL; gv = gv->next) {
         if (gv->type == DETECT_FLOWBITS && gv->idx == idx) {
@@ -627,13 +627,9 @@ static int FlowBitsTestSig07(void)
     }
     FAIL_IF(result);
 
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 
-    if(gv) GenericVarFree(gv);
     FLOW_DESTROY(&f);
 
     SCFree(p);
@@ -664,7 +660,7 @@ static int FlowBitsTestSig08(void)
     Flow f;
     GenericVar flowvar, *gv = NULL;
     int result = 0;
-    int idx = 0;
+    uint32_t idx = 0;
 
     memset(p, 0, SIZE_OF_PACKET);
     memset(&th_v, 0, sizeof(th_v));
@@ -692,14 +688,14 @@ static int FlowBitsTestSig08(void)
     s = s->next  = SigInit(de_ctx,"alert ip any any -> any any (msg:\"Flowbit unset\"; flowbits:toggle,myflow2; sid:11;)");
     FAIL_IF_NULL(s);
 
+    idx = VarNameStoreSetupAdd("myflow", VAR_TYPE_FLOW_BIT);
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
 
-    idx = VariableNameGetIdx(de_ctx, "myflow", VAR_TYPE_FLOW_BIT);
-
     gv = p->flow->flowvar;
+    FAIL_IF_NULL(gv);
 
     for ( ; gv != NULL; gv = gv->next) {
         if (gv->type == DETECT_FLOWBITS && gv->idx == idx) {
@@ -708,13 +704,9 @@ static int FlowBitsTestSig08(void)
     }
     FAIL_IF(result);
 
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 
-    if(gv) GenericVarFree(gv);
     FLOW_DESTROY(&f);
 
     SCFree(p);

diff --git a/src/detect-flowbits.h b/src/detect-flowbits.h
index 8961da3fd8980e498c9d8c8af394d1708e37aafb..7d50edb83542c342eb146d534a84220d2dce1407 100644 (file)
--- a/src/detect-flowbits.h
+++ b/src/detect-flowbits.h
@@ -34,7 +34,7 @@
 #define DETECT_FLOWBITS_CMD_MAX      6
 
 typedef struct DetectFlowbitsData_ {
-    uint16_t idx;
+    uint32_t idx;
     uint8_t cmd;
 } DetectFlowbitsData;
 

diff --git a/src/detect-flowint.c b/src/detect-flowint.c
index 6ceb7ca39e7f6757f2ad39e45d41df46902e8b03..76bcb1a5a7b9c971bfd55b544e9adad701b9d30c 100644 (file)
--- a/src/detect-flowint.c
+++ b/src/detect-flowint.c
@@ -104,7 +104,7 @@ int DetectFlowintMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx,
      * return zero(not match).
      */
     if (sfd->targettype == FLOWINT_TARGET_VAR) {
-        uint16_t tvar_idx = VariableNameGetIdx(det_ctx->de_ctx, sfd->target.tvar.name, VAR_TYPE_FLOW_INT);
+        uint32_t tvar_idx = VarNameStoreLookupByName(sfd->target.tvar.name, VAR_TYPE_FLOW_INT);
 
         fvt = FlowVarGet(p->flow, tvar_idx);
             /* We don't have that variable initialized yet */
@@ -326,8 +326,8 @@ DetectFlowintData *DetectFlowintParse(DetectEngineCtx *de_ctx, char *rawstr)
         SCLogError(SC_ERR_MEM_ALLOC, "malloc from strdup failed");
         goto error;
     }
-    if (de_ctx != NULL)
-        sfd->idx = VariableNameGetIdx(de_ctx, varname, VAR_TYPE_FLOW_INT);
+    sfd->idx = VarNameStoreSetupAdd(varname, VAR_TYPE_FLOW_INT);
+    SCLogDebug("sfd->name %s id %u", sfd->name, sfd->idx);
     sfd->modifier = modifier;
 
     pcre_free_substring(varname);

diff --git a/src/detect-flowint.h b/src/detect-flowint.h
index 9a18efd50d318ef32fca27c55b07c6c73ca4769f..9caf68258cc7f0a5fa69a1743adcc0beb6ab9df8 100644 (file)
--- a/src/detect-flowint.h
+++ b/src/detect-flowint.h
@@ -64,7 +64,7 @@ typedef struct DetectFlowintData_ {
     * against the target */
     char *name;
     /* Internal id of the var */
-    uint16_t idx;
+    uint32_t idx;
 
     /* The modifier/operation/condition we are
     * going to execute */

diff --git a/src/detect-flowvar.c b/src/detect-flowvar.c
index 46741016b6969c367338fcc2151897e7f9172edc..c5f838bb6c31242565ae56323283c45a92d1da7f 100644 (file)
--- a/src/detect-flowvar.c
+++ b/src/detect-flowvar.c
@@ -166,7 +166,7 @@ static int DetectFlowvarSetup (DetectEngineCtx *de_ctx, Signature *s, char *raws
     fd->name = SCStrdup(varname);
     if (unlikely(fd->name == NULL))
         goto error;
-    fd->idx = VariableNameGetIdx(de_ctx, varname, VAR_TYPE_FLOW_VAR);
+    fd->idx = VarNameStoreSetupAdd(varname, VAR_TYPE_FLOW_VAR);
 
     /* Okay so far so good, lets get this into a SigMatch
      * and put it in the Signature. */
@@ -194,7 +194,7 @@ error:
 
 
 /** \brief Store flowvar in det_ctx so we can exec it post-match */
-int DetectFlowvarStoreMatch(DetectEngineThreadCtx *det_ctx, uint16_t idx,
+int DetectFlowvarStoreMatch(DetectEngineThreadCtx *det_ctx, uint32_t idx,
         uint8_t *buffer, uint16_t len, int type)
 {
     DetectFlowvarList *fs = det_ctx->flowvarlist;
@@ -229,7 +229,7 @@ int DetectFlowvarStoreMatch(DetectEngineThreadCtx *det_ctx, uint16_t idx,
 /** \brief Setup a post-match for flowvar storage
  *  We're piggyback riding the DetectFlowvarData struct
  */
-int DetectFlowvarPostMatchSetup(Signature *s, uint16_t idx)
+int DetectFlowvarPostMatchSetup(Signature *s, uint32_t idx)
 {
     SigMatch *sm = NULL;
     DetectFlowvarData *fv = NULL;

diff --git a/src/detect-flowvar.h b/src/detect-flowvar.h
index c8081385b87a0182dac3fbff219da31f08b45f40..32222372b4bf9f075f07545c625be745acc0dbc7 100644 (file)
--- a/src/detect-flowvar.h
+++ b/src/detect-flowvar.h
@@ -26,7 +26,7 @@
 
 typedef struct DetectFlowvarData_ {
     char *name;
-    uint16_t idx;
+    uint32_t idx;
     uint8_t *content;
     uint8_t content_len;
     uint8_t flags;
@@ -35,8 +35,8 @@ typedef struct DetectFlowvarData_ {
 /* prototypes */
 void DetectFlowvarRegister (void);
 
-int DetectFlowvarPostMatchSetup(Signature *s, uint16_t idx);
-int DetectFlowvarStoreMatch(DetectEngineThreadCtx *, uint16_t, uint8_t *, uint16_t, int);
+int DetectFlowvarPostMatchSetup(Signature *s, uint32_t idx);
+int DetectFlowvarStoreMatch(DetectEngineThreadCtx *, uint32_t, uint8_t *, uint16_t, int);
 
 /* For use only by DetectFlowvarProcessList() */
 void DetectFlowvarProcessListInternal(DetectFlowvarList *fs, Flow *f);

diff --git a/src/detect-hostbits.c b/src/detect-hostbits.c
index e2fa800c6f68f192d13dfb0dbf260d358e125ebb..bf1b728824d14ae1951849bda1d4ab161512aa95 100644 (file)
--- a/src/detect-hostbits.c
+++ b/src/detect-hostbits.c
@@ -385,7 +385,7 @@ int DetectHostbitSetup (DetectEngineCtx *de_ctx, Signature *s, char *rawstr)
     if (unlikely(cd == NULL))
         goto error;
 
-    cd->idx = VariableNameGetIdx(de_ctx, fb_name, VAR_TYPE_HOST_BIT);
+    cd->idx = VarNameStoreSetupAdd(fb_name, VAR_TYPE_HOST_BIT);
     cd->cmd = fb_cmd;
     cd->tracker = hb_dir;
     cd->type = VAR_TYPE_HOST_BIT;
@@ -752,7 +752,7 @@ static int HostBitsTestSig04(void)
     s = de_ctx->sig_list = SigInit(de_ctx,"alert ip any any -> any any (msg:\"isset option\"; hostbits:isset,fbt; content:\"GET \"; sid:1;)");
     FAIL_IF_NULL(s);
 
-    idx = VariableNameGetIdx(de_ctx, "fbt", VAR_TYPE_HOST_BIT);
+    idx = VarNameStoreSetupAdd("fbt", VAR_TYPE_HOST_BIT);
     FAIL_IF(idx != 1);
 
     SigGroupBuild(de_ctx);
@@ -760,9 +760,6 @@ static int HostBitsTestSig04(void)
 
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
 
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
     HostBitsTestShutdown();

diff --git a/src/detect-lua.c b/src/detect-lua.c
index 87d3e55f9b9a66d43873aa02770e225fe37cbb4a..e08490c11df3e0fb7d8950c8b1ff658d7c399ce2 100644 (file)
--- a/src/detect-lua.c
+++ b/src/detect-lua.c
@@ -776,7 +776,7 @@ static int DetectLuaSetupPrime(DetectEngineCtx *de_ctx, DetectLuaData *ld)
                         goto error;
                     }
 
-                    uint16_t idx = VariableNameGetIdx(de_ctx, (char *)value, VAR_TYPE_FLOW_VAR);
+                    uint32_t idx = VarNameStoreSetupAdd((char *)value, VAR_TYPE_FLOW_VAR);
                     ld->flowvar[ld->flowvars++] = idx;
                     SCLogDebug("script uses flowvar %u with script id %u", idx, ld->flowvars - 1);
                 }
@@ -798,7 +798,7 @@ static int DetectLuaSetupPrime(DetectEngineCtx *de_ctx, DetectLuaData *ld)
                         goto error;
                     }
 
-                    uint16_t idx = VariableNameGetIdx(de_ctx, (char *)value, VAR_TYPE_FLOW_INT);
+                    uint32_t idx = VarNameStoreSetupAdd((char *)value, VAR_TYPE_FLOW_INT);
                     ld->flowint[ld->flowints++] = idx;
                     SCLogDebug("script uses flowint %u with script id %u", idx, ld->flowints - 1);
                 }

diff --git a/src/detect-lua.h b/src/detect-lua.h
index 16e451e9109099534c4b8cb67c4942dcafeb4cef..224d6225f305be2fcce88a6867f6cf7918852f5b 100644 (file)
--- a/src/detect-lua.h
+++ b/src/detect-lua.h
@@ -42,10 +42,10 @@ typedef struct DetectLuaData {
     uint32_t flags;
     AppProto alproto;
     char *buffername; /* buffer name in case of a single buffer */
-    uint16_t flowint[DETECT_LUAJIT_MAX_FLOWINTS];
+    uint32_t flowint[DETECT_LUAJIT_MAX_FLOWINTS];
     uint16_t flowints;
-    uint16_t flowvar[DETECT_LUAJIT_MAX_FLOWVARS];
     uint16_t flowvars;
+    uint32_t flowvar[DETECT_LUAJIT_MAX_FLOWVARS];
     uint32_t sid;
     uint32_t rev;
     uint32_t gid;

diff --git a/src/detect-pcre.c b/src/detect-pcre.c
index cea56672ba7167b2f6e89405ab2d98ba90321cfb..3537538842042dc345fd4dbe3646eef90edeabb2 100644 (file)
--- a/src/detect-pcre.c
+++ b/src/detect-pcre.c
@@ -657,9 +657,9 @@ static int DetectPcreParseCapture(char *regexstr, DetectEngineCtx *de_ctx, Detec
     }
     if (pd->capname != NULL) {
         if (pd->flags & DETECT_PCRE_CAPTURE_PKT)
-            pd->capidx = VariableNameGetIdx(de_ctx, (char *)pd->capname, VAR_TYPE_PKT_VAR);
+            pd->capidx = VarNameStoreSetupAdd((char *)pd->capname, VAR_TYPE_PKT_VAR);
         else if (pd->flags & DETECT_PCRE_CAPTURE_FLOW)
-            pd->capidx = VariableNameGetIdx(de_ctx, (char *)pd->capname, VAR_TYPE_FLOW_VAR);
+            pd->capidx = VarNameStoreSetupAdd((char *)pd->capname, VAR_TYPE_FLOW_VAR);
     }
 
     SCLogDebug("pd->capname %s", pd->capname);

diff --git a/src/detect-pcre.h b/src/detect-pcre.h
index 50b93b052833105e68340fc110d50e2a25f6d7c3..3f9ce7557f8cce2610f65e41af4c89784f0ca335 100644 (file)
--- a/src/detect-pcre.h
+++ b/src/detect-pcre.h
@@ -39,7 +39,7 @@ typedef struct DetectPcreData_ {
     pcre_extra *sd;
     int opts;
     uint16_t flags;
-    uint16_t capidx;
+    uint32_t capidx;
     char *capname;
 } DetectPcreData;
 

diff --git a/src/detect-xbits.c b/src/detect-xbits.c
index 664f9451a3d15f669f0f6958659599a3673f81f6..a6264ea26709f1c983ab4608610f81a849bac564 100644 (file)
--- a/src/detect-xbits.c
+++ b/src/detect-xbits.c
@@ -304,7 +304,7 @@ static int DetectXbitParse(DetectEngineCtx *de_ctx,
     if (unlikely(cd == NULL))
         return -1;
 
-    cd->idx = VariableNameGetIdx(de_ctx, fb_name, var_type);
+    cd->idx = VarNameStoreSetupAdd(fb_name, var_type);
     cd->cmd = fb_cmd;
     cd->tracker = hb_dir;
     cd->type = var_type;

diff --git a/src/detect-xbits.h b/src/detect-xbits.h
index 6794af2d9c56f6e1aa40bdc3d6ab04969fa8e1ee..4f34ef0dfe1e165f7fee6df4d4568200292592d1 100644 (file)
--- a/src/detect-xbits.h
+++ b/src/detect-xbits.h
@@ -40,7 +40,7 @@
 #define DETECT_XBITS_EXPIRE_DEFAULT 30
 
 typedef struct DetectXbitsData_ {
-    uint16_t idx;
+    uint32_t idx;
     uint8_t cmd;
     uint8_t tracker;
     uint32_t expire;

diff --git a/src/detect.c b/src/detect.c
index 177daeeae3e79c560f50f4d4aa806daeb69894cb..bbfa9ab94b1c1737f35a3e4f3cb1021f9d0c8a9b 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -890,7 +890,7 @@ static void AlertDebugLogModeSyncFlowbitsNamesToPacketStruct(Packet *p, DetectEn
         }
 
         FlowBit *fb = (FlowBit *) gv;
-        const char *name = VariableIdxGetName(de_ctx, fb->idx, VAR_TYPE_FLOW_BIT);
+        const char *name = VarNameStoreLookupById(fb->idx, VAR_TYPE_FLOW_BIT);
         if (name != NULL) {
             p->debuglog_flowbits_names[i] = SCStrdup(name);
             if (p->debuglog_flowbits_names[i] == NULL) {
@@ -3858,6 +3858,10 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
 #endif
     SCFree(de_ctx->app_mpms);
     de_ctx->app_mpms = NULL;
+
+    if (!DetectEngineMultiTenantEnabled()) {
+        VarNameStoreActivateStaging();
+    }
     return 0;
 }
 

diff --git a/src/detect.h b/src/detect.h
index 88c31ddd3cdcb9d82b36227a6cd41cc9059dff8c..fd609f056b77a698a8f1342b605fa6de3c7b8de3 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -47,6 +47,8 @@
 
 #include "stream.h"
 
+#include "util-var-name.h"
+
 #define DETECT_MAX_RULE_SIZE 8192
 
 /* forward declarations for the structures from detect-engine-sigorder.h */
@@ -495,7 +497,7 @@ typedef struct DetectReplaceList_ {
 /** list for flowvar store candidates, to be stored from
  *  post-match function */
 typedef struct DetectFlowvarList_ {
-    uint16_t idx;                       /**< flowvar name idx */
+    uint32_t idx;                       /**< flowvar name idx */
     uint16_t len;                       /**< data len */
     int type;                           /**< type of store candidate POSTMATCH or ALWAYS */
     uint8_t *buffer;                    /**< alloc'd buffer, may be freed by
@@ -618,10 +620,6 @@ typedef struct DetectEngineCtx_ {
 
     HashListTable *mpm_hash_table;
 
-    HashListTable *variable_names;
-    HashListTable *variable_idxs;
-    uint16_t variable_names_idx;
-
     /* hash table used to cull out duplicate sigs */
     HashListTable *dup_sig_hash_table;
 

diff --git a/src/flow-bit.c b/src/flow-bit.c
index 533bef5cc9df9454c0b4b97334e12aeb0657f736..3662611ec8cf5a67044686623319412624d6f55d 100644 (file)
--- a/src/flow-bit.c
+++ b/src/flow-bit.c
@@ -43,7 +43,7 @@
 #include "util-unittest.h"
 
 /* get the flowbit with idx from the flow */
-static FlowBit *FlowBitGet(Flow *f, uint16_t idx)
+static FlowBit *FlowBitGet(Flow *f, uint32_t idx)
 {
     GenericVar *gv = f->flowvar;
     for ( ; gv != NULL; gv = gv->next) {
@@ -56,7 +56,7 @@ static FlowBit *FlowBitGet(Flow *f, uint16_t idx)
 }
 
 /* add a flowbit to the flow */
-static void FlowBitAdd(Flow *f, uint16_t idx)
+static void FlowBitAdd(Flow *f, uint32_t idx)
 {
     FlowBit *fb = FlowBitGet(f, idx);
     if (fb == NULL) {
@@ -71,7 +71,7 @@ static void FlowBitAdd(Flow *f, uint16_t idx)
     }
 }
 
-static void FlowBitRemove(Flow *f, uint16_t idx)
+static void FlowBitRemove(Flow *f, uint32_t idx)
 {
     FlowBit *fb = FlowBitGet(f, idx);
     if (fb == NULL)
@@ -81,17 +81,17 @@ static void FlowBitRemove(Flow *f, uint16_t idx)
     FlowBitFree(fb);
 }
 
-void FlowBitSet(Flow *f, uint16_t idx)
+void FlowBitSet(Flow *f, uint32_t idx)
 {
     FlowBitAdd(f, idx);
 }
 
-void FlowBitUnset(Flow *f, uint16_t idx)
+void FlowBitUnset(Flow *f, uint32_t idx)
 {
     FlowBitRemove(f, idx);
 }
 
-void FlowBitToggle(Flow *f, uint16_t idx)
+void FlowBitToggle(Flow *f, uint32_t idx)
 {
     FlowBit *fb = FlowBitGet(f, idx);
     if (fb != NULL) {
@@ -101,7 +101,7 @@ void FlowBitToggle(Flow *f, uint16_t idx)
     }
 }
 
-int FlowBitIsset(Flow *f, uint16_t idx)
+int FlowBitIsset(Flow *f, uint32_t idx)
 {
     int r = 0;
 
@@ -113,7 +113,7 @@ int FlowBitIsset(Flow *f, uint16_t idx)
     return r;
 }
 
-int FlowBitIsnotset(Flow *f, uint16_t idx)
+int FlowBitIsnotset(Flow *f, uint32_t idx)
 {
     int r = 0;
 

diff --git a/src/flow-bit.h b/src/flow-bit.h
index 633b5ee3d9a9b160c2ef41cc95e94fa42465909e..e6161606fc42bbc9e3f1ab007e18b046188b5cb4 100644 (file)
--- a/src/flow-bit.h
+++ b/src/flow-bit.h
@@ -29,7 +29,7 @@
 
 typedef struct FlowBit_ {
     uint8_t type; /* type, DETECT_FLOWBITS in this case */
-    uint16_t idx; /* name idx */
+    uint32_t idx; /* name idx */
     GenericVar *next; /* right now just implement this as a list,
                        * in the long run we have think of something
                        * faster. */
@@ -38,10 +38,10 @@ typedef struct FlowBit_ {
 void FlowBitFree(FlowBit *);
 void FlowBitRegisterTests(void);
 
-void FlowBitSet(Flow *, uint16_t);
-void FlowBitUnset(Flow *, uint16_t);
-void FlowBitToggle(Flow *, uint16_t);
-int FlowBitIsset(Flow *, uint16_t);
-int FlowBitIsnotset(Flow *, uint16_t);
+void FlowBitSet(Flow *, uint32_t);
+void FlowBitUnset(Flow *, uint32_t);
+void FlowBitToggle(Flow *, uint32_t);
+int FlowBitIsset(Flow *, uint32_t);
+int FlowBitIsnotset(Flow *, uint32_t);
 #endif /* __FLOW_BIT_H__ */
 

diff --git a/src/flow-var.c b/src/flow-var.c
index 96b3ad3a84ef3b29c0b394eea482f9ea2ca94942..d929d55874451a487352329acc576ff4a47eaece 100644 (file)
--- a/src/flow-var.c
+++ b/src/flow-var.c
@@ -51,7 +51,7 @@ static void FlowVarUpdateInt(FlowVar *fv, uint32_t value)
  *  \note flow is not locked by this function, caller is
  *        responsible
  */
-FlowVar *FlowVarGet(Flow *f, uint16_t idx)
+FlowVar *FlowVarGet(Flow *f, uint32_t idx)
 {
     GenericVar *gv = f->flowvar;
 
@@ -64,7 +64,7 @@ FlowVar *FlowVarGet(Flow *f, uint16_t idx)
 }
 
 /* add a flowvar to the flow, or update it */
-void FlowVarAddStrNoLock(Flow *f, uint16_t idx, uint8_t *value, uint16_t size)
+void FlowVarAddStrNoLock(Flow *f, uint32_t idx, uint8_t *value, uint16_t size)
 {
     FlowVar *fv = FlowVarGet(f, idx);
     if (fv == NULL) {
@@ -86,13 +86,13 @@ void FlowVarAddStrNoLock(Flow *f, uint16_t idx, uint8_t *value, uint16_t size)
 }
 
 /* add a flowvar to the flow, or update it */
-void FlowVarAddStr(Flow *f, uint16_t idx, uint8_t *value, uint16_t size)
+void FlowVarAddStr(Flow *f, uint32_t idx, uint8_t *value, uint16_t size)
 {
     FlowVarAddStrNoLock(f, idx, value, size);
 }
 
 /* add a flowvar to the flow, or update it */
-void FlowVarAddIntNoLock(Flow *f, uint16_t idx, uint32_t value)
+void FlowVarAddIntNoLock(Flow *f, uint32_t idx, uint32_t value)
 {
     FlowVar *fv = FlowVarGet(f, idx);
     if (fv == NULL) {
@@ -113,7 +113,7 @@ void FlowVarAddIntNoLock(Flow *f, uint16_t idx, uint32_t value)
 }
 
 /* add a flowvar to the flow, or update it */
-void FlowVarAddInt(Flow *f, uint16_t idx, uint32_t value)
+void FlowVarAddInt(Flow *f, uint32_t idx, uint32_t value)
 {
     FlowVarAddIntNoLock(f, idx, value);
 }
@@ -144,7 +144,7 @@ void FlowVarPrint(GenericVar *gv)
         FlowVar *fv = (FlowVar *)gv;
 
         if (fv->datatype == FLOWVAR_TYPE_STR) {
-            SCLogDebug("Name idx \"%" PRIu16 "\", Value \"", fv->idx);
+            SCLogDebug("Name idx \"%" PRIu32 "\", Value \"", fv->idx);
             for (u = 0; u < fv->data.fv_str.value_len; u++) {
                 if (isprint(fv->data.fv_str.value[u]))
                     SCLogDebug("%c", fv->data.fv_str.value[u]);
@@ -153,7 +153,7 @@ void FlowVarPrint(GenericVar *gv)
             }
             SCLogDebug("\", Len \"%" PRIu16 "\"\n", fv->data.fv_str.value_len);
         } else if (fv->datatype == FLOWVAR_TYPE_INT) {
-            SCLogDebug("Name idx \"%" PRIu16 "\", Value \"%" PRIu32 "\"", fv->idx,
+            SCLogDebug("Name idx \"%" PRIu32 "\", Value \"%" PRIu32 "\"", fv->idx,
                     fv->data.fv_int.value);
         } else {
             SCLogDebug("Unknown data type at flowvars\n");

diff --git a/src/flow-var.h b/src/flow-var.h
index e45d803043a7ac57ab4df12753f0d69c66cf5a11..0f7eeeb9266f655778704cdcfec6325077f0bc9d 100644 (file)
--- a/src/flow-var.h
+++ b/src/flow-var.h
@@ -47,7 +47,7 @@ typedef struct FlowVarTypeInt_ {
 /** Generic Flowvar Structure */
 typedef struct FlowVar_ {
     uint8_t type;       /* type, DETECT_FLOWVAR in this case */
-    uint16_t idx;       /* name idx */
+    uint32_t idx;       /* name idx */
     GenericVar *next;   /* right now just implement this as a list,
                          * in the long run we have think of something
                          * faster. */
@@ -61,11 +61,11 @@ typedef struct FlowVar_ {
 
 /** Flowvar Interface API */
 
-void FlowVarAddStrNoLock(Flow *, uint16_t, uint8_t *, uint16_t);
-void FlowVarAddStr(Flow *, uint16_t, uint8_t *, uint16_t);
-void FlowVarAddIntNoLock(Flow *, uint16_t, uint32_t);
-void FlowVarAddInt(Flow *, uint16_t, uint32_t);
-FlowVar *FlowVarGet(Flow *, uint16_t);
+void FlowVarAddStrNoLock(Flow *, uint32_t, uint8_t *, uint16_t);
+void FlowVarAddStr(Flow *, uint32_t, uint8_t *, uint16_t);
+void FlowVarAddIntNoLock(Flow *, uint32_t, uint32_t);
+void FlowVarAddInt(Flow *, uint32_t, uint32_t);
+FlowVar *FlowVarGet(Flow *, uint32_t);
 void FlowVarFree(FlowVar *);
 void FlowVarPrint(GenericVar *);
 

diff --git a/src/host-bit.c b/src/host-bit.c
index 9c4083b02eda1e5d9fce21d8a1602d7217119963..757654f7be729842617b6bcca98ad0df5a355cbd 100644 (file)
--- a/src/host-bit.c
+++ b/src/host-bit.c
@@ -78,7 +78,7 @@ int HostBitsTimedoutCheck(Host *h, struct timeval *ts)
 }
 
 /* get the bit with idx from the host */
-static XBit *HostBitGet(Host *h, uint16_t idx)
+static XBit *HostBitGet(Host *h, uint32_t idx)
 {
     GenericVar *gv = HostGetStorageById(h, host_bit_id);
     for ( ; gv != NULL; gv = gv->next) {
@@ -91,7 +91,7 @@ static XBit *HostBitGet(Host *h, uint16_t idx)
 }
 
 /* add a flowbit to the flow */
-static void HostBitAdd(Host *h, uint16_t idx, uint32_t expire)
+static void HostBitAdd(Host *h, uint32_t idx, uint32_t expire)
 {
     XBit *fb = HostBitGet(h, idx);
     if (fb == NULL) {
@@ -114,7 +114,7 @@ static void HostBitAdd(Host *h, uint16_t idx, uint32_t expire)
     }
 }
 
-static void HostBitRemove(Host *h, uint16_t idx)
+static void HostBitRemove(Host *h, uint32_t idx)
 {
     XBit *fb = HostBitGet(h, idx);
     if (fb == NULL)
@@ -128,7 +128,7 @@ static void HostBitRemove(Host *h, uint16_t idx)
     }
 }
 
-void HostBitSet(Host *h, uint16_t idx, uint32_t expire)
+void HostBitSet(Host *h, uint32_t idx, uint32_t expire)
 {
     XBit *fb = HostBitGet(h, idx);
     if (fb == NULL) {
@@ -136,7 +136,7 @@ void HostBitSet(Host *h, uint16_t idx, uint32_t expire)
     }
 }
 
-void HostBitUnset(Host *h, uint16_t idx)
+void HostBitUnset(Host *h, uint32_t idx)
 {
     XBit *fb = HostBitGet(h, idx);
     if (fb != NULL) {
@@ -144,7 +144,7 @@ void HostBitUnset(Host *h, uint16_t idx)
     }
 }
 
-void HostBitToggle(Host *h, uint16_t idx, uint32_t expire)
+void HostBitToggle(Host *h, uint32_t idx, uint32_t expire)
 {
     XBit *fb = HostBitGet(h, idx);
     if (fb != NULL) {
@@ -154,7 +154,7 @@ void HostBitToggle(Host *h, uint16_t idx, uint32_t expire)
     }
 }
 
-int HostBitIsset(Host *h, uint16_t idx, uint32_t ts)
+int HostBitIsset(Host *h, uint32_t idx, uint32_t ts)
 {
     XBit *fb = HostBitGet(h, idx);
     if (fb != NULL) {
@@ -167,7 +167,7 @@ int HostBitIsset(Host *h, uint16_t idx, uint32_t ts)
     return 0;
 }
 
-int HostBitIsnotset(Host *h, uint16_t idx, uint32_t ts)
+int HostBitIsnotset(Host *h, uint32_t idx, uint32_t ts)
 {
     XBit *fb = HostBitGet(h, idx);
     if (fb == NULL) {

diff --git a/src/host-bit.h b/src/host-bit.h
index d4bba11abf5bc96630f359e9a1084dedc826cd49..f125ed09cb51901b59b17a6acc2e974828e64b2e 100644 (file)
--- a/src/host-bit.h
+++ b/src/host-bit.h
@@ -33,9 +33,9 @@ void HostBitRegisterTests(void);
 int HostHasHostBits(Host *host);
 int HostBitsTimedoutCheck(Host *h, struct timeval *ts);
 
-void HostBitSet(Host *, uint16_t, uint32_t);
-void HostBitUnset(Host *, uint16_t);
-void HostBitToggle(Host *, uint16_t, uint32_t);
-int HostBitIsset(Host *, uint16_t, uint32_t);
-int HostBitIsnotset(Host *, uint16_t, uint32_t);
+void HostBitSet(Host *, uint32_t, uint32_t);
+void HostBitUnset(Host *, uint32_t);
+void HostBitToggle(Host *, uint32_t, uint32_t);
+int HostBitIsset(Host *, uint32_t, uint32_t);
+int HostBitIsnotset(Host *, uint32_t, uint32_t);
 #endif /* __HOST_BIT_H__ */

diff --git a/src/ippair-bit.c b/src/ippair-bit.c
index b69fec2fd8d7825074be7aa6e41ee943439a8d39..c617371630d55d2ac99a11c69fabf0ba803302e9 100644 (file)
--- a/src/ippair-bit.c
+++ b/src/ippair-bit.c
@@ -78,7 +78,7 @@ int IPPairBitsTimedoutCheck(IPPair *h, struct timeval *ts)
 }
 
 /* get the bit with idx from the ippair */
-static XBit *IPPairBitGet(IPPair *h, uint16_t idx)
+static XBit *IPPairBitGet(IPPair *h, uint32_t idx)
 {
     GenericVar *gv = IPPairGetStorageById(h, ippair_bit_id);
     for ( ; gv != NULL; gv = gv->next) {
@@ -91,7 +91,7 @@ static XBit *IPPairBitGet(IPPair *h, uint16_t idx)
 }
 
 /* add a flowbit to the flow */
-static void IPPairBitAdd(IPPair *h, uint16_t idx, uint32_t expire)
+static void IPPairBitAdd(IPPair *h, uint32_t idx, uint32_t expire)
 {
     XBit *fb = IPPairBitGet(h, idx);
     if (fb == NULL) {
@@ -114,7 +114,7 @@ static void IPPairBitAdd(IPPair *h, uint16_t idx, uint32_t expire)
     }
 }
 
-static void IPPairBitRemove(IPPair *h, uint16_t idx)
+static void IPPairBitRemove(IPPair *h, uint32_t idx)
 {
     XBit *fb = IPPairBitGet(h, idx);
     if (fb == NULL)
@@ -127,7 +127,7 @@ static void IPPairBitRemove(IPPair *h, uint16_t idx)
     }
 }
 
-void IPPairBitSet(IPPair *h, uint16_t idx, uint32_t expire)
+void IPPairBitSet(IPPair *h, uint32_t idx, uint32_t expire)
 {
     XBit *fb = IPPairBitGet(h, idx);
     if (fb == NULL) {
@@ -135,7 +135,7 @@ void IPPairBitSet(IPPair *h, uint16_t idx, uint32_t expire)
     }
 }
 
-void IPPairBitUnset(IPPair *h, uint16_t idx)
+void IPPairBitUnset(IPPair *h, uint32_t idx)
 {
     XBit *fb = IPPairBitGet(h, idx);
     if (fb != NULL) {
@@ -143,7 +143,7 @@ void IPPairBitUnset(IPPair *h, uint16_t idx)
     }
 }
 
-void IPPairBitToggle(IPPair *h, uint16_t idx, uint32_t expire)
+void IPPairBitToggle(IPPair *h, uint32_t idx, uint32_t expire)
 {
     XBit *fb = IPPairBitGet(h, idx);
     if (fb != NULL) {
@@ -153,7 +153,7 @@ void IPPairBitToggle(IPPair *h, uint16_t idx, uint32_t expire)
     }
 }
 
-int IPPairBitIsset(IPPair *h, uint16_t idx, uint32_t ts)
+int IPPairBitIsset(IPPair *h, uint32_t idx, uint32_t ts)
 {
     XBit *fb = IPPairBitGet(h, idx);
     if (fb != NULL) {
@@ -167,7 +167,7 @@ int IPPairBitIsset(IPPair *h, uint16_t idx, uint32_t ts)
     return 0;
 }
 
-int IPPairBitIsnotset(IPPair *h, uint16_t idx, uint32_t ts)
+int IPPairBitIsnotset(IPPair *h, uint32_t idx, uint32_t ts)
 {
     XBit *fb = IPPairBitGet(h, idx);
     if (fb == NULL) {

diff --git a/src/ippair-bit.h b/src/ippair-bit.h
index 44a0ac469c7e7205bad2759479756bb50e897e2b..cbc7cb739be82d94a8036d7983ab0877c7dfeed2 100644 (file)
--- a/src/ippair-bit.h
+++ b/src/ippair-bit.h
@@ -33,10 +33,10 @@ void IPPairBitRegisterTests(void);
 int IPPairHasBits(IPPair *host);
 int IPPairBitsTimedoutCheck(IPPair *h, struct timeval *ts);
 
-void IPPairBitSet(IPPair *, uint16_t, uint32_t);
-void IPPairBitUnset(IPPair *, uint16_t);
-void IPPairBitToggle(IPPair *, uint16_t, uint32_t);
-int IPPairBitIsset(IPPair *, uint16_t, uint32_t);
-int IPPairBitIsnotset(IPPair *, uint16_t, uint32_t);
+void IPPairBitSet(IPPair *, uint32_t, uint32_t);
+void IPPairBitUnset(IPPair *, uint32_t);
+void IPPairBitToggle(IPPair *, uint32_t, uint32_t);
+int IPPairBitIsset(IPPair *, uint32_t, uint32_t);
+int IPPairBitIsnotset(IPPair *, uint32_t, uint32_t);
 
 #endif /* __IPPAIR_BIT_H__ */

diff --git a/src/util-var-name.c b/src/util-var-name.c
index 73f87d3dadedfc0acc62a26dca99b5728c7348f9..276b8723dc2e9c1fd23b6d6fe73dacdebfaf7804 100644 (file)
--- a/src/util-var-name.c
+++ b/src/util-var-name.c
@@ -26,12 +26,43 @@
 #include "suricata-common.h"
 #include "detect.h"
 #include "util-hashlist.h"
+#include "util-var-name.h"
+
+/* the way this can be used w/o locking lookups:
+ * - Lookups use only g_varnamestore_current which is read only
+ * - Detection setups a new ctx in staging, which will include the 'current'
+ *   entries keeping ID's stable
+ * - Detection hot swaps staging into current after a new detect engine was
+ *   created. Current remains available through 'old'.
+ * - When detect reload is complete (threads are all moved over), 'old' can
+ *   be freed.
+ */
+
+typedef struct VarNameStore_ {
+    HashListTable *names;
+    HashListTable *ids;
+    uint32_t max_id;
+    uint32_t de_ctx_version;    /**< de_ctx version 'owning' this */
+} VarNameStore;
+
+static int initialized = 0;
+/* currently VarNameStore that is READ ONLY. This way lookups can
+ * be done w/o locking or synchronization */
+SC_ATOMIC_DECLARE(VarNameStore *, g_varnamestore_current);
+
+/* old VarNameStore on the way out */
+static VarNameStore *g_varnamestore_old = NULL;
+
+/* new VarNameStore that is being prepared. Multiple DetectLoader threads
+ * may be updating it so a lock is used for synchronization. */
+static VarNameStore *g_varnamestore_staging = NULL;
+static SCMutex g_varnamestore_staging_m = SCMUTEX_INITIALIZER;
 
 /** \brief Name2idx mapping structure for flowbits, flowvars and pktvars. */
 typedef struct VariableName_ {
     char *name;
     uint8_t type; /* flowbit, pktvar, etc */
-    uint16_t idx;
+    uint32_t idx;
 } VariableName;
 
 static uint32_t VariableNameHash(HashListTable *ht, void *buf, uint16_t buflen)
@@ -98,45 +129,49 @@ static void VariableNameFree(void *data)
 }
 
 /** \brief Initialize the Name idx hash.
- *  \param de_ctx Ptr to the detection engine ctx.
- *  \retval -1 in case of error
- *  \retval 0 in case of success
  */
-int VariableNameInitHash(DetectEngineCtx *de_ctx)
+static VarNameStore *VarNameStoreInit(void)
 {
-    de_ctx->variable_names = HashListTableInit(4096, VariableNameHash, VariableNameCompare, VariableNameFree);
-    if (de_ctx->variable_names == NULL)
-        return -1;
+    VarNameStore *v = SCCalloc(1, sizeof(*v));
+    if (v == NULL)
+        return NULL;
+
+    v->names = HashListTableInit(4096, VariableNameHash, VariableNameCompare, VariableNameFree);
+    if (v->names == NULL) {
+        SCFree(v);
+        return NULL;
+    }
 
-    de_ctx->variable_idxs = HashListTableInit(4096, VariableIdxHash, VariableIdxCompare, NULL);
-    if (de_ctx->variable_idxs == NULL)
-        return -1;
+    v->ids = HashListTableInit(4096, VariableIdxHash, VariableIdxCompare, NULL);
+    if (v->ids == NULL) {
+        HashListTableFree(v->names);
+        SCFree(v);
+        return NULL;
+    }
 
-    de_ctx->variable_names_idx = 0;
-    return 0;
+    v->max_id = 0;
+    return v;
 }
 
-void VariableNameFreeHash(DetectEngineCtx *de_ctx)
+static void VarNameStoreDoFree(VarNameStore *v)
 {
-    if (de_ctx->variable_names != NULL) {
-        HashListTableFree(de_ctx->variable_names);
-        HashListTableFree(de_ctx->variable_idxs);
-        de_ctx->variable_names = NULL;
-        de_ctx->variable_idxs = NULL;
+    if (v) {
+        HashListTableFree(v->names);
+        HashListTableFree(v->ids);
+        SCFree(v);
     }
-
-    return;
 }
 
+
 /** \brief Get a name idx for a name. If the name is already used reuse the idx.
  *  \param name nul terminated string with the name
  *  \param type variable type
  *  \retval 0 in case of error
  *  \retval idx the idx or 0
  */
-uint16_t VariableNameGetIdx(DetectEngineCtx *de_ctx, const char *name, enum VarTypes type)
+static uint32_t VariableNameGetIdx(VarNameStore *v, const char *name, enum VarTypes type)
 {
-    uint16_t idx = 0;
+    uint32_t idx = 0;
 
     VariableName *fn = SCMalloc(sizeof(VariableName));
     if (unlikely(fn == NULL))
@@ -149,13 +184,14 @@ uint16_t VariableNameGetIdx(DetectEngineCtx *de_ctx, const char *name, enum VarT
     if (fn->name == NULL)
         goto error;
 
-    VariableName *lookup_fn = (VariableName *)HashListTableLookup(de_ctx->variable_names, (void *)fn, 0);
+    VariableName *lookup_fn = (VariableName *)HashListTableLookup(v->names, (void *)fn, 0);
     if (lookup_fn == NULL) {
-        de_ctx->variable_names_idx++;
+        v->max_id++;
 
-        idx = fn->idx = de_ctx->variable_names_idx;
-        HashListTableAdd(de_ctx->variable_names, (void *)fn, 0);
-        HashListTableAdd(de_ctx->variable_idxs, (void *)fn, 0);
+        idx = fn->idx = v->max_id;
+        HashListTableAdd(v->names, (void *)fn, 0);
+        HashListTableAdd(v->ids, (void *)fn, 0);
+        SCLogDebug("new registration %s id %u type %u", fn->name, fn->idx, fn->type);
     } else {
         idx = lookup_fn->idx;
         VariableNameFree(fn);
@@ -167,13 +203,15 @@ error:
     return 0;
 }
 
+#if 0
 /** \brief Get a name from the idx.
  *  \param idx index of the variable whose name is to be fetched
  *  \param type variable type
  *  \retval NULL in case of error
  *  \retval name of the variable if successful.
+ *  \todo no alloc on lookup
  */
-const char *VariableIdxGetName(DetectEngineCtx *de_ctx, uint16_t idx, enum VarTypes type)
+static const char *VariableIdxGetName(VarNameStore *v, uint32_t idx, enum VarTypes type)
 {
     VariableName *fn = SCMalloc(sizeof(VariableName));
     if (unlikely(fn == NULL))
@@ -185,7 +223,7 @@ const char *VariableIdxGetName(DetectEngineCtx *de_ctx, uint16_t idx, enum VarTy
     fn->type = type;
     fn->idx = idx;
 
-    VariableName *lookup_fn = (VariableName *)HashListTableLookup(de_ctx->variable_idxs, (void *)fn, 0);
+    VariableName *lookup_fn = (VariableName *)HashListTableLookup(v->ids, (void *)fn, 0);
     if (lookup_fn != NULL) {
         name = SCStrdup(lookup_fn->name);
         if (unlikely(name == NULL))
@@ -201,3 +239,141 @@ error:
     VariableNameFree(fn);
     return NULL;
 }
+#endif
+/** \brief setup staging store. Include current store if there is one.
+ */
+int VarNameStoreSetupStaging(uint32_t de_ctx_version)
+{
+    SCMutexLock(&g_varnamestore_staging_m);
+
+    if (!initialized) {
+        SC_ATOMIC_INIT(g_varnamestore_current);
+        initialized = 1;
+    }
+
+    if (g_varnamestore_staging != NULL &&
+        g_varnamestore_staging->de_ctx_version == de_ctx_version) {
+        SCMutexUnlock(&g_varnamestore_staging_m);
+        return 0;
+    }
+
+    VarNameStore *nv = VarNameStoreInit();
+    if (nv == NULL) {
+        SCMutexUnlock(&g_varnamestore_staging_m);
+        return -1;
+    }
+    g_varnamestore_staging = nv;
+    nv->de_ctx_version = de_ctx_version;
+
+    VarNameStore *current = SC_ATOMIC_GET(g_varnamestore_current);
+    if (current) {
+        /* add all entries from the current hash into this new one. */
+        HashListTableBucket *b = HashListTableGetListHead(current->names);
+        while (b) {
+            VariableName *var = HashListTableGetListData(b);
+
+            VariableName *newvar = SCCalloc(1, sizeof(*newvar));
+            BUG_ON(newvar == NULL);
+            memcpy(newvar, var, sizeof(*newvar));
+            newvar->name = SCStrdup(var->name);
+            BUG_ON(newvar->name == NULL);
+
+            HashListTableAdd(nv->names, (void *)newvar, 0);
+            HashListTableAdd(nv->ids, (void *)newvar, 0);
+            nv->max_id = MAX(nv->max_id, newvar->idx);
+            SCLogDebug("xfer %s id %u type %u", newvar->name, newvar->idx, newvar->type);
+
+            b = HashListTableGetListNext(b);
+        }
+    }
+
+    SCLogDebug("set up staging with detect engine ver %u", nv->de_ctx_version);
+    SCMutexUnlock(&g_varnamestore_staging_m);
+    return 0;
+}
+
+const char *VarNameStoreLookupById(const uint32_t id, const enum VarTypes type)
+{
+    VarNameStore *current = SC_ATOMIC_GET(g_varnamestore_current);
+    BUG_ON(current == NULL);
+    VariableName lookup = { NULL, type, id };
+    VariableName *found = (VariableName *)HashListTableLookup(current->ids, (void *)&lookup, 0);
+    if (found == NULL) {
+        return NULL;
+    }
+    return found->name;
+}
+
+uint32_t VarNameStoreLookupByName(const char *name, const enum VarTypes type)
+{
+    VarNameStore *current = SC_ATOMIC_GET(g_varnamestore_current);
+    BUG_ON(current == NULL);
+    VariableName lookup = { (char *)name, type, 0 };
+    VariableName *found = (VariableName *)HashListTableLookup(current->names, (void *)&lookup, 0);
+    if (found == NULL) {
+        return 0;
+    }
+    SCLogDebug("found %u for %s type %u", found->idx, name, type);
+    return found->idx;
+}
+
+/** \brief add to staging or return existing id if already in there */
+uint32_t VarNameStoreSetupAdd(const char *name, const enum VarTypes type)
+{
+    uint32_t id;
+    SCMutexLock(&g_varnamestore_staging_m);
+    id = VariableNameGetIdx(g_varnamestore_staging, name, type);
+    SCMutexUnlock(&g_varnamestore_staging_m);
+    return id;
+}
+
+void VarNameStoreActivateStaging(void)
+{
+    SCMutexLock(&g_varnamestore_staging_m);
+    if (g_varnamestore_old) {
+        VarNameStoreDoFree(g_varnamestore_old);
+        g_varnamestore_old = NULL;
+    }
+    g_varnamestore_old = SC_ATOMIC_GET(g_varnamestore_current);
+    SC_ATOMIC_SET(g_varnamestore_current, g_varnamestore_staging);
+    g_varnamestore_staging = NULL;
+    SCMutexUnlock(&g_varnamestore_staging_m);
+}
+
+void VarNameStoreFreeOld(void)
+{
+    SCMutexLock(&g_varnamestore_staging_m);
+    SCLogDebug("freeing g_varnamestore_old %p", g_varnamestore_old);
+    if (g_varnamestore_old) {
+        VarNameStoreDoFree(g_varnamestore_old);
+        g_varnamestore_old = NULL;
+    }
+    SCMutexUnlock(&g_varnamestore_staging_m);
+}
+
+void VarNameStoreFree(uint32_t de_ctx_version)
+{
+    SCLogDebug("freeing detect engine version %u", de_ctx_version);
+    SCMutexLock(&g_varnamestore_staging_m);
+    if (g_varnamestore_old && g_varnamestore_old->de_ctx_version == de_ctx_version) {
+        VarNameStoreDoFree(g_varnamestore_old);
+        g_varnamestore_old = NULL;
+        SCLogDebug("freeing detect engine version %u: old done", de_ctx_version);
+    }
+
+    /* if at this point we have a staging area which matches our version
+     * we didn't complete the setup and are cleaning up the mess. */
+    if (g_varnamestore_staging && g_varnamestore_staging->de_ctx_version == de_ctx_version) {
+        VarNameStoreDoFree(g_varnamestore_staging);
+        g_varnamestore_staging = NULL;
+        SCLogDebug("freeing detect engine version %u: staging done", de_ctx_version);
+    }
+
+    VarNameStore *current = SC_ATOMIC_GET(g_varnamestore_current);
+    if (current && current->de_ctx_version == de_ctx_version) {
+        VarNameStoreDoFree(current);
+        SC_ATOMIC_SET(g_varnamestore_current, NULL);
+        SCLogDebug("freeing detect engine version %u: current done", de_ctx_version);
+    }
+    SCMutexUnlock(&g_varnamestore_staging_m);
+}

diff --git a/src/util-var-name.h b/src/util-var-name.h
index 7d2c527a31aff650cd2b59e6d4a28e87d0be1b41..c620d74eb5a49cf50ee4aa01005f09d37ca8ad91 100644 (file)
--- a/src/util-var-name.h
+++ b/src/util-var-name.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2016 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -24,13 +24,13 @@
 #ifndef __UTIL_VAR_NAME_H__
 #define __UTIL_VAR_NAME_H__
 
-int VariableNameInitHash(DetectEngineCtx *);
-void VariableNameFreeHash(DetectEngineCtx *);
-
-uint16_t VariableNameGetIdx(DetectEngineCtx *,
-        const char *name, enum VarTypes type);
-const char *VariableIdxGetName(DetectEngineCtx *,
-        uint16_t id, enum VarTypes type);
+int VarNameStoreSetupStaging(uint32_t de_ctx_version);
+const char *VarNameStoreLookupById(const uint32_t id, const enum VarTypes type);
+uint32_t VarNameStoreLookupByName(const char *name, const enum VarTypes type);
+uint32_t VarNameStoreSetupAdd(const char *name, const enum VarTypes type);
+void VarNameStoreActivateStaging(void);
+void VarNameStoreFreeOld(void);
+void VarNameStoreFree(uint32_t de_ctx_version);
 
 #endif
 

diff --git a/src/util-var.h b/src/util-var.h
index 39f2698dc08804f457e18d640dbc0ee2391bf780..3deeaf747912500cdaf0c1113745e1b0f0e84361 100644 (file)
--- a/src/util-var.h
+++ b/src/util-var.h
@@ -46,13 +46,13 @@ enum VarTypes {
 
 typedef struct GenericVar_ {
     uint8_t type;
-    uint16_t idx;
+    uint32_t idx;
     struct GenericVar_ *next;
 } GenericVar;
 
 typedef struct XBit_ {
     uint8_t type;       /* type, DETECT_XBITS in this case */
-    uint16_t idx;       /* name idx */
+    uint32_t idx;       /* name idx */
     GenericVar *next;
     uint32_t expire;
 } XBit;