From: Victor Julien <victor@inliniac.net>
Date: Sun, 6 Nov 2016 21:43:41 +0000 (-0500)
Subject: lua: support key/value flowvars in lua
X-Git-Tag: suricata-4.0.0-beta1~315
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=99517cbd53a378a22f2e14063a5ceb00e095d4b9;p=thirdparty%2Fsuricata.git

lua: support key/value flowvars in lua
---

diff --git a/src/detect-flowvar.c b/src/detect-flowvar.c
index 920ecf6510..bd7ff4038c 100644
--- a/src/detect-flowvar.c
+++ b/src/detect-flowvar.c
@@ -303,7 +303,7 @@ static int DetectFlowvarPostMatch(ThreadVars *tv,
             //PrintRawDataFp(stdout, fs->buffer, fs->len);
 
             if (fs->type == DETECT_VAR_TYPE_FLOW_POSTMATCH && p && p->flow) {
-                FlowVarAddStrNoLock(p->flow, fs->idx, fs->buffer, fs->len);
+                FlowVarAddIdValue(p->flow, fs->idx, fs->buffer, fs->len);
                 /* memory at fs->buffer is now the responsibility of
                  * the flowvar code. */
             } else if (fs->type == DETECT_VAR_TYPE_PKT_POSTMATCH && fs->key && p) {
diff --git a/src/detect-lua-extensions.c b/src/detect-lua-extensions.c
index 06a11569a5..91d1f071e4 100644
--- a/src/detect-lua-extensions.c
+++ b/src/detect-lua-extensions.c
@@ -72,308 +72,340 @@
 #include "util-lua-dnp3.h"
 
 static const char luaext_key_ld[] = "suricata:luadata";
-static const char luaext_key_det_ctx[] = "suricata:det_ctx";
 
-static int LuaGetFlowvar(lua_State *luastate)
+/* hack to please scan-build. Even though LuaCallbackError *always*
+ * returns 2, scan-build doesn't accept it and and generates false
+ * positives */
+#define LUA_ERROR(msg)                  \
+    LuaCallbackError(luastate, (msg));  \
+    return 2
+
+static int GetLuaData(lua_State *luastate, DetectLuaData **ret_ld)
 {
-    uint16_t idx;
-    int id;
-    Flow *f;
-    FlowVar *fv;
-    DetectLuaData *ld;
+    *ret_ld = NULL;
 
-    /* need lua data for id -> idx conversion */
+    DetectLuaData *ld;
     lua_pushlightuserdata(luastate, (void *)&luaext_key_ld);
     lua_gettable(luastate, LUA_REGISTRYINDEX);
     ld = lua_touserdata(luastate, -1);
-    SCLogDebug("ld %p", ld);
     if (ld == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no ld");
-        return 2;
+        LUA_ERROR("internal error: no ld");
     }
+    *ret_ld = ld;
+    return 0;
+}
 
-    /* need flow and lock hint */
-    f = LuaStateGetFlow(luastate);
+static int GetFlow(lua_State *luastate, Flow **ret_f)
+{
+    Flow *f = LuaStateGetFlow(luastate);
     if (f == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow");
-        return 2;
+        LUA_ERROR("no flow");
     }
+    *ret_f = f;
+    return 0;
+}
+
+static int GetFlowVarById(lua_State *luastate, Flow *f,
+        FlowVar **ret_fv, _Bool fv_may_be_null, uint32_t *ret_idx)
+{
+    DetectLuaData *ld = NULL;
+    if (ret_idx)
+        *ret_idx = 0;
+    *ret_fv = NULL;
+
+    /* need lua data for id -> idx conversion */
+    int ret = GetLuaData(luastate, &ld);
+    if (ret != 0)
+        return ret;
 
-    /* need flowvar idx */
     if (!lua_isnumber(luastate, 1)) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "1st arg not a number");
-        return 2;
+        LUA_ERROR("flowvar id not a number");
     }
-    id = lua_tonumber(luastate, 1);
+    int id = lua_tonumber(luastate, 1);
     if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWVARS) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowvar id out of range");
-        return 2;
+        LUA_ERROR("flowvar id out of range");
     }
-    idx = ld->flowvar[id];
+    uint32_t idx = ld->flowvar[id];
     if (idx == 0) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowvar id uninitialized");
-        return 2;
+        LUA_ERROR("flowvar id uninitialized");
+    }
+    FlowVar *fv = FlowVarGet(f, idx);
+    if (!fv_may_be_null && fv == NULL) {
+        LUA_ERROR("no flow var");
+    }
+    *ret_fv = fv;
+    if (ret_idx)
+        *ret_idx = idx;
+    return 0;
+}
+
+static int GetFlowVarByKey(lua_State *luastate, Flow *f, FlowVar **ret_fv)
+{
+    *ret_fv = NULL;
+
+    if (!lua_isstring(luastate, 1)) {
+        LUA_ERROR("flowvar key not a string");
+    }
+    const char *keystr = lua_tostring(luastate, 1);
+    if (keystr == NULL) {
+        LUA_ERROR("key is null");
+    }
+    if (!lua_isnumber(luastate, 2)) {
+        LUA_ERROR("key length not specified");
+    }
+    int keylen = lua_tonumber(luastate, 2);
+    if (keylen < 0 || keylen > 0xff) {
+        LUA_ERROR("key len out of range: max 256");
     }
 
-    fv = FlowVarGet(f, idx);
+    FlowVar *fv = FlowVarGetByKey(f, (const uint8_t *)keystr, keylen);
     if (fv == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow var");
-        return 2;
+        LUA_ERROR("no flow var");
     }
+    *ret_fv = fv;
+    return 0;
+}
 
-    LuaPushStringBuffer(luastate, (const uint8_t *)fv->data.fv_str.value,
-            (size_t)fv->data.fv_str.value_len);
+static int GetFlowIntById(lua_State *luastate, Flow *f,
+        FlowVar **ret_fv, _Bool fv_may_be_null, uint32_t *ret_idx)
+{
+    DetectLuaData *ld = NULL;
+    if (ret_idx)
+        *ret_idx = 0;
+    *ret_fv = NULL;
 
-    return 1;
+    /* need lua data for id -> idx conversion */
+    int ret = GetLuaData(luastate, &ld);
+    if (ret != 0)
+        return ret;
+
+    if (!lua_isnumber(luastate, 1)) {
+        LUA_ERROR("flowvar id not a number");
+    }
+    int id = lua_tonumber(luastate, 1);
+    if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWVARS) {
+        LUA_ERROR("flowvar id out of range");
+    }
+    uint32_t idx = ld->flowint[id];
+    if (idx == 0) {
+        LUA_ERROR("flowvar id uninitialized");
+    }
+    FlowVar *fv = FlowVarGet(f, idx);
+    if (!fv_may_be_null && fv == NULL) {
+        LUA_ERROR("no flow var");
+    }
+    *ret_fv = fv;
+    if (ret_idx)
+        *ret_idx = idx;
+    return 0;
+}
 
+static int LuaGetFlowvar(lua_State *luastate)
+{
+    Flow *f;
+    FlowVar *fv;
+    int ret;
+
+    /* need flow */
+    ret = GetFlow(luastate, &f);
+    if (ret != 0)
+        return ret;
+
+    if (lua_isnumber(luastate, 1)) {
+        ret = GetFlowVarById(luastate, f, &fv, FALSE, NULL);
+        if (ret != 0 || fv == NULL)
+            return ret;
+    } else if (lua_isstring(luastate, 1)) {
+        ret = GetFlowVarByKey(luastate, f, &fv);
+        if (ret != 0 || fv == NULL)
+            return ret;
+    } else {
+        LUA_ERROR("invalid data type as first argument");
+    }
+
+    LuaPushStringBuffer(luastate,
+            (const uint8_t *)fv->data.fv_str.value,
+            (size_t)fv->data.fv_str.value_len);
+    return 1;
 }
 
-int LuaSetFlowvar(lua_State *luastate)
+static int LuaSetFlowvarById(lua_State *luastate)
 {
-    uint16_t idx;
-    int id;
+    uint32_t idx = 0;
     Flow *f;
     const char *str;
     int len;
     uint8_t *buffer;
-    DetectEngineThreadCtx *det_ctx;
-    DetectLuaData *ld;
+    FlowVar *fv = NULL;
 
-    /* need lua data for id -> idx conversion */
-    lua_pushlightuserdata(luastate, (void *)&luaext_key_ld);
-    lua_gettable(luastate, LUA_REGISTRYINDEX);
-    ld = lua_touserdata(luastate, -1);
-    SCLogDebug("ld %p", ld);
-    if (ld == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no ld");
-        return 2;
+    /* need flow */
+    int ret = GetFlow(luastate, &f);
+    if (ret != 0)
+        return ret;
+
+    ret = GetFlowVarById(luastate, f, &fv, TRUE, &idx);
+    if (ret != 0)
+        return ret;
+
+    if (!lua_isstring(luastate, 2)) {
+        LUA_ERROR("buffer not a string");
+    }
+    str = lua_tostring(luastate, 2);
+    if (str == NULL) {
+        LUA_ERROR("buffer is null");
     }
 
-    /* need det_ctx */
-    lua_pushlightuserdata(luastate, (void *)&luaext_key_det_ctx);
-    lua_gettable(luastate, LUA_REGISTRYINDEX);
-    det_ctx = lua_touserdata(luastate, -1);
-    SCLogDebug("det_ctx %p", det_ctx);
-    if (det_ctx == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no det_ctx");
-        return 2;
+    if (!lua_isnumber(luastate, 3)) {
+        LUA_ERROR("buffer length not specified");
+    }
+    len = lua_tonumber(luastate, 3);
+    if (len < 0 || len > 0xffff) {
+        LUA_ERROR("len out of range: max 64k");
     }
 
-    /* need flow and lock hint */
-    f = LuaStateGetFlow(luastate);
-    if (f == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow");
-        return 2;
+    buffer = SCMalloc(len+1);
+    if (unlikely(buffer == NULL)) {
+        LUA_ERROR("out of memory");
     }
+    memcpy(buffer, str, len);
+    buffer[len] = '\0';
 
-    /* need flowvar idx */
-    if (!lua_isnumber(luastate, 1)) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "1st arg not a number");
-        return 2;
+    FlowVarAddIdValue(f, idx, buffer, len);
+    return 0;
+}
+
+static int LuaSetFlowvarByKey(lua_State *luastate)
+{
+    Flow *f;
+    const char *str;
+    int len;
+    uint8_t *buffer;
+
+    /* need flow */
+    int ret = GetFlow(luastate, &f);
+    if (ret != 0)
+        return ret;
+
+    const char *keystr = NULL;
+    int keylen = 0;
+
+    keystr = lua_tostring(luastate, 1);
+    if (keystr == NULL) {
+        LUA_ERROR("key is null");
     }
-    id = lua_tonumber(luastate, 1);
-    if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWVARS) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowvar id out of range");
-        return 2;
+    if (!lua_isnumber(luastate, 2)) {
+        LUA_ERROR("key length not specified");
+    }
+    keylen = lua_tonumber(luastate, 2);
+    if (keylen < 0 || keylen > 0xff) {
+        LUA_ERROR("key len out of range: max 256");
     }
 
-    if (!lua_isstring(luastate, 2)) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "2nd arg not a string");
-        return 2;
+    if (!lua_isstring(luastate, 3)) {
+        LUA_ERROR("buffer not a string");
     }
-    str = lua_tostring(luastate, 2);
+    str = lua_tostring(luastate, 3);
     if (str == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "null string");
-        return 2;
+        LUA_ERROR("buffer is null");
     }
 
-    if (!lua_isnumber(luastate, 3)) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "3rd arg not a number");
-        return 2;
+    if (!lua_isnumber(luastate, 4)) {
+        LUA_ERROR("buffer length not specified");
     }
-    len = lua_tonumber(luastate, 3);
+    len = lua_tonumber(luastate, 4);
     if (len < 0 || len > 0xffff) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "len out of range: max 64k");
-        return 2;
-    }
-
-    idx = ld->flowvar[id];
-    if (idx == 0) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowvar id uninitialized");
-        return 2;
+        LUA_ERROR("len out of range: max 64k");
     }
 
     buffer = SCMalloc(len+1);
     if (unlikely(buffer == NULL)) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "out of memory");
-        return 2;
+        LUA_ERROR("out of memory");
     }
     memcpy(buffer, str, len);
     buffer[len] = '\0';
 
-    FlowVarAddStr(f, idx, buffer, len);
+    uint8_t *keybuf = SCMalloc(keylen+1);
+    if (unlikely(keybuf == NULL)) {
+        SCFree(buffer);
+        LUA_ERROR("out of memory");
+    }
+    memcpy(keybuf, keystr, keylen);
+    keybuf[keylen] = '\0';
+    FlowVarAddKeyValue(f, keybuf, keylen, buffer, len);
 
-    //SCLogInfo("stored:");
-    //PrintRawDataFp(stdout,buffer,len);
     return 0;
 }
 
+static int LuaSetFlowvar(lua_State *luastate)
+{
+    if (lua_isnumber(luastate, 1)) {
+        return LuaSetFlowvarById(luastate);
+    } else {
+        return LuaSetFlowvarByKey(luastate);
+    }
+}
+
 static int LuaGetFlowint(lua_State *luastate)
 {
-    uint16_t idx;
-    int id;
     Flow *f;
     FlowVar *fv;
-    DetectLuaData *ld;
     uint32_t number;
 
-    /* need lua data for id -> idx conversion */
-    lua_pushlightuserdata(luastate, (void *)&luaext_key_ld);
-    lua_gettable(luastate, LUA_REGISTRYINDEX);
-    ld = lua_touserdata(luastate, -1);
-    SCLogDebug("ld %p", ld);
-    if (ld == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no ld");
-        return 2;
-    }
-
-    /* need flow and lock hint */
-    f = LuaStateGetFlow(luastate);
-    if (f == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow");
-        return 2;
-    }
+    /* need flow */
+    int ret = GetFlow(luastate, &f);
+    if (ret != 0)
+        return ret;
 
-    /* need flowint idx */
-    if (!lua_isnumber(luastate, 1)) {
-        SCLogDebug("1st arg not a number");
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "1st arg not a number");
-        return 2;
-    }
-    id = lua_tonumber(luastate, 1);
-    if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWINTS) {
-        SCLogDebug("id %d", id);
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id out of range");
-        return 2;
-    }
-    idx = ld->flowint[id];
-    if (idx == 0) {
-        SCLogDebug("idx %u", idx);
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id uninitialized");
-        return 2;
-    }
+    ret = GetFlowIntById(luastate, f, &fv, FALSE, NULL);
+    if (ret != 0)
+        return ret;
 
-    /* lookup var */
-    fv = FlowVarGet(f, idx);
-    if (fv == NULL) {
-        SCLogDebug("fv NULL");
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow var");
-        return 2;
-    }
     number = fv->data.fv_int.value;
 
     /* return value through luastate, as a luanumber */
     lua_pushnumber(luastate, (lua_Number)number);
-    SCLogDebug("retrieved flow:%p idx:%u value:%u", f, idx, number);
-
     return 1;
 
 }
 
-int LuaSetFlowint(lua_State *luastate)
+static int LuaSetFlowint(lua_State *luastate)
 {
-    uint16_t idx;
-    int id;
+    uint32_t idx;
     Flow *f;
-    DetectEngineThreadCtx *det_ctx;
     DetectLuaData *ld;
-    uint32_t number;
-    lua_Number luanumber;
 
     /* need lua data for id -> idx conversion */
-    lua_pushlightuserdata(luastate, (void *)&luaext_key_ld);
-    lua_gettable(luastate, LUA_REGISTRYINDEX);
-    ld = lua_touserdata(luastate, -1);
-    SCLogDebug("ld %p", ld);
-    if (ld == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no ld");
-        return 2;
-    }
-
-    /* need det_ctx */
-    lua_pushlightuserdata(luastate, (void *)&luaext_key_det_ctx);
-    lua_gettable(luastate, LUA_REGISTRYINDEX);
-    det_ctx = lua_touserdata(luastate, -1);
-    SCLogDebug("det_ctx %p", det_ctx);
-    if (det_ctx == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no det_ctx");
-        return 2;
-    }
+    int ret = GetLuaData(luastate, &ld);
+    if (ret != 0)
+        return ret;
 
-    /* need flow and lock hint */
-    f = LuaStateGetFlow(luastate);
-    if (f == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow");
-        return 2;
-    }
+    /* need flow */
+    ret = GetFlow(luastate, &f);
+    if (ret != 0)
+        return ret;
 
     /* need flowint idx */
     if (!lua_isnumber(luastate, 1)) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "1st arg not a number");
-        return 2;
+        LUA_ERROR("1st arg not a number");
     }
-    id = lua_tonumber(luastate, 1);
+    int id = lua_tonumber(luastate, 1);
     if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWVARS) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id out of range");
-        return 2;
+        LUA_ERROR("flowint id out of range");
     }
 
     if (!lua_isnumber(luastate, 2)) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "2nd arg not a number");
-        return 2;
+        LUA_ERROR("2nd arg not a number");
     }
-    luanumber = lua_tonumber(luastate, 2);
+    lua_Number luanumber = lua_tonumber(luastate, 2);
     if (luanumber < 0 || id > (double)UINT_MAX) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "value out of range, value must be unsigned 32bit int");
-        return 2;
+        LUA_ERROR("value out of range, "
+                "value must be unsigned 32bit int");
     }
-    number = (uint32_t)luanumber;
+    uint32_t number = (uint32_t)luanumber;
 
     idx = ld->flowint[id];
     if (idx == 0) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id uninitialized");
-        return 2;
+        LUA_ERROR("flowint id uninitialized");
     }
 
     FlowVarAddInt(f, idx, number);
@@ -384,56 +416,20 @@ int LuaSetFlowint(lua_State *luastate)
 
 static int LuaIncrFlowint(lua_State *luastate)
 {
-    uint16_t idx;
-    int id;
+    uint32_t idx;
     Flow *f;
     FlowVar *fv;
-    DetectLuaData *ld;
     uint32_t number;
 
-    /* need lua data for id -> idx conversion */
-    lua_pushlightuserdata(luastate, (void *)&luaext_key_ld);
-    lua_gettable(luastate, LUA_REGISTRYINDEX);
-    ld = lua_touserdata(luastate, -1);
-    SCLogDebug("ld %p", ld);
-    if (ld == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no ld");
-        return 2;
-    }
-
-    /* need flow and lock hint */
-    f = LuaStateGetFlow(luastate);
-    if (f == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow");
-        return 2;
-    }
+    /* need flow */
+    int ret = GetFlow(luastate, &f);
+    if (ret != 0)
+        return ret;
 
-    /* need flowint idx */
-    if (!lua_isnumber(luastate, 1)) {
-        SCLogDebug("1st arg not a number");
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "1st arg not a number");
-        return 2;
-    }
-    id = lua_tonumber(luastate, 1);
-    if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWINTS) {
-        SCLogDebug("id %d", id);
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id out of range");
-        return 2;
-    }
-    idx = ld->flowint[id];
-    if (idx == 0) {
-        SCLogDebug("idx %u", idx);
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id uninitialized");
-        return 2;
-    }
+    ret = GetFlowIntById(luastate, f, &fv, TRUE, &idx);
+    if (ret != 0)
+        return ret;
 
-    /* lookup var */
-    fv = FlowVarGet(f, idx);
     if (fv == NULL) {
         number = 1;
     } else {
@@ -446,63 +442,26 @@ static int LuaIncrFlowint(lua_State *luastate)
     /* return value through luastate, as a luanumber */
     lua_pushnumber(luastate, (lua_Number)number);
     SCLogDebug("incremented flow:%p idx:%u value:%u", f, idx, number);
-
     return 1;
 
 }
 
 static int LuaDecrFlowint(lua_State *luastate)
 {
-    uint16_t idx;
-    int id;
+    uint32_t idx;
     Flow *f;
     FlowVar *fv;
-    DetectLuaData *ld;
     uint32_t number;
 
-    /* need lua data for id -> idx conversion */
-    lua_pushlightuserdata(luastate, (void *)&luaext_key_ld);
-    lua_gettable(luastate, LUA_REGISTRYINDEX);
-    ld = lua_touserdata(luastate, -1);
-    SCLogDebug("ld %p", ld);
-    if (ld == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "internal error: no ld");
-        return 2;
-    }
+    /* need flow */
+    int ret = GetFlow(luastate, &f);
+    if (ret != 0)
+        return ret;
 
-    /* need flow and lock hint */
-    f = LuaStateGetFlow(luastate);
-    if (f == NULL) {
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "no flow");
-        return 2;
-    }
+    ret = GetFlowIntById(luastate, f, &fv, TRUE, &idx);
+    if (ret != 0)
+        return ret;
 
-    /* need flowint idx */
-    if (!lua_isnumber(luastate, 1)) {
-        SCLogDebug("1st arg not a number");
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "1st arg not a number");
-        return 2;
-    }
-    id = lua_tonumber(luastate, 1);
-    if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWINTS) {
-        SCLogDebug("id %d", id);
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id out of range");
-        return 2;
-    }
-    idx = ld->flowint[id];
-    if (idx == 0) {
-        SCLogDebug("idx %u", idx);
-        lua_pushnil(luastate);
-        lua_pushstring(luastate, "flowint id uninitialized");
-        return 2;
-    }
-
-    /* lookup var */
-    fv = FlowVarGet(f, idx);
     if (fv == NULL) {
         number = 0;
     } else {
@@ -515,7 +474,6 @@ static int LuaDecrFlowint(lua_State *luastate)
     /* return value through luastate, as a luanumber */
     lua_pushnumber(luastate, (lua_Number)number);
     SCLogDebug("decremented flow:%p idx:%u value:%u", f, idx, number);
-
     return 1;
 
 }
@@ -530,11 +488,6 @@ void LuaExtensionsMatchSetup(lua_State *lua_state, DetectLuaData *ld, DetectEngi
     lua_pushlightuserdata(lua_state, (void *)ld);
     lua_settable(lua_state, LUA_REGISTRYINDEX);
 
-    /* detection engine thread ctx */
-    lua_pushlightuserdata(lua_state, (void *)&luaext_key_det_ctx);
-    lua_pushlightuserdata(lua_state, (void *)det_ctx);
-    lua_settable(lua_state, LUA_REGISTRYINDEX);
-
     LuaStateSetFlow(lua_state, f);
 
     if (det_ctx->tx_id_set) {
diff --git a/src/flow-var.c b/src/flow-var.c
index d929d55874..d4506b9b03 100644
--- a/src/flow-var.c
+++ b/src/flow-var.c
@@ -47,6 +47,27 @@ static void FlowVarUpdateInt(FlowVar *fv, uint32_t value)
     fv->data.fv_int.value = value;
 }
 
+/** \brief get the flowvar with index 'idx' from the flow
+ *  \note flow is not locked by this function, caller is
+ *        responsible
+ */
+FlowVar *FlowVarGetByKey(Flow *f, const uint8_t *key, uint16_t keylen)
+{
+    GenericVar *gv = f->flowvar;
+
+    for ( ; gv != NULL; gv = gv->next) {
+        if (gv->type == DETECT_FLOWVAR && gv->idx == 0) {
+
+            FlowVar *fv = (FlowVar *)gv;
+            if (fv->keylen == keylen && memcmp(key, fv->key, keylen) == 0) {
+                return fv;
+            }
+        }
+    }
+
+    return NULL;
+}
+
 /** \brief get the flowvar with index 'idx' from the flow
  *  \note flow is not locked by this function, caller is
  *        responsible
@@ -64,11 +85,30 @@ FlowVar *FlowVarGet(Flow *f, uint32_t idx)
 }
 
 /* add a flowvar to the flow, or update it */
-void FlowVarAddStrNoLock(Flow *f, uint32_t idx, uint8_t *value, uint16_t size)
+void FlowVarAddKeyValue(Flow *f, uint8_t *key, uint16_t keysize, uint8_t *value, uint16_t size)
+{
+    FlowVar *fv = SCCalloc(1, sizeof(FlowVar));
+    if (unlikely(fv == NULL))
+        return;
+
+    fv->type = DETECT_FLOWVAR;
+    fv->datatype = FLOWVAR_TYPE_STR;
+    fv->idx = 0;
+    fv->data.fv_str.value = value;
+    fv->data.fv_str.value_len = size;
+    fv->key = key;
+    fv->keylen = keysize;
+    fv->next = NULL;
+
+    GenericVarAppend(&f->flowvar, (GenericVar *)fv);
+}
+
+/* add a flowvar to the flow, or update it */
+void FlowVarAddIdValue(Flow *f, uint32_t idx, uint8_t *value, uint16_t size)
 {
     FlowVar *fv = FlowVarGet(f, idx);
     if (fv == NULL) {
-        fv = SCMalloc(sizeof(FlowVar));
+        fv = SCCalloc(1, sizeof(FlowVar));
         if (unlikely(fv == NULL))
             return;
 
@@ -85,12 +125,6 @@ void FlowVarAddStrNoLock(Flow *f, uint32_t idx, uint8_t *value, uint16_t size)
     }
 }
 
-/* add a flowvar to the flow, or update it */
-void FlowVarAddStr(Flow *f, uint32_t idx, uint8_t *value, uint16_t size)
-{
-    FlowVarAddStrNoLock(f, idx, value, size);
-}
-
 /* add a flowvar to the flow, or update it */
 void FlowVarAddIntNoLock(Flow *f, uint32_t idx, uint32_t value)
 {
diff --git a/src/flow-var.h b/src/flow-var.h
index 0f7eeeb926..7d0d4edb87 100644
--- a/src/flow-var.h
+++ b/src/flow-var.h
@@ -56,16 +56,19 @@ typedef struct FlowVar_ {
         FlowVarTypeStr fv_str;
         FlowVarTypeInt fv_int;
     } data;
-
+    uint8_t *key;
+    uint16_t keylen;
 } FlowVar;
 
 /** Flowvar Interface API */
 
-void FlowVarAddStrNoLock(Flow *, uint32_t, uint8_t *, uint16_t);
-void FlowVarAddStr(Flow *, uint32_t, uint8_t *, uint16_t);
+void FlowVarAddIdValue(Flow *, uint32_t id, uint8_t *value, uint16_t size);
+void FlowVarAddKeyValue(Flow *f, uint8_t *key, uint16_t keysize, uint8_t *value, uint16_t size);
+
 void FlowVarAddIntNoLock(Flow *, uint32_t, uint32_t);
 void FlowVarAddInt(Flow *, uint32_t, uint32_t);
 FlowVar *FlowVarGet(Flow *, uint32_t);
+FlowVar *FlowVarGetByKey(Flow *f, const uint8_t *key, uint16_t keylen);
 void FlowVarFree(FlowVar *);
 void FlowVarPrint(GenericVar *);
 
diff --git a/src/output-json.c b/src/output-json.c
index 7db6262d33..95f9b0167d 100644
--- a/src/output-json.c
+++ b/src/output-json.c
@@ -160,7 +160,7 @@ static void JsonAddFlowvars(const Flow *f, json_t *js_vars)
     while (gv != NULL) {
         if (gv->type == DETECT_FLOWVAR || gv->type == DETECT_FLOWINT) {
             FlowVar *fv = (FlowVar *)gv;
-            if (fv->datatype == FLOWVAR_TYPE_STR) {
+            if (fv->datatype == FLOWVAR_TYPE_STR && fv->key == NULL) {
                 const char *varname = VarNameStoreLookupById(fv->idx, VAR_TYPE_FLOW_VAR);
                 if (varname) {
                     if (js_flowvars == NULL) {
@@ -179,6 +179,29 @@ static void JsonAddFlowvars(const Flow *f, json_t *js_vars)
                     json_object_set_new(js_flowvars, varname,
                             json_string((char *)printable_buf));
                 }
+            } else if (fv->datatype == FLOWVAR_TYPE_STR && fv->key != NULL) {
+                if (js_flowvars == NULL) {
+                    js_flowvars = json_object();
+                    if (js_flowvars == NULL)
+                        break;
+                }
+
+                uint8_t keybuf[fv->keylen + 1];
+                uint32_t offset = 0;
+                PrintStringsToBuffer(keybuf, &offset,
+                        sizeof(keybuf),
+                        fv->key, fv->keylen);
+
+                uint32_t len = fv->data.fv_str.value_len;
+                uint8_t printable_buf[len + 1];
+                offset = 0;
+                PrintStringsToBuffer(printable_buf, &offset,
+                        sizeof(printable_buf),
+                        fv->data.fv_str.value, fv->data.fv_str.value_len);
+
+                json_object_set_new(js_flowvars, (const char *)keybuf,
+                        json_string((char *)printable_buf));
+
             } else if (fv->datatype == FLOWVAR_TYPE_INT) {
                 const char *varname = VarNameStoreLookupById(fv->idx, VAR_TYPE_FLOW_INT);
                 if (varname) {