datajson: limit impact of feature for non user

The det_ctx structure was inflated by the additoin of the array to
handle JSON context. This commit updates the code to use a growing
buffer and limit the impact.

diff --git a/src/decode.c b/src/decode.c
index 9acbd0ab24809ea4c25295a1d953d32429f83be1..ad4ea19bd81ee6f4c41aebc97683599d72541637 100644 (file)
--- a/src/decode.c
+++ b/src/decode.c
@@ -154,6 +154,7 @@ void PacketAlertRecycle(PacketAlert *pa_array, uint16_t cnt)
         struct PacketContextData *current_json = pa_array[i].json_info;
         while (current_json) {
             struct PacketContextData *next_json = current_json->next;
+            SCFree(current_json->json_string);
             SCFree(current_json);
             current_json = next_json;
         }
@@ -169,6 +170,7 @@ void PacketAlertFree(PacketAlert *pa_array)
         struct PacketContextData *allocated_json = pa_array[i].json_info;
         while (allocated_json) {
             struct PacketContextData *next_json = allocated_json->next;
+            SCFree(allocated_json->json_string);
             SCFree(allocated_json);
             allocated_json = next_json;
         }

diff --git a/src/detect-dataset.c b/src/detect-dataset.c
index c41d2be7ad7a35475b8d0125b07e0c2ad533576a..54e3be28c709fba16900e76816eb729913f6a046 100644 (file)
--- a/src/detect-dataset.c
+++ b/src/detect-dataset.c
@@ -81,12 +81,17 @@ static int DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, const Detec
             if (r.json.len > 0) {
                 /* we need to add 3 on length check for the added quotes and colon when
                 building the json string */
-                if ((det_ctx->json_content_len < SIG_JSON_CONTENT_ARRAY_LEN) &&
-                        (r.json.len + strlen(sd->json_key) + 3 < SIG_JSON_CONTENT_ITEM_LEN)) {
+                if (r.json.len + strlen(sd->json_key) + 3 < SIG_JSON_CONTENT_ITEM_LEN) {
+                    if (DetectEngineThreadCtxGetJsonContext(det_ctx) < 0) {
+                        DatajsonUnlockElt(&r);
+                        return 0;
+                    }
                     snprintf(det_ctx->json_content[det_ctx->json_content_len].json_content,
                             SIG_JSON_CONTENT_ITEM_LEN, "\"%s\":%s", sd->json_key, r.json.value);
                     det_ctx->json_content[det_ctx->json_content_len].id = sd->id;
                     det_ctx->json_content_len++;
+                    SCLogDebug("Added json content %u (alloc length %u)", det_ctx->json_content_len,
+                            det_ctx->json_content_capacity);
                 }
             }
             DatajsonUnlockElt(&r);

diff --git a/src/detect-engine-alert.c b/src/detect-engine-alert.c
index 709e41c2ef3d8a3f8c38923b944b2eabc997201c..2ac0038964d6532ce57eedf99705b326bfa0e338 100644 (file)
--- a/src/detect-engine-alert.c
+++ b/src/detect-engine-alert.c
@@ -291,15 +291,24 @@ static inline int PacketAlertSetContext(
     if (det_ctx->json_content_len) {
         /* We have some JSON attached in the current detection so let's try
            to see if some need to be used for current signature. */
-        struct PacketContextData *current_json = SCCalloc(1, sizeof(struct PacketContextData));
-        if (current_json == NULL) {
-            /* Allocation error, let's return now */
-            return -1;
-        }
-        pa->json_info = current_json;
-        for (size_t i = 0; i < det_ctx->json_content_len; i++) {
+        struct PacketContextData *current_json = NULL;
+        for (uint8_t i = 0; i < det_ctx->json_content_len; i++) {
             if (s == det_ctx->json_content[i].id) {
-                if (current_json->json_string != NULL) {
+                SCLogDebug("signature %p, content index %u", s, i);
+                if (current_json == NULL) {
+                    /* Allocate the first one */
+                    current_json = SCCalloc(1, sizeof(struct PacketContextData));
+                    if (current_json == NULL) {
+                        /* Allocation error, let's return now */
+                        return -1;
+                    }
+                    if (pa->json_info == NULL) {
+                        /* If this is the first one, set it */
+                        pa->json_info = current_json;
+                    }
+                    current_json->next = NULL;
+                } else {
+                    /* Allocate the next one */
                     struct PacketContextData *next_json =
                             SCCalloc(1, sizeof(struct PacketContextData));
                     if (next_json) {
@@ -311,7 +320,9 @@ static inline int PacketAlertSetContext(
                         return -1;
                     }
                 }
-                current_json->json_string = det_ctx->json_content[i].json_content;
+                current_json->json_string = SCStrdup(det_ctx->json_content[i].json_content);
+                SCLogDebug("json content %u, value '%s' (%p)", (unsigned int)i,
+                        current_json->json_string, s);
             }
         }
     }

diff --git a/src/detect-engine.c b/src/detect-engine.c
index bb9b41572df9e42c97db58feb99cd22ff47cadd1..b70f01da5804a8d318d7bb6a3a75c69c333e606a 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -3416,6 +3416,10 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
     /* Register a counter for Lua memory limit errors. */
     det_ctx->lua_memory_limit_errors = StatsRegisterCounter("detect.lua.memory_limit_errors", tv);
 
+    det_ctx->json_content = NULL;
+    det_ctx->json_content_capacity = 0;
+    det_ctx->json_content_len = 0;
+
 #ifdef PROFILING
     det_ctx->counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
     det_ctx->counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
@@ -3585,6 +3589,12 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
 #endif
     }
 
+    if (det_ctx->json_content) {
+        SCFree(det_ctx->json_content);
+        det_ctx->json_content = NULL;
+        det_ctx->json_content_capacity = 0;
+    }
+
     AppLayerDecoderEventsFreeEvents(&det_ctx->decoder_events);
     PrefilterPktNonPFStatsDump();
     SCFree(det_ctx);
@@ -5051,6 +5061,30 @@ void SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc fn, void *a
     DetectEngineDeReference(&de_ctx);
 }
 
+int DetectEngineThreadCtxGetJsonContext(DetectEngineThreadCtx *det_ctx)
+{
+    if (det_ctx->json_content_len > SIG_JSON_CONTENT_ARRAY_LEN - 1) {
+        SCLogDebug("json content length %u exceeds maximum %u", det_ctx->json_content_len,
+                SIG_JSON_CONTENT_ARRAY_LEN);
+        return -1;
+    }
+    if (det_ctx->json_content_len >= det_ctx->json_content_capacity) {
+        if (det_ctx->json_content_capacity == 0) {
+            det_ctx->json_content_capacity = 1;
+        } else {
+            det_ctx->json_content_capacity *= 2;
+        }
+        void *tmp = SCRealloc(
+                det_ctx->json_content, det_ctx->json_content_capacity * sizeof(SigJsonContent));
+        if (unlikely(tmp == NULL)) {
+            return -1;
+        }
+        SCLogDebug("reallocated json content array to %u items", det_ctx->json_content_capacity);
+        det_ctx->json_content = tmp;
+    }
+    return 0;
+}
+
 /*************************************Unittest*********************************/
 
 #ifdef UNITTESTS

diff --git a/src/detect-engine.h b/src/detect-engine.h
index f6b233ba03e97435347a434e74d04a7e8c562df6..0ba8322415b1bec48de5680d39ab5cbcdc3a06a5 100644 (file)
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -79,6 +79,7 @@ DetectEngineCtx *DetectEngineCtxInit(void);
 DetectEngineCtx *DetectEngineCtxInitStubForDD(void);
 DetectEngineCtx *DetectEngineCtxInitStubForMT(void);
 void DetectEngineCtxFree(DetectEngineCtx *);
+int DetectEngineThreadCtxGetJsonContext(DetectEngineThreadCtx *det_ctx);
 
 int DetectRegisterThreadCtxGlobalFuncs(const char *name,
         void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *));

diff --git a/src/detect-pcre.c b/src/detect-pcre.c
index 604b556a800f128a1d3115a617abfe78353fd479..4914db5548cd4c7312cbe9179bc56908f3f59555 100644 (file)
--- a/src/detect-pcre.c
+++ b/src/detect-pcre.c
@@ -168,8 +168,11 @@ static void DetectAlertStoreMatch(DetectEngineThreadCtx *det_ctx, const Signatur
 
     SCLogDebug("json key: %s", json_key);
     /* Setup the data*/
-    if ((det_ctx->json_content_len < SIG_JSON_CONTENT_ARRAY_LEN) &&
-            (capture_len + strlen(json_key) + 5 < SIG_JSON_CONTENT_ITEM_LEN)) {
+    if (capture_len + strlen(json_key) + 5 < SIG_JSON_CONTENT_ITEM_LEN) {
+        if (DetectEngineThreadCtxGetJsonContext(det_ctx) < 0) {
+            SCFree(str_ptr);
+            return;
+        }
         SCJsonBuilder *js = SCJbNewObject();
         if (unlikely(js == NULL)) {
             SCFree(str_ptr);

diff --git a/src/detect.h b/src/detect.h
index a5e9e1527738bdba62d8fa695150f3404b1bd435..5d06bd0937885a8b62f1f7c8a0cfdb66818d3af0 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -1278,7 +1278,9 @@ typedef struct DetectEngineThreadCtx_ {
     /* byte_* values */
     uint64_t *byte_values;
 
-    size_t json_content_len;
+    SigJsonContent *json_content;
+    uint8_t json_content_capacity;
+    uint8_t json_content_len;
 
     /* counter for the filestore array below -- up here for cache reasons. */
     uint16_t filestore_cnt;
@@ -1382,7 +1384,6 @@ typedef struct DetectEngineThreadCtx_ {
     /** stat of lua memory limit errors. */
     uint16_t lua_memory_limit_errors;
 
-    SigJsonContent json_content[SIG_JSON_CONTENT_ARRAY_LEN];
 #ifdef DEBUG
     uint64_t pkt_stream_add_cnt;
     uint64_t payload_mpm_cnt;

diff --git a/src/output-json-alert.c b/src/output-json-alert.c
index 2705933998a891c44f711806f9a284ed31561ceb..184370a65edc009d363d1f502732ed58f2749266 100644 (file)
--- a/src/output-json-alert.c
+++ b/src/output-json-alert.c
@@ -257,6 +257,7 @@ void AlertJsonHeader(const Packet *p, const PacketAlert *pa, SCJsonBuilder *js,
         SCJbOpenObject(js, "context");
         const struct PacketContextData *json_info = pa->json_info;
         while (json_info) {
+            SCLogDebug("JSON string '{%s}'", json_info->json_string);
             /* The string is valid json as it is validated by JANSSON
                during parsing and included later via a format string */
             SCJbSetFormatted(js, json_info->json_string);