datajson: add remove_key option to dataset

This option allows to remove the key corresponding to the match
value from the JSON object before creating the JSON object that
will be added to the `extra` data.

For example, matching on the following JSON on the `ip` key:

```json
{"ip": "10.16.1.11", "test": "success", "context":3}
```

with a match like:

```
dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip;
```

will produce the following:

```json
"extra": {
  "src_ip": {
    "ip": "10.16.1.11",
    "test": "success",
    "context": 3
  }
```

if we add the `remove_key` option to the match:

```
dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip, remove_key;
```

it will produce the following:

```json
"extra": {
  "src_ip": {
    "test": "success",
    "context": 3
  }
```

The option is set to false by default.

Ticket: #7372

diff --git a/src/datajson.c b/src/datajson.c
index f805d89541dee1beb1a32e359aec98295eaf13a5..7df205cd877536d25f7d047025e561bad46be251 100644 (file)
--- a/src/datajson.c
+++ b/src/datajson.c
@@ -347,8 +347,12 @@ static uint32_t DatajsonAddStringElement(Dataset *set, json_t *value, char *json
 
     *found = true;
 
-    const char *val = json_string_value(key);
+    char val[DATAJSON_JSON_LENGTH];
+    strlcpy(val, json_string_value(key), DATAJSON_JSON_LENGTH - 1);
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -408,6 +412,9 @@ static uint32_t DatajsonAddMd5Element(Dataset *set, json_t *value, char *json_ke
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -467,6 +474,9 @@ static uint32_t DatajsonAddSha256Element(Dataset *set, json_t *value, char *json
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -521,6 +531,9 @@ static uint32_t DatajsonAddIpv4Element(Dataset *set, json_t *value, char *json_k
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -577,6 +590,9 @@ static uint32_t DatajsonAddIPv6Element(Dataset *set, json_t *value, char *json_k
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -616,7 +632,8 @@ static int DatajsonLoadIPv6(Dataset *set, char *json_key, char *array_key, Datas
 }
 
 Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap,
-        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format)
+        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format,
+        bool remove_key)
 {
     uint64_t default_memcap = 0;
     uint32_t default_hashsize = 0;
@@ -665,6 +682,7 @@ Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load,
 
     strlcpy(set->name, name, sizeof(set->name));
     set->type = type;
+    set->remove_key = remove_key;
     if (load && strlen(load)) {
         strlcpy(set->load, load, sizeof(set->load));
         SCLogDebug("set \'%s\' loading \'%s\' from \'%s\'", set->name, load, set->load);

diff --git a/src/datajson.h b/src/datajson.h
index 92116961ca6b1c292bd9157d0b175501b24706a3..f6f93a0e1edf54a6906416b66c4eae90a439347a 100644 (file)
--- a/src/datajson.h
+++ b/src/datajson.h
@@ -42,7 +42,8 @@ typedef struct DataJsonResultType {
 /* Common functions */
 
 Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap,
-        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format);
+        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format,
+        bool remove_key);
 
 DataJsonResultType DatajsonLookup(Dataset *set, const uint8_t *data, const uint32_t data_len);
 

diff --git a/src/datasets.h b/src/datasets.h
index 752f0fa0832232d32c957094ac655e832676baba..66d3026f5b6b8d080e7f250e56451bdd27fe5380 100644 (file)
--- a/src/datasets.h
+++ b/src/datasets.h
@@ -50,6 +50,7 @@ typedef struct Dataset {
     uint32_t id;
     bool from_yaml;                     /* Mark whether the set was retrieved from YAML */
     bool hidden;                        /* Mark the old sets hidden in case of reload */
+    bool remove_key;                    /* Mark that value key should be removed from extra data */
     THashTableContext *hash;
 
     char load[PATH_MAX];

diff --git a/src/detect-dataset.c b/src/detect-dataset.c
index 5b0cf237f4ce17c5843edade9d960e88752c3d9a..1485c4b2e0c8270d0322744f22c4451635e8c656 100644 (file)
--- a/src/detect-dataset.c
+++ b/src/detect-dataset.c
@@ -162,7 +162,7 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
         enum DatasetTypes *type, char *load, size_t load_size, char *save, size_t save_size,
         uint64_t *memcap, uint32_t *hashsize, DatasetFormats *format, char *value_key,
         size_t value_key_size, char *array_key, size_t array_key_size, char *enrichment_key,
-        size_t enrichment_key_size)
+        size_t enrichment_key_size, bool *remove_key)
 {
     bool cmd_set = false;
     bool name_set = false;
@@ -209,10 +209,12 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
             name_set = true;
         } else {
             if (val == NULL) {
-                return -1;
-            }
-
-            if (strcmp(key, "type") == 0) {
+                /* only non fixed place option without value is remove_key */
+                if (strcmp(key, "remove_key") == 0) {
+                    *remove_key = true;
+                } else
+                    return -1;
+            } else if (strcmp(key, "type") == 0) {
                 SCLogDebug("type %s", val);
 
                 if (strcmp(val, "md5") == 0) {
@@ -455,6 +457,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     char value_key[SIG_JSON_CONTENT_KEY_LEN] = "";
     char array_key[SIG_JSON_CONTENT_KEY_LEN] = "";
     char enrichment_key[SIG_JSON_CONTENT_KEY_LEN] = "";
+    bool remove_key = false;
 
     if (DetectBufferGetActiveList(de_ctx, s) == -1) {
         SCLogError("datasets are only supported for sticky buffers");
@@ -470,7 +473,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name, sizeof(name), &type, load,
                 sizeof(load), save, sizeof(save), &memcap, &hashsize, &format, value_key,
                 sizeof(value_key), array_key, sizeof(array_key), enrichment_key,
-                sizeof(enrichment_key))) {
+                sizeof(enrichment_key), &remove_key)) {
         return -1;
     }
 
@@ -529,11 +532,11 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     Dataset *set = NULL;
 
     if (format == DATASET_FORMAT_JSON) {
-        set = DatajsonGet(
-                name, type, load, memcap, hashsize, value_key, array_key, DATASET_FORMAT_JSON);
+        set = DatajsonGet(name, type, load, memcap, hashsize, value_key, array_key,
+                DATASET_FORMAT_JSON, remove_key);
     } else if (format == DATASET_FORMAT_JSONLINE) {
-        set = DatajsonGet(
-                name, type, load, memcap, hashsize, value_key, NULL, DATASET_FORMAT_JSONLINE);
+        set = DatajsonGet(name, type, load, memcap, hashsize, value_key, NULL,
+                DATASET_FORMAT_JSONLINE, remove_key);
     } else {
         set = DatasetGet(name, type, save, load, memcap, hashsize);
     }