From: Eric Leblond Date: Sat, 29 Mar 2025 07:49:12 +0000 (+0100) Subject: datajson: add remove_key option to dataset X-Git-Tag: suricata-8.0.0-rc1~62 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3dde17fb75d131013ca586d82a184d3194245feb;p=thirdparty%2Fsuricata.git datajson: add remove_key option to dataset This option allows to remove the key corresponding to the match value from the JSON object before creating the JSON object that will be added to the `extra` data. For example, matching on the following JSON on the `ip` key: ```json {"ip": "10.16.1.11", "test": "success", "context":3} ``` with a match like: ``` dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip; ``` will produce the following: ```json "extra": { "src_ip": { "ip": "10.16.1.11", "test": "success", "context": 3 } ``` if we add the `remove_key` option to the match: ``` dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip, remove_key; ``` it will produce the following: ```json "extra": { "src_ip": { "test": "success", "context": 3 } ``` The option is set to false by default. Ticket: #7372 --- diff --git a/src/datajson.c b/src/datajson.c index f805d89541..7df205cd87 100644 --- a/src/datajson.c +++ b/src/datajson.c @@ -347,8 +347,12 @@ static uint32_t DatajsonAddStringElement(Dataset *set, json_t *value, char *json *found = true; - const char *val = json_string_value(key); + char val[DATAJSON_JSON_LENGTH]; + strlcpy(val, json_string_value(key), DATAJSON_JSON_LENGTH - 1); DataJsonType elt = { .value = NULL, .len = 0 }; + if (set->remove_key) { + json_object_del(value, json_key); + } elt.value = json_dumps(value, JSON_COMPACT); elt.len = strlen(elt.value); @@ -408,6 +412,9 @@ static uint32_t DatajsonAddMd5Element(Dataset *set, json_t *value, char *json_ke return 0; } DataJsonType elt = { .value = NULL, .len = 0 }; + if (set->remove_key) { + json_object_del(value, json_key); + } elt.value = json_dumps(value, JSON_COMPACT); elt.len = strlen(elt.value); @@ -467,6 +474,9 @@ static uint32_t DatajsonAddSha256Element(Dataset *set, json_t *value, char *json return 0; } DataJsonType elt = { .value = NULL, .len = 0 }; + if (set->remove_key) { + json_object_del(value, json_key); + } elt.value = json_dumps(value, JSON_COMPACT); elt.len = strlen(elt.value); @@ -521,6 +531,9 @@ static uint32_t DatajsonAddIpv4Element(Dataset *set, json_t *value, char *json_k return 0; } DataJsonType elt = { .value = NULL, .len = 0 }; + if (set->remove_key) { + json_object_del(value, json_key); + } elt.value = json_dumps(value, JSON_COMPACT); elt.len = strlen(elt.value); @@ -577,6 +590,9 @@ static uint32_t DatajsonAddIPv6Element(Dataset *set, json_t *value, char *json_k return 0; } DataJsonType elt = { .value = NULL, .len = 0 }; + if (set->remove_key) { + json_object_del(value, json_key); + } elt.value = json_dumps(value, JSON_COMPACT); elt.len = strlen(elt.value); @@ -616,7 +632,8 @@ static int DatajsonLoadIPv6(Dataset *set, char *json_key, char *array_key, Datas } Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap, - uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format) + uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format, + bool remove_key) { uint64_t default_memcap = 0; uint32_t default_hashsize = 0; @@ -665,6 +682,7 @@ Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, strlcpy(set->name, name, sizeof(set->name)); set->type = type; + set->remove_key = remove_key; if (load && strlen(load)) { strlcpy(set->load, load, sizeof(set->load)); SCLogDebug("set \'%s\' loading \'%s\' from \'%s\'", set->name, load, set->load); diff --git a/src/datajson.h b/src/datajson.h index 92116961ca..f6f93a0e1e 100644 --- a/src/datajson.h +++ b/src/datajson.h @@ -42,7 +42,8 @@ typedef struct DataJsonResultType { /* Common functions */ Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap, - uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format); + uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format, + bool remove_key); DataJsonResultType DatajsonLookup(Dataset *set, const uint8_t *data, const uint32_t data_len); diff --git a/src/datasets.h b/src/datasets.h index 752f0fa083..66d3026f5b 100644 --- a/src/datasets.h +++ b/src/datasets.h @@ -50,6 +50,7 @@ typedef struct Dataset { uint32_t id; bool from_yaml; /* Mark whether the set was retrieved from YAML */ bool hidden; /* Mark the old sets hidden in case of reload */ + bool remove_key; /* Mark that value key should be removed from extra data */ THashTableContext *hash; char load[PATH_MAX]; diff --git a/src/detect-dataset.c b/src/detect-dataset.c index 5b0cf237f4..1485c4b2e0 100644 --- a/src/detect-dataset.c +++ b/src/detect-dataset.c @@ -162,7 +162,7 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam enum DatasetTypes *type, char *load, size_t load_size, char *save, size_t save_size, uint64_t *memcap, uint32_t *hashsize, DatasetFormats *format, char *value_key, size_t value_key_size, char *array_key, size_t array_key_size, char *enrichment_key, - size_t enrichment_key_size) + size_t enrichment_key_size, bool *remove_key) { bool cmd_set = false; bool name_set = false; @@ -209,10 +209,12 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam name_set = true; } else { if (val == NULL) { - return -1; - } - - if (strcmp(key, "type") == 0) { + /* only non fixed place option without value is remove_key */ + if (strcmp(key, "remove_key") == 0) { + *remove_key = true; + } else + return -1; + } else if (strcmp(key, "type") == 0) { SCLogDebug("type %s", val); if (strcmp(val, "md5") == 0) { @@ -455,6 +457,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst char value_key[SIG_JSON_CONTENT_KEY_LEN] = ""; char array_key[SIG_JSON_CONTENT_KEY_LEN] = ""; char enrichment_key[SIG_JSON_CONTENT_KEY_LEN] = ""; + bool remove_key = false; if (DetectBufferGetActiveList(de_ctx, s) == -1) { SCLogError("datasets are only supported for sticky buffers"); @@ -470,7 +473,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name, sizeof(name), &type, load, sizeof(load), save, sizeof(save), &memcap, &hashsize, &format, value_key, sizeof(value_key), array_key, sizeof(array_key), enrichment_key, - sizeof(enrichment_key))) { + sizeof(enrichment_key), &remove_key)) { return -1; } @@ -529,11 +532,11 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst Dataset *set = NULL; if (format == DATASET_FORMAT_JSON) { - set = DatajsonGet( - name, type, load, memcap, hashsize, value_key, array_key, DATASET_FORMAT_JSON); + set = DatajsonGet(name, type, load, memcap, hashsize, value_key, array_key, + DATASET_FORMAT_JSON, remove_key); } else if (format == DATASET_FORMAT_JSONLINE) { - set = DatajsonGet( - name, type, load, memcap, hashsize, value_key, NULL, DATASET_FORMAT_JSONLINE); + set = DatajsonGet(name, type, load, memcap, hashsize, value_key, NULL, + DATASET_FORMAT_JSONLINE, remove_key); } else { set = DatasetGet(name, type, save, load, memcap, hashsize); }