From: Victor Julien Date: Thu, 7 Dec 2017 10:29:06 +0000 (+0100) Subject: eve/json: introduce community flow id X-Git-Tag: suricata-4.1.0-rc2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c4d8508f511030d728962541ffab10cf172de948;p=thirdparty%2Fsuricata.git eve/json: introduce community flow id Add support for community flow id, meant to give a records a predictable flow id that can be used to match records to output of other tools. Takes a 'seed' that needs to be same across sensors and tools to make the id less predictable. --- diff --git a/src/output-json-email-common.h b/src/output-json-email-common.h index 3f35a56f37..70deaf9e57 100644 --- a/src/output-json-email-common.h +++ b/src/output-json-email-common.h @@ -24,6 +24,7 @@ #ifndef __OUTPUT_JSON_EMAIL_COMMON_H__ #define __OUTPUT_JSON_EMAIL_COMMON_H__ +#ifdef HAVE_LIBJANSSON typedef struct OutputJsonEmailCtx_ { LogFileCtx *file_ctx; uint32_t flags; /** Store mode */ @@ -31,8 +32,6 @@ typedef struct OutputJsonEmailCtx_ { OutputJsonCommonSettings cfg; } OutputJsonEmailCtx; - -#ifdef HAVE_LIBJANSSON typedef struct JsonEmailLogThread_ { OutputJsonEmailCtx *emaillog_ctx; MemBuffer *buffer; @@ -40,8 +39,8 @@ typedef struct JsonEmailLogThread_ { TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id); json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id); -#endif void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx); +#endif /* HAVE_LIBJANSSON */ #endif /* __OUTPUT_JSON_EMAIL_COMMON_H__ */ diff --git a/src/output-json.c b/src/output-json.c index 87bcec30b6..ab488b9c70 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -59,6 +59,7 @@ #include "util-log-redis.h" #include "util-device.h" #include "util-validate.h" +#include "util-crypt.h" #include "flow-var.h" #include "flow-bit.h" @@ -90,6 +91,7 @@ void OutputJsonRegister (void) #define MAX_JSON_SIZE 2048 static void OutputJsonDeInitCtx(OutputCtx *); +static void CreateJSONCommunityFlowId(json_t *js, const Flow *f, const uint16_t seed); static const char *TRAFFIC_ID_PREFIX = "traffic/id/"; static const char *TRAFFIC_LABEL_PREFIX = "traffic/label/"; @@ -391,6 +393,9 @@ void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, if (cfg->include_metadata) { JsonAddMetadata(p, f, js); } + if (cfg->include_community_id && f != NULL) { + CreateJSONCommunityFlowId(js, f, cfg->community_id_seed); + } } /** \brief jsonify tcp flags field @@ -546,6 +551,124 @@ void JsonFiveTuple(const Packet *p, enum OutputJsonLogDirection dir, json_t *js) json_object_set_new(js, "proto", json_string(proto)); } +static void CreateJSONCommunityFlowIdv4(json_t *js, const Flow *f, + const uint16_t seed) +{ + struct { + uint16_t seed; + uint32_t src; + uint32_t dst; + uint8_t proto; + uint8_t pad0; + uint16_t sp; + uint16_t dp; + } __attribute__((__packed__)) ipv4; + + uint32_t src = f->src.addr_data32[0]; + uint32_t dst = f->dst.addr_data32[0]; + uint16_t sp = f->sp; + if (f->proto == IPPROTO_ICMP) + sp = f->icmp_s.type; + sp = htons(sp); + uint16_t dp = f->dp; + if (f->proto == IPPROTO_ICMP) + dp = f->icmp_d.type; + dp = htons(dp); + + ipv4.seed = htons(seed); + if (ntohl(src) < ntohl(dst) || (src == dst && sp < dp)) { + ipv4.src = src; + ipv4.dst = dst; + ipv4.sp = sp; + ipv4.dp = dp; + } else { + ipv4.src = dst; + ipv4.dst = src; + ipv4.sp = dp; + ipv4.dp = sp; + } + ipv4.proto = f->proto; + ipv4.pad0 = 0; + + uint8_t hash[20]; + if (ComputeSHA1((const uint8_t *)&ipv4, sizeof(ipv4), hash, sizeof(hash)) == 1) { + unsigned char base64buf[64] = "1:"; + unsigned long out_len = sizeof(base64buf) - 2; + if (Base64Encode(hash, sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) { + json_object_set_new(js, "community_id", json_string((const char *)base64buf)); + } + } +} + +static inline bool FlowHashRawAddressIPv6LtU32(const uint32_t *a, const uint32_t *b) +{ + for (int i = 0; i < 4; i++) { + if (a[i] < b[i]) + return true; + if (a[i] > b[i]) + break; + } + + return false; +} + +static void CreateJSONCommunityFlowIdv6(json_t *js, const Flow *f, + const uint16_t seed) +{ + struct { + uint16_t seed; + uint32_t src[4]; + uint32_t dst[4]; + uint8_t proto; + uint8_t pad0; + uint16_t sp; + uint16_t dp; + } __attribute__((__packed__)) ipv6; + + uint16_t sp = f->sp; + if (f->proto == IPPROTO_ICMPV6) + sp = f->icmp_s.type; + sp = htons(sp); + uint16_t dp = f->dp; + if (f->proto == IPPROTO_ICMPV6) + dp = f->icmp_d.type; + dp = htons(dp); + + ipv6.seed = htons(seed); + if (FlowHashRawAddressIPv6LtU32(f->src.addr_data32, f->dst.addr_data32) || + ((memcmp(&f->src, &f->dst, sizeof(f->src)) == 0) && sp < dp)) + { + memcpy(&ipv6.src, &f->src.addr_data32, 16); + memcpy(&ipv6.dst, &f->dst.addr_data32, 16); + ipv6.sp = sp; + ipv6.dp = dp; + } else { + memcpy(&ipv6.src, &f->dst.addr_data32, 16); + memcpy(&ipv6.dst, &f->src.addr_data32, 16); + ipv6.sp = dp; + ipv6.dp = sp; + } + ipv6.proto = f->proto; + ipv6.pad0 = 0; + + uint8_t hash[20]; + if (ComputeSHA1((const uint8_t *)&ipv6, sizeof(ipv6), hash, sizeof(hash)) == 1) { + unsigned char base64buf[64] = "1:"; + unsigned long out_len = sizeof(base64buf) - 2; + if (Base64Encode(hash, sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) { + json_object_set_new(js, "community_id", json_string((const char *)base64buf)); + } + } +} + +static void CreateJSONCommunityFlowId(json_t *js, const Flow *f, const uint16_t seed) +{ + if (f->flags & FLOW_IPV4) + return CreateJSONCommunityFlowIdv4(js, f, seed); + else if (f->flags & FLOW_IPV6) + return CreateJSONCommunityFlowIdv6(js, f, seed); +} + void CreateJSONFlowId(json_t *js, const Flow *f) { if (f == NULL) @@ -885,6 +1008,26 @@ OutputInitResult OutputJsonInitCtx(ConfNode *conf) json_ctx->cfg.include_metadata = true; } + /* See if we want to enable the community id */ + const ConfNode *community_id = ConfNodeLookupChild(conf, "community-id"); + if (community_id && community_id->val && ConfValIsTrue(community_id->val)) { + SCLogConfig("Enabling eve community_id logging."); + json_ctx->cfg.include_community_id = true; + } else { + json_ctx->cfg.include_community_id = false; + } + const char *cid_seed = ConfNodeLookupChildValue(conf, "community-id-seed"); + if (cid_seed != NULL) { + if (ByteExtractStringUint16(&json_ctx->cfg.community_id_seed, + 10, 0, cid_seed) == -1) + { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "Failed to initialize JSON output, " + "invalid community-id-seed: %s", cid_seed); + exit(EXIT_FAILURE); + } + } + /* Do we have a global eve xff configuration? */ const ConfNode *xff = ConfNodeLookupChild(conf, "xff"); if (xff != NULL) { diff --git a/src/output-json.h b/src/output-json.h index 140ab65cc0..5a4ed908bb 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -66,6 +66,8 @@ TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data); typedef struct OutputJsonCommonSettings_ { bool include_metadata; + bool include_community_id; + uint16_t community_id_seed; } OutputJsonCommonSettings; /* diff --git a/suricata.yaml.in b/suricata.yaml.in index 03777f32ed..99023c67b5 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -102,6 +102,19 @@ outputs: pcap-file: false + # Community Flow ID + # Adds a 'community_id' field to EVE records. These are meant to give + # a records a predictable flow id that can be used to match records to + # output of other tools such as Bro. + # + # Takes a 'seed' that needs to be same across sensors and tools + # to make the id less predictable. + + # enable/disable the community id feature. + community-id: false + # Seed value for the ID output. Valid values are 0-65535. + community-id-seed: 0 + # HTTP X-Forwarded-For support by adding an extra field or overwriting # the source or destination IP address (depending on flow direction) # with the one reported in the X-Forwarded-For HTTP header. This is