From: Jason Ish Date: Wed, 29 Apr 2020 17:45:22 +0000 (-0600) Subject: flow/eve: convert to jsonbuilder X-Git-Tag: suricata-6.0.0-beta1~379 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=baf272375724d2467b48b770785bae042fb1c022;p=thirdparty%2Fsuricata.git flow/eve: convert to jsonbuilder --- diff --git a/src/output-json-alert.c b/src/output-json-alert.c index b9b973f182..129beb5304 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -557,12 +557,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) if (p->flow) { EveAddAppProto(p->flow, jb); if (json_output_ctx->flags & LOG_JSON_FLOW) { - hjs = json_object(); - if (hjs != NULL) { - JsonAddFlow(p->flow, hjs); - jb_set_jsont(jb, "flow", hjs); - json_decref(hjs); - } + jb_open_object(jb, "flow"); + EveAddFlow(p->flow, jb); + jb_close(jb); } } diff --git a/src/output-json-flow.c b/src/output-json-flow.c index a3c26c17b5..2efaf353f4 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -62,15 +62,16 @@ typedef struct JsonFlowLogThread_ { MemBuffer *buffer; } JsonFlowLogThread; -static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type) +static JsonBuilder *CreateEveHeaderFromFlow(const Flow *f, const char *event_type) { char timebuf[64]; char srcip[46] = {0}, dstip[46] = {0}; Port sp, dp; - json_t *js = json_object(); - if (unlikely(js == NULL)) + JsonBuilder *jb = jb_new_object(); + if (unlikely(jb == NULL)) { return NULL; + } struct timeval tv; memset(&tv, 0x00, sizeof(tv)); @@ -108,9 +109,9 @@ static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type) } /* time */ - json_object_set_new(js, "timestamp", json_string(timebuf)); + jb_set_string(jb, "timestamp", timebuf); - CreateJSONFlowId(js, (const Flow *)f); + CreateEveFlowId(jb, (const Flow *)f); #if 0 // TODO /* sensor id */ @@ -120,84 +121,57 @@ static json_t *CreateJSONHeaderFromFlow(const Flow *f, const char *event_type) /* input interface */ if (f->livedev) { - json_object_set_new(js, "in_iface", json_string(f->livedev->dev)); + jb_set_string(jb, "in_iface", f->livedev->dev); } if (event_type) { - json_object_set_new(js, "event_type", json_string(event_type)); + jb_set_string(jb, "event_type", event_type); } /* vlan */ if (f->vlan_idx > 0) { - json_t *js_vlan = json_array(); - json_array_append_new(js_vlan, json_integer(f->vlan_id[0])); + jb_open_array(jb, "vlan"); + jb_append_uint(jb, f->vlan_id[0]); if (f->vlan_idx > 1) { - json_array_append_new(js_vlan, json_integer(f->vlan_id[1])); + jb_append_uint(jb, f->vlan_id[1]); } - json_object_set_new(js, "vlan", js_vlan); + jb_close(jb); } /* tuple */ - json_object_set_new(js, "src_ip", json_string(srcip)); + jb_set_string(jb, "src_ip", srcip); switch(f->proto) { case IPPROTO_ICMP: break; case IPPROTO_UDP: case IPPROTO_TCP: case IPPROTO_SCTP: - json_object_set_new(js, "src_port", json_integer(sp)); + jb_set_uint(jb, "src_port", sp); break; } - json_object_set_new(js, "dest_ip", json_string(dstip)); + jb_set_string(jb, "dest_ip", dstip); switch(f->proto) { case IPPROTO_ICMP: break; case IPPROTO_UDP: case IPPROTO_TCP: case IPPROTO_SCTP: - json_object_set_new(js, "dest_port", json_integer(dp)); + jb_set_uint(jb, "dest_port", dp); break; } - json_object_set_new(js, "proto", json_string(proto)); + jb_set_string(jb, "proto", proto); switch (f->proto) { case IPPROTO_ICMP: case IPPROTO_ICMPV6: - json_object_set_new(js, "icmp_type", - json_integer(f->icmp_s.type)); - json_object_set_new(js, "icmp_code", - json_integer(f->icmp_s.code)); + jb_set_uint(jb, "icmp_type", f->icmp_s.type); + jb_set_uint(jb, "icmp_code", f->icmp_s.code); if (f->tosrcpktcnt) { - json_object_set_new(js, "response_icmp_type", - json_integer(f->icmp_d.type)); - json_object_set_new(js, "response_icmp_code", - json_integer(f->icmp_d.code)); + jb_set_uint(jb, "response_icmp_type", f->icmp_d.type); + jb_set_uint(jb, "response_icmp_code", f->icmp_d.code); } break; } - return js; -} - -void JsonAddAppProto(Flow *f, json_t *js) -{ - json_object_set_new(js, "app_proto", - json_string(AppProtoToString(f->alproto))); - if (f->alproto_ts != f->alproto) { - json_object_set_new(js, "app_proto_ts", - json_string(AppProtoToString(f->alproto_ts))); - } - if (f->alproto_tc != f->alproto) { - json_object_set_new(js, "app_proto_tc", - json_string(AppProtoToString(f->alproto_tc))); - } - if (f->alproto_orig != f->alproto && f->alproto_orig != ALPROTO_UNKNOWN) { - json_object_set_new(js, "app_proto_orig", - json_string(AppProtoToString(f->alproto_orig))); - } - if (f->alproto_expect != f->alproto && f->alproto_expect != ALPROTO_UNKNOWN) { - json_object_set_new(js, "app_proto_expected", - json_string(AppProtoToString(f->alproto_expect))); - } - + return jb; } void EveAddAppProto(Flow *f, JsonBuilder *js) @@ -221,68 +195,51 @@ void EveAddAppProto(Flow *f, JsonBuilder *js) } -void JsonAddFlow(Flow *f, json_t *js) +void EveAddFlow(Flow *f, JsonBuilder *js) { FlowBypassInfo *fc = FlowGetStorageById(f, GetFlowBypassInfoID()); if (fc) { - json_object_set_new(js, "pkts_toserver", - json_integer(f->todstpktcnt + fc->todstpktcnt)); - json_object_set_new(js, "pkts_toclient", - json_integer(f->tosrcpktcnt + fc->tosrcpktcnt)); - json_object_set_new(js, "bytes_toserver", - json_integer(f->todstbytecnt + fc->todstbytecnt)); - json_object_set_new(js, "bytes_toclient", - json_integer(f->tosrcbytecnt + fc->tosrcbytecnt)); - json_t *bjs = json_object(); - if (bjs != NULL) { - json_object_set_new(bjs, "pkts_toserver", - json_integer(fc->todstpktcnt)); - json_object_set_new(bjs, "pkts_toclient", - json_integer(fc->tosrcpktcnt)); - json_object_set_new(bjs, "bytes_toserver", - json_integer(fc->todstbytecnt)); - json_object_set_new(bjs, "bytes_toclient", - json_integer(fc->tosrcbytecnt)); - json_object_set_new(js, "bypassed", bjs); - } + jb_set_uint(js, "pkts_toserver", f->todstpktcnt + fc->todstpktcnt); + jb_set_uint(js, "pkts_toclient", f->tosrcpktcnt + fc->tosrcpktcnt); + jb_set_uint(js, "bytes_toserver", f->todstbytecnt + fc->todstbytecnt); + jb_set_uint(js, "bytes_toclient", f->tosrcbytecnt + fc->tosrcbytecnt); + + jb_open_object(js, "bypassed"); + jb_set_uint(js, "pkts_toserver", fc->todstpktcnt); + jb_set_uint(js, "pkts_toclient", fc->tosrcpktcnt); + jb_set_uint(js, "bytes_toserver", fc->todstbytecnt); + jb_set_uint(js, "bytes_toclient", fc->tosrcbytecnt); + jb_close(js); } else { - json_object_set_new(js, "pkts_toserver", - json_integer(f->todstpktcnt)); - json_object_set_new(js, "pkts_toclient", - json_integer(f->tosrcpktcnt)); - json_object_set_new(js, "bytes_toserver", - json_integer(f->todstbytecnt)); - json_object_set_new(js, "bytes_toclient", - json_integer(f->tosrcbytecnt)); + jb_set_uint(js, "pkts_toserver", f->todstpktcnt); + jb_set_uint(js, "pkts_toclient", f->tosrcpktcnt); + jb_set_uint(js, "bytes_toserver", f->todstbytecnt); + jb_set_uint(js, "bytes_toclient", f->tosrcbytecnt); } char timebuf1[64]; CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1)); - json_object_set_new(js, "start", json_string(timebuf1)); + jb_set_string(js, "start", timebuf1); } -/* JSON format logging */ -static void JsonFlowLogJSON(JsonFlowLogThread *aft, json_t *js, Flow *f) +/* Eve format logging */ +static void EveFlowLogJSON(JsonFlowLogThread *aft, JsonBuilder *jb, Flow *f) { LogJsonFileCtx *flow_ctx = aft->flowlog_ctx; - json_t *hjs = json_object(); - if (hjs == NULL) { - return; - } - JsonAddAppProto(f, js); - JsonAddFlow(f, hjs); + EveAddAppProto(f, jb); + jb_open_object(jb, "flow"); + EveAddFlow(f, jb); char timebuf2[64]; CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2)); - json_object_set_new(hjs, "end", json_string(timebuf2)); + jb_set_string(jb, "end", timebuf2); int32_t age = f->lastts.tv_sec - f->startts.tv_sec; - json_object_set_new(hjs, "age", - json_integer(age)); + jb_set_uint(jb, "age", age); if (f->flow_end_flags & FLOW_END_FLAG_EMERGENCY) - json_object_set_new(hjs, "emergency", json_true()); + jb_set_bool(jb, "emergency", true); const char *state = NULL; if (f->flow_end_flags & FLOW_END_FLAG_STATE_NEW) state = "new"; @@ -295,13 +252,11 @@ static void JsonFlowLogJSON(JsonFlowLogThread *aft, json_t *js, Flow *f) int flow_state = SC_ATOMIC_GET(f->flow_state); switch (flow_state) { case FLOW_STATE_LOCAL_BYPASSED: - json_object_set_new(hjs, "bypass", - json_string("local")); + jb_set_string(jb, "bypass", "local"); break; #ifdef CAPTURE_OFFLOAD case FLOW_STATE_CAPTURE_BYPASSED: - json_object_set_new(hjs, "bypass", - json_string("capture")); + jb_set_string(jb, "bypass", "capture"); break; #endif default: @@ -311,8 +266,7 @@ static void JsonFlowLogJSON(JsonFlowLogThread *aft, json_t *js, Flow *f) } } - json_object_set_new(hjs, "state", - json_string(state)); + jb_set_string(jb, "state", state); const char *reason = NULL; if (f->flow_end_flags & FLOW_END_FLAG_TIMEOUT) @@ -322,40 +276,37 @@ static void JsonFlowLogJSON(JsonFlowLogThread *aft, json_t *js, Flow *f) else if (f->flow_end_flags & FLOW_END_FLAG_SHUTDOWN) reason = "shutdown"; - json_object_set_new(hjs, "reason", - json_string(reason)); + jb_set_string(jb, "reason", reason); - json_object_set_new(hjs, "alerted", json_boolean(FlowHasAlerts(f))); + jb_set_bool(jb, "alerted", FlowHasAlerts(f)); if (f->flags & FLOW_WRONG_THREAD) - json_object_set_new(hjs, "wrong_thread", json_true()); + jb_set_bool(jb, "wrong_thread", true); - json_object_set_new(js, "flow", hjs); + /* Close flow. */ + jb_close(jb); - JsonAddCommonOptions(&flow_ctx->cfg, NULL, f, js); + EveAddCommonOptions(&flow_ctx->cfg, NULL, f, jb); /* TCP */ if (f->proto == IPPROTO_TCP) { - json_t *tjs = json_object(); - if (tjs == NULL) { - return; - } + jb_open_object(jb, "tcp"); TcpSession *ssn = f->protoctx; char hexflags[3]; snprintf(hexflags, sizeof(hexflags), "%02x", ssn ? ssn->tcp_packet_flags : 0); - json_object_set_new(tjs, "tcp_flags", json_string(hexflags)); + jb_set_string(jb, "tcp_flags", hexflags); snprintf(hexflags, sizeof(hexflags), "%02x", ssn ? ssn->client.tcp_flags : 0); - json_object_set_new(tjs, "tcp_flags_ts", json_string(hexflags)); + jb_set_string(jb, "tcp_flags_ts", hexflags); snprintf(hexflags, sizeof(hexflags), "%02x", ssn ? ssn->server.tcp_flags : 0); - json_object_set_new(tjs, "tcp_flags_tc", json_string(hexflags)); + jb_set_string(jb, "tcp_flags_tc", hexflags); - JsonTcpFlags(ssn ? ssn->tcp_packet_flags : 0, tjs); + EveTcpFlags(ssn ? ssn->tcp_packet_flags : 0, jb); if (ssn) { const char *tcp_state = NULL; @@ -397,14 +348,15 @@ static void JsonFlowLogJSON(JsonFlowLogThread *aft, json_t *js, Flow *f) tcp_state = "closed"; break; } - json_object_set_new(tjs, "state", json_string(tcp_state)); + jb_set_string(jb, "state", tcp_state); if (ssn->client.flags & STREAMTCP_STREAM_FLAG_GAP) - json_object_set_new(tjs, "gap_ts", json_true()); + jb_set_bool(jb, "gap_ts", true); if (ssn->server.flags & STREAMTCP_STREAM_FLAG_GAP) - json_object_set_new(tjs, "gap_tc", json_true()); + jb_set_bool(jb, "gap_tc", true); } - json_object_set_new(js, "tcp", tjs); + /* Close tcp. */ + jb_close(jb); } } @@ -416,17 +368,15 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) /* reset */ MemBufferReset(jhl->buffer); - json_t *js = CreateJSONHeaderFromFlow(f, "flow"); - if (unlikely(js == NULL)) + JsonBuilder *jb = CreateEveHeaderFromFlow(f, "flow"); + if (unlikely(jb == NULL)) { return TM_ECODE_OK; + } - JsonFlowLogJSON(jhl, js, f); - - OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer); - json_object_del(js, "http"); + EveFlowLogJSON(jhl, jb, f); - json_object_clear(js); - json_decref(js); + OutputJsonBuilderBuffer(jb, jhl->flowlog_ctx->file_ctx, &jhl->buffer); + jb_free(jb); SCReturnInt(TM_ECODE_OK); } diff --git a/src/output-json-flow.h b/src/output-json-flow.h index ffdeafef5b..871968ac5e 100644 --- a/src/output-json-flow.h +++ b/src/output-json-flow.h @@ -25,8 +25,7 @@ #define __OUTPUT_JSON_FLOW_H__ void JsonFlowLogRegister(void); -void JsonAddFlow(Flow *f, json_t *js); -void JsonAddAppProto(Flow *f, json_t *js); +void EveAddFlow(Flow *f, JsonBuilder *js); void EveAddAppProto(Flow *f, JsonBuilder *js); #endif /* __OUTPUT_JSON_FLOW_H__ */ diff --git a/src/output-json.c b/src/output-json.c index 11aa6eefe2..fc0caafc72 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -694,6 +694,29 @@ void JsonTcpFlags(uint8_t flags, json_t *js) json_object_set_new(js, "cwr", json_true()); } +/** \brief jsonify tcp flags field + * Only add 'true' fields in an attempt to keep things reasonably compact. + */ +void EveTcpFlags(const uint8_t flags, JsonBuilder *js) +{ + if (flags & TH_SYN) + jb_set_bool(js, "syn", true); + if (flags & TH_FIN) + jb_set_bool(js, "fin", true); + if (flags & TH_RST) + jb_set_bool(js, "rst", true); + if (flags & TH_PUSH) + jb_set_bool(js, "psh", true); + if (flags & TH_ACK) + jb_set_bool(js, "ack", true); + if (flags & TH_URG) + jb_set_bool(js, "urg", true); + if (flags & TH_ECN) + jb_set_bool(js, "ecn", true); + if (flags & TH_CWR) + jb_set_bool(js, "cwr", true); +} + void JsonAddrInfoInit(const Packet *p, enum OutputJsonLogDirection dir, JsonAddrInfo *addr) { char srcip[46] = {0}, dstip[46] = {0}; diff --git a/src/output-json.h b/src/output-json.h index 01c0070232..63f5350478 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -72,6 +72,7 @@ int OutputJSONMemBufferCallback(const char *str, size_t size, void *data); void CreateJSONFlowId(json_t *js, const Flow *f); void CreateEveFlowId(JsonBuilder *js, const Flow *f); void JsonTcpFlags(uint8_t flags, json_t *js); +void EveTcpFlags(uint8_t flags, JsonBuilder *js); void JsonPacket(const Packet *p, json_t *js, unsigned long max_length); void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length); void JsonFiveTuple(const Packet *, enum OutputJsonLogDirection, json_t *);