From: Giuseppe Longo Date: Wed, 2 Aug 2017 13:55:01 +0000 (+0200) Subject: output-json-dns: add new output formats for v2 X-Git-Tag: suricata-4.1.0-beta1~53 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=756bed06a8f9d879fdd2b138a168223f3096698d;p=thirdparty%2Fsuricata.git output-json-dns: add new output formats for v2 This adds two new output formats that permits to reduce the number of line logged for a dns answer because actually an event is logged for each answer. With this patch, only an event that contains all the answers is logged. The formats are named 'detailed' and 'grouped'. The first format provides a list of answers with the following fields: - rrname - rrdata - ttl - rdata The second format provides a list of record data grouped by their type. The output below is an example of the formats: { "timestamp": "2017-11-29T10:27:18.148282+0100", "flow_id": 268864910185905, "in_iface": "wlp2s0", "event_type": "dns", "src_ip": "192.168.1.254", "src_port": 53, "dest_ip": "192.168.1.176", "dest_port": 52609, "proto": "UDP", "dns": { "type": "answer", "id": 3654, "rcode": "NOERROR", "answers": [ { "rrname": "wordpress.org", "rrtype": "A", "ttl": 544, "rdata": "66.155.40.249" }, { "rrname": "wordpress.org", "rrtype": "A", "ttl": 544, "rdata": "66.155.40.250" } ], "grouped": { "A": [ "66.155.40.249", "66.155.40.250" ] } } } --- diff --git a/src/output-json-dns.c b/src/output-json-dns.c index 1ab31e66a9..dca4c7fff9 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -123,7 +123,11 @@ #define LOG_ANY BIT_U64(58) #define LOG_URI BIT_U64(59) -#define LOG_ALL_RRTYPES (~(uint64_t)(LOG_QUERIES|LOG_ANSWERS)) +#define LOG_FORMAT_GROUPED BIT_U64(60) +#define LOG_FORMAT_DETAILED BIT_U64(61) + +#define LOG_FORMAT_ALL (LOG_FORMAT_GROUPED|LOG_FORMAT_DETAILED) +#define LOG_ALL_RRTYPES (~(uint64_t)(LOG_QUERIES|LOG_ANSWERS|LOG_FORMAT_DETAILED|LOG_FORMAT_GROUPED)) typedef enum { DNS_RRTYPE_A = 0, @@ -402,26 +406,13 @@ static int DNSRRTypeEnabled(uint16_t type, uint64_t flags) #endif #ifndef HAVE_RUST -static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, - uint64_t tx_id, DNSQueryEntry *entry) __attribute__((nonnull)); - -static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, - uint64_t tx_id, DNSQueryEntry *entry) +static json_t *OutputQuery(DNSTransaction *tx, uint64_t tx_id, DNSQueryEntry *entry) { - SCLogDebug("got a DNS request and now logging !!"); - - if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) { - return; - } - json_t *djs = json_object(); if (djs == NULL) { - return; + return NULL; } - /* reset */ - MemBufferReset(aft->buffer); - /* type */ json_object_set_new(djs, "type", json_string("query")); @@ -444,6 +435,26 @@ static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, /* tx id (tx counter) */ json_object_set_new(djs, "tx_id", json_integer(tx_id)); + return djs; +} + +static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, + uint64_t tx_id, DNSQueryEntry *entry) +{ + SCLogDebug("got a DNS request and now logging !!"); + + if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) { + return; + } + + json_t *djs = OutputQuery(tx, tx_id, entry); + if (djs == NULL) { + return; + } + + /* reset */ + MemBufferReset(aft->buffer); + /* dns */ json_object_set_new(js, "dns", djs); OutputJSONBuffer(js, aft->dnslog_ctx->file_ctx, &aft->buffer); @@ -452,10 +463,242 @@ static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, #endif #ifndef HAVE_RUST -static void OutputAnswer(LogDnsLogThread *aft, json_t *djs, - DNSTransaction *tx, DNSAnswerEntry *entry) __attribute__((nonnull)); -static void OutputAnswer(LogDnsLogThread *aft, json_t *djs, +static json_t *DnsParseSshFpType(DNSAnswerEntry *entry, uint8_t *ptr) +{ + /* get algo and type */ + uint8_t algo = *ptr; + uint8_t fptype = *(ptr+1); + + /* turn fp raw buffer into a nice :-separate hex string */ + uint16_t fp_len = (entry->data_len - 2); + uint8_t *dptr = ptr+2; + + /* c-string for ':' separated hex and trailing \0. */ + uint32_t output_len = fp_len * 3 + 1; + char hexstring[output_len]; + memset(hexstring, 0x00, output_len); + + uint16_t x; + for (x = 0; x < fp_len; x++) { + char one[4]; + snprintf(one, sizeof(one), x == fp_len - 1 ? "%02x" : "%02x:", dptr[x]); + strlcat(hexstring, one, output_len); + } + + /* wrap the whole thing in it's own structure */ + json_t *hjs = json_object(); + if (hjs == NULL) { + return NULL; + } + + json_object_set_new(hjs, "fingerprint", json_string(hexstring)); + json_object_set_new(hjs, "algo", json_integer(algo)); + json_object_set_new(hjs, "type", json_integer(fptype)); + + return hjs; +} + +static void OutputAnswerDetailed(DNSAnswerEntry *entry, json_t *js) +{ + do { + json_t *jdata = json_object(); + if (jdata == NULL) { + return; + } + + /* query */ + if (entry->fqdn_len > 0) { + char *c; + c = BytesToString((uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)), + entry->fqdn_len); + if (c != NULL) { + json_object_set_new(jdata, "rrname", json_string(c)); + SCFree(c); + } + } + + /* name */ + char record[16] = ""; + DNSCreateTypeString(entry->type, record, sizeof(record)); + json_object_set_new(jdata, "rrtype", json_string(record)); + + /* ttl */ + json_object_set_new(jdata, "ttl", json_integer(entry->ttl)); + + uint8_t *ptr = (uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)+ entry->fqdn_len); + if (entry->type == DNS_RECORD_TYPE_A && entry->data_len == 4) { + char a[16] = ""; + PrintInet(AF_INET, (const void *)ptr, a, sizeof(a)); + json_object_set_new(jdata, "rdata", json_string(a)); + } else if (entry->type == DNS_RECORD_TYPE_AAAA && entry->data_len == 16) { + char a[46] = ""; + PrintInet(AF_INET6, (const void *)ptr, a, sizeof(a)); + json_object_set_new(jdata, "rdata", json_string(a)); + } else if (entry->data_len == 0) { + json_object_set_new(jdata, "rdata", json_string("")); + } else if (entry->type == DNS_RECORD_TYPE_TXT || entry->type == DNS_RECORD_TYPE_CNAME || + entry->type == DNS_RECORD_TYPE_MX || entry->type == DNS_RECORD_TYPE_PTR || + entry->type == DNS_RECORD_TYPE_NS) { + if (entry->data_len != 0) { + char buffer[256] = ""; + uint16_t copy_len = entry->data_len < (sizeof(buffer) - 1) ? + entry->data_len : sizeof(buffer) - 1; + memcpy(buffer, ptr, copy_len); + buffer[copy_len] = '\0'; + json_object_set_new(jdata, "rdata", json_string(buffer)); + } else { + json_object_set_new(jdata, "rdata", json_string("")); + } + } else if (entry->type == DNS_RECORD_TYPE_SSHFP) { + if (entry->data_len > 2) { + json_t *hjs = DnsParseSshFpType(entry, ptr); + if (hjs != NULL) { + json_object_set_new(jdata, "sshfp", hjs); + } + } + } + json_array_append_new(js, jdata); + } while ((entry = TAILQ_NEXT(entry, next))); +} + +static void OutputAnswerGrouped(DNSAnswerEntry *entry, json_t *js) +{ + struct { + #define ENTRY_TYPE_A 0 + #define ENTRY_TYPE_AAAA 1 + #define ENTRY_TYPE_TXT 2 + #define ENTRY_TYPE_CNAME 3 + #define ENTRY_TYPE_MX 4 + #define ENTRY_TYPE_PTR 5 + #define ENTRY_TYPE_NS 6 + #define ENTRY_TYPE_SSHFP 7 + #define ENTRY_TYPE_MAX 8 + const char *name; + json_t *value; + } dns_rtypes[] = { + { "A", NULL }, + { "AAAA", NULL }, + { "TXT", NULL }, + { "CNAME", NULL }, + { "MX", NULL }, + { "PTR", NULL }, + { "NS", NULL }, + { "SSHFP", NULL } + }; + + int i; + json_t *jrdata = json_object(); + if (jrdata == NULL) { + return; + } + + do { + uint8_t *ptr = (uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)+ entry->fqdn_len); + if (entry->type == DNS_RECORD_TYPE_A && entry->data_len == 4) { + char a[16] = ""; + if (dns_rtypes[ENTRY_TYPE_A].value == NULL) { + dns_rtypes[ENTRY_TYPE_A].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_A].value == NULL) { + goto out; + } + } + PrintInet(AF_INET, (const void *)ptr, a, sizeof(a)); + json_array_append_new(dns_rtypes[ENTRY_TYPE_A].value, json_string(a)); + } else if (entry->type == DNS_RECORD_TYPE_AAAA && entry->data_len == 16) { + char a[46] = ""; + if (dns_rtypes[ENTRY_TYPE_AAAA].value == NULL) { + dns_rtypes[ENTRY_TYPE_AAAA].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_AAAA].value == NULL) { + goto out; + } + } + PrintInet(AF_INET6, (const void *)ptr, a, sizeof(a)); + json_array_append_new(dns_rtypes[ENTRY_TYPE_AAAA].value, json_string(a)); + } else if (entry->data_len == 0) { + json_object_set_new(js, "rdata", json_string("")); + } else if (entry->type == DNS_RECORD_TYPE_TXT || entry->type == DNS_RECORD_TYPE_CNAME || + entry->type == DNS_RECORD_TYPE_MX || entry->type == DNS_RECORD_TYPE_PTR || + entry->type == DNS_RECORD_TYPE_NS) { + if (entry->data_len != 0) { + char buffer[256] = ""; + uint16_t copy_len = entry->data_len < (sizeof(buffer) - 1) ? + entry->data_len : sizeof(buffer) - 1; + memcpy(buffer, ptr, copy_len); + buffer[copy_len] = '\0'; + + if (entry->type == DNS_RECORD_TYPE_TXT) { + if (dns_rtypes[ENTRY_TYPE_TXT].value == NULL) { + dns_rtypes[ENTRY_TYPE_TXT].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_TXT].value == NULL) { + goto out; + } + } + json_array_append_new(dns_rtypes[ENTRY_TYPE_TXT].value, json_string(buffer)); + } else if (entry->type == DNS_RECORD_TYPE_CNAME) { + if (dns_rtypes[ENTRY_TYPE_CNAME].value == NULL) { + dns_rtypes[ENTRY_TYPE_CNAME].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_CNAME].value == NULL) { + goto out; + } + } + json_array_append_new(dns_rtypes[ENTRY_TYPE_CNAME].value, json_string(buffer)); + } else if (entry->type == DNS_RECORD_TYPE_MX) { + if (dns_rtypes[ENTRY_TYPE_MX].value == NULL) { + dns_rtypes[ENTRY_TYPE_MX].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_MX].value == NULL) { + goto out; + } + } + json_array_append_new(dns_rtypes[ENTRY_TYPE_MX].value, json_string(buffer)); + } else if (entry->type == DNS_RECORD_TYPE_PTR) { + if (dns_rtypes[ENTRY_TYPE_PTR].value == NULL) { + dns_rtypes[ENTRY_TYPE_PTR].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_PTR].value == NULL) { + goto out; + } + } + json_array_append_new(dns_rtypes[ENTRY_TYPE_PTR].value, json_string(buffer)); + } else if (entry->type == DNS_RECORD_TYPE_NS) { + if (dns_rtypes[ENTRY_TYPE_NS].value == NULL) { + dns_rtypes[ENTRY_TYPE_NS].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_NS].value == NULL) { + goto out; + } + } + json_array_append_new(dns_rtypes[ENTRY_TYPE_NS].value, json_string(buffer)); + } + } else { + json_object_set_new(js, "rdata", json_string("")); + } + } else if (entry->type == DNS_RECORD_TYPE_SSHFP) { + if (entry->data_len > 2) { + json_t *hjs = DnsParseSshFpType(entry, ptr); + if (hjs != NULL) { + if (dns_rtypes[ENTRY_TYPE_SSHFP].value == NULL) { + dns_rtypes[ENTRY_TYPE_SSHFP].value = json_array(); + if (dns_rtypes[ENTRY_TYPE_SSHFP].value == NULL) { + goto out; + } + } + json_array_append_new(dns_rtypes[ENTRY_TYPE_SSHFP].value, hjs); + } + } + } + } while ((entry = TAILQ_NEXT(entry, next))); + +out: + for (i = 0; i < ENTRY_TYPE_MAX; i++) { + if (dns_rtypes[i].value != NULL) { + json_object_set_new(jrdata, dns_rtypes[i].name, dns_rtypes[i].value); + dns_rtypes[i].value = NULL; + } + } + + json_object_set_new(js, "grouped", jrdata); +} + +static void OutputAnswerV1(LogDnsLogThread *aft, json_t *djs, DNSTransaction *tx, DNSAnswerEntry *entry) { if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) { @@ -538,33 +781,8 @@ static void OutputAnswer(LogDnsLogThread *aft, json_t *djs, } } else if (entry->type == DNS_RECORD_TYPE_SSHFP) { if (entry->data_len > 2) { - /* get algo and type */ - uint8_t algo = *ptr; - uint8_t fptype = *(ptr+1); - - /* turn fp raw buffer into a nice :-separate hex string */ - uint16_t fp_len = (entry->data_len - 2); - uint8_t *dptr = ptr+2; - - /* c-string for ':' separated hex and trailing \0. */ - uint32_t output_len = fp_len * 3 + 1; - char hexstring[output_len]; - memset(hexstring, 0x00, output_len); - - uint16_t x; - for (x = 0; x < fp_len; x++) { - char one[4]; - snprintf(one, sizeof(one), x == fp_len - 1 ? "%02x" : "%02x:", dptr[x]); - strlcat(hexstring, one, output_len); - } - - /* wrap the whole thing in it's own structure */ - json_t *hjs = json_object(); + json_t *hjs = DnsParseSshFpType(entry, ptr); if (hjs != NULL) { - json_object_set_new(hjs, "fingerprint", json_string(hexstring)); - json_object_set_new(hjs, "algo", json_integer(algo)); - json_object_set_new(hjs, "type", json_integer(fptype)); - json_object_set_new(js, "sshfp", hjs); } } @@ -578,6 +796,50 @@ static void OutputAnswer(LogDnsLogThread *aft, json_t *djs, return; } + +static void OutputAnswerV2(LogDnsLogThread *aft, json_t *djs, + DNSTransaction *tx, DNSAnswerEntry *entry) +{ + if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) { + return; + } + + json_t *js = json_object(); + if (js == NULL) + return; + + /* type */ + json_object_set_new(js, "type", json_string("answer")); + + /* id */ + json_object_set_new(js, "id", json_integer(tx->tx_id)); + + /* rcode */ + char rcode[16] = ""; + DNSCreateRcodeString(tx->rcode, rcode, sizeof(rcode)); + json_object_set_new(js, "rcode", json_string(rcode)); + + if (aft->dnslog_ctx->flags & LOG_FORMAT_DETAILED) { + json_t *jarray = json_array(); + if (jarray == NULL) + return; + + OutputAnswerDetailed(entry, jarray); + json_object_set_new(js, "answers", jarray); + } + + if (aft->dnslog_ctx->flags & LOG_FORMAT_GROUPED) { + OutputAnswerGrouped(entry, js); + } + + /* reset */ + MemBufferReset(aft->buffer); + json_object_set_new(djs, "dns", js); + OutputJSONBuffer(djs, aft->dnslog_ctx->file_ctx, &aft->buffer); + json_object_del(djs, "dns"); + + return; +} #endif #ifndef HAVE_RUST @@ -642,13 +904,20 @@ static void LogAnswers(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, uin } DNSAnswerEntry *entry = NULL; - TAILQ_FOREACH(entry, &tx->answer_list, next) { - OutputAnswer(aft, js, tx, entry); + if (aft->dnslog_ctx->version == DNS_VERSION_2) { + entry = TAILQ_FIRST(&tx->answer_list); + if (entry) { + OutputAnswerV2(aft, js, tx, entry); + } + } else { + TAILQ_FOREACH(entry, &tx->answer_list, next) { + OutputAnswerV1(aft, js, tx, entry); + } } entry = NULL; TAILQ_FOREACH(entry, &tx->authority_list, next) { - OutputAnswer(aft, js, tx, entry); + OutputAnswerV1(aft, js, tx, entry); } } @@ -911,6 +1180,23 @@ static void JsonDnsLogInitFilters(LogDnsFileCtx *dnslog_ctx, ConfNode *conf) JsonDnsLogParseConfig(dnslog_ctx, conf, "query", "answer", "custom"); } else { JsonDnsLogParseConfig(dnslog_ctx, conf, "requests", "responses", "types"); + + if (dnslog_ctx->flags & LOG_ANSWERS) { + ConfNode *format; + if ((format = ConfNodeLookupChild(conf, "formats")) != NULL) { + dnslog_ctx->flags &= ~LOG_FORMAT_ALL; + ConfNode *field; + TAILQ_FOREACH(field, &format->head, next) { + if (strcasecmp(field->val, "detailed") == 0) { + dnslog_ctx->flags |= LOG_FORMAT_DETAILED; + } else if (strcasecmp(field->val, "grouped") == 0) { + dnslog_ctx->flags |= LOG_FORMAT_GROUPED; + } + } + } else { + dnslog_ctx->flags |= LOG_FORMAT_ALL; + } + } } } }