]> git.ipfire.org Git - thirdparty/suricata.git/commitdiff
output-json-dns: add new output formats for v2
authorGiuseppe Longo <glongo@stamus-networks.com>
Wed, 2 Aug 2017 13:55:01 +0000 (15:55 +0200)
committerVictor Julien <victor@inliniac.net>
Wed, 14 Mar 2018 21:29:39 +0000 (22:29 +0100)
This adds two new output formats that permits to reduce
the number of line logged for a dns answer because
actually an event is logged for each answer.
With this patch, only an event that contains all the answers
is logged.

The formats are named 'detailed' and 'grouped'.

The first format provides a list of answers with
the following fields:
- rrname
- rrdata
- ttl
- rdata

The second format provides a list of record data grouped
by their type.

The output below is an example of the formats:

{
  "timestamp": "2017-11-29T10:27:18.148282+0100",
  "flow_id": 268864910185905,
  "in_iface": "wlp2s0",
  "event_type": "dns",
  "src_ip": "192.168.1.254",
  "src_port": 53,
  "dest_ip": "192.168.1.176",
  "dest_port": 52609,
  "proto": "UDP",
  "dns": {
    "type": "answer",
    "id": 3654,
    "rcode": "NOERROR",
    "answers": [
      {
        "rrname": "wordpress.org",
        "rrtype": "A",
        "ttl": 544,
        "rdata": "66.155.40.249"
      },
      {
        "rrname": "wordpress.org",
        "rrtype": "A",
        "ttl": 544,
        "rdata": "66.155.40.250"
      }
    ],
    "grouped": {
      "A": [
        "66.155.40.249",
        "66.155.40.250"
      ]
    }
  }
}

src/output-json-dns.c

index 1ab31e66a90be03676adf51a671e707093787d74..dca4c7fff97bba0a64b8923e074e06adf6b19894 100644 (file)
 #define LOG_ANY        BIT_U64(58)
 #define LOG_URI        BIT_U64(59)
 
-#define LOG_ALL_RRTYPES (~(uint64_t)(LOG_QUERIES|LOG_ANSWERS))
+#define LOG_FORMAT_GROUPED     BIT_U64(60)
+#define LOG_FORMAT_DETAILED    BIT_U64(61)
+
+#define LOG_FORMAT_ALL (LOG_FORMAT_GROUPED|LOG_FORMAT_DETAILED)
+#define LOG_ALL_RRTYPES (~(uint64_t)(LOG_QUERIES|LOG_ANSWERS|LOG_FORMAT_DETAILED|LOG_FORMAT_GROUPED))
 
 typedef enum {
     DNS_RRTYPE_A = 0,
@@ -402,26 +406,13 @@ static int DNSRRTypeEnabled(uint16_t type, uint64_t flags)
 #endif
 
 #ifndef HAVE_RUST
-static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx,
-        uint64_t tx_id, DNSQueryEntry *entry) __attribute__((nonnull));
-
-static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx,
-        uint64_t tx_id, DNSQueryEntry *entry)
+static json_t *OutputQuery(DNSTransaction *tx, uint64_t tx_id, DNSQueryEntry *entry)
 {
-    SCLogDebug("got a DNS request and now logging !!");
-
-    if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) {
-        return;
-    }
-
     json_t *djs = json_object();
     if (djs == NULL) {
-        return;
+        return NULL;
     }
 
-    /* reset */
-    MemBufferReset(aft->buffer);
-
     /* type */
     json_object_set_new(djs, "type", json_string("query"));
 
@@ -444,6 +435,26 @@ static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx,
     /* tx id (tx counter) */
     json_object_set_new(djs, "tx_id", json_integer(tx_id));
 
+    return djs;
+}
+
+static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx,
+        uint64_t tx_id, DNSQueryEntry *entry)
+{
+    SCLogDebug("got a DNS request and now logging !!");
+
+    if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) {
+        return;
+    }
+
+    json_t *djs = OutputQuery(tx, tx_id, entry);
+    if (djs == NULL) {
+        return;
+    }
+
+    /* reset */
+    MemBufferReset(aft->buffer);
+
     /* dns */
     json_object_set_new(js, "dns", djs);
     OutputJSONBuffer(js, aft->dnslog_ctx->file_ctx, &aft->buffer);
@@ -452,10 +463,242 @@ static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx,
 #endif
 
 #ifndef HAVE_RUST
-static void OutputAnswer(LogDnsLogThread *aft, json_t *djs,
-        DNSTransaction *tx, DNSAnswerEntry *entry) __attribute__((nonnull));
 
-static void OutputAnswer(LogDnsLogThread *aft, json_t *djs,
+static json_t *DnsParseSshFpType(DNSAnswerEntry *entry, uint8_t *ptr)
+{
+    /* get algo and type */
+    uint8_t algo = *ptr;
+    uint8_t fptype = *(ptr+1);
+
+    /* turn fp raw buffer into a nice :-separate hex string */
+    uint16_t fp_len = (entry->data_len - 2);
+    uint8_t *dptr = ptr+2;
+
+    /* c-string for ':' separated hex and trailing \0. */
+    uint32_t output_len = fp_len * 3 + 1;
+    char hexstring[output_len];
+    memset(hexstring, 0x00, output_len);
+
+    uint16_t x;
+    for (x = 0; x < fp_len; x++) {
+        char one[4];
+        snprintf(one, sizeof(one), x == fp_len - 1 ? "%02x" : "%02x:", dptr[x]);
+        strlcat(hexstring, one, output_len);
+    }
+
+    /* wrap the whole thing in it's own structure */
+    json_t *hjs = json_object();
+    if (hjs == NULL) {
+        return NULL;
+    }
+
+    json_object_set_new(hjs, "fingerprint", json_string(hexstring));
+    json_object_set_new(hjs, "algo", json_integer(algo));
+    json_object_set_new(hjs, "type", json_integer(fptype));
+
+    return hjs;
+}
+
+static void OutputAnswerDetailed(DNSAnswerEntry *entry, json_t *js)
+{
+    do {
+        json_t *jdata = json_object();
+        if (jdata == NULL) {
+            return;
+        }
+
+        /* query */
+        if (entry->fqdn_len > 0) {
+            char *c;
+            c = BytesToString((uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)),
+                    entry->fqdn_len);
+            if (c != NULL) {
+                json_object_set_new(jdata, "rrname", json_string(c));
+                SCFree(c);
+            }
+        }
+
+        /* name */
+        char record[16] = "";
+        DNSCreateTypeString(entry->type, record, sizeof(record));
+        json_object_set_new(jdata, "rrtype", json_string(record));
+
+        /* ttl */
+        json_object_set_new(jdata, "ttl", json_integer(entry->ttl));
+
+        uint8_t *ptr = (uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)+ entry->fqdn_len);
+        if (entry->type == DNS_RECORD_TYPE_A && entry->data_len == 4) {
+            char a[16] = "";
+            PrintInet(AF_INET, (const void *)ptr, a, sizeof(a));
+            json_object_set_new(jdata, "rdata", json_string(a));
+        } else if (entry->type == DNS_RECORD_TYPE_AAAA && entry->data_len == 16) {
+            char a[46] = "";
+            PrintInet(AF_INET6, (const void *)ptr, a, sizeof(a));
+            json_object_set_new(jdata, "rdata", json_string(a));
+        } else if (entry->data_len == 0) {
+            json_object_set_new(jdata, "rdata", json_string(""));
+        } else if (entry->type == DNS_RECORD_TYPE_TXT || entry->type == DNS_RECORD_TYPE_CNAME ||
+                entry->type == DNS_RECORD_TYPE_MX || entry->type == DNS_RECORD_TYPE_PTR ||
+                entry->type == DNS_RECORD_TYPE_NS) {
+            if (entry->data_len != 0) {
+                char buffer[256] = "";
+                uint16_t copy_len = entry->data_len < (sizeof(buffer) - 1) ?
+                    entry->data_len : sizeof(buffer) - 1;
+                memcpy(buffer, ptr, copy_len);
+                buffer[copy_len] = '\0';
+                json_object_set_new(jdata, "rdata", json_string(buffer));
+            } else {
+                json_object_set_new(jdata, "rdata", json_string(""));
+            }
+        } else if (entry->type == DNS_RECORD_TYPE_SSHFP) {
+            if (entry->data_len > 2) {
+                json_t *hjs = DnsParseSshFpType(entry, ptr);
+                if (hjs != NULL) {
+                   json_object_set_new(jdata, "sshfp", hjs);
+                }
+            }
+        }
+        json_array_append_new(js, jdata);
+    } while ((entry = TAILQ_NEXT(entry, next)));
+}
+
+static void OutputAnswerGrouped(DNSAnswerEntry *entry, json_t *js)
+{
+    struct {
+        #define ENTRY_TYPE_A        0
+        #define ENTRY_TYPE_AAAA     1
+        #define ENTRY_TYPE_TXT      2
+        #define ENTRY_TYPE_CNAME    3
+        #define ENTRY_TYPE_MX       4
+        #define ENTRY_TYPE_PTR      5
+        #define ENTRY_TYPE_NS       6
+        #define ENTRY_TYPE_SSHFP    7
+        #define ENTRY_TYPE_MAX      8
+        const char *name;
+        json_t *value;
+    } dns_rtypes[] = {
+        { "A",      NULL },
+        { "AAAA",   NULL },
+        { "TXT",    NULL },
+        { "CNAME",  NULL },
+        { "MX",     NULL },
+        { "PTR",    NULL },
+        { "NS",     NULL },
+        { "SSHFP",  NULL }
+    };
+
+    int i;
+    json_t *jrdata = json_object();
+    if (jrdata == NULL) {
+        return;
+    }
+
+    do {
+        uint8_t *ptr = (uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)+ entry->fqdn_len);
+        if (entry->type == DNS_RECORD_TYPE_A && entry->data_len == 4) {
+            char a[16] = "";
+            if (dns_rtypes[ENTRY_TYPE_A].value == NULL) {
+                dns_rtypes[ENTRY_TYPE_A].value = json_array();
+                if (dns_rtypes[ENTRY_TYPE_A].value == NULL) {
+                    goto out;
+                }
+            }
+            PrintInet(AF_INET, (const void *)ptr, a, sizeof(a));
+            json_array_append_new(dns_rtypes[ENTRY_TYPE_A].value, json_string(a));
+        } else if (entry->type == DNS_RECORD_TYPE_AAAA && entry->data_len == 16) {
+            char a[46] = "";
+            if (dns_rtypes[ENTRY_TYPE_AAAA].value == NULL) {
+                dns_rtypes[ENTRY_TYPE_AAAA].value = json_array();
+                if (dns_rtypes[ENTRY_TYPE_AAAA].value == NULL) {
+                    goto out;
+                }
+            }
+            PrintInet(AF_INET6, (const void *)ptr, a, sizeof(a));
+            json_array_append_new(dns_rtypes[ENTRY_TYPE_AAAA].value, json_string(a));
+        } else if (entry->data_len == 0) {
+            json_object_set_new(js, "rdata", json_string(""));
+        } else if (entry->type == DNS_RECORD_TYPE_TXT || entry->type == DNS_RECORD_TYPE_CNAME ||
+                entry->type == DNS_RECORD_TYPE_MX || entry->type == DNS_RECORD_TYPE_PTR ||
+                entry->type == DNS_RECORD_TYPE_NS) {
+            if (entry->data_len != 0) {
+                char buffer[256] = "";
+                uint16_t copy_len = entry->data_len < (sizeof(buffer) - 1) ?
+                    entry->data_len : sizeof(buffer) - 1;
+                memcpy(buffer, ptr, copy_len);
+                buffer[copy_len] = '\0';
+
+                if (entry->type == DNS_RECORD_TYPE_TXT) {
+                    if (dns_rtypes[ENTRY_TYPE_TXT].value == NULL) {
+                        dns_rtypes[ENTRY_TYPE_TXT].value = json_array();
+                        if (dns_rtypes[ENTRY_TYPE_TXT].value == NULL) {
+                            goto out;
+                        }
+                    }
+                    json_array_append_new(dns_rtypes[ENTRY_TYPE_TXT].value, json_string(buffer));
+                } else if (entry->type == DNS_RECORD_TYPE_CNAME) {
+                    if (dns_rtypes[ENTRY_TYPE_CNAME].value == NULL) {
+                        dns_rtypes[ENTRY_TYPE_CNAME].value = json_array();
+                        if (dns_rtypes[ENTRY_TYPE_CNAME].value == NULL) {
+                            goto out;
+                        }
+                    }
+                    json_array_append_new(dns_rtypes[ENTRY_TYPE_CNAME].value, json_string(buffer));
+                } else if (entry->type == DNS_RECORD_TYPE_MX) {
+                    if (dns_rtypes[ENTRY_TYPE_MX].value == NULL) {
+                        dns_rtypes[ENTRY_TYPE_MX].value = json_array();
+                        if (dns_rtypes[ENTRY_TYPE_MX].value == NULL) {
+                            goto out;
+                        }
+                    }
+                    json_array_append_new(dns_rtypes[ENTRY_TYPE_MX].value, json_string(buffer));
+                } else if (entry->type == DNS_RECORD_TYPE_PTR) {
+                    if (dns_rtypes[ENTRY_TYPE_PTR].value == NULL) {
+                        dns_rtypes[ENTRY_TYPE_PTR].value = json_array();
+                        if (dns_rtypes[ENTRY_TYPE_PTR].value == NULL) {
+                            goto out;
+                        }
+                    }
+                    json_array_append_new(dns_rtypes[ENTRY_TYPE_PTR].value, json_string(buffer));
+                } else if (entry->type == DNS_RECORD_TYPE_NS) {
+                    if (dns_rtypes[ENTRY_TYPE_NS].value == NULL) {
+                        dns_rtypes[ENTRY_TYPE_NS].value = json_array();
+                        if (dns_rtypes[ENTRY_TYPE_NS].value == NULL) {
+                            goto out;
+                        }
+                    }
+                    json_array_append_new(dns_rtypes[ENTRY_TYPE_NS].value, json_string(buffer));
+                }
+            } else {
+                json_object_set_new(js, "rdata", json_string(""));
+            }
+        } else if (entry->type == DNS_RECORD_TYPE_SSHFP) {
+            if (entry->data_len > 2) {
+                json_t *hjs = DnsParseSshFpType(entry, ptr);
+                if (hjs != NULL) {
+                    if (dns_rtypes[ENTRY_TYPE_SSHFP].value == NULL) {
+                        dns_rtypes[ENTRY_TYPE_SSHFP].value = json_array();
+                        if (dns_rtypes[ENTRY_TYPE_SSHFP].value == NULL) {
+                            goto out;
+                        }
+                    }
+                    json_array_append_new(dns_rtypes[ENTRY_TYPE_SSHFP].value, hjs);
+                }
+            }
+        }
+    } while ((entry = TAILQ_NEXT(entry, next)));
+
+out:
+    for (i = 0; i < ENTRY_TYPE_MAX; i++) {
+        if (dns_rtypes[i].value != NULL) {
+            json_object_set_new(jrdata, dns_rtypes[i].name, dns_rtypes[i].value);
+            dns_rtypes[i].value = NULL;
+        }
+    }
+
+    json_object_set_new(js, "grouped", jrdata);
+}
+
+static void OutputAnswerV1(LogDnsLogThread *aft, json_t *djs,
         DNSTransaction *tx, DNSAnswerEntry *entry)
 {
     if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) {
@@ -538,33 +781,8 @@ static void OutputAnswer(LogDnsLogThread *aft, json_t *djs,
         }
     } else if (entry->type == DNS_RECORD_TYPE_SSHFP) {
         if (entry->data_len > 2) {
-            /* get algo and type */
-            uint8_t algo = *ptr;
-            uint8_t fptype = *(ptr+1);
-
-            /* turn fp raw buffer into a nice :-separate hex string */
-            uint16_t fp_len = (entry->data_len - 2);
-            uint8_t *dptr = ptr+2;
-
-            /* c-string for ':' separated hex and trailing \0. */
-            uint32_t output_len = fp_len * 3 + 1;
-            char hexstring[output_len];
-            memset(hexstring, 0x00, output_len);
-
-            uint16_t x;
-            for (x = 0; x < fp_len; x++) {
-                char one[4];
-                snprintf(one, sizeof(one), x == fp_len - 1 ? "%02x" : "%02x:", dptr[x]);
-                strlcat(hexstring, one, output_len);
-            }
-
-            /* wrap the whole thing in it's own structure */
-            json_t *hjs = json_object();
+            json_t *hjs = DnsParseSshFpType(entry, ptr);
             if (hjs != NULL) {
-                json_object_set_new(hjs, "fingerprint", json_string(hexstring));
-                json_object_set_new(hjs, "algo", json_integer(algo));
-                json_object_set_new(hjs, "type", json_integer(fptype));
-
                 json_object_set_new(js, "sshfp", hjs);
             }
         }
@@ -578,6 +796,50 @@ static void OutputAnswer(LogDnsLogThread *aft, json_t *djs,
 
     return;
 }
+
+static void OutputAnswerV2(LogDnsLogThread *aft, json_t *djs,
+        DNSTransaction *tx, DNSAnswerEntry *entry)
+{
+    if (!DNSRRTypeEnabled(entry->type, aft->dnslog_ctx->flags)) {
+        return;
+    }
+
+    json_t *js = json_object();
+    if (js == NULL)
+        return;
+
+    /* type */
+    json_object_set_new(js, "type", json_string("answer"));
+
+    /* id */
+    json_object_set_new(js, "id", json_integer(tx->tx_id));
+
+    /* rcode */
+    char rcode[16] = "";
+    DNSCreateRcodeString(tx->rcode, rcode, sizeof(rcode));
+    json_object_set_new(js, "rcode", json_string(rcode));
+
+    if (aft->dnslog_ctx->flags & LOG_FORMAT_DETAILED) {
+        json_t *jarray = json_array();
+        if (jarray == NULL)
+            return;
+
+        OutputAnswerDetailed(entry, jarray);
+        json_object_set_new(js, "answers", jarray);
+    }
+
+    if (aft->dnslog_ctx->flags & LOG_FORMAT_GROUPED) {
+        OutputAnswerGrouped(entry, js);
+    }
+
+    /* reset */
+    MemBufferReset(aft->buffer);
+    json_object_set_new(djs, "dns", js);
+    OutputJSONBuffer(djs, aft->dnslog_ctx->file_ctx, &aft->buffer);
+    json_object_del(djs, "dns");
+
+    return;
+}
 #endif
 
 #ifndef HAVE_RUST
@@ -642,13 +904,20 @@ static void LogAnswers(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, uin
     }
 
     DNSAnswerEntry *entry = NULL;
-    TAILQ_FOREACH(entry, &tx->answer_list, next) {
-        OutputAnswer(aft, js, tx, entry);
+    if (aft->dnslog_ctx->version == DNS_VERSION_2) {
+        entry = TAILQ_FIRST(&tx->answer_list);
+        if (entry) {
+            OutputAnswerV2(aft, js, tx, entry);
+        }
+    } else {
+        TAILQ_FOREACH(entry, &tx->answer_list, next) {
+            OutputAnswerV1(aft, js, tx, entry);
+        }
     }
 
     entry = NULL;
     TAILQ_FOREACH(entry, &tx->authority_list, next) {
-        OutputAnswer(aft, js, tx, entry);
+        OutputAnswerV1(aft, js, tx, entry);
     }
 
 }
@@ -911,6 +1180,23 @@ static void JsonDnsLogInitFilters(LogDnsFileCtx *dnslog_ctx, ConfNode *conf)
             JsonDnsLogParseConfig(dnslog_ctx, conf, "query", "answer", "custom");
         } else {
             JsonDnsLogParseConfig(dnslog_ctx, conf, "requests", "responses", "types");
+
+            if (dnslog_ctx->flags & LOG_ANSWERS) {
+                ConfNode *format;
+                if ((format = ConfNodeLookupChild(conf, "formats")) != NULL) {
+                    dnslog_ctx->flags &= ~LOG_FORMAT_ALL;
+                    ConfNode *field;
+                    TAILQ_FOREACH(field, &format->head, next) {
+                        if (strcasecmp(field->val, "detailed") == 0) {
+                            dnslog_ctx->flags |= LOG_FORMAT_DETAILED;
+                        } else if (strcasecmp(field->val, "grouped") == 0) {
+                            dnslog_ctx->flags |= LOG_FORMAT_GROUPED;
+                        }
+                    }
+                } else {
+                    dnslog_ctx->flags |= LOG_FORMAT_ALL;
+                }
+            }
         }
     }
 }