From: Tom DeCanio Date: Tue, 5 Nov 2013 04:33:51 +0000 (-0800) Subject: First cut at united .yaml configuration X-Git-Tag: suricata-2.0rc1~107 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=ce6b07b1b9dd2b71883ad09b9126d251c8e1dcbd;p=thirdparty%2Fsuricata.git First cut at united .yaml configuration --- diff --git a/src/alert-json.c b/src/alert-json.c index 51ecab3063..5903a70e55 100644 --- a/src/alert-json.c +++ b/src/alert-json.c @@ -47,7 +47,8 @@ #include "util-syslog.h" #include "output.h" -#include "alert-json.h" +#include "output-dnslog.h" +#include "output-httplog.h" #include "util-byte.h" #include "util-privs.h" @@ -57,6 +58,8 @@ #include "util-buffer.h" #include "util-logopenfile.h" +#include "alert-json.h" + /*#undef HAVE_LIBJANSSON for testing without messing with config */ #ifndef HAVE_LIBJANSSON /** Handle the case where no JSON support is compiled in. @@ -75,7 +78,12 @@ void TmModuleAlertJsonRegister (void) { tmm_modules[TMM_ALERTJSON].Func = AlertJson; tmm_modules[TMM_ALERTJSON].ThreadDeinit = AlertJsonThreadDeinit; tmm_modules[TMM_ALERTJSON].RegisterTests = AlertJsonRegisterTests; -} + + /* enable the logger for the app layer */ + AppLayerRegisterLogger(ALPROTO_DNS_UDP); + AppLayerRegisterLogger(ALPROTO_DNS_TCP); + + AppLayerRegisterLogger(ALPROTO_HTTP); OutputCtx *AlertJsonInitCtx(ConfNode *conf) { @@ -106,7 +114,7 @@ void AlertJsonRegisterTests (void) { #include -#define DEFAULT_LOG_FILENAME "json.log" +#define DEFAULT_LOG_FILENAME "eve.json" #define DEFAULT_ALERT_SYSLOG_FACILITY_STR "local0" #define DEFAULT_ALERT_SYSLOG_FACILITY LOG_LOCAL0 #define DEFAULT_ALERT_SYSLOG_LEVEL LOG_INFO @@ -137,7 +145,7 @@ void TmModuleAlertJsonRegister (void) { tmm_modules[TMM_ALERTJSON].RegisterTests = AlertJsonRegisterTests; tmm_modules[TMM_ALERTJSON].cap_flags = 0; - OutputRegisterModule(MODULE_NAME, "json-alert", AlertJsonInitCtx); + OutputRegisterModule(MODULE_NAME, "eve-log", AlertJsonInitCtx); } /* Default Sensor ID value */ @@ -149,33 +157,125 @@ enum json_output { ALERT_FILE, ALERT_UNIX_STREAM }; static enum json_output json_out = ALERT_FILE; +#define OUTPUT_ALERTS (1<<0) +#define OUTPUT_DNS (1<<1) +#define OUTPUT_HTTP (1<<2) + +static uint32_t outputFlags = 0; + enum json_format { COMPACT, INDENT }; static enum json_format format = COMPACT; -typedef struct AlertJsonThread_ { - /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ - LogFileCtx* file_ctx; +json_t *CreateJSONHeader(Packet *p, int direction_sensative) +{ + char timebuf[64]; + char srcip[46], dstip[46]; + Port sp, dp; + + json_t *js = json_object(); + if (unlikely(js == NULL)) + return NULL; + + CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); + + srcip[0] = '\0'; + dstip[0] = '\0'; + if (direction_sensative) { + if ((PKT_IS_TOCLIENT(p))) { + if (PKT_IS_IPV4(p)) { + PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); + } else if (PKT_IS_IPV6(p)) { + PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); + } + sp = p->sp; + dp = p->dp; + } else { + if (PKT_IS_IPV4(p)) { + PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), dstip, sizeof(dstip)); + } else if (PKT_IS_IPV6(p)) { + PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), dstip, sizeof(dstip)); + } + sp = p->dp; + dp = p->sp; + } + } else { + if (PKT_IS_IPV4(p)) { + PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); + } else if (PKT_IS_IPV6(p)) { + PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); + } + sp = p->sp; + dp = p->dp; + } + + char proto[16] = ""; + if (SCProtoNameValid(IPV4_GET_IPPROTO(p)) == TRUE) { + strlcpy(proto, known_proto[IPV4_GET_IPPROTO(p)], sizeof(proto)); + } else { + snprintf(proto, sizeof(proto), "PROTO:%03" PRIu32, IPV4_GET_IPPROTO(p)); + } + + /* time & tx */ + json_object_set_new(js, "time", json_string(timebuf)); + + /* sensor id */ + if (sensor_id >= 0) + json_object_set_new(js, "sensor-id", json_integer(sensor_id)); + - MemBuffer *buffer; -} AlertJsonThread; + /* tuple */ + json_object_set_new(js, "srcip", json_string(srcip)); + json_object_set_new(js, "sp", json_integer(sp)); + json_object_set_new(js, "dstip", json_string(dstip)); + json_object_set_new(js, "dp", json_integer(dp)); + json_object_set_new(js, "proto", json_string(proto)); + + return js; +} + +TmEcode OutputJSON(json_t *js, void *data, uint64_t *count) +{ + AlertJsonThread *aft = (AlertJsonThread *)data; + MemBuffer *buffer = (MemBuffer *)aft->buffer; + char *js_s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); + if (unlikely(js_s == NULL)) + return TM_ECODE_OK; + + SCMutexLock(&aft->file_ctx->fp_mutex); + if (json_out == ALERT_FILE) { + MemBufferWriteString(buffer, "%s\n", js_s); + (void)MemBufferPrintToFPAsString(buffer, aft->file_ctx->fp); + fflush(aft->file_ctx->fp); + } else { + syslog(alert_syslog_level, "%s", js_s); + } + *count += 1; + SCMutexUnlock(&aft->file_ctx->fp_mutex); + return TM_ECODE_OK; +} TmEcode AlertJsonIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { AlertJsonThread *aft = (AlertJsonThread *)data; + MemBuffer *buffer = (MemBuffer *)aft->buffer; int i; - char timebuf[64]; char *action = "Pass"; if (p->alerts.cnt == 0) return TM_ECODE_OK; - MemBufferReset(aft->buffer); + MemBufferReset(buffer); - CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); + json_t *js = CreateJSONHeader(p, 0); + if (unlikely(js == NULL)) + return TM_ECODE_OK; - char srcip[16], dstip[16]; - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); for (i = 0; i < p->alerts.cnt; i++) { PacketAlert *pa = &p->alerts.alerts[i]; if (unlikely(pa->s == NULL)) { @@ -188,38 +288,12 @@ TmEcode AlertJsonIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pa action = "wDrop"; } - char proto[16] = ""; - if (SCProtoNameValid(IPV4_GET_IPPROTO(p)) == TRUE) { - strlcpy(proto, known_proto[IPV4_GET_IPPROTO(p)], sizeof(proto)); - } else { - snprintf(proto, sizeof(proto), "PROTO:%03" PRIu32, IPV4_GET_IPPROTO(p)); - } - - json_t *js = json_object(); - if (js == NULL) - return TM_ECODE_OK; - json_t *ajs = json_object(); if (ajs == NULL) { free(js); return TM_ECODE_OK; } - /* time & tx */ - json_object_set_new(js, "time", json_string(timebuf)); - - /* sensor id */ - if (sensor_id >= 0) - json_object_set_new(js, "sensor-id", json_integer(sensor_id)); - - - /* tuple */ - json_object_set_new(js, "srcip", json_string(srcip)); - json_object_set_new(js, "sp", json_integer(p->sp)); - json_object_set_new(js, "dstip", json_string(dstip)); - json_object_set_new(js, "dp", json_integer(p->dp)); - json_object_set_new(js, "proto", json_string(proto)); - json_object_set_new(ajs, "action", json_string(action)); json_object_set_new(ajs, "gid", json_integer(pa->s->gid)); json_object_set_new(ajs, "id", json_integer(pa->s->id)); @@ -233,24 +307,11 @@ TmEcode AlertJsonIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pa /* alert */ json_object_set_new(js, "alert", ajs); - char *js_s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - if (unlikely(js_s == NULL)) - return TM_ECODE_OK; - - SCMutexLock(&aft->file_ctx->fp_mutex); - if (json_out == ALERT_FILE) { - MemBufferWriteString(aft->buffer, "%s\n", js_s); - (void)MemBufferPrintToFPAsString(aft->buffer, aft->file_ctx->fp); - fflush(aft->file_ctx->fp); - } else { - syslog(alert_syslog_level, "%s", js_s); - } - aft->file_ctx->alerts++; - SCMutexUnlock(&aft->file_ctx->fp_mutex); - free(ajs); - free(js); - free(js_s); + OutputJSON(js, aft, &aft->file_ctx->alerts); + json_object_del(js, "alert"); } + json_object_clear(js); + free(js); return TM_ECODE_OK; } @@ -258,20 +319,19 @@ TmEcode AlertJsonIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pa TmEcode AlertJsonIPv6(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { AlertJsonThread *aft = (AlertJsonThread *)data; + MemBuffer *buffer = (MemBuffer *)aft->buffer; int i; - char timebuf[64]; char *action = "Pass"; if (p->alerts.cnt == 0) return TM_ECODE_OK; - MemBufferReset(aft->buffer); + MemBufferReset(buffer); - CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); + json_t *js = CreateJSONHeader(p, 0); + if (unlikely(js == NULL)) + return TM_ECODE_OK; - char srcip[46], dstip[46]; - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); for (i = 0; i < p->alerts.cnt; i++) { PacketAlert *pa = &p->alerts.alerts[i]; if (unlikely(pa->s == NULL)) { @@ -284,33 +344,12 @@ TmEcode AlertJsonIPv6(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pa action = "wDrop"; } - char proto[16] = ""; - if (SCProtoNameValid(IP_GET_IPPROTO(p)) == TRUE) { - strlcpy(proto, known_proto[IP_GET_IPPROTO(p)], sizeof(proto)); - } else { - snprintf(proto, sizeof(proto), "PROTO:%03" PRIu32, IP_GET_IPPROTO(p)); - } - - json_t *js = json_object(); - if (js == NULL) - return TM_ECODE_OK; - json_t *ajs = json_object(); if (ajs == NULL) { free(js); return TM_ECODE_OK; } - /* time & tx */ - json_object_set_new(js, "time", json_string(timebuf)); - - /* tuple */ - json_object_set_new(js, "srcip", json_string(srcip)); - json_object_set_new(js, "sp", json_integer(p->sp)); - json_object_set_new(js, "dstip", json_string(dstip)); - json_object_set_new(js, "dp", json_integer(p->dp)); - json_object_set_new(js, "proto", json_string(proto)); - json_object_set_new(ajs, "action", json_string(action)); json_object_set_new(ajs, "gid", json_integer(pa->s->gid)); json_object_set_new(ajs, "id", json_integer(pa->s->id)); @@ -324,26 +363,11 @@ TmEcode AlertJsonIPv6(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pa /* alert */ json_object_set_new(js, "alert", ajs); - SCMutexLock(&aft->file_ctx->fp_mutex); - if (json_out == ALERT_FILE) { - char *s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - MemBufferWriteString(aft->buffer, "%s", s); - MemBufferWriteString(aft->buffer, "\n"); - free(s); - (void)MemBufferPrintToFPAsString(aft->buffer, aft->file_ctx->fp); - fflush(aft->file_ctx->fp); - } else { - char *js_s; - js_s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - if (js_s) { - syslog(alert_syslog_level, "%s", js_s); - free(js_s); - } - } - aft->file_ctx->alerts++; - SCMutexUnlock(&aft->file_ctx->fp_mutex); - free(js); + OutputJSON(js, aft, &aft->file_ctx->alerts); + json_object_del(js, "alert"); } + json_object_clear(js); + free(js); return TM_ECODE_OK; } @@ -351,14 +375,16 @@ TmEcode AlertJsonIPv6(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pa TmEcode AlertJsonDecoderEvent(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { AlertJsonThread *aft = (AlertJsonThread *)data; + MemBuffer *buffer = (MemBuffer *)aft->buffer; int i; char timebuf[64]; char *action = "Pass"; + json_t *js; if (p->alerts.cnt == 0) return TM_ECODE_OK; - MemBufferReset(aft->buffer); + MemBufferReset(buffer); CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); @@ -377,7 +403,7 @@ TmEcode AlertJsonDecoderEvent(ThreadVars *tv, Packet *p, void *data, PacketQueue char buf[(32 * 3) + 1]; PrintRawLineHexBuf(buf, sizeof(buf), GET_PKT_DATA(p), GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32); - json_t *js = json_object(); + js = json_object(); if (js == NULL) return TM_ECODE_OK; @@ -409,25 +435,8 @@ TmEcode AlertJsonDecoderEvent(ThreadVars *tv, Packet *p, void *data, PacketQueue /* alert */ json_object_set_new(js, "alert", ajs); - - SCMutexLock(&aft->file_ctx->fp_mutex); - if (json_out == ALERT_FILE) { - char *s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - MemBufferWriteString(aft->buffer, "%s", s); - MemBufferWriteString(aft->buffer, "\n"); - free(s); - (void)MemBufferPrintToFPAsString(aft->buffer, aft->file_ctx->fp); - fflush(aft->file_ctx->fp); - } else { - char *js_s; - js_s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - if (js_s) { - syslog(alert_syslog_level, "%s", js_s); - free(js_s); - } - } - aft->file_ctx->alerts++; - SCMutexUnlock(&aft->file_ctx->fp_mutex); + OutputJSON(js, aft, &aft->file_ctx->alerts); + json_object_clear(js); free(js); } @@ -436,12 +445,25 @@ TmEcode AlertJsonDecoderEvent(ThreadVars *tv, Packet *p, void *data, PacketQueue TmEcode AlertJson (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { - if (PKT_IS_IPV4(p)) { - return AlertJsonIPv4(tv, p, data, pq, postpq); - } else if (PKT_IS_IPV6(p)) { - return AlertJsonIPv6(tv, p, data, pq, postpq); - } else if (p->events.cnt > 0) { - return AlertJsonDecoderEvent(tv, p, data, pq, postpq); + if (outputFlags & OUTPUT_ALERTS) { + + if (PKT_IS_IPV4(p)) { + AlertJsonIPv4(tv, p, data, pq, postpq); + } else if (PKT_IS_IPV6(p)) { + AlertJsonIPv6(tv, p, data, pq, postpq); + } else if (p->events.cnt > 0) { + AlertJsonDecoderEvent(tv, p, data, pq, postpq); + } + } + + if (outputFlags & OUTPUT_DNS) { + if (OutputDnsNeedsLog(p)) { + OutputDnsLog(tv, p, data, pq, postpq); + } + } + + if (outputFlags & OUTPUT_HTTP) { + OutputHttpLog(tv, p, data, pq, postpq); } return TM_ECODE_OK; @@ -466,7 +488,6 @@ TmEcode AlertJsonThreadInit(ThreadVars *t, void *initdata, void **data) } /** Use the Ouptut Context (file pointer and mutex) */ - //aft->ctx = ((OutputCtx *)initdata)->data; aft->file_ctx = ((OutputCtx *)initdata)->data; *data = (void *)aft; @@ -503,7 +524,7 @@ OutputCtx *AlertJsonInitCtx(ConfNode *conf) { LogFileCtx *logfile_ctx = LogFileNewCtx(); if (logfile_ctx == NULL) { - SCLogDebug("AlertJsonInitCtx: Could not create nnew LogFileCtx"); + SCLogDebug("AlertJsonInitCtx: Could not create new LogFileCtx"); return NULL; } @@ -589,7 +610,32 @@ OutputCtx *AlertJsonInitCtx(ConfNode *conf) "invalid sensor-is: %s", sensor_id_s); exit(EXIT_FAILURE); } - //sensor_id = htonl(sensor_id); + } + + ConfNode *outputs, *output; + outputs = ConfNodeLookupChild(conf, "types"); + if (outputs) { + /* + * TODO: make this more general with some sort of + * registration capability + */ + TAILQ_FOREACH(output, &outputs->head, next) { + if (strcmp(output->val, "alert") == 0) { + SCLogDebug("Enabling alert output"); + outputFlags |= OUTPUT_ALERTS; + continue; + } + if (strcmp(output->val, "dns") == 0) { + SCLogDebug("Enabling DNS output"); + outputFlags |= OUTPUT_DNS; + continue; + } + if (strcmp(output->val, "http") == 0) { + SCLogDebug("Enabling HTTP output"); + outputFlags |= OUTPUT_HTTP; + continue; + } + } } } diff --git a/src/alert-json.h b/src/alert-json.h index 1317f5b22d..ed10407056 100644 --- a/src/alert-json.h +++ b/src/alert-json.h @@ -24,9 +24,25 @@ #ifndef __ALERT_JSON_H__ #define __ALERT_JSON_H__ +json_t *CreateJSONHeader(Packet *p, int direction_sensative); +TmEcode OutputJSON(json_t *js, void *data, uint64_t *count); + void TmModuleAlertJsonRegister (void); void TmModuleAlertJsonIPv4Register (void); void TmModuleAlertJsonPv6Register (void); OutputCtx *AlertJsonInitCtx(ConfNode *); +typedef struct AlertJsonThread_ { + /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ + LogFileCtx* file_ctx; + + void *buffer; /* pointer to MemBuffer */ + + uint64_t alert_cnt; + uint64_t dns_cnt; + uint64_t http_cnt; + uint32_t http_flags; +} AlertJsonThread; + #endif /* __ALERT_JSON_H__ */ + diff --git a/src/output-dnslog.c b/src/output-dnslog.c index 98f0b5c2fe..61b2e1c6a9 100644 --- a/src/output-dnslog.c +++ b/src/output-dnslog.c @@ -18,9 +18,9 @@ /** * \file * - * \author Victor Julien + * \author Tom DeCanio * - * Implements dns logging portion of the engine. + * Implements JSON DNS logging portion of the engine. */ #include "suricata-common.h" @@ -48,69 +48,15 @@ #include "util-logopenfile.h" #include "util-time.h" +#include "alert-json.h" + #ifdef HAVE_LIBJANSSON #include -#define DEFAULT_DNS_SYSLOG_FACILITY_STR "local0" -#define DEFAULT_DNS_SYSLOG_FACILITY LOG_LOCAL0 -#define DEFAULT_DNS_SYSLOG_LEVEL LOG_INFO - -#ifndef OS_WIN32 -static int dns_syslog_level = DEFAULT_DNS_SYSLOG_LEVEL; -#endif -#endif - -#define DEFAULT_LOG_FILENAME "dns.log" - -#define MODULE_NAME "DnsJson" - -#define OUTPUT_BUFFER_SIZE 65535 - /* we can do query logging as well, but it's disabled for now as the * TX id handling doesn't expect it */ #define QUERY 0 -TmEcode DnsJson (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); -TmEcode DnsJsonIPv4(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); -TmEcode DnsJsonIPv6(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); -TmEcode DnsJsonThreadInit(ThreadVars *, void *, void **); -TmEcode DnsJsonThreadDeinit(ThreadVars *, void *); -void DnsJsonExitPrintStats(ThreadVars *, void *); -static void DnsJsonDeInitCtx(OutputCtx *); - -void TmModuleDnsJsonRegister (void) { - tmm_modules[TMM_LOGDNSJSON].name = MODULE_NAME; - tmm_modules[TMM_LOGDNSJSON].ThreadInit = DnsJsonThreadInit; - tmm_modules[TMM_LOGDNSJSON].Func = DnsJson; - tmm_modules[TMM_LOGDNSJSON].ThreadExitPrintStats = DnsJsonExitPrintStats; - tmm_modules[TMM_LOGDNSJSON].ThreadDeinit = DnsJsonThreadDeinit; - tmm_modules[TMM_LOGDNSJSON].RegisterTests = NULL; - tmm_modules[TMM_LOGDNSJSON].cap_flags = 0; - - OutputRegisterModule(MODULE_NAME, "dns-log-json", DnsJsonInitCtx); - - /* enable the logger for the app layer */ - AppLayerRegisterLogger(ALPROTO_DNS_UDP); - AppLayerRegisterLogger(ALPROTO_DNS_TCP); - SCLogDebug("registered %s", MODULE_NAME); -} - -typedef struct LogDnsFileCtx_ { - LogFileCtx *file_ctx; - uint32_t flags; /** Store mode */ -} LogDnsFileCtx; - -#define LOG_DNS_DEFAULT 0 -#define LOG_DNS_JSON_SYSLOG 2 /* JSON output via syslog */ - -typedef struct LogDnsLogThread_ { - LogDnsFileCtx *dnslog_ctx; - /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ - uint32_t dns_cnt; - - MemBuffer *buffer; -} LogDnsLogThread; - static void CreateTypeString(uint16_t type, char *str, size_t str_size) { if (type == DNS_RECORD_TYPE_A) { snprintf(str, str_size, "A"); @@ -139,17 +85,13 @@ static void CreateTypeString(uint16_t type, char *str, size_t str_size) { } } -static void LogQuery(LogDnsLogThread *aft, char *timebuf, char *srcip, char *dstip, Port sp, Port dp, char *proto, DNSTransaction *tx, DNSQueryEntry *entry) { - LogDnsFileCtx *hlog = aft->dnslog_ctx; +static void LogQuery(AlertJsonThread/*LogDnsLogThread*/ *aft, json_t *js, /*char *timebuf, char *srcip, char *dstip, Port sp, Port dp, char *proto, */ DNSTransaction *tx, DNSQueryEntry *entry) { + MemBuffer *buffer = (MemBuffer *)aft->buffer; SCLogDebug("got a DNS request and now logging !!"); /* reset */ - MemBufferReset(aft->buffer); - - json_t *js = json_object(); - if (js == NULL) - return; + MemBufferReset(buffer); json_t *djs = json_object(); if (djs == NULL) { @@ -157,16 +99,6 @@ static void LogQuery(LogDnsLogThread *aft, char *timebuf, char *srcip, char *dst return; } - /* time & tx */ - json_object_set_new(js, "time", json_string(timebuf)); - - /* tuple */ - json_object_set_new(js, "srcip", json_string(srcip)); - json_object_set_new(js, "sp", json_integer(sp)); - json_object_set_new(js, "dstip", json_string(dstip)); - json_object_set_new(js, "dp", json_integer(dp)); - json_object_set_new(js, "proto", json_string(proto)); - /* type */ json_object_set_new(djs, "type", json_string("query")); @@ -188,120 +120,10 @@ static void LogQuery(LogDnsLogThread *aft, char *timebuf, char *srcip, char *dst /* dns */ json_object_set_new(js, "dns", djs); - char *s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - MemBufferWriteString(aft->buffer, "%s", s); - free(s); - free(djs); - free(js); - - aft->dns_cnt++; - - SCMutexLock(&hlog->file_ctx->fp_mutex); -#ifdef HAVE_LIBJANSSON - if (hlog->flags & LOG_DNS_JSON_SYSLOG) { - syslog(dns_syslog_level, "%s", (char *)aft->buffer->buffer); - } else { - MemBufferWriteString(aft->buffer, "\n"); -#endif - (void)MemBufferPrintToFPAsString(aft->buffer, hlog->file_ctx->fp); - fflush(hlog->file_ctx->fp); -#ifdef HAVE_LIBJANSSON - } -#endif - SCMutexUnlock(&hlog->file_ctx->fp_mutex); + OutputJSON(js, aft, &aft->dns_cnt); + json_object_del(js, "dns"); } -#if 0 -static void LogAnswer(LogDnsLogThread *aft, char *timebuf, char *srcip, char *dstip, Port sp, Port dp, DNSTransaction *tx, DNSAnswerEntry *entry) { - LogDnsFileCtx *hlog = aft->dnslog_ctx; - - SCLogDebug("got a DNS response and now logging !!"); - - /* reset */ - MemBufferReset(aft->buffer); - - json_t *js = json_object(); - if (js == NULL) - return; - - json_t *djs = json_object(); - if (djs == NULL) { - free(js); - return; - } - - /* time & tx */ - json_object_set_new(js, "time", json_string(timebuf)); - - /* tuple */ - json_object_set_new(js, "srcip", json_string(srcip)); - json_object_set_new(js, "sp", json_integer(sp)); - json_object_set_new(js, "dstip", json_string(dstip)); - json_object_set_new(js, "dp", json_integer(dp)); - - /* type */ - json_object_set_new(djs, "type", json_string("answer")); - /* id */ - json_object_set_new(djs, "id", json_integer(tx->tx_id)); - - if (entry != NULL) { - /* query */ - if (entry->fqdn_len > 0) { - char *c; - json_object_set_new(djs, "query", - json_string(c = strndup( - (char *)((char *)entry + sizeof(DNSAnswerEntry)), - entry->fqdn_len))); - if (c) free(c); - } - - /* name */ - char record[16] = ""; - CreateTypeString(entry->type, record, sizeof(record)); - json_object_set_new(djs, "record", json_string(record)); - - /* ttl */ - json_object_set_new(djs, "ttl", json_integer(entry->ttl)); - - uint8_t *ptr = (uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)+ entry->fqdn_len); - if (entry->type == DNS_RECORD_TYPE_A) { - char a[16] = ""; - PrintInet(AF_INET, (const void *)ptr, a, sizeof(a)); - json_object_set_new(djs, "addr", json_string(a)); - } else if (entry->type == DNS_RECORD_TYPE_AAAA) { - char a[46] = ""; - PrintInet(AF_INET6, (const void *)ptr, a, sizeof(a)); - json_object_set_new(djs, "addr", json_string(a)); - } else if (entry->data_len == 0) { - json_object_set_new(djs, "addr", json_string("")); - } - } - - /* dns */ - json_object_set_new(js, "dns", djs); - char *s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - MemBufferWriteString(aft->buffer, "%s", s); - free(s); - free(djs); - free(js); - - aft->dns_cnt++; - - SCMutexLock(&hlog->file_ctx->fp_mutex); -#ifdef HAVE_LIBJANSSON - if (hlog->flags & LOG_DNS_JSON_SYSLOG) { - syslog(dns_syslog_level, "%s", (char *)aft->buffer->buffer); - } else { - MemBufferWriteString(aft->buffer, "\n"); -#endif - (void)MemBufferPrintToFPAsString(aft->buffer, hlog->file_ctx->fp); - fflush(hlog->file_ctx->fp); -#ifdef HAVE_LIBJANSSON - } -#endif - SCMutexUnlock(&hlog->file_ctx->fp_mutex); -} -#endif static void AppendAnswer(json_t *djs, DNSTransaction *tx, DNSAnswerEntry *entry) { json_t *js = json_object(); if (js == NULL) @@ -348,17 +170,13 @@ static void AppendAnswer(json_t *djs, DNSTransaction *tx, DNSAnswerEntry *entry) json_array_append_new(djs, js); } -static void LogAnswers(LogDnsLogThread *aft, char *timebuf, char *srcip, char *dstip, Port sp, Port dp, char *proto, DNSTransaction *tx) { - LogDnsFileCtx *hlog = aft->dnslog_ctx; +static void LogAnswers(AlertJsonThread/*LogDnsLogThread*/ *aft, json_t *js, /*char *timebuf, char *srcip, char *dstip, Port sp, Port dp, char *proto,*/ DNSTransaction *tx) { + MemBuffer *buffer = (MemBuffer *)aft->buffer; SCLogDebug("got a DNS response and now logging !!"); /* reset */ - MemBufferReset(aft->buffer); - - json_t *js = json_object(); - if (js == NULL) - return; + MemBufferReset(buffer); json_t *djs = json_array(); if (djs == NULL) { @@ -366,17 +184,6 @@ static void LogAnswers(LogDnsLogThread *aft, char *timebuf, char *srcip, char *d return; } - /* time & tx */ - json_object_set_new(js, "time", json_string(timebuf)); - - /* tuple */ - json_object_set_new(js, "srcip", json_string(srcip)); - json_object_set_new(js, "sp", json_integer(sp)); - json_object_set_new(js, "dstip", json_string(dstip)); - json_object_set_new(js, "dp", json_integer(dp)); - json_object_set_new(js, "proto", json_string(proto)); - -#if 1 if (tx->no_such_name) { AppendAnswer(djs, tx, NULL); } @@ -390,70 +197,11 @@ static void LogAnswers(LogDnsLogThread *aft, char *timebuf, char *srcip, char *d TAILQ_FOREACH(entry, &tx->authority_list, next) { AppendAnswer(djs, tx, entry); } -#else - /* type */ - json_object_set_new(djs, "type", json_string("answer")); - - /* id */ - json_object_set_new(djs, "id", json_integer(tx->tx_id)); - - if (entry != NULL) { - /* query */ - if (entry->fqdn_len > 0) { - char *c; - json_object_set_new(djs, "query", - json_string(c = strndup( - (char *)((char *)entry + sizeof(DNSAnswerEntry)), - entry->fqdn_len))); - if (c) free(c); - } - - /* name */ - char record[16] = ""; - CreateTypeString(entry->type, record, sizeof(record)); - json_object_set_new(djs, "record", json_string(record)); - - /* ttl */ - json_object_set_new(djs, "ttl", json_integer(entry->ttl)); - - uint8_t *ptr = (uint8_t *)((uint8_t *)entry + sizeof(DNSAnswerEntry)+ entry->fqdn_len); - if (entry->type == DNS_RECORD_TYPE_A) { - char a[16] = ""; - PrintInet(AF_INET, (const void *)ptr, a, sizeof(a)); - json_object_set_new(djs, "addr", json_string(a)); - } else if (entry->type == DNS_RECORD_TYPE_AAAA) { - char a[46] = ""; - PrintInet(AF_INET6, (const void *)ptr, a, sizeof(a)); - json_object_set_new(djs, "addr", json_string(a)); - } else if (entry->data_len == 0) { - json_object_set_new(djs, "addr", json_string("")); - } - } -#endif /* dns */ json_object_set_new(js, "dns", djs); - char *s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - MemBufferWriteString(aft->buffer, "%s", s); - free(s); - free(djs); - free(js); - - aft->dns_cnt++; - - SCMutexLock(&hlog->file_ctx->fp_mutex); -#ifdef HAVE_LIBJANSSON - if (hlog->flags & LOG_DNS_JSON_SYSLOG) { - syslog(dns_syslog_level, "%s", (char *)aft->buffer->buffer); - } else { - MemBufferWriteString(aft->buffer, "\n"); -#endif - (void)MemBufferPrintToFPAsString(aft->buffer, hlog->file_ctx->fp); - fflush(hlog->file_ctx->fp); -#ifdef HAVE_LIBJANSSON - } -#endif - SCMutexUnlock(&hlog->file_ctx->fp_mutex); + OutputJSON(js, aft, &aft->dns_cnt); + json_object_del(js, "dns"); } static TmEcode DnsJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, @@ -461,8 +209,7 @@ static TmEcode DnsJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQue { SCEnter(); - LogDnsLogThread *aft = (LogDnsLogThread *)data; - char timebuf[64]; + AlertJsonThread *aft = (AlertJsonThread *)data; /* no flow, no htp state */ if (p->flow == NULL) { @@ -489,48 +236,9 @@ static TmEcode DnsJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQue //int tx_progress_done_value_ts = AppLayerGetAlstateProgressCompletionStatus(proto, 0); //int tx_progress_done_value_tc = AppLayerGetAlstateProgressCompletionStatus(proto, 1); - SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt); - CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); - - char srcip[46], dstip[46]; - Port sp, dp; - if ((PKT_IS_TOCLIENT(p))) { - switch (ipproto) { - case AF_INET: - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); - break; - case AF_INET6: - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); - break; - default: - goto end; - } - sp = p->sp; - dp = p->dp; - } else { - switch (ipproto) { - case AF_INET: - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), dstip, sizeof(dstip)); - break; - case AF_INET6: - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), dstip, sizeof(dstip)); - break; - default: - goto end; - } - sp = p->dp; - dp = p->sp; - } - char proto_s[16]; - if (SCProtoNameValid(IPV4_GET_IPPROTO(p)) == TRUE) { - strlcpy(proto_s, known_proto[IPV4_GET_IPPROTO(p)], sizeof(proto_s)); - } else { - snprintf(proto_s, sizeof(proto), "PROTO:%03" PRIu32, IPV4_GET_IPPROTO(p)); - } + json_t *js = CreateJSONHeader(p, 1); + if (unlikely(js == NULL)) + return TM_ECODE_OK; #if QUERY if (PKT_IS_TOSERVER(p)) { @@ -553,26 +261,10 @@ static TmEcode DnsJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQue DNSQueryEntry *query = NULL; TAILQ_FOREACH(query, &tx->query_list, next) { - LogQuery(aft, timebuf, dstip, srcip, dp, sp, proto_s, tx, query); - } - -#if 1 - LogAnswers(aft, timebuf, srcip, dstip, sp, dp, proto_s, tx); -#else - if (tx->no_such_name) { - LogAnswer(aft, timebuf, srcip, dstip, sp, dp, tx, NULL); + LogQuery(aft, js, /*timebuf, dstip, srcip, dp, sp, proto_s,*/ tx, query); } - DNSAnswerEntry *entry = NULL; - TAILQ_FOREACH(entry, &tx->answer_list, next) { - LogAnswer(aft, timebuf, srcip, dstip, sp, dp, tx, entry); - } - - entry = NULL; - TAILQ_FOREACH(entry, &tx->authority_list, next) { - LogAnswer(aft, timebuf, srcip, dstip, sp, dp, tx, entry); - } -#endif + LogAnswers(aft, js, /*timebuf, srcip, dstip, sp, dp, proto_s,*/ tx); SCLogDebug("calling AppLayerTransactionUpdateLoggedId"); AppLayerTransactionUpdateLogId(ALPROTO_DNS_UDP, p->flow); @@ -584,147 +276,44 @@ end: SCReturnInt(TM_ECODE_OK); } -TmEcode DnsJsonIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) -{ - return DnsJsonIPWrapper(tv, p, data, pq, postpq, AF_INET); -} - -TmEcode DnsJsonIPv6(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) -{ - return DnsJsonIPWrapper(tv, p, data, pq, postpq, AF_INET6); -} - -TmEcode DnsJson (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) +int OutputDnsNeedsLog(Packet *p) { SCEnter(); - SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt); /* no flow, no htp state */ if (p->flow == NULL) { - SCReturnInt(TM_ECODE_OK); + SCReturnInt(0); } if (!(PKT_IS_UDP(p)) && !(PKT_IS_TCP(p))) { - SCReturnInt(TM_ECODE_OK); - } - - if (PKT_IS_IPV4(p)) { - int r = DnsJsonIPv4(tv, p, data, pq, postpq); - SCReturnInt(r); - } else if (PKT_IS_IPV6(p)) { - int r = DnsJsonIPv6(tv, p, data, pq, postpq); - SCReturnInt(r); + SCReturnInt(0); } - SCReturnInt(TM_ECODE_OK); -} - -TmEcode DnsJsonThreadInit(ThreadVars *t, void *initdata, void **data) -{ - LogDnsLogThread *aft = SCMalloc(sizeof(LogDnsLogThread)); - if (unlikely(aft == NULL)) - return TM_ECODE_FAILED; - memset(aft, 0, sizeof(LogDnsLogThread)); - - if(initdata == NULL) - { - SCLogDebug("Error getting context for DNSLog. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + /* check if we have DNS state or not */ + FLOWLOCK_WRLOCK(p->flow); + uint16_t proto = AppLayerGetProtoFromPacket(p); + if (proto != ALPROTO_DNS_UDP && proto != ALPROTO_DNS_TCP) { + SCLogDebug("proto not ALPROTO_DNS_UDP: %u", proto); + goto end; } - aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); - if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + DNSState *dns_state = (DNSState *)AppLayerGetProtoStateFromPacket(p); + if (dns_state == NULL) { + SCLogDebug("no dns state, so no request logging"); + goto end; } +end: + FLOWLOCK_UNLOCK(p->flow); - /* Use the Ouptut Context (file pointer and mutex) */ - aft->dnslog_ctx= ((OutputCtx *)initdata)->data; - - *data = (void *)aft; - return TM_ECODE_OK; + SCReturnInt(1); } -TmEcode DnsJsonThreadDeinit(ThreadVars *t, void *data) +TmEcode OutputDnsLog(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { - LogDnsLogThread *aft = (LogDnsLogThread *)data; - if (aft == NULL) { - return TM_ECODE_OK; - } - - MemBufferFree(aft->buffer); - /* clear memory */ - memset(aft, 0, sizeof(LogDnsLogThread)); + SCEnter(); - SCFree(aft); - return TM_ECODE_OK; -} + DnsJsonIPWrapper(tv, p, data, pq, postpq, AF_INET); -void DnsJsonExitPrintStats(ThreadVars *tv, void *data) { - LogDnsLogThread *aft = (LogDnsLogThread *)data; - if (aft == NULL) { - return; - } - - SCLogInfo("DNS logger logged %" PRIu32 " requests", aft->dns_cnt); + SCReturnInt(TM_ECODE_OK); } - -/** \brief Create a new dns log LogFileCtx. - * \param conf Pointer to ConfNode containing this loggers configuration. - * \return NULL if failure, LogFileCtx* to the file_ctx if succesful - * */ -OutputCtx *DnsJsonInitCtx(ConfNode *conf) -{ - LogFileCtx* file_ctx = LogFileNewCtx(); - - if(file_ctx == NULL) { - SCLogError(SC_ERR_DNS_LOG_GENERIC, "couldn't create new file_ctx"); - return NULL; - } - - if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME) < 0) { - LogFileFreeCtx(file_ctx); - return NULL; - } - - LogDnsFileCtx *dnslog_ctx = SCMalloc(sizeof(LogDnsFileCtx)); - if (unlikely(dnslog_ctx == NULL)) { - LogFileFreeCtx(file_ctx); - return NULL; - } - memset(dnslog_ctx, 0x00, sizeof(LogDnsFileCtx)); - - dnslog_ctx->file_ctx = file_ctx; - -#ifdef HAVE_LIBJANSSON - const char *json = ConfNodeLookupChildValue(conf, "json"); - if (json) { - if (strcmp(json, "syslog") == 0) { - dnslog_ctx->flags |= LOG_DNS_JSON_SYSLOG; - } - } #endif - - OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); - if (unlikely(output_ctx == NULL)) { - LogFileFreeCtx(file_ctx); - SCFree(dnslog_ctx); - return NULL; - } - - output_ctx->data = dnslog_ctx; - output_ctx->DeInit = DnsJsonDeInitCtx; - - SCLogDebug("DNS json output initialized"); - - return output_ctx; -} - -static void DnsJsonDeInitCtx(OutputCtx *output_ctx) -{ - LogDnsFileCtx *dnslog_ctx = (LogDnsFileCtx *)output_ctx->data; - LogFileFreeCtx(dnslog_ctx->file_ctx); - SCFree(dnslog_ctx); - SCFree(output_ctx); -} diff --git a/src/output-dnslog.h b/src/output-dnslog.h index f06b2588d5..bcbac67acd 100644 --- a/src/output-dnslog.h +++ b/src/output-dnslog.h @@ -27,6 +27,8 @@ void TmModuleDnsJsonRegister (void); void TmModuleDnsJsonIPv4Register (void); void TmModuleDnsJsonIPv6Register (void); +int OutputDnsNeedsLog(Packet *p); +TmEcode OutputDnsLog(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq); OutputCtx *DnsJsonInitCtx(ConfNode *); #endif /* __OUTPUT_DNSLOG_H__ */ diff --git a/src/output-httplog.c b/src/output-httplog.c index c698299f21..7b3d1a0998 100644 --- a/src/output-httplog.c +++ b/src/output-httplog.c @@ -18,10 +18,9 @@ /** * \file * - * \author Victor Julien - * \author Ignacio Sanchez + * \author Tom DeCanio * - * Implements http logging portion of the engine. + * Implements HTTP JSON logging portion of the engine. */ #include "suricata-common.h" @@ -48,66 +47,11 @@ #include "util-proto-name.h" #include "util-logopenfile.h" #include "util-time.h" +#include "alert-json.h" #ifdef HAVE_LIBJANSSON #include -#define DEFAULT_HTTP_SYSLOG_FACILITY_STR "local0" -#define DEFAULT_HTTP_SYSLOG_FACILITY LOG_LOCAL0 -#define DEFAULT_HTTP_SYSLOG_LEVEL LOG_INFO - -#ifndef OS_WIN32 -static int http_syslog_level = DEFAULT_HTTP_SYSLOG_LEVEL; -#endif -#endif - -#define DEFAULT_LOG_FILENAME "http.log" - -#define MODULE_NAME "HttpJson" - -#define OUTPUT_BUFFER_SIZE 65535 - -TmEcode HttpJson (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); -TmEcode HttpJsonIPv4(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); -TmEcode HttpJsonIPv6(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); -TmEcode HttpJsonThreadInit(ThreadVars *, void *, void **); -TmEcode HttpJsonThreadDeinit(ThreadVars *, void *); -void HttpJsonExitPrintStats(ThreadVars *, void *); -static void HttpJsonDeInitCtx(OutputCtx *); - -void TmModuleHttpJsonRegister (void) { - tmm_modules[TMM_LOGHTTPJSON].name = MODULE_NAME; - tmm_modules[TMM_LOGHTTPJSON].ThreadInit = HttpJsonThreadInit; - tmm_modules[TMM_LOGHTTPJSON].Func = HttpJson; - tmm_modules[TMM_LOGHTTPJSON].ThreadExitPrintStats = HttpJsonExitPrintStats; - tmm_modules[TMM_LOGHTTPJSON].ThreadDeinit = HttpJsonThreadDeinit; - tmm_modules[TMM_LOGHTTPJSON].RegisterTests = NULL; - tmm_modules[TMM_LOGHTTPJSON].cap_flags = 0; - - OutputRegisterModule(MODULE_NAME, "http-log-json", HttpJsonInitCtx); - - /* enable the logger for the app layer */ - AppLayerRegisterLogger(ALPROTO_HTTP); -} - -void TmModuleHttpJsonIPv4Register (void) { - tmm_modules[TMM_LOGHTTPJSON4].name = "HttpJsonIPv4"; - tmm_modules[TMM_LOGHTTPJSON4].ThreadInit = HttpJsonThreadInit; - tmm_modules[TMM_LOGHTTPJSON4].Func = HttpJsonIPv4; - tmm_modules[TMM_LOGHTTPJSON4].ThreadExitPrintStats = HttpJsonExitPrintStats; - tmm_modules[TMM_LOGHTTPJSON4].ThreadDeinit = HttpJsonThreadDeinit; - tmm_modules[TMM_LOGHTTPJSON4].RegisterTests = NULL; -} - -void TmModuleHttpJsonIPv6Register (void) { - tmm_modules[TMM_LOGHTTPJSON6].name = "HttpJsonIPv6"; - tmm_modules[TMM_LOGHTTPJSON6].ThreadInit = HttpJsonThreadInit; - tmm_modules[TMM_LOGHTTPJSON6].Func = HttpJsonIPv6; - tmm_modules[TMM_LOGHTTPJSON6].ThreadExitPrintStats = HttpJsonExitPrintStats; - tmm_modules[TMM_LOGHTTPJSON6].ThreadDeinit = HttpJsonThreadDeinit; - tmm_modules[TMM_LOGHTTPJSON6].RegisterTests = NULL; -} - #define LOG_HTTP_MAXN_NODES 64 #define LOG_HTTP_NODE_STRLEN 256 #define LOG_HTTP_NODE_MAXOUTPUTLEN 8192 @@ -185,10 +129,13 @@ static uint32_t GetCookieValue(uint8_t *rawcookies, uint32_t rawcookies_len, cha } /* Custom format logging */ -static void LogHttpLogJSONCustom(LogHttpLogThread *aft, htp_tx_t *tx, const struct timeval *ts, - char *srcip, Port sp, char *dstip, Port dp) +static void LogHttpLogJSONCustom(AlertJsonThread *aft, json_t *js, htp_tx_t *tx, const struct timeval *ts /*, + char *srcip, Port sp, char *dstip, Port dp*/) { +#if 0 LogHttpFileCtx *httplog_ctx = aft->httplog_ctx; +#endif +#if 0 uint32_t i; uint32_t datalen; char buf[128]; @@ -227,6 +174,7 @@ static void LogHttpLogJSONCustom(LogHttpLogThread *aft, htp_tx_t *tx, const stru PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, (uint8_t *)buf,strlen(buf)); break; +#ifdef NOTYET case LOG_HTTP_CF_CLIENT_IP: /* CLIENT IP ADDRESS */ PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, @@ -245,6 +193,7 @@ static void LogHttpLogJSONCustom(LogHttpLogThread *aft, htp_tx_t *tx, const stru /* SERVER PORT */ MemBufferWriteString(aft->buffer, "%" PRIu16 "", dp); break; +#endif case LOG_HTTP_CF_REQUEST_METHOD: /* METHOD */ if (tx->request_method != NULL) { @@ -376,38 +325,20 @@ static void LogHttpLogJSONCustom(LogHttpLogThread *aft, htp_tx_t *tx, const stru } } MemBufferWriteString(aft->buffer, "\n"); +#endif } #ifdef HAVE_LIBJANSSON /* JSON format logging */ -static void LogHttpLogJSON(LogHttpLogThread *aft, htp_tx_t *tx, char * timebuf, +static void LogHttpLogJSON(AlertJsonThread *aft, json_t *js, htp_tx_t *tx /*, char * timebuf, char *srcip, Port sp, char *dstip, Port dp, - char *proto) + char *proto*/) { - LogHttpFileCtx *hlog = aft->httplog_ctx; - json_t *js = json_object(); - if (js == NULL) - return; -#if 0 - json_t *hjs = js; -#else json_t *hjs = json_object(); if (hjs == NULL) { free(js); return; } -#endif - - /* time */ - json_object_set_new(js, "time", json_string(timebuf)); - - /* tuple */ - json_object_set_new(js, "srcip", json_string(srcip)); - json_object_set_new(js, "sp", json_integer(sp)); - json_object_set_new(js, "dstip", json_string(dstip)); - json_object_set_new(js, "dp", json_integer(dp)); - json_object_set_new(js, "proto", json_string(proto)); - char *c; /* hostname */ @@ -471,7 +402,7 @@ static void LogHttpLogJSON(LogHttpLogThread *aft, htp_tx_t *tx, char * timebuf, if (c) free(c); } - if (hlog->flags & LOG_HTTP_EXTENDED) { + if (aft->http_flags & LOG_HTTP_EXTENDED) { /* referer */ htp_header_t *h_referer = NULL; if (tx->request_headers != NULL) { @@ -521,11 +452,6 @@ static void LogHttpLogJSON(LogHttpLogThread *aft, htp_tx_t *tx, char * timebuf, } json_object_set_new(js, "http", hjs); - char *s = json_dumps(js, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII); - MemBufferWriteRaw(aft->buffer, s, strlen(s)); - free(s); - free(hjs); - free(js); } #endif @@ -590,7 +516,7 @@ static void LogHttpLogExtended(LogHttpLogThread *aft, htp_tx_t *tx) } static TmEcode HttpJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, - PacketQueue *postpq, int ipproto) + PacketQueue *postpq/*, int ipproto*/) { SCEnter(); @@ -601,10 +527,8 @@ static TmEcode HttpJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQu int tx_progress = 0; int tx_progress_done_value_ts = 0; int tx_progress_done_value_tc = 0; - LogHttpLogThread *aft = (LogHttpLogThread *)data; - LogHttpFileCtx *hlog = aft->httplog_ctx; - char timebuf[64]; - char proto_s[16]; + AlertJsonThread *aft = (AlertJsonThread *)data; + MemBuffer *buffer = (MemBuffer *)aft->buffer; /* no flow, no htp state */ if (p->flow == NULL) { @@ -628,46 +552,9 @@ static TmEcode HttpJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQu tx_progress_done_value_ts = AppLayerGetAlstateProgressCompletionStatus(ALPROTO_HTTP, 0); tx_progress_done_value_tc = AppLayerGetAlstateProgressCompletionStatus(ALPROTO_HTTP, 1); - CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); - - char srcip[46], dstip[46]; - Port sp, dp; - if ((PKT_IS_TOSERVER(p))) { - switch (ipproto) { - case AF_INET: - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); - break; - case AF_INET6: - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); - break; - default: - goto end; - } - sp = p->sp; - dp = p->dp; - } else { - switch (ipproto) { - case AF_INET: - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), dstip, sizeof(dstip)); - break; - case AF_INET6: - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), dstip, sizeof(dstip)); - break; - default: - goto end; - } - sp = p->dp; - dp = p->sp; - } - if (SCProtoNameValid(IPV4_GET_IPPROTO(p)) == TRUE) { - strlcpy(proto_s, known_proto[IPV4_GET_IPPROTO(p)], sizeof(proto_s)); - } else { - snprintf(proto_s, sizeof(proto), "PROTO:%03" PRIu32, IPV4_GET_IPPROTO(p)); - } + json_t *js = CreateJSONHeader(p, 1); + if (unlikely(js == NULL)) + return TM_ECODE_OK; for (; tx_id < total_txs; tx_id++) { @@ -690,36 +577,21 @@ static TmEcode HttpJsonIPWrapper(ThreadVars *tv, Packet *p, void *data, PacketQu SCLogDebug("got a HTTP request and now logging !!"); /* reset */ - MemBufferReset(aft->buffer); + MemBufferReset(buffer); - if (hlog->flags & LOG_HTTP_CUSTOM) { - LogHttpLogJSONCustom(aft, tx, &p->ts, srcip, sp, dstip, dp); - //} else if (hlog->flags & LOG_HTTP_JSON) { + if (aft->http_flags & LOG_HTTP_CUSTOM) { + LogHttpLogJSONCustom(aft, js, tx, &p->ts/*, srcip, sp, dstip, dp*/); } else { - LogHttpLogJSON(aft, tx, timebuf, srcip, sp, dstip, dp, proto_s); + LogHttpLogJSON(aft, js, tx /*, timebuf, srcip, sp, dstip, dp, proto_s*/); } - aft->uri_cnt ++; - - SCMutexLock(&hlog->file_ctx->fp_mutex); -#ifdef HAVE_LIBJANSSON - if (hlog->flags & LOG_HTTP_JSON_SYSLOG) { - syslog(http_syslog_level, "%s", (char *)aft->buffer->buffer); - } else { - //if (hlog->flags & LOG_HTTP_JSON) { - if (TRUE) { - MemBufferWriteString(aft->buffer, "\n"); - } -#endif - (void)MemBufferPrintToFPAsString(aft->buffer, hlog->file_ctx->fp); - fflush(hlog->file_ctx->fp); -#ifdef HAVE_LIBJANSSON - } -#endif - SCMutexUnlock(&hlog->file_ctx->fp_mutex); + OutputJSON(js, aft, &aft->http_cnt); + json_object_del(js, "http"); AppLayerTransactionUpdateLogId(ALPROTO_HTTP, p->flow); } + json_object_clear(js); + free(js); end: FLOWLOCK_UNLOCK(p->flow); @@ -727,241 +599,10 @@ end: } -TmEcode HttpJsonIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) -{ - return HttpJsonIPWrapper(tv, p, data, pq, postpq, AF_INET); -} - -TmEcode HttpJsonIPv6(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) -{ - return HttpJsonIPWrapper(tv, p, data, pq, postpq, AF_INET6); -} - -TmEcode HttpJson (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) +TmEcode OutputHttpLog (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { SCEnter(); - - /* no flow, no htp state */ - if (p->flow == NULL) { - SCReturnInt(TM_ECODE_OK); - } - - if (!(PKT_IS_TCP(p))) { - SCReturnInt(TM_ECODE_OK); - } - - if (PKT_IS_IPV4(p)) { - SCReturnInt(HttpJsonIPv4(tv, p, data, pq, postpq)); - } else if (PKT_IS_IPV6(p)) { - SCReturnInt(HttpJsonIPv6(tv, p, data, pq, postpq)); - } - + HttpJsonIPWrapper(tv, p, data, pq, postpq); SCReturnInt(TM_ECODE_OK); } - -TmEcode HttpJsonThreadInit(ThreadVars *t, void *initdata, void **data) -{ - LogHttpLogThread *aft = SCMalloc(sizeof(LogHttpLogThread)); - if (unlikely(aft == NULL)) - return TM_ECODE_FAILED; - memset(aft, 0, sizeof(LogHttpLogThread)); - - if(initdata == NULL) - { - SCLogDebug("Error getting context for HTTPLog. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; - } - - aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); - if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; - } - - /* Use the Ouptut Context (file pointer and mutex) */ - aft->httplog_ctx= ((OutputCtx *)initdata)->data; - - *data = (void *)aft; - return TM_ECODE_OK; -} - -TmEcode HttpJsonThreadDeinit(ThreadVars *t, void *data) -{ - LogHttpLogThread *aft = (LogHttpLogThread *)data; - if (aft == NULL) { - return TM_ECODE_OK; - } - - MemBufferFree(aft->buffer); - /* clear memory */ - memset(aft, 0, sizeof(LogHttpLogThread)); - - SCFree(aft); - return TM_ECODE_OK; -} - -void HttpJsonExitPrintStats(ThreadVars *tv, void *data) { - LogHttpLogThread *aft = (LogHttpLogThread *)data; - if (aft == NULL) { - return; - } - - SCLogInfo("HTTP logger logged %" PRIu32 " requests", aft->uri_cnt); -} - -/** \brief Create a new http log LogFileCtx. - * \param conf Pointer to ConfNode containing this loggers configuration. - * \return NULL if failure, LogFileCtx* to the file_ctx if succesful - * */ -OutputCtx *HttpJsonInitCtx(ConfNode *conf) -{ - LogFileCtx* file_ctx = LogFileNewCtx(); - const char *p, *np; - uint32_t n; - if(file_ctx == NULL) { - SCLogError(SC_ERR_HTTP_LOG_GENERIC, "couldn't create new file_ctx"); - return NULL; - } - - if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME) < 0) { - LogFileFreeCtx(file_ctx); - return NULL; - } - - LogHttpFileCtx *httplog_ctx = SCMalloc(sizeof(LogHttpFileCtx)); - if (unlikely(httplog_ctx == NULL)) { - LogFileFreeCtx(file_ctx); - return NULL; - } - memset(httplog_ctx, 0x00, sizeof(LogHttpFileCtx)); - - httplog_ctx->file_ctx = file_ctx; - httplog_ctx->cf_n=0; - - const char *extended = ConfNodeLookupChildValue(conf, "extended"); - const char *custom = ConfNodeLookupChildValue(conf, "custom"); - const char *customformat = ConfNodeLookupChildValue(conf, "customformat"); - const char *json = ConfNodeLookupChildValue(conf, "json"); - - /* If custom logging format is selected, lets parse it */ - if (custom != NULL && customformat != NULL && ConfValIsTrue(custom)) { - p=customformat; - httplog_ctx->flags |= LOG_HTTP_CUSTOM; - for (httplog_ctx->cf_n = 0; httplog_ctx->cf_n < LOG_HTTP_MAXN_NODES-1 && p && *p != '\0'; - httplog_ctx->cf_n++){ - httplog_ctx->cf_nodes[httplog_ctx->cf_n] = SCMalloc(sizeof(LogHttpCustomFormatNode)); - httplog_ctx->cf_nodes[httplog_ctx->cf_n]->maxlen=0; - if (httplog_ctx->cf_nodes[httplog_ctx->cf_n] == NULL) { - for (n = 0; n < httplog_ctx->cf_n; n++) { - SCFree(httplog_ctx->cf_nodes[n]); - } - LogFileFreeCtx(file_ctx); - SCFree(httplog_ctx); - return NULL; - } - if (*p != '%'){ - /* Literal found in format string */ - httplog_ctx->cf_nodes[httplog_ctx->cf_n]->type = LOG_HTTP_CF_LITERAL; - np = strchr(p, '%'); - if (np == NULL){ - n = LOG_HTTP_NODE_STRLEN-2; - np = NULL; /* End */ - }else{ - n = np-p; - } - strlcpy(httplog_ctx->cf_nodes[httplog_ctx->cf_n]->data,p,n+1); - p = np; - } else { - /* Non Literal found in format string */ - p++; - if (*p == '[') { /* Check if maxlength has been specified (ie: [25]) */ - p++; - np = strchr(p, ']'); - if (np != NULL) { - if (np-p > 0 && np-p < 10){ - long maxlen = strtol(p,NULL,10); - if (maxlen > 0 && maxlen < LOG_HTTP_NODE_MAXOUTPUTLEN) { - httplog_ctx->cf_nodes[httplog_ctx->cf_n]->maxlen = (uint32_t) maxlen; - } - } else { - goto parsererror; - } - p = np + 1; - } else { - goto parsererror; - } - } - if (*p == '{') { /* Simple format char */ - np = strchr(p, '}'); - if (np != NULL && np-p > 1 && np-p < LOG_HTTP_NODE_STRLEN-2) { - p++; - n = np-p; - strlcpy(httplog_ctx->cf_nodes[httplog_ctx->cf_n]->data, p, n+1); - p = np; - } else { - goto parsererror; - } - p++; - } else { - httplog_ctx->cf_nodes[httplog_ctx->cf_n]->data[0] = '\0'; - } - httplog_ctx->cf_nodes[httplog_ctx->cf_n]->type = *p; - if (*p == '%'){ - httplog_ctx->cf_nodes[httplog_ctx->cf_n]->type = LOG_HTTP_CF_LITERAL; - strlcpy(httplog_ctx->cf_nodes[httplog_ctx->cf_n]->data, "%", 2); - } - p++; - } - } - } else { - if (extended == NULL) { - httplog_ctx->flags |= LOG_HTTP_DEFAULT; - } else { - if (ConfValIsTrue(extended)) { - httplog_ctx->flags |= LOG_HTTP_EXTENDED; - } - } - } -#ifdef HAVE_LIBJANSSON - if (json) { - if (strcmp(json, "syslog") == 0) { - httplog_ctx->flags |= LOG_HTTP_JSON_SYSLOG; - } - } #endif - - OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); - if (unlikely(output_ctx == NULL)) { - goto parsererror; - } - - output_ctx->data = httplog_ctx; - output_ctx->DeInit = HttpJsonDeInitCtx; - - SCLogDebug("HTTP log output initialized"); - - return output_ctx; - -parsererror: - for (n = 0;n < httplog_ctx->cf_n;n++) { - SCFree(httplog_ctx->cf_nodes[n]); - } - LogFileFreeCtx(file_ctx); - SCFree(httplog_ctx); - SCLogError(SC_ERR_INVALID_ARGUMENT,"Syntax error in custom http log format string."); - return NULL; - -} - -static void HttpJsonDeInitCtx(OutputCtx *output_ctx) -{ - LogHttpFileCtx *httplog_ctx = (LogHttpFileCtx *)output_ctx->data; - uint32_t i; - for (i = 0; i < httplog_ctx->cf_n; i++) { - SCFree(httplog_ctx->cf_nodes[i]); - } - LogFileFreeCtx(httplog_ctx->file_ctx); - SCFree(httplog_ctx); - SCFree(output_ctx); -} diff --git a/src/output-httplog.h b/src/output-httplog.h index a6974c8014..16c912f910 100644 --- a/src/output-httplog.h +++ b/src/output-httplog.h @@ -23,6 +23,7 @@ #ifndef __OUTPUT_HTTPLOG_H__ #define __OUTPUT_HTTPLOG_H__ +TmEcode OutputHttpLog (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq); void TmModuleHttpJsonRegister (void); void TmModuleHttpJsonIPv4Register (void);