From: Jeff Lucovsky Date: Tue, 28 Jul 2020 13:09:27 +0000 (-0400) Subject: output/json: Multi-threaded EVE logging support X-Git-Tag: suricata-6.0.0-beta1~63 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=30ae98f658a19c4e74f161f2e77922277517cb0a;p=thirdparty%2Fsuricata.git output/json: Multi-threaded EVE logging support This commit modifies the JSON loggers with changes necessary to support multi-threaded EVE output. Each "thread-init" function sets up the per-thread log file context for subsequent calls to the JSON output to buffer function. --- diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 61262e64a3..ca1d0523a3 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2013-2014 Open Information Security Foundation +/* Copyright (C) 2013-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -35,6 +35,7 @@ #include "threadvars.h" #include "util-debug.h" +#include "util-logopenfile.h" #include "util-misc.h" #include "util-unittest.h" #include "util-unittest-helper.h" @@ -716,37 +717,47 @@ static int JsonAlertLogCondition(ThreadVars *tv, const Packet *p) static TmEcode JsonAlertLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonAlertLogThread *aft = SCMalloc(sizeof(JsonAlertLogThread)); + JsonAlertLogThread *aft = SCCalloc(1, sizeof(JsonAlertLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonAlertLogThread)); - if(initdata == NULL) + + if (initdata == NULL) { SCLogDebug("Error getting context for EveLogAlert. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->json_buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } /** Use the Output Context (file pointer and mutex) */ AlertJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data; - aft->file_ctx = json_output_ctx->file_ctx; - aft->json_output_ctx = json_output_ctx; aft->payload_buffer = MemBufferCreateNew(json_output_ctx->payload_buffer_size); if (aft->payload_buffer == NULL) { - MemBufferFree(aft->json_buffer); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } + aft->file_ctx = LogFileEnsureExists(json_output_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; + } + + aft->json_output_ctx = json_output_ctx; *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->json_buffer != NULL) { + MemBufferFree(aft->json_buffer); + } + if (aft->payload_buffer != NULL) { + MemBufferFree(aft->payload_buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonAlertLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-anomaly.c b/src/output-json-anomaly.c index 3f2fec40cb..7beea83b56 100644 --- a/src/output-json-anomaly.c +++ b/src/output-json-anomaly.c @@ -287,23 +287,32 @@ static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, voi if (initdata == NULL) { SCLogDebug("Error getting context for EveLogAnomaly. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->json_buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } /** Use the Output Context (file pointer and mutex) */ AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data; - aft->file_ctx = json_output_ctx->file_ctx; + + aft->file_ctx = LogFileEnsureExists(json_output_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; + } aft->json_output_ctx = json_output_ctx; *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->json_buffer != NULL) { + MemBufferFree(aft->json_buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-common.c b/src/output-json-common.c index 8165a96077..42fc00f4ce 100644 --- a/src/output-json-common.c +++ b/src/output-json-common.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2018 Open Information Security Foundation +/* Copyright (C) 2018-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -85,13 +85,25 @@ TmEcode JsonLogThreadInit(ThreadVars *t, const void *initdata, void **data) thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->ctx = ((OutputCtx *)initdata)->data; + LogFileEnsureExists(thread->ctx->file_ctx, t->id); + thread->file_ctx = LogFileEnsureExists(thread->ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } + *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-dcerpc.c b/src/output-json-dcerpc.c index f27eef49b8..76737bd78f 100644 --- a/src/output-json-dcerpc.c +++ b/src/output-json-dcerpc.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2017-2018 Open Information Security Foundation +/* Copyright (C) 2017-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -58,7 +58,7 @@ static int JsonDCERPCLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-dhcp.c b/src/output-json-dhcp.c index 07e3989e66..273ba806a5 100644 --- a/src/output-json-dhcp.c +++ b/src/output-json-dhcp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015 Open Information Security Foundation +/* Copyright (C) 2015-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -56,6 +56,7 @@ typedef struct LogDHCPFileCtx_ { typedef struct LogDHCPLogThread_ { LogDHCPFileCtx *dhcplog_ctx; MemBuffer *buffer; + LogFileCtx *file_ctx; } LogDHCPLogThread; static int JsonDHCPLogger(ThreadVars *tv, void *thread_data, @@ -79,7 +80,7 @@ static int JsonDHCPLogger(ThreadVars *tv, void *thread_data, EveAddCommonOptions(&thread->dhcplog_ctx->cfg, p, f, js); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(js, thread->dhcplog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer); jb_free(js); return TM_ECODE_OK; @@ -133,20 +134,29 @@ static TmEcode JsonDHCPLogThreadInit(ThreadVars *t, const void *initdata, void * if (initdata == NULL) { SCLogDebug("Error getting context for EveLogDHCP. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->dhcplog_ctx = ((OutputCtx *)initdata)->data; - *data = (void *)thread; + thread->file_ctx = LogFileEnsureExists(thread->dhcplog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } + *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (unlikely(thread->buffer != NULL)) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonDHCPLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-dnp3.c b/src/output-json-dnp3.c index 7fe1f67e70..0e558bb98d 100644 --- a/src/output-json-dnp3.c +++ b/src/output-json-dnp3.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015 Open Information Security Foundation +/* Copyright (C) 2015-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -51,6 +51,7 @@ typedef struct LogDNP3FileCtx_ { } LogDNP3FileCtx; typedef struct LogDNP3LogThread_ { + LogFileCtx *file_ctx; LogDNP3FileCtx *dnp3log_ctx; MemBuffer *buffer; } LogDNP3LogThread; @@ -231,7 +232,7 @@ static int JsonDNP3LoggerToServer(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogRequest(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->dnp3log_ctx->file_ctx, &buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &buffer); jb_free(js); } @@ -258,7 +259,7 @@ static int JsonDNP3LoggerToClient(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogResponse(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->dnp3log_ctx->file_ctx, &buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &buffer); jb_free(js); } @@ -314,20 +315,30 @@ static TmEcode JsonDNP3LogThreadInit(ThreadVars *t, const void *initdata, void * if (initdata == NULL) { SCLogDebug("Error getting context for DNP3. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->dnp3log_ctx = ((OutputCtx *)initdata)->data; + thread->file_ctx = LogFileEnsureExists(thread->dnp3log_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } + *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonDNP3LogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-dns.c b/src/output-json-dns.c index f6d59cc897..a97021a8bb 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -266,6 +266,7 @@ typedef struct LogDnsFileCtx_ { typedef struct LogDnsLogThread_ { LogDnsFileCtx *dnslog_ctx; /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ + LogFileCtx *file_ctx; MemBuffer *buffer; } LogDnsLogThread; @@ -330,7 +331,7 @@ static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(td->buffer); - OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer); + OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer); jb_free(jb); } @@ -361,7 +362,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data, rs_dns_log_json_answer(txptr, td->dnslog_ctx->flags, jb); jb_close(jb); MemBufferReset(td->buffer); - OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer); + OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer); jb_free(jb); } } else { @@ -382,7 +383,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data, jb_set_object(jb, "dns", answer); MemBufferReset(td->buffer); - OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer); + OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer); jb_free(jb); } /* Log authorities. */ @@ -402,7 +403,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data, jb_set_object(jb, "dns", answer); MemBufferReset(td->buffer); - OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer); + OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer); jb_free(jb); } } @@ -412,29 +413,37 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data, static TmEcode LogDnsLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - LogDnsLogThread *aft = SCMalloc(sizeof(LogDnsLogThread)); + LogDnsLogThread *aft = SCCalloc(1, sizeof(LogDnsLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(LogDnsLogThread)); if(initdata == NULL) { SCLogDebug("Error getting context for EveLogDNS. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } /* Use the Ouptut Context (file pointer and mutex) */ aft->dnslog_ctx= ((OutputCtx *)initdata)->data; + aft->file_ctx = LogFileEnsureExists(aft->dnslog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; + } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode LogDnsLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-drop.c b/src/output-json-drop.c index 8182cfc1d1..ecfbb41e94 100644 --- a/src/output-json-drop.c +++ b/src/output-json-drop.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -67,6 +67,7 @@ typedef struct JsonDropOutputCtx_ { } JsonDropOutputCtx; typedef struct JsonDropLogThread_ { + LogFileCtx *file_ctx; JsonDropOutputCtx *drop_ctx; MemBuffer *buffer; } JsonDropLogThread; @@ -173,7 +174,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) } } - OutputJsonBuilderBuffer(js, aft->drop_ctx->file_ctx, &aft->buffer); + OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->buffer); jb_free(js); return TM_ECODE_OK; @@ -181,29 +182,37 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) static TmEcode JsonDropLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonDropLogThread *aft = SCMalloc(sizeof(JsonDropLogThread)); + JsonDropLogThread *aft = SCCalloc(1, sizeof(JsonDropLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(*aft)); if(initdata == NULL) { SCLogDebug("Error getting context for EveLogDrop. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } /** Use the Ouptut Context (file pointer and mutex) */ aft->drop_ctx = ((OutputCtx *)initdata)->data; + aft->file_ctx = LogFileEnsureExists(aft->drop_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; + } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonDropLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-email-common.h b/src/output-json-email-common.h index 62aca5901c..a725099139 100644 --- a/src/output-json-email-common.h +++ b/src/output-json-email-common.h @@ -32,6 +32,7 @@ typedef struct OutputJsonEmailCtx_ { } OutputJsonEmailCtx; typedef struct JsonEmailLogThread_ { + LogFileCtx *file_ctx; OutputJsonEmailCtx *emaillog_ctx; MemBuffer *buffer; } JsonEmailLogThread; diff --git a/src/output-json-file.c b/src/output-json-file.c index cfe158552d..8e03b27ed8 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -77,6 +77,7 @@ typedef struct OutputFileCtx_ { typedef struct JsonFileLogThread_ { OutputFileCtx *filelog_ctx; + LogFileCtx *file_ctx; MemBuffer *buffer; } JsonFileLogThread; @@ -203,7 +204,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, } MemBufferReset(aft->buffer); - OutputJsonBuilderBuffer(js, aft->filelog_ctx->file_ctx, &aft->buffer); + OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->buffer); jb_free(js); } @@ -224,29 +225,37 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, static TmEcode JsonFileLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonFileLogThread *aft = SCMalloc(sizeof(JsonFileLogThread)); + JsonFileLogThread *aft = SCCalloc(1, sizeof(JsonFileLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonFileLogThread)); if(initdata == NULL) { SCLogDebug("Error getting context for EveLogFile. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } - /* Use the Ouptut Context (file pointer and mutex) */ - aft->filelog_ctx = ((OutputCtx *)initdata)->data; - aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; + } + + /* Use the Ouptut Context (file pointer and mutex) */ + aft->filelog_ctx = ((OutputCtx *)initdata)->data; + aft->file_ctx = LogFileEnsureExists(aft->filelog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-flow.c b/src/output-json-flow.c index 8faccc73da..3d47f42eae 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -60,6 +60,7 @@ typedef struct LogJsonFileCtx_ { typedef struct JsonFlowLogThread_ { LogJsonFileCtx *flowlog_ctx; /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ + LogFileCtx *file_ctx; MemBuffer *buffer; } JsonFlowLogThread; @@ -333,12 +334,12 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) JsonBuilder *jb = CreateEveHeaderFromFlow(f); if (unlikely(jb == NULL)) { - return TM_ECODE_OK; + SCReturnInt(TM_ECODE_OK); } EveFlowLogJSON(jhl, jb, f); - OutputJsonBuilderBuffer(jb, jhl->flowlog_ctx->file_ctx, &jhl->buffer); + OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer); jb_free(jb); SCReturnInt(TM_ECODE_OK); @@ -425,29 +426,38 @@ static OutputInitResult OutputFlowLogInitSub(ConfNode *conf, OutputCtx *parent_c static TmEcode JsonFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonFlowLogThread *aft = SCMalloc(sizeof(JsonFlowLogThread)); + JsonFlowLogThread *aft = SCCalloc(1, sizeof(JsonFlowLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonFlowLogThread)); if(initdata == NULL) { SCLogDebug("Error getting context for EveLogFlow. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } - /* Use the Ouptut Context (file pointer and mutex) */ + /* Use the Outptut Context (file pointer and mutex) */ aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; + } + + aft->file_ctx = LogFileEnsureExists(aft->flowlog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonFlowLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-ftp.c b/src/output-json-ftp.c index 8d3f67457f..619f4148b3 100644 --- a/src/output-json-ftp.c +++ b/src/output-json-ftp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2017 Open Information Security Foundation +/* Copyright (C) 2017-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -54,6 +54,7 @@ typedef struct LogFTPFileCtx_ { } LogFTPFileCtx; typedef struct LogFTPLogThread_ { + LogFileCtx *file_ctx; LogFTPFileCtx *ftplog_ctx; MemBuffer *buffer; } LogFTPLogThread; @@ -175,7 +176,7 @@ static int JsonFTPLogger(ThreadVars *tv, void *thread_data, } MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->ftplog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); } @@ -234,20 +235,30 @@ static TmEcode JsonFTPLogThreadInit(ThreadVars *t, const void *initdata, void ** if (initdata == NULL) { SCLogDebug("Error getting context for EveLogFTP. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->ftplog_ctx = ((OutputCtx *)initdata)->data; + thread->file_ctx = LogFileEnsureExists(thread->ftplog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } + *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonFTPLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-http.c b/src/output-json-http.c index 78b479f285..695e34cb61 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -66,6 +66,7 @@ typedef struct LogHttpFileCtx_ { typedef struct JsonHttpLogThread_ { LogHttpFileCtx *httplog_ctx; + LogFileCtx *file_ctx; /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ uint32_t uri_cnt; @@ -538,7 +539,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } } - OutputJsonBuilderBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer); + OutputJsonBuilderBuffer(js, jhl->file_ctx, &jhl->buffer); jb_free(js); SCReturnInt(TM_ECODE_OK); @@ -741,29 +742,38 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c static TmEcode JsonHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonHttpLogThread *aft = SCMalloc(sizeof(JsonHttpLogThread)); + JsonHttpLogThread *aft = SCCalloc(1, sizeof(JsonHttpLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonHttpLogThread)); if(initdata == NULL) { SCLogDebug("Error getting context for EveLogHTTP. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } - /* Use the Ouptut Context (file pointer and mutex) */ + /* Use the Output Context (file pointer and mutex) */ aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; + } + + aft->file_ctx = LogFileEnsureExists(aft->httplog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-ikev2.c b/src/output-json-ikev2.c index 8beb71a7dc..3b4577ee62 100644 --- a/src/output-json-ikev2.c +++ b/src/output-json-ikev2.c @@ -55,6 +55,7 @@ typedef struct LogIKEv2FileCtx_ { } LogIKEv2FileCtx; typedef struct LogIKEv2LogThread_ { + LogFileCtx *file_ctx; LogIKEv2FileCtx *ikev2log_ctx; MemBuffer *buffer; } LogIKEv2LogThread; @@ -79,7 +80,7 @@ static int JsonIKEv2Logger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->ikev2log_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); return TM_ECODE_OK; @@ -135,20 +136,29 @@ static TmEcode JsonIKEv2LogThreadInit(ThreadVars *t, const void *initdata, void if (initdata == NULL) { SCLogDebug("Error getting context for EveLogIKEv2. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->ikev2log_ctx = ((OutputCtx *)initdata)->data; - *data = (void *)thread; + thread->file_ctx = LogFileEnsureExists(thread->ikev2log_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } + *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonIKEv2LogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-krb5.c b/src/output-json-krb5.c index 3141caed24..616932fd9a 100644 --- a/src/output-json-krb5.c +++ b/src/output-json-krb5.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2018 Open Information Security Foundation +/* Copyright (C) 2018-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -55,6 +55,7 @@ typedef struct LogKRB5FileCtx_ { } LogKRB5FileCtx; typedef struct LogKRB5LogThread_ { + LogFileCtx *file_ctx; LogKRB5FileCtx *krb5log_ctx; MemBuffer *buffer; } LogKRB5LogThread; @@ -79,7 +80,7 @@ static int JsonKRB5Logger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->krb5log_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); return TM_ECODE_OK; @@ -136,20 +137,29 @@ static TmEcode JsonKRB5LogThreadInit(ThreadVars *t, const void *initdata, void * if (initdata == NULL) { SCLogDebug("Error getting context for EveLogKRB5. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->krb5log_ctx = ((OutputCtx *)initdata)->data; + thread->file_ctx = LogFileEnsureExists(thread->krb5log_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonKRB5LogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-metadata.c b/src/output-json-metadata.c index 43e4f56c0f..415f24de4f 100644 --- a/src/output-json-metadata.c +++ b/src/output-json-metadata.c @@ -106,30 +106,37 @@ static int JsonMetadataLogCondition(ThreadVars *tv, const Packet *p) static TmEcode JsonMetadataLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonMetadataLogThread *aft = SCMalloc(sizeof(JsonMetadataLogThread)); + JsonMetadataLogThread *aft = SCCalloc(1, sizeof(JsonMetadataLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonMetadataLogThread)); - if(initdata == NULL) - { + + if(initdata == NULL) { SCLogDebug("Error getting context for EveLogMetadata. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->json_buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } /** Use the Output Context (file pointer and mutex) */ MetadataJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data; - aft->file_ctx = json_output_ctx->file_ctx; + aft->file_ctx = LogFileEnsureExists(json_output_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; + } aft->json_output_ctx = json_output_ctx; *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->json_buffer != NULL) { + MemBufferFree(aft->json_buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonMetadataLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index a244858f0b..51acf10e3e 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -55,6 +55,7 @@ typedef struct LogJsonFileCtx_ { } LogJsonFileCtx; typedef struct JsonNetFlowLogThread_ { + LogFileCtx *file_ctx; LogJsonFileCtx *flowlog_ctx; /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ @@ -287,7 +288,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToServer(jhl, jb, f); EveAddCommonOptions(&netflow_ctx->cfg, NULL, f, jb); - OutputJsonBuilderBuffer(jb, jhl->flowlog_ctx->file_ctx, &jhl->buffer); + OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer); jb_free(jb); /* only log a response record if we actually have seen response packets */ @@ -299,7 +300,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToClient(jhl, jb, f); EveAddCommonOptions(&netflow_ctx->cfg, NULL, f, jb); - OutputJsonBuilderBuffer(jb, jhl->flowlog_ctx->file_ctx, &jhl->buffer); + OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer); jb_free(jb); } SCReturnInt(TM_ECODE_OK); @@ -386,16 +387,13 @@ static OutputInitResult OutputNetFlowLogInitSub(ConfNode *conf, OutputCtx *paren static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonNetFlowLogThread *aft = SCMalloc(sizeof(JsonNetFlowLogThread)); + JsonNetFlowLogThread *aft = SCCalloc(1, sizeof(JsonNetFlowLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonNetFlowLogThread)); - if(initdata == NULL) - { + if(initdata == NULL) { SCLogDebug("Error getting context for EveLogNetflow. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } /* Use the Ouptut Context (file pointer and mutex) */ @@ -403,12 +401,23 @@ static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, voi aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; + } + + aft->file_ctx = LogFileEnsureExists(aft->flowlog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonNetFlowLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-nfs.c b/src/output-json-nfs.c index cdd94d9d18..2799a3f49f 100644 --- a/src/output-json-nfs.c +++ b/src/output-json-nfs.c @@ -96,7 +96,7 @@ static int JsonNFSLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); return TM_ECODE_OK; } diff --git a/src/output-json-rdp.c b/src/output-json-rdp.c index bb302c1fc9..fce31b7f2f 100644 --- a/src/output-json-rdp.c +++ b/src/output-json-rdp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 Open Information Security Foundation +/* Copyright (C) 2019-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -51,6 +51,7 @@ typedef struct LogRdpFileCtx_ { typedef struct LogRdpLogThread_ { LogRdpFileCtx *rdplog_ctx; + LogFileCtx *file_ctx; MemBuffer *buffer; } LogRdpLogThread; @@ -69,7 +70,7 @@ static int JsonRdpLogger(ThreadVars *tv, void *thread_data, return TM_ECODE_FAILED; } MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(js, thread->rdplog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer); jb_free(js); return TM_ECODE_OK; @@ -126,14 +127,24 @@ static TmEcode JsonRdpLogThreadInit(ThreadVars *t, const void *initdata, void ** thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->rdplog_ctx = ((OutputCtx *)initdata)->data; - *data = (void *)thread; + thread->file_ctx = LogFileEnsureExists(thread->rdplog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } + *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonRdpLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-rfb.c b/src/output-json-rfb.c index 45313a80b4..dc117c28f4 100644 --- a/src/output-json-rfb.c +++ b/src/output-json-rfb.c @@ -53,6 +53,7 @@ typedef struct LogRFBFileCtx_ { typedef struct LogRFBLogThread_ { LogRFBFileCtx *rfblog_ctx; + LogFileCtx *file_ctx; MemBuffer *buffer; } LogRFBLogThread; @@ -84,7 +85,7 @@ static int JsonRFBLogger(ThreadVars *tv, void *thread_data, } MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(js, thread->rfblog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer); jb_free(js); return TM_ECODE_OK; @@ -142,14 +143,24 @@ static TmEcode JsonRFBLogThreadInit(ThreadVars *t, const void *initdata, void ** thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->rfblog_ctx = ((OutputCtx *)initdata)->data; + thread->file_ctx = LogFileEnsureExists(thread->rfblog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonRFBLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-sip.c b/src/output-json-sip.c index 7400944b50..a0576995d3 100644 --- a/src/output-json-sip.c +++ b/src/output-json-sip.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2018 Open Information Security Foundation +/* Copyright (C) 2018-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -55,6 +55,7 @@ typedef struct LogSIPFileCtx_ { } LogSIPFileCtx; typedef struct LogSIPLogThread_ { + LogFileCtx *file_ctx; LogSIPFileCtx *siplog_ctx; MemBuffer *buffer; } LogSIPLogThread; @@ -87,7 +88,7 @@ static int JsonSIPLogger(ThreadVars *tv, void *thread_data, } MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(js, thread->siplog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer); jb_free(js); return TM_ECODE_OK; @@ -145,20 +146,30 @@ static TmEcode JsonSIPLogThreadInit(ThreadVars *t, const void *initdata, void ** if (initdata == NULL) { SCLogDebug("Error getting context for EveLogSIP. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->siplog_ctx = ((OutputCtx *)initdata)->data; + + thread->file_ctx = LogFileEnsureExists(thread->siplog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonSIPLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-smb.c b/src/output-json-smb.c index f56cb0244a..fa8a2a79aa 100644 --- a/src/output-json-smb.c +++ b/src/output-json-smb.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2017-2018 Open Information Security Foundation +/* Copyright (C) 2017-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -77,7 +77,7 @@ static int JsonSMBLogger(ThreadVars *tv, void *thread_data, EveAddCommonOptions(&thread->ctx->cfg, p, f, jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-smtp.c b/src/output-json-smtp.c index 3d56894c79..cccf7ce81e 100644 --- a/src/output-json-smtp.c +++ b/src/output-json-smtp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2015 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -89,7 +89,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl jb_close(jb); if (EveEmailLogJson(jhl, jb, p, f, state, tx, tx_id) == TM_ECODE_OK) { - OutputJsonBuilderBuffer(jb, jhl->emaillog_ctx->file_ctx, &jhl->buffer); + OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer); } jb_free(jb); @@ -206,29 +206,36 @@ static OutputInitResult OutputSmtpLogInitSub(ConfNode *conf, OutputCtx *parent_c static TmEcode JsonSmtpLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonEmailLogThread *aft = SCMalloc(sizeof(JsonEmailLogThread)); + JsonEmailLogThread *aft = SCCalloc(1, sizeof(JsonEmailLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonEmailLogThread)); - if(initdata == NULL) - { + if(initdata == NULL) { SCLogDebug("Error getting context for EveLogSMTP. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } - /* Use the Ouptut Context (file pointer and mutex) */ + /* Use the Output Context (file pointer and mutex) */ aft->emaillog_ctx = ((OutputCtx *)initdata)->data; aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } + aft->file_ctx = LogFileEnsureExists(aft->emaillog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; + } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonSmtpLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-snmp.c b/src/output-json-snmp.c index 3b34179ad9..4e9599ec67 100644 --- a/src/output-json-snmp.c +++ b/src/output-json-snmp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2018-2019 Open Information Security Foundation +/* Copyright (C) 2018-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -55,6 +55,7 @@ typedef struct LogSNMPFileCtx_ { } LogSNMPFileCtx; typedef struct LogSNMPLogThread_ { + LogFileCtx *file_ctx; LogSNMPFileCtx *snmplog_ctx; MemBuffer *buffer; } LogSNMPLogThread; @@ -79,7 +80,7 @@ static int JsonSNMPLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->snmplog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); return TM_ECODE_OK; @@ -135,20 +136,30 @@ static TmEcode JsonSNMPLogThreadInit(ThreadVars *t, const void *initdata, void * if (initdata == NULL) { SCLogDebug("Error getting context for EveLogSNMP. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->snmplog_ctx = ((OutputCtx *)initdata)->data; - *data = (void *)thread; + thread->file_ctx = LogFileEnsureExists(thread->snmplog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } + + *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonSNMPLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-ssh.c b/src/output-json-ssh.c index dd74a6a051..cc9210fb06 100644 --- a/src/output-json-ssh.c +++ b/src/output-json-ssh.c @@ -61,6 +61,7 @@ typedef struct OutputSshCtx_ { typedef struct JsonSshLogThread_ { OutputSshCtx *sshlog_ctx; + LogFileCtx *file_ctx; MemBuffer *buffer; } JsonSshLogThread; @@ -103,22 +104,32 @@ static TmEcode JsonSshLogThreadInit(ThreadVars *t, const void *initdata, void ** return TM_ECODE_FAILED; } - JsonSshLogThread *aft = SCMalloc(sizeof(JsonSshLogThread)); + JsonSshLogThread *aft = SCCalloc(1, sizeof(JsonSshLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonSshLogThread)); - /* Use the Ouptut Context (file pointer and mutex) */ + /* Use the Output Context (file pointer and mutex) */ aft->sshlog_ctx = ((OutputCtx *)initdata)->data; aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; + } + + aft->file_ctx = LogFileEnsureExists(aft->sshlog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonSshLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-stats.c b/src/output-json-stats.c index 666ec1763b..fae2fd2f29 100644 --- a/src/output-json-stats.c +++ b/src/output-json-stats.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 Open Information Security Foundation +/* Copyright (C) 2014-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -70,6 +70,7 @@ typedef struct OutputStatsCtx_ { typedef struct JsonStatsLogThread_ { OutputStatsCtx *statslog_ctx; + LogFileCtx *file_ctx; MemBuffer *buffer; } JsonStatsLogThread; @@ -313,7 +314,7 @@ static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable * json_object_set_new(js, "stats", js_stats); - OutputJSONBuffer(js, aft->statslog_ctx->file_ctx, &aft->buffer); + OutputJSONBuffer(js, aft->file_ctx, &aft->buffer); MemBufferReset(aft->buffer); json_object_clear(js_stats); @@ -326,29 +327,38 @@ static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable * static TmEcode JsonStatsLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonStatsLogThread *aft = SCMalloc(sizeof(JsonStatsLogThread)); + JsonStatsLogThread *aft = SCCalloc(1, sizeof(JsonStatsLogThread)); if (unlikely(aft == NULL)) return TM_ECODE_FAILED; - memset(aft, 0, sizeof(JsonStatsLogThread)); if(initdata == NULL) { SCLogDebug("Error getting context for EveLogStats. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } - /* Use the Ouptut Context (file pointer and mutex) */ - aft->statslog_ctx = ((OutputCtx *)initdata)->data; - aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; + } + + /* Use the Output Context (file pointer and mutex) */ + aft->statslog_ctx = ((OutputCtx *)initdata)->data; + + aft->file_ctx = LogFileEnsureExists(aft->statslog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonStatsLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-template-rust.c b/src/output-json-template-rust.c index b8cb4d826f..c5832bb2fd 100644 --- a/src/output-json-template-rust.c +++ b/src/output-json-template-rust.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2018 Open Information Security Foundation +/* Copyright (C) 2018-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -61,6 +61,7 @@ typedef struct LogTemplateFileCtx_ { typedef struct LogTemplateLogThread_ { LogTemplateFileCtx *templatelog_ctx; + LogFileCtx *file_ctx; MemBuffer *buffer; } LogTemplateLogThread; @@ -82,7 +83,7 @@ static int JsonTemplateLogger(ThreadVars *tv, void *thread_data, jb_close(js); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(js, thread->templatelog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer); jb_free(js); return TM_ECODE_OK; @@ -137,20 +138,29 @@ static TmEcode JsonTemplateLogThreadInit(ThreadVars *t, const void *initdata, vo if (initdata == NULL) { SCLogDebug("Error getting context for EveLogTemplate. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->templatelog_ctx = ((OutputCtx *)initdata)->data; + thread->file_ctx = LogFileEnsureExists(thread->templatelog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-template.c b/src/output-json-template.c index 8ab25e1977..d4dd139888 100644 --- a/src/output-json-template.c +++ b/src/output-json-template.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015 Open Information Security Foundation +/* Copyright (C) 2015-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -59,6 +59,7 @@ typedef struct LogTemplateFileCtx_ { } LogTemplateFileCtx; typedef struct LogTemplateLogThread_ { + LogFileCtx *file_ctx; LogTemplateFileCtx *templatelog_ctx; MemBuffer *buffer; } LogTemplateLogThread; @@ -94,7 +95,7 @@ static int JsonTemplateLogger(ThreadVars *tv, void *thread_data, jb_close(js); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(js, thread->templatelog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer); jb_free(js); return TM_ECODE_OK; @@ -145,20 +146,29 @@ static TmEcode JsonTemplateLogThreadInit(ThreadVars *t, const void *initdata, vo if (initdata == NULL) { SCLogDebug("Error getting context for EveLogTemplate. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->templatelog_ctx = ((OutputCtx *)initdata)->data; + thread->file_ctx = LogFileEnsureExists(thread->templatelog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-tftp.c b/src/output-json-tftp.c index 3a47cfe4cc..7c444fe3d9 100644 --- a/src/output-json-tftp.c +++ b/src/output-json-tftp.c @@ -57,6 +57,7 @@ typedef struct LogTFTPFileCtx_ { } LogTFTPFileCtx; typedef struct LogTFTPLogThread_ { + LogFileCtx *file_ctx; LogTFTPFileCtx *tftplog_ctx; MemBuffer *buffer; } LogTFTPLogThread; @@ -79,7 +80,7 @@ static int JsonTFTPLogger(ThreadVars *tv, void *thread_data, EveAddCommonOptions(&thread->tftplog_ctx->cfg, p, f, jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread->tftplog_ctx->file_ctx, &thread->buffer); + OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer); jb_free(jb); return TM_ECODE_OK; @@ -135,20 +136,29 @@ static TmEcode JsonTFTPLogThreadInit(ThreadVars *t, const void *initdata, void * if (initdata == NULL) { SCLogDebug("Error getting context for EveLogTFTP. \"initdata\" is NULL."); - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (unlikely(thread->buffer == NULL)) { - SCFree(thread); - return TM_ECODE_FAILED; + goto error_exit; } thread->tftplog_ctx = ((OutputCtx *)initdata)->data; + thread->file_ctx = LogFileEnsureExists(thread->tftplog_ctx->file_ctx, t->id); + if (!thread->file_ctx) { + goto error_exit; + } *data = (void *)thread; return TM_ECODE_OK; + +error_exit: + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_FAILED; } static TmEcode JsonTFTPLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json-tls.c b/src/output-json-tls.c index dc589d84fe..a4ddd7a9d8 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -106,6 +106,7 @@ typedef struct OutputTlsCtx_ { typedef struct JsonTlsLogThread_ { + LogFileCtx *file_ctx; OutputTlsCtx *tlslog_ctx; MemBuffer *buffer; } JsonTlsLogThread; @@ -440,7 +441,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, /* Close the tls object. */ jb_close(js); - OutputJsonBuilderBuffer(js, tls_ctx->file_ctx, &aft->buffer); + OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->buffer); jb_free(js); return 0; @@ -448,30 +449,37 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, static TmEcode JsonTlsLogThreadInit(ThreadVars *t, const void *initdata, void **data) { - JsonTlsLogThread *aft = SCMalloc(sizeof(JsonTlsLogThread)); + JsonTlsLogThread *aft = SCCalloc(1, sizeof(JsonTlsLogThread)); if (unlikely(aft == NULL)) { return TM_ECODE_FAILED; } - memset(aft, 0, sizeof(JsonTlsLogThread)); - if (initdata == NULL) { SCLogDebug("Error getting context for eve-log tls 'initdata' argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } - /* use the Output Context (file pointer and mutex) */ - aft->tlslog_ctx = ((OutputCtx *)initdata)->data; - aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE); if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; + goto error_exit; } + /* use the Output Context (file pointer and mutex) */ + aft->tlslog_ctx = ((OutputCtx *)initdata)->data; + + aft->file_ctx = LogFileEnsureExists(aft->tlslog_ctx->file_ctx, t->id); + if (!aft->file_ctx) { + goto error_exit; + } *data = (void *)aft; return TM_ECODE_OK; + +error_exit: + if (aft->buffer != NULL) { + MemBufferFree(aft->buffer); + } + SCFree(aft); + return TM_ECODE_FAILED; } static TmEcode JsonTlsLogThreadDeinit(ThreadVars *t, void *data) diff --git a/src/output-json.c b/src/output-json.c index bfca393faf..e3c2381ea1 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -1022,6 +1022,17 @@ OutputInitResult OutputJsonInitCtx(ConfNode *conf) json_ctx->json_out == LOGFILE_TYPE_UNIX_DGRAM || json_ctx->json_out == LOGFILE_TYPE_UNIX_STREAM) { + if (json_ctx->json_out == LOGFILE_TYPE_FILE) { + /* Threaded file output */ + const ConfNode *threaded = ConfNodeLookupChild(conf, "threaded"); + if (threaded && threaded->val && ConfValIsTrue(threaded->val)) { + SCLogConfig("Enabling threaded eve logging."); + json_ctx->file_ctx->threaded = true; + } else { + json_ctx->file_ctx->threaded = false; + } + } + if (SCConfLogOpenGeneric(conf, json_ctx->file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) { LogFileFreeCtx(json_ctx->file_ctx); SCFree(json_ctx); @@ -1029,6 +1040,7 @@ OutputInitResult OutputJsonInitCtx(ConfNode *conf) return result; } OutputRegisterFileRotationFlag(&json_ctx->file_ctx->rotation_flag); + } #ifndef OS_WIN32 else if (json_ctx->json_out == LOGFILE_TYPE_SYSLOG) { diff --git a/src/output-json.h b/src/output-json.h index 86bcc074b0..57515d0fbc 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -105,6 +105,7 @@ typedef struct OutputJsonCtx_ { typedef struct OutputJsonThreadCtx_ { OutputJsonCtx *ctx; + LogFileCtx *file_ctx; MemBuffer *buffer; } OutputJsonThreadCtx; diff --git a/suricata.yaml.in b/suricata.yaml.in index 80928e04cf..fca2fd2608 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -84,6 +84,9 @@ outputs: enabled: @e_enable_evelog@ filetype: regular #regular|syslog|unix_dgram|unix_stream|redis filename: eve.json + # Enable for multi-threaded eve.json output; output files are suffixed + # with an identifier, e.g., eve.json.9. + #threaded: false #prefix: "@cee: " # prefix to prepend to each log entry # the following are valid when type: syslog above #identity: "suricata"