From e98346b555b39e689a1c2c6d0e643d0846ff61f6 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 31 Oct 2014 23:37:04 +0100 Subject: [PATCH] Introduce stats log API, convert existing output Convert regular 'stats.log' output to this new API. In addition to the current stats value, also give the last value. This makes it easy to display the difference. --- src/Makefile.am | 2 + src/counters.c | 317 +++++++++++++++------------------------- src/counters.h | 21 --- src/log-stats.c | 231 +++++++++++++++++++++++++++++ src/log-stats.h | 29 ++++ src/output-stats.c | 234 +++++++++++++++++++++++++++++ src/output-stats.h | 53 +++++++ src/output.c | 70 +++++++++ src/output.h | 8 + src/runmodes.c | 9 +- src/suricata.c | 4 + src/tm-modules.c | 2 + src/tm-threads-common.h | 2 + 13 files changed, 757 insertions(+), 225 deletions(-) create mode 100644 src/log-stats.c create mode 100644 src/log-stats.h create mode 100644 src/output-stats.c create mode 100644 src/output-stats.h diff --git a/src/Makefile.am b/src/Makefile.am index d788afdc57..6fac36e750 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,6 +215,7 @@ log-file.c log-file.h \ log-filestore.c log-filestore.h \ log-httplog.c log-httplog.h \ log-pcap.c log-pcap.h \ +log-stats.c log-stats.h \ log-tcp-data.c log-tcp-data.h \ log-tlslog.c log-tlslog.h \ output.c output.h \ @@ -234,6 +235,7 @@ output-json-ssh.c output-json-ssh.h \ output-json-tls.c output-json-tls.h \ output-lua.c output-lua.h \ output-packet.c output-packet.h \ +output-stats.c output-stats.h \ output-streaming.c output-streaming.h \ output-tx.c output-tx.h \ output-json.c output-json.h \ diff --git a/src/counters.c b/src/counters.c index 9aae1fe52d..5ea17c55e2 100644 --- a/src/counters.c +++ b/src/counters.c @@ -19,6 +19,7 @@ * \file * * \author Anoop Saldanha + * \author Victor Julien * * Performance counters */ @@ -43,14 +44,33 @@ /* Used to parse the interval for Timebased counters */ #define SC_PERF_PCRE_TIMEBASED_INTERVAL "^(?:(\\d+)([shm]))(?:(\\d+)([shm]))?(?:(\\d+)([shm]))?$" +/* Time interval for syncing the local counters with the global ones */ +#define SC_PERF_WUT_TTS 3 + +/* Time interval at which the mgmt thread o/p the stats */ +#define SC_PERF_MGMTT_TTS 8 + +static void *stats_thread_data = NULL; static SCPerfOPIfaceContext *sc_perf_op_ctx = NULL; static time_t sc_start_time; /** refresh interval in seconds */ static uint32_t sc_counter_tts = SC_PERF_MGMTT_TTS; /** is the stats counter enabled? */ static char sc_counter_enabled = TRUE; -/** append or overwrite? 1: append, 0: overwrite */ -static char sc_counter_append = TRUE; + +static int SCPerfOutputCounterFileIface(ThreadVars *tv); + +/** stats table is filled each interval and passed to the + * loggers. Initialized at first use. */ +static StatsTable stats_table = { NULL, 0, 0, {0 , 0}}; + +/** + * \brief The output interface dispatcher for the counter api + */ +void SCPerfOutputCounters(ThreadVars *tv) +{ + SCPerfOutputCounterFileIface(tv); +} /** * \brief Adds a value of type uint64_t to the local counter. @@ -126,65 +146,6 @@ void SCPerfCounterSetUI64(uint16_t id, SCPerfCounterArray *pca, return; } -/** - * \brief Get the filename with path to the stats log file. - * - * This function returns a string containing the log filename. It uses - * allocated memory simply to drop into the existing code a little better - * where a SCStrdup was used. So as before, it is up to the caller to free - * the memory. - * - * \retval An allocated string containing the log filename on success or NULL on - * failure. - */ -static char *SCPerfGetLogFilename(ConfNode *stats) -{ - char *log_dir = NULL; - char *log_filename = NULL; - const char* filename = NULL; - - log_dir = ConfigGetLogDirectory(); - - if ( (log_filename = SCMalloc(PATH_MAX)) == NULL) { - return NULL; - } - - if (stats != NULL) { - filename = ConfNodeLookupChildValue(stats, "filename"); - if (filename == NULL) { - filename = SC_PERF_DEFAULT_LOG_FILENAME; - } - } else { - filename = SC_PERF_DEFAULT_LOG_FILENAME; - } - - if (snprintf(log_filename, PATH_MAX, "%s/%s", log_dir, - filename) < 0) { - SCLogError(SC_ERR_SPRINTF, "Sprintf Error"); - SCFree(log_filename); - return NULL; - } - - return log_filename; -} - -/** - * \brief Reopen the log file. - * - * \retval 1 if successful, otherwise 0. - */ -static int SCPerfFileReopen(SCPerfOPIfaceContext *sc_perf_op_ctx) -{ - fclose(sc_perf_op_ctx->fp); - if ((sc_perf_op_ctx->fp = fopen(sc_perf_op_ctx->file, "w+")) == NULL) { - SCLogError(SC_ERR_FOPEN, "Failed to reopen file \"%s\"." - "Stats logging will now be disabled.", - sc_perf_op_ctx->file); - return 0; - } - return 1; -} - /** * \brief Initializes the output interface context * @@ -193,73 +154,14 @@ static int SCPerfFileReopen(SCPerfOPIfaceContext *sc_perf_op_ctx) static void SCPerfInitOPCtx(void) { SCEnter(); - - ConfNode *root = ConfGetNode("outputs"); - ConfNode *node = NULL; - ConfNode *stats = NULL; - if (root != NULL) { - TAILQ_FOREACH(node, &root->head, next) { - if (strncmp(node->val, "stats", 5) == 0) { - stats = node->head.tqh_first; - } - } - } - /* Check if the stats module is enabled or not */ - if (stats != NULL) { - const char *enabled = ConfNodeLookupChildValue(stats, "enabled"); - if (enabled != NULL && ConfValIsFalse(enabled)) { - sc_counter_enabled = FALSE; - SCLogDebug("Stats module has been disabled"); - SCReturn; - } - const char *interval = ConfNodeLookupChildValue(stats, "interval"); - if (interval != NULL) - sc_counter_tts = (uint32_t) atoi(interval); - - const char *append = ConfNodeLookupChildValue(stats, "append"); - if (append != NULL) - sc_counter_append = ConfValIsTrue(append); - } - - /* Store the engine start time */ - time(&sc_start_time); - if ( (sc_perf_op_ctx = SCMalloc(sizeof(SCPerfOPIfaceContext))) == NULL) { SCLogError(SC_ERR_FATAL, "Fatal error encountered in SCPerfInitOPCtx. Exiting..."); exit(EXIT_FAILURE); } memset(sc_perf_op_ctx, 0, sizeof(SCPerfOPIfaceContext)); - sc_perf_op_ctx->iface = SC_PERF_IFACE_FILE; - - if ( (sc_perf_op_ctx->file = SCPerfGetLogFilename(stats)) == NULL) { - SCLogInfo("Error retrieving Perf Counter API output file path"); - } - - char *mode; - if (sc_counter_append) - mode = "a+"; - else - mode = "w+"; - - if ( (sc_perf_op_ctx->fp = fopen(sc_perf_op_ctx->file, mode)) == NULL) { - SCLogError(SC_ERR_FOPEN, "fopen error opening file \"%s\". Resorting " - "to using the standard output for output", - sc_perf_op_ctx->file); - - SCFree(sc_perf_op_ctx->file); - - /* Let us use the standard output for output */ - sc_perf_op_ctx->fp = stdout; - if ( (sc_perf_op_ctx->file = SCStrdup("stdout")) == NULL) { - SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory"); - exit(EXIT_FAILURE); - } - } - else { - /* File opened, register for rotation notification. */ - OutputRegisterFileRotationFlag(&sc_perf_op_ctx->rotation_flag); - } + /* Store the engine start time */ + time(&sc_start_time); /* init the lock used by SCPerfClubTMInst */ if (SCMutexInit(&sc_perf_op_ctx->pctmi_lock, NULL) != 0) { @@ -285,14 +187,6 @@ static void SCPerfReleaseOPCtx() SCPerfClubTMInst *temp = NULL; pctmi = sc_perf_op_ctx->pctmi; - OutputUnregisterFileRotationFlag(&sc_perf_op_ctx->rotation_flag); - - if (sc_perf_op_ctx->fp != NULL) - fclose(sc_perf_op_ctx->fp); - - if (sc_perf_op_ctx->file != NULL) - SCFree(sc_perf_op_ctx->file); - while (pctmi != NULL) { if (pctmi->tm_name != NULL) SCFree(pctmi->tm_name); @@ -308,6 +202,12 @@ static void SCPerfReleaseOPCtx() SCFree(sc_perf_op_ctx); sc_perf_op_ctx = NULL; + /* free stats table */ + if (stats_table.stats != NULL) { + SCFree(stats_table.stats); + memset(&stats_table, 0, sizeof(stats_table)); + } + return; } @@ -349,6 +249,17 @@ static void *SCPerfMgmtThread(void *arg) return NULL; } + TmModule *tm = &tmm_modules[TMM_STATSLOGGER]; + BUG_ON(tm->ThreadInit == NULL); + int r = tm->ThreadInit(tv_local, NULL, &stats_thread_data); + if (r != 0 || stats_thread_data == NULL) { + SCLogError(SC_ERR_THREAD_INIT, "Perf Counter API " + "ThreadInit failed"); + TmThreadsSetFlag(tv_local, THV_CLOSED | THV_RUNNING_DONE); + return NULL; + } + SCLogDebug("stats_thread_data %p", &stats_thread_data); + TmThreadsSetFlag(tv_local, THV_INIT_DONE); while (run) { if (TmThreadsCheckFlag(tv_local, THV_PAUSE)) { @@ -364,7 +275,7 @@ static void *SCPerfMgmtThread(void *arg) SCCtrlCondTimedwait(tv_local->ctrl_cond, tv_local->ctrl_mutex, &cond_time); SCCtrlMutexUnlock(tv_local->ctrl_mutex); - SCPerfOutputCounters(); + SCPerfOutputCounters(tv_local); if (TmThreadsCheckFlag(tv_local, THV_KILL)) { run = 0; @@ -633,68 +544,91 @@ static uint64_t SCPerfOutputCalculateCounterValue(SCPerfCounter *pc) return pc->value; } + /** * \brief The file output interface for the Perf Counter api */ -static int SCPerfOutputCounterFileIface() +static int SCPerfOutputCounterFileIface(ThreadVars *tv) { - SCPerfClubTMInst *pctmi = NULL; - SCPerfCounter *pc = NULL; + const SCPerfClubTMInst *pctmi = NULL; + const SCPerfCounter *pc = NULL; SCPerfCounter **pc_heads = NULL; uint64_t ui64_temp = 0; uint64_t ui64_result = 0; - struct timeval tval; - struct tm *tms; - uint32_t u = 0; int flag = 0; + void *td = stats_thread_data; - if (sc_perf_op_ctx->fp == NULL) { - SCLogDebug("perf_op_ctx->fp is NULL"); - return 0; - } + if (stats_table.nstats == 0) { + uint32_t nstats = 0; - if (sc_perf_op_ctx->rotation_flag) { - SCLogDebug("Rotating log file"); - sc_perf_op_ctx->rotation_flag = 0; - if (!SCPerfFileReopen(sc_perf_op_ctx)) { - /* Rotation failed, error already logged. */ - return 0; + pctmi = sc_perf_op_ctx->pctmi; + while (pctmi != NULL) { + if (pctmi->size == 0) { + pctmi = pctmi->next; + continue; + } + + if ((pc_heads = SCMalloc(pctmi->size * sizeof(SCPerfCounter *))) == NULL) + return 0; + memset(pc_heads, 0, pctmi->size * sizeof(SCPerfCounter *)); + + for (u = 0; u < pctmi->size; u++) { + pc_heads[u] = pctmi->head[u]->head; + SCMutexLock(&pctmi->head[u]->m); + } + + flag = 1; + while (flag) { + if (pc_heads[0] == NULL) + break; + + for (u = 0; u < pctmi->size; u++) { + if (pc_heads[u] != NULL) + pc_heads[u] = pc_heads[u]->next; + if (pc_heads[u] == NULL) + flag = 0; + } + + /* count */ + nstats++; + } + + for (u = 0; u < pctmi->size; u++) + SCMutexUnlock(&pctmi->head[u]->m); + + pctmi = pctmi->next; + SCFree(pc_heads); + + } + if (nstats == 0) { + SCLogError(SC_ERR_PERF_STATS_NOT_INIT, "no counters registered"); + return -1; + } + + stats_table.nstats = nstats; + stats_table.stats = SCCalloc(stats_table.nstats, sizeof(StatsRecord)); + if (stats_table.stats == NULL) { + stats_table.nstats = 0; + SCLogError(SC_ERR_MEM_ALLOC, "could not alloc memory for stats"); + return -1; } + + stats_table.start_time = sc_start_time; } + StatsRecord *table = stats_table.stats; - memset(&tval, 0, sizeof(struct timeval)); - - gettimeofday(&tval, NULL); - struct tm local_tm; - tms = SCLocalTime(tval.tv_sec, &local_tm); - - /* Calculate the Engine uptime */ - int up_time = (int)difftime(tval.tv_sec, sc_start_time); - int sec = up_time % 60; // Seconds in a minute - int in_min = up_time / 60; - int min = in_min % 60; // Minutes in a hour - int in_hours = in_min / 60; - int hours = in_hours % 24; // Hours in a day - int days = in_hours / 24; - - fprintf(sc_perf_op_ctx->fp, "----------------------------------------------" - "---------------------\n"); - fprintf(sc_perf_op_ctx->fp, "Date: %" PRId32 "/%" PRId32 "/%04d -- " - "%02d:%02d:%02d (uptime: %"PRId32"d, %02dh %02dm %02ds)\n", - tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900, tms->tm_hour, - tms->tm_min, tms->tm_sec, days, hours, min, sec); - fprintf(sc_perf_op_ctx->fp, "----------------------------------------------" - "---------------------\n"); - fprintf(sc_perf_op_ctx->fp, "%-25s | %-25s | %-s\n", "Counter", "TM Name", - "Value"); - fprintf(sc_perf_op_ctx->fp, "----------------------------------------------" - "---------------------\n"); + int table_i = 0; pctmi = sc_perf_op_ctx->pctmi; while (pctmi != NULL) { + if (pctmi->size == 0) { + pctmi = pctmi->next; + continue; + } + if ((pc_heads = SCMalloc(pctmi->size * sizeof(SCPerfCounter *))) == NULL) return 0; memset(pc_heads, 0, pctmi->size * sizeof(SCPerfCounter *)); @@ -722,20 +656,24 @@ static int SCPerfOutputCounterFileIface() flag = 0; } - fprintf(sc_perf_op_ctx->fp, "%-25s | %-25s | %-" PRIu64 "\n", - pc->cname, pctmi->tm_name, ui64_result); + /* store in the table */ + table[table_i].name = pc->cname; + table[table_i].tm_name = pctmi->tm_name; + table[table_i].pvalue = table[table_i].value; + table[table_i].value = ui64_result; + table_i++; } for (u = 0; u < pctmi->size; u++) SCMutexUnlock(&pctmi->head[u]->m); pctmi = pctmi->next; - SCFree(pc_heads); - fflush(sc_perf_op_ctx->fp); } + /* invoke logger(s) */ + OutputStatsLog(tv, td, &stats_table); return 1; } @@ -1294,29 +1232,6 @@ double SCPerfGetLocalCounterValue(uint16_t id, SCPerfCounterArray *pca) return pca->head[id].ui64_cnt; } -/** - * \brief The output interface dispatcher for the counter api - */ -void SCPerfOutputCounters() -{ - switch (sc_perf_op_ctx->iface) { - case SC_PERF_IFACE_FILE: - SCPerfOutputCounterFileIface(); - - break; - case SC_PERF_IFACE_CONSOLE: - /* yet to be implemented */ - - break; - case SC_PERF_IFACE_SYSLOG: - /* yet to be implemented */ - - break; - } - - return; -} - /** * \brief Releases the resources alloted by the Perf Counter API */ diff --git a/src/counters.h b/src/counters.h index 123a34f723..a1617d4037 100644 --- a/src/counters.h +++ b/src/counters.h @@ -27,12 +27,6 @@ /* forward declaration of the ThreadVars structure */ struct ThreadVars_; -/* Time interval for syncing the local counters with the global ones */ -#define SC_PERF_WUT_TTS 3 - -/* Time interval at which the mgmt thread o/p the stats */ -#define SC_PERF_MGMTT_TTS 8 - /** * \brief Data type for different kind of Perf counters that can be registered */ @@ -147,21 +141,8 @@ typedef struct SCPerfClubTMInst_ { * \brief Holds the output interface context for the counter api */ typedef struct SCPerfOPIfaceContext_ { - /* the iface to be used for output */ - uint32_t iface; - - /* the file to be used if the output interface used is SC_PERF_IFACE_FILE */ - char *file; - - /* more interfaces to be supported later. For now just a file */ - FILE *fp; - SCPerfClubTMInst *pctmi; SCMutex pctmi_lock; - - /* Flag set on file rotation notification. */ - int rotation_flag; - } SCPerfOPIfaceContext; /* the initialization functions */ @@ -186,8 +167,6 @@ SCPerfCounterArray * SCPerfGetAllCountersArray(SCPerfContext *); int SCPerfUpdateCounterArray(SCPerfCounterArray *, SCPerfContext *); double SCPerfGetLocalCounterValue(uint16_t, SCPerfCounterArray *); -void SCPerfOutputCounters(void); - /* functions used to free the resources alloted by the Perf counter API */ void SCPerfReleaseResources(void); void SCPerfReleasePerfCounterS(SCPerfCounter *); diff --git a/src/log-stats.c b/src/log-stats.c new file mode 100644 index 0000000000..fafd4c2852 --- /dev/null +++ b/src/log-stats.c @@ -0,0 +1,231 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#include "suricata-common.h" +#include "debug.h" +#include "detect.h" +#include "pkt-var.h" +#include "conf.h" + +#include "threads.h" +#include "threadvars.h" +#include "tm-threads.h" + +#include "util-print.h" +#include "util-unittest.h" + +#include "util-debug.h" + +#include "output.h" +#include "log-stats.h" +#include "util-privs.h" +#include "util-buffer.h" + +#include "util-logopenfile.h" +#include "util-time.h" + +#define DEFAULT_LOG_FILENAME "stats.log" +#define MODULE_NAME "LogStatsLog" +#define OUTPUT_BUFFER_SIZE 65535 + +TmEcode LogStatsLogThreadInit(ThreadVars *, void *, void **); +TmEcode LogStatsLogThreadDeinit(ThreadVars *, void *); +void LogStatsLogExitPrintStats(ThreadVars *, void *); +static void LogStatsLogDeInitCtx(OutputCtx *); + +typedef struct LogStatsFileCtx_ { + LogFileCtx *file_ctx; + uint32_t flags; /** Store mode */ +} LogStatsFileCtx; + +typedef struct LogStatsLogThread_ { + LogStatsFileCtx *statslog_ctx; + MemBuffer *buffer; +} LogStatsLogThread; + +int LogStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st) +{ + SCEnter(); + LogStatsLogThread *aft = (LogStatsLogThread *)thread_data; + + struct timeval tval; + struct tm *tms; + + gettimeofday(&tval, NULL); + struct tm local_tm; + tms = SCLocalTime(tval.tv_sec, &local_tm); + + /* Calculate the Engine uptime */ + int up_time = (int)difftime(tval.tv_sec, st->start_time); + int sec = up_time % 60; // Seconds in a minute + int in_min = up_time / 60; + int min = in_min % 60; // Minutes in a hour + int in_hours = in_min / 60; + int hours = in_hours % 24; // Hours in a day + int days = in_hours / 24; + + MemBufferWriteString(aft->buffer, "----------------------------------------------" + "---------------------\n"); + MemBufferWriteString(aft->buffer, "Date: %" PRId32 "/%" PRId32 "/%04d -- " + "%02d:%02d:%02d (uptime: %"PRId32"d, %02dh %02dm %02ds)\n", + tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900, tms->tm_hour, + tms->tm_min, tms->tm_sec, days, hours, min, sec); + MemBufferWriteString(aft->buffer, "----------------------------------------------" + "---------------------\n"); + MemBufferWriteString(aft->buffer, "%-25s | %-25s | %-s\n", "Counter", "TM Name", + "Value"); + MemBufferWriteString(aft->buffer, "----------------------------------------------" + "---------------------\n"); + + uint32_t u = 0; + for (u = 0; u < st->nstats; u++) { + if (st->stats[u].name == NULL) + break; + MemBufferWriteString(aft->buffer, "%-25s | %-25s | %-" PRIu64 "\n", + st->stats[u].name, st->stats[u].tm_name, st->stats[u].value); + } + + SCMutexLock(&aft->statslog_ctx->file_ctx->fp_mutex); + aft->statslog_ctx->file_ctx->Write((const char *)MEMBUFFER_BUFFER(aft->buffer), + MEMBUFFER_OFFSET(aft->buffer), aft->statslog_ctx->file_ctx); + SCMutexUnlock(&aft->statslog_ctx->file_ctx->fp_mutex); + + MemBufferReset(aft->buffer); + + SCReturnInt(0); +} + +TmEcode LogStatsLogThreadInit(ThreadVars *t, void *initdata, void **data) +{ + LogStatsLogThread *aft = SCMalloc(sizeof(LogStatsLogThread)); + if (unlikely(aft == NULL)) + return TM_ECODE_FAILED; + memset(aft, 0, sizeof(LogStatsLogThread)); + + if(initdata == NULL) + { + SCLogDebug("Error getting context for HTTPLog. \"initdata\" argument NULL"); + SCFree(aft); + return TM_ECODE_FAILED; + } + + aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); + if (aft->buffer == NULL) { + SCFree(aft); + return TM_ECODE_FAILED; + } + + /* Use the Ouptut Context (file pointer and mutex) */ + aft->statslog_ctx= ((OutputCtx *)initdata)->data; + + *data = (void *)aft; + return TM_ECODE_OK; +} + +TmEcode LogStatsLogThreadDeinit(ThreadVars *t, void *data) +{ + LogStatsLogThread *aft = (LogStatsLogThread *)data; + if (aft == NULL) { + return TM_ECODE_OK; + } + + MemBufferFree(aft->buffer); + /* clear memory */ + memset(aft, 0, sizeof(LogStatsLogThread)); + + SCFree(aft); + return TM_ECODE_OK; +} + +void LogStatsLogExitPrintStats(ThreadVars *tv, void *data) +{ + LogStatsLogThread *aft = (LogStatsLogThread *)data; + if (aft == NULL) { + return; + } +} + +/** \brief Create a new http log LogFileCtx. + * \param conf Pointer to ConfNode containing this loggers configuration. + * \return NULL if failure, LogFileCtx* to the file_ctx if succesful + * */ +OutputCtx *LogStatsLogInitCtx(ConfNode *conf) +{ + LogFileCtx *file_ctx = LogFileNewCtx(); + if (file_ctx == NULL) { + SCLogError(SC_ERR_HTTP_LOG_GENERIC, "couldn't create new file_ctx"); + return NULL; + } + + if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME) < 0) { + LogFileFreeCtx(file_ctx); + return NULL; + } + + LogStatsFileCtx *statslog_ctx = SCMalloc(sizeof(LogStatsFileCtx)); + if (unlikely(statslog_ctx == NULL)) { + LogFileFreeCtx(file_ctx); + return NULL; + } + memset(statslog_ctx, 0x00, sizeof(LogStatsFileCtx)); + + statslog_ctx->file_ctx = file_ctx; + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (unlikely(output_ctx == NULL)) { + LogFileFreeCtx(file_ctx); + SCFree(statslog_ctx); + return NULL; + } + + output_ctx->data = statslog_ctx; + output_ctx->DeInit = LogStatsLogDeInitCtx; + + SCLogDebug("STATS log output initialized"); + + OutputRegisterFileRotationFlag(&file_ctx->rotation_flag); + return output_ctx; +} + +static void LogStatsLogDeInitCtx(OutputCtx *output_ctx) +{ + LogStatsFileCtx *statslog_ctx = (LogStatsFileCtx *)output_ctx->data; + OutputUnregisterFileRotationFlag(&statslog_ctx->file_ctx->rotation_flag); + LogFileFreeCtx(statslog_ctx->file_ctx); + SCFree(statslog_ctx); + SCFree(output_ctx); +} + +void TmModuleLogStatsLogRegister (void) +{ + tmm_modules[TMM_LOGSTATSLOG].name = MODULE_NAME; + tmm_modules[TMM_LOGSTATSLOG].ThreadInit = LogStatsLogThreadInit; + tmm_modules[TMM_LOGSTATSLOG].ThreadExitPrintStats = LogStatsLogExitPrintStats; + tmm_modules[TMM_LOGSTATSLOG].ThreadDeinit = LogStatsLogThreadDeinit; + tmm_modules[TMM_LOGSTATSLOG].RegisterTests = NULL; + tmm_modules[TMM_LOGSTATSLOG].cap_flags = 0; + tmm_modules[TMM_LOGSTATSLOG].flags = TM_FLAG_LOGAPI_TM; + + OutputRegisterStatsModule(MODULE_NAME, "stats", LogStatsLogInitCtx, LogStatsLogger); +} diff --git a/src/log-stats.h b/src/log-stats.h new file mode 100644 index 0000000000..957104f3c7 --- /dev/null +++ b/src/log-stats.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __LOG_STATS_H__ +#define __LOG_STATS_H__ + +void TmModuleLogStatsLogRegister (void); + +#endif /* __LOG_HTTPLOG_H__ */ diff --git a/src/output-stats.c b/src/output-stats.c new file mode 100644 index 0000000000..df6fa1b09b --- /dev/null +++ b/src/output-stats.c @@ -0,0 +1,234 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Stats Logger Output registration functions + */ + +#include "suricata-common.h" +#include "tm-modules.h" +#include "output-stats.h" + +typedef struct OutputLoggerThreadStore_ { + void *thread_data; + struct OutputLoggerThreadStore_ *next; +} OutputLoggerThreadStore; + +/** per thread data for this module, contains a list of per thread + * data for the packet loggers. */ +typedef struct OutputLoggerThreadData_ { + OutputLoggerThreadStore *store; +} OutputLoggerThreadData; + +/* logger instance, a module + a output ctx, + * it's perfectly valid that have multiple instances of the same + * log module (e.g. http.log) with different output ctx'. */ +typedef struct OutputStatsLogger_ { + StatsLogger LogFunc; + OutputCtx *output_ctx; + struct OutputStatsLogger_ *next; + const char *name; + TmmId module_id; +} OutputStatsLogger; + +static OutputStatsLogger *list = NULL; + +int OutputRegisterStatsLogger(const char *name, StatsLogger LogFunc, OutputCtx *output_ctx) +{ + int module_id = TmModuleGetIdByName(name); + if (module_id < 0) + return -1; + + OutputStatsLogger *op = SCMalloc(sizeof(*op)); + if (op == NULL) + return -1; + memset(op, 0x00, sizeof(*op)); + + op->LogFunc = LogFunc; + op->output_ctx = output_ctx; + op->name = name; + op->module_id = (TmmId) module_id; + + if (list == NULL) + list = op; + else { + OutputStatsLogger *t = list; + while (t->next) + t = t->next; + t->next = op; + } + + SCLogDebug("OutputRegisterStatsLogger happy"); + return 0; +} + +TmEcode OutputStatsLog(ThreadVars *tv, void *thread_data, StatsTable *st) +{ + BUG_ON(thread_data == NULL); + BUG_ON(list == NULL); + + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputStatsLogger *logger = list; + OutputLoggerThreadStore *store = op_thread_data->store; + + BUG_ON(logger == NULL && store != NULL); + BUG_ON(logger != NULL && store == NULL); + BUG_ON(logger == NULL && store == NULL); + + logger = list; + store = op_thread_data->store; + while (logger && store) { + BUG_ON(logger->LogFunc == NULL); + + logger->LogFunc(tv, store->thread_data, st); + + logger = logger->next; + store = store->next; + + BUG_ON(logger == NULL && store != NULL); + BUG_ON(logger != NULL && store == NULL); + } + + return TM_ECODE_OK; +} + +/** \brief thread init for the tx logger + * This will run the thread init functions for the individual registered + * loggers */ +static TmEcode OutputStatsLogThreadInit(ThreadVars *tv, void *initdata, void **data) +{ + OutputLoggerThreadData *td = SCMalloc(sizeof(*td)); + if (td == NULL) + return TM_ECODE_FAILED; + memset(td, 0x00, sizeof(*td)); + + *data = (void *)td; + + SCLogDebug("OutputStatsLogThreadInit happy (*data %p)", *data); + + OutputStatsLogger *logger = list; + while (logger) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadInit) { + void *retptr = NULL; + if (tm_module->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) { + OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts)); +/* todo */ BUG_ON(ts == NULL); + memset(ts, 0x00, sizeof(*ts)); + + /* store thread handle */ + ts->thread_data = retptr; + + if (td->store == NULL) { + td->store = ts; + } else { + OutputLoggerThreadStore *tmp = td->store; + while (tmp->next != NULL) + tmp = tmp->next; + tmp->next = ts; + } + + SCLogDebug("%s is now set up", logger->name); + } + } + + logger = logger->next; + } + + SCLogDebug("OutputStatsLogThreadInit happy (*data %p)", *data); + return TM_ECODE_OK; +} + +static TmEcode OutputStatsLogThreadDeinit(ThreadVars *tv, void *thread_data) +{ + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputLoggerThreadStore *store = op_thread_data->store; + OutputStatsLogger *logger = list; + + while (logger && store) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadDeinit) { + tm_module->ThreadDeinit(tv, store->thread_data); + } + + OutputLoggerThreadStore *next_store = store->next; + SCFree(store); + store = next_store; + logger = logger->next; + } + return TM_ECODE_OK; +} + +static void OutputStatsLogExitPrintStats(ThreadVars *tv, void *thread_data) +{ + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputLoggerThreadStore *store = op_thread_data->store; + OutputStatsLogger *logger = list; + + while (logger && store) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadExitPrintStats) { + tm_module->ThreadExitPrintStats(tv, store->thread_data); + } + + logger = logger->next; + store = store->next; + } +} + +void TmModuleStatsLoggerRegister (void) +{ + tmm_modules[TMM_STATSLOGGER].name = "__stats_logger__"; + tmm_modules[TMM_STATSLOGGER].ThreadInit = OutputStatsLogThreadInit; + //tmm_modules[TMM_STATSLOGGER].Func = OutputStatsLog; + tmm_modules[TMM_STATSLOGGER].ThreadExitPrintStats = OutputStatsLogExitPrintStats; + tmm_modules[TMM_STATSLOGGER].ThreadDeinit = OutputStatsLogThreadDeinit; + tmm_modules[TMM_STATSLOGGER].cap_flags = 0; +} + +void OutputStatsShutdown(void) +{ + OutputStatsLogger *logger = list; + while (logger) { + OutputStatsLogger *next_logger = logger->next; + SCFree(logger); + logger = next_logger; + } + list = NULL; +} diff --git a/src/output-stats.h b/src/output-stats.h new file mode 100644 index 0000000000..adb3d11761 --- /dev/null +++ b/src/output-stats.h @@ -0,0 +1,53 @@ +/* Copyright (C) 2007-2013 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Stats Logger Output registration functions + */ + +#ifndef __OUTPUT_STATS_H__ +#define __OUTPUT_STATS_H__ + +typedef struct StatsRecord_ { + const char *name; + const char *tm_name; + uint64_t value; /**< total value */ + uint64_t pvalue; /**< prev value (may be higher for memuse counters) */ +} StatsRecord; + +typedef struct StatsTable_ { + StatsRecord *stats; + uint32_t nstats; + time_t start_time; + struct timeval ts; +} StatsTable; + +TmEcode OutputStatsLog(ThreadVars *tv, void *thread_data, StatsTable *st); + +typedef int (*StatsLogger)(ThreadVars *, void *thread_data, const StatsTable *); + +int OutputRegisterStatsLogger(const char *name, StatsLogger LogFunc, OutputCtx *); + +void TmModuleStatsLoggerRegister (void); + +void OutputStatsShutdown(void); + +#endif /* __OUTPUT_STATS_H__ */ diff --git a/src/output.c b/src/output.c index 9229eb17b4..1146b7b226 100644 --- a/src/output.c +++ b/src/output.c @@ -497,6 +497,76 @@ error: exit(EXIT_FAILURE); } +/** + * \brief Register a stats data output module. + * + * This function will register an output module so it can be + * configured with the configuration file. + * + * \retval Returns 0 on success, -1 on failure. + */ +void +OutputRegisterStatsModule(const char *name, const char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), StatsLogger StatsLogFunc) +{ + if (unlikely(StatsLogFunc == NULL)) { + goto error; + } + + OutputModule *module = SCCalloc(1, sizeof(*module)); + if (unlikely(module == NULL)) { + goto error; + } + + module->name = name; + module->conf_name = conf_name; + module->InitFunc = InitFunc; + module->StatsLogFunc = StatsLogFunc; + TAILQ_INSERT_TAIL(&output_modules, module, entries); + + SCLogDebug("Stats logger \"%s\" registered.", name); + return; +error: + SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting..."); + exit(EXIT_FAILURE); +} + +/** + * \brief Register a stats data output sub-module. + * + * This function will register an output module so it can be + * configured with the configuration file. + * + * \retval Returns 0 on success, -1 on failure. + */ +void +OutputRegisterStatsSubModule(const char *parent_name, const char *name, + const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), + StatsLogger StatsLogFunc) +{ + if (unlikely(StatsLogFunc == NULL)) { + goto error; + } + + OutputModule *module = SCCalloc(1, sizeof(*module)); + if (unlikely(module == NULL)) { + goto error; + } + + module->name = name; + module->conf_name = conf_name; + module->parent_name = parent_name; + module->InitSubFunc = InitFunc; + module->StatsLogFunc = StatsLogFunc; + TAILQ_INSERT_TAIL(&output_modules, module, entries); + + SCLogDebug("Stats logger \"%s\" registered.", name); + return; +error: + SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting..."); + exit(EXIT_FAILURE); +} + /** * \brief Get an output module by name. * diff --git a/src/output.h b/src/output.h index 85092c3141..9a4add0efb 100644 --- a/src/output.h +++ b/src/output.h @@ -36,6 +36,7 @@ #include "output-filedata.h" #include "output-flow.h" #include "output-streaming.h" +#include "output-stats.h" typedef struct OutputModule_ { const char *name; @@ -51,6 +52,7 @@ typedef struct OutputModule_ { FiledataLogger FiledataLogFunc; FlowLogger FlowLogFunc; StreamingLogger StreamingLogFunc; + StatsLogger StatsLogFunc; AppProto alproto; enum OutputStreamingType stream_type; @@ -98,6 +100,12 @@ void OutputRegisterStreamingSubModule(const char *parent_name, const char *name, const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), StreamingLogger StreamingLogFunc, enum OutputStreamingType stream_type); +void OutputRegisterStatsModule(const char *name, const char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), StatsLogger StatsLogFunc); +void OutputRegisterStatsSubModule(const char *parent_name, const char *name, + const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), + StatsLogger StatsLogFunc); + OutputModule *OutputGetModuleByConfName(const char *name); void OutputDeregisterAll(void); diff --git a/src/runmodes.c b/src/runmodes.c index 847fadcb10..76f9022f86 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -447,6 +447,7 @@ void RunModeShutDown(void) OutputTxShutdown(); OutputFileShutdown(); OutputFiledataShutdown(); + OutputStatsShutdown(); /* Close any log files. */ RunModeOutput *output; @@ -490,6 +491,11 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu OutputRegisterFlowLogger(module->name, module->FlowLogFunc, output_ctx); return; } + /* stats logger doesn't run in the packet path */ + if (module->StatsLogFunc) { + OutputRegisterStatsLogger(module->name, module->StatsLogFunc, output_ctx); + return; + } TmModule *tm_module = TmModuleGetByName(module->name); if (tm_module == NULL) { @@ -636,9 +642,6 @@ void RunModeInitializeOutputs(void) TAILQ_FOREACH(output, &outputs->head, next) { - if (strcmp(output->val, "stats") == 0) - continue; - output_config = ConfNodeLookupChild(output, output->val); if (output_config == NULL) { /* Shouldn't happen. */ diff --git a/src/suricata.c b/src/suricata.c index d0c084a76c..e7053ab965 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -96,6 +96,7 @@ #include "output-json-smtp.h" #include "log-filestore.h" #include "log-tcp-data.h" +#include "log-stats.h" #include "output-json.h" @@ -892,6 +893,8 @@ void RegisterAllModules() TmModuleJsonDnsLogRegister(); /* tcp streaming data */ TmModuleLogTcpDataLogRegister(); + /* log stats */ + TmModuleLogStatsLogRegister(); TmModuleJsonAlertLogRegister(); /* flow/netflow */ @@ -904,6 +907,7 @@ void RegisterAllModules() TmModuleFileLoggerRegister(); TmModuleFiledataLoggerRegister(); TmModuleStreamingLoggerRegister(); + TmModuleStatsLoggerRegister(); TmModuleDebugList(); /* nflog */ TmModuleReceiveNFLOGRegister(); diff --git a/src/tm-modules.c b/src/tm-modules.c index 8503f02686..19caeca727 100644 --- a/src/tm-modules.c +++ b/src/tm-modules.c @@ -250,6 +250,7 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE (TMM_DECODEAFP); CASE_CODE (TMM_PACKETLOGGER); CASE_CODE (TMM_TXLOGGER); + CASE_CODE (TMM_STATSLOGGER); CASE_CODE (TMM_FILELOGGER); CASE_CODE (TMM_FILEDATALOGGER); CASE_CODE (TMM_STREAMINGLOGGER); @@ -267,6 +268,7 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE (TMM_FLOWMANAGER); CASE_CODE (TMM_FLOWRECYCLER); CASE_CODE (TMM_LUALOG); + CASE_CODE (TMM_LOGSTATSLOG); CASE_CODE (TMM_SIZE); } diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index 0137e10279..059ba7b12d 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -81,6 +81,7 @@ typedef enum { TMM_DECODENAPATECH, TMM_PACKETLOGGER, TMM_TXLOGGER, + TMM_STATSLOGGER, TMM_FILELOGGER, TMM_FILEDATALOGGER, TMM_STREAMINGLOGGER, @@ -96,6 +97,7 @@ typedef enum { TMM_DECODENFLOG, TMM_JSONFLOWLOG, TMM_JSONNETFLOWLOG, + TMM_LOGSTATSLOG, TMM_FLOWMANAGER, TMM_FLOWRECYCLER, -- 2.47.3