]> git.ipfire.org Git - thirdparty/suricata.git/commitdiff
Introduce 'file' logging API
authorVictor Julien <victor@inliniac.net>
Wed, 15 Jan 2014 12:22:47 +0000 (13:22 +0100)
committerVictor Julien <victor@inliniac.net>
Mon, 27 Jan 2014 14:20:59 +0000 (15:20 +0100)
This patch introduces a new logging API for logging extracted file info.
It allows for registration of a callback that is called once per file:
when it's considered 'closed'.

Users of this API register their Log Function through:
    OutputRegisterFileModule()

The API uses a magic settings globally. This might be changed later.

src/Makefile.am
src/output-file.c [new file with mode: 0644]
src/output-file.h [new file with mode: 0644]
src/output.c
src/output.h
src/runmodes.c
src/suricata.c
src/tm-threads-common.h

index 0bfb6f9db7d32c9954423370fffb52f69210932e..86b7c123a6774fb86ff1d667fd090b47ae68a287 100644 (file)
@@ -214,6 +214,7 @@ log-httplog.c log-httplog.h \
 log-pcap.c log-pcap.h \
 log-tlslog.c log-tlslog.h \
 output.c output.h \
+output-file.c output-file.h \
 output-packet.c output-packet.h \
 output-tx.c output-tx.h \
 packet-queue.c packet-queue.h \
diff --git a/src/output-file.c b/src/output-file.c
new file mode 100644 (file)
index 0000000..382d959
--- /dev/null
@@ -0,0 +1,273 @@
+/* Copyright (C) 2007-2014 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ *
+ * AppLayer File Logger Output registration functions
+ */
+
+#include "suricata-common.h"
+#include "tm-modules.h"
+#include "output-file.h"
+#include "app-layer.h"
+#include "app-layer-parser.h"
+#include "detect-filemagic.h"
+
+typedef struct OutputLoggerThreadStore_ {
+    void *thread_data;
+    struct OutputLoggerThreadStore_ *next;
+} OutputLoggerThreadStore;
+
+/** per thread data for this module, contains a list of per thread
+ *  data for the packet loggers. */
+typedef struct OutputLoggerThreadData_ {
+    OutputLoggerThreadStore *store;
+} OutputLoggerThreadData;
+
+/* logger instance, a module + a output ctx,
+ * it's perfectly valid that have multiple instances of the same
+ * log module (e.g. http.log) with different output ctx'. */
+typedef struct OutputFileLogger_ {
+    FileLogger LogFunc;
+    OutputCtx *output_ctx;
+    struct OutputFileLogger_ *next;
+    const char *name;
+} OutputFileLogger;
+
+static OutputFileLogger *list = NULL;
+
+int OutputRegisterFileLogger(char *name, FileLogger LogFunc, OutputCtx *output_ctx) {
+    OutputFileLogger *op = SCMalloc(sizeof(*op));
+    if (op == NULL)
+        return -1;
+    memset(op, 0x00, sizeof(*op));
+
+    op->LogFunc = LogFunc;
+    op->output_ctx = output_ctx;
+    op->name = SCStrdup(name);
+    if (op->name == NULL) {
+        SCFree(op);
+        return -1;
+    }
+
+    if (list == NULL)
+        list = op;
+    else {
+        OutputFileLogger *t = list;
+        while (t->next)
+            t = t->next;
+        t->next = op;
+    }
+
+    SCLogDebug("OutputRegisterTxLogger happy");
+    return 0;
+}
+
+static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq) {
+    BUG_ON(thread_data == NULL);
+    BUG_ON(list == NULL);
+
+    OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
+    OutputFileLogger *logger = list;
+    OutputLoggerThreadStore *store = op_thread_data->store;
+
+    BUG_ON(logger == NULL && store != NULL);
+    BUG_ON(logger != NULL && store == NULL);
+    BUG_ON(logger == NULL && store == NULL);
+
+    uint8_t flags = 0;
+    Flow * const f = p->flow;
+
+    /* no flow, no files */
+    if (f == NULL) {
+        SCReturnInt(TM_ECODE_OK);
+    }
+
+    if (p->flowflags & FLOW_PKT_TOCLIENT)
+        flags |= STREAM_TOCLIENT;
+    else
+        flags |= STREAM_TOSERVER;
+
+    int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
+    int file_trunc = 0;
+
+    FLOWLOCK_WRLOCK(f); // < need write lock for FilePrune below
+    file_trunc = StreamTcpReassembleDepthReached(p);
+
+    FileContainer *ffc = AppLayerParserGetFiles(p->proto, f->alproto,
+                                                f->alstate, flags);
+    SCLogDebug("ffc %p", ffc);
+    if (ffc != NULL) {
+        File *ff;
+        for (ff = ffc->head; ff != NULL; ff = ff->next) {
+            if (ff->flags & FILE_LOGGED)
+                continue;
+
+            SCLogDebug("ff %p", ff);
+
+            if (file_trunc && ff->state < FILE_STATE_CLOSED)
+                ff->state = FILE_STATE_TRUNCATED;
+
+            if (file_close && ff->state < FILE_STATE_CLOSED)
+                ff->state = FILE_STATE_TRUNCATED;
+
+            if (ff->state == FILE_STATE_CLOSED    ||
+                ff->state == FILE_STATE_TRUNCATED ||
+                ff->state == FILE_STATE_ERROR)
+            {
+                int file_logged = 0;
+
+                if (FileForceMagic() && ff->magic == NULL) {
+                    FilemagicGlobalLookup(ff);
+                }
+
+                logger = list;
+                store = op_thread_data->store;
+                while (logger && store) {
+                    BUG_ON(logger->LogFunc == NULL);
+
+                    SCLogDebug("logger %p", logger);
+                    logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff);
+                    file_logged = 1;
+
+                    logger = logger->next;
+                    store = store->next;
+
+                    BUG_ON(logger == NULL && store != NULL);
+                    BUG_ON(logger != NULL && store == NULL);
+                }
+
+                if (file_logged) {
+                    ff->flags |= FILE_LOGGED;
+                }
+            }
+        }
+
+        FilePrune(ffc);
+    }
+
+    FLOWLOCK_UNLOCK(f);
+    return TM_ECODE_OK;
+}
+
+/** \brief thread init for the tx logger
+ *  This will run the thread init functions for the individual registered
+ *  loggers */
+static TmEcode OutputFileLogThreadInit(ThreadVars *tv, void *initdata, void **data) {
+    OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
+    if (td == NULL)
+        return TM_ECODE_FAILED;
+    memset(td, 0x00, sizeof(*td));
+
+    *data = (void *)td;
+
+    SCLogDebug("OutputFileLogThreadInit happy (*data %p)", *data);
+
+    OutputFileLogger *logger = list;
+    while (logger) {
+        TmModule *tm_module = TmModuleGetByName((char *)logger->name);
+        if (tm_module == NULL) {
+            SCLogError(SC_ERR_INVALID_ARGUMENT,
+                    "TmModuleGetByName for %s failed", logger->name);
+            exit(EXIT_FAILURE);
+        }
+
+        if (tm_module->ThreadInit) {
+            void *retptr = NULL;
+            if (tm_module->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
+                OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
+/* todo */      BUG_ON(ts == NULL);
+                memset(ts, 0x00, sizeof(*ts));
+
+                /* store thread handle */
+                ts->thread_data = retptr;
+
+                if (td->store == NULL) {
+                    td->store = ts;
+                } else {
+                    OutputLoggerThreadStore *tmp = td->store;
+                    while (tmp->next != NULL)
+                        tmp = tmp->next;
+                    tmp->next = ts;
+                }
+
+                SCLogDebug("%s is now set up", logger->name);
+            }
+        }
+
+        logger = logger->next;
+    }
+
+    return TM_ECODE_OK;
+}
+
+static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data) {
+    OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
+    OutputLoggerThreadStore *store = op_thread_data->store;
+    OutputFileLogger *logger = list;
+
+    while (logger && store) {
+        TmModule *tm_module = TmModuleGetByName((char *)logger->name);
+        if (tm_module == NULL) {
+            SCLogError(SC_ERR_INVALID_ARGUMENT,
+                    "TmModuleGetByName for %s failed", logger->name);
+            exit(EXIT_FAILURE);
+        }
+
+        if (tm_module->ThreadDeinit) {
+            tm_module->ThreadDeinit(tv, store->thread_data);
+        }
+
+        logger = logger->next;
+        store = store->next;
+    }
+    return TM_ECODE_OK;
+}
+
+static void OutputFileLogExitPrintStats(ThreadVars *tv, void *thread_data) {
+    OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
+    OutputLoggerThreadStore *store = op_thread_data->store;
+    OutputFileLogger *logger = list;
+
+    while (logger && store) {
+        TmModule *tm_module = TmModuleGetByName((char *)logger->name);
+        if (tm_module == NULL) {
+            SCLogError(SC_ERR_INVALID_ARGUMENT,
+                    "TmModuleGetByName for %s failed", logger->name);
+            exit(EXIT_FAILURE);
+        }
+
+        if (tm_module->ThreadExitPrintStats) {
+            tm_module->ThreadExitPrintStats(tv, store->thread_data);
+        }
+
+        logger = logger->next;
+        store = store->next;
+    }
+}
+
+void TmModuleFileLoggerRegister (void) {
+    tmm_modules[TMM_FILELOGGER].name = "__file_logger__";
+    tmm_modules[TMM_FILELOGGER].ThreadInit = OutputFileLogThreadInit;
+    tmm_modules[TMM_FILELOGGER].Func = OutputFileLog;
+    tmm_modules[TMM_FILELOGGER].ThreadExitPrintStats = OutputFileLogExitPrintStats;
+    tmm_modules[TMM_FILELOGGER].ThreadDeinit = OutputFileLogThreadDeinit;
+    tmm_modules[TMM_FILELOGGER].cap_flags = 0;
+}
diff --git a/src/output-file.h b/src/output-file.h
new file mode 100644 (file)
index 0000000..cc17abf
--- /dev/null
@@ -0,0 +1,44 @@
+/* Copyright (C) 2007-2014 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ *
+ * AppLayer File Logger Output registration functions
+ */
+
+#ifndef __OUTPUT_FILE_H__
+#define __OUTPUT_FILE_H__
+
+#include "decode.h"
+#include "util-file.h"
+
+/** packet logger function pointer type */
+typedef int (*FileLogger)(ThreadVars *, void *thread_data, const Packet *, const File *);
+
+/** packet logger condition function pointer type,
+ *  must return true for packets that should be logged
+ */
+//typedef int (*TxLogCondition)(ThreadVars *, const Packet *);
+
+int OutputRegisterFileLogger(char *name, FileLogger LogFunc, OutputCtx *);
+
+void TmModuleFileLoggerRegister (void);
+
+#endif /* __OUTPUT_FILE_H__ */
index 571c2ac55099b5c25006186d1fa52f211ca5ed27..cf921e4a5b21027d809677b740a801ac2eae12b2 100644 (file)
@@ -148,6 +148,44 @@ error:
     exit(EXIT_FAILURE);
 }
 
+/**
+ * \brief Register a file output module.
+ *
+ * This function will register an output module so it can be
+ * configured with the configuration file.
+ *
+ * \retval Returns 0 on success, -1 on failure.
+ */
+void
+OutputRegisterFileModule(char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc)
+{
+    if (unlikely(FileLogFunc == NULL)) {
+        goto error;
+    }
+
+    OutputModule *module = SCCalloc(1, sizeof(*module));
+    if (unlikely(module == NULL)) {
+        goto error;
+    }
+
+    module->name = SCStrdup(name);
+    if (unlikely(module->name == NULL))
+        goto error;
+    module->conf_name = SCStrdup(conf_name);
+    if (unlikely(module->conf_name == NULL))
+        goto error;
+    module->InitFunc = InitFunc;
+    module->FileLogFunc = FileLogFunc;
+    TAILQ_INSERT_TAIL(&output_modules, module, entries);
+
+    SCLogDebug("File logger \"%s\" registered.", name);
+    return;
+error:
+    SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting...");
+    exit(EXIT_FAILURE);
+}
+
 /**
  * \brief Get an output module by name.
  *
index c8d2fa8f9af7ace8a5deec02a5c8e7eefae42d65..c4c843256f265dbe302e6ae699b5e0bc10fa1d33 100644 (file)
@@ -32,6 +32,7 @@
 
 #include "output-packet.h"
 #include "output-tx.h"
+#include "output-file.h"
 
 typedef struct OutputModule_ {
     char *name;
@@ -41,6 +42,7 @@ typedef struct OutputModule_ {
     PacketLogger PacketLogFunc;
     PacketLogCondition PacketConditionFunc;
     TxLogger TxLogFunc;
+    FileLogger FileLogFunc;
     uint16_t alproto;
 
     TAILQ_ENTRY(OutputModule_) entries;
@@ -51,11 +53,11 @@ void OutputRegisterModule(char *, char *, OutputCtx *(*)(ConfNode *));
 void OutputRegisterPacketModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *),
     PacketLogger LogFunc, PacketLogCondition ConditionFunc);
-void
-OutputRegisterTxModule(char *name, char *conf_name,
+void OutputRegisterTxModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), uint16_t alproto,
     TxLogger TxLogFunc);
-
+void OutputRegisterFileModule(char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc);
 
 OutputModule *OutputGetModuleByConfName(char *name);
 void OutputDeregisterAll(void);
index 96c9cc0583ab900e7c0bd5ea72b420d5e9054ef4..766c0320c673c1e355c477b26190aa394f85098a 100644 (file)
@@ -425,6 +425,7 @@ void RunModeInitializeOutputs(void)
     TmModule *tm_module;
     TmModule *pkt_logger_module = NULL;
     TmModule *tx_logger_module = NULL;
+    TmModule *file_logger_module = NULL;
     const char *enabled;
 
     TAILQ_FOREACH(output, &outputs->head, next) {
@@ -530,6 +531,27 @@ void RunModeInitializeOutputs(void)
                 TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
                 SCLogDebug("__tx_logger__ added");
             }
+        } else if (module->FileLogFunc) {
+            SCLogDebug("%s is a file logger", module->name);
+            OutputRegisterFileLogger(module->name, module->FileLogFunc, output_ctx);
+
+            /* need one instance of the tx logger module */
+            if (file_logger_module == NULL) {
+                file_logger_module = TmModuleGetByName("__file_logger__");
+                if (file_logger_module == NULL) {
+                    SCLogError(SC_ERR_INVALID_ARGUMENT,
+                            "TmModuleGetByName for __file_logger__ failed");
+                    exit(EXIT_FAILURE);
+                }
+
+                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+                if (unlikely(runmode_output == NULL))
+                    return;
+                runmode_output->tm_module = file_logger_module;
+                runmode_output->output_ctx = NULL;
+                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+                SCLogDebug("__file_logger__ added");
+            }
         } else {
             SCLogDebug("%s is a regular logger", module->name);
 
index 2f06f31d4dd34abee0d9c2b05b04dd7dde4bc6b2..230a1190bce2cd2254e70cafa9bf70ca43ded1de 100644 (file)
 #include "output.h"
 #include "output-packet.h"
 #include "output-tx.h"
+#include "output-file.h"
+
 #include "util-privs.h"
 
 #include "tmqh-packetpool.h"
@@ -800,6 +802,7 @@ void RegisterAllModules()
     TmModuleLogDnsLogRegister();
     TmModulePacketLoggerRegister();
     TmModuleTxLoggerRegister();
+    TmModuleFileLoggerRegister();
     TmModuleDebugList();
 
 }
index 4c13525658111442c531ad74c9c40c0d74dffa04..6ee213649d812d9435c6bf5dc472ebc019741391 100644 (file)
@@ -81,6 +81,7 @@ typedef enum {
     TMM_DECODENAPATECH,
     TMM_PACKETLOGGER,
     TMM_TXLOGGER,
+    TMM_FILELOGGER,
     TMM_SIZE,
 } TmmId;