jansson file log: make file log module

Turn the libjansson based file logger into a file module, as a child
of eve-log.

diff --git a/src/output-json-file.c b/src/output-json-file.c
index 6eb33c46f691e2a2060169fd38788daa4b69b50d..218a0f89d1cc10a80d7108d0e75dd9991f8a75c7 100644 (file)
--- a/src/output-json-file.c
+++ b/src/output-json-file.c
@@ -64,10 +64,16 @@
 #include <jansson.h>
 
 typedef struct OutputFileCtx_ {
+    LogFileCtx *file_ctx;
     uint32_t file_cnt;
 } OutputFileCtx;
 
-static json_t *LogFileMetaGetUri(Packet *p, File *ff) {
+typedef struct JsonFileLogThread_ {
+    OutputFileCtx *filelog_ctx;
+    MemBuffer *buffer;
+} JsonFileLogThread;
+
+static json_t *LogFileMetaGetUri(const Packet *p, const File *ff) {
     HtpState *htp_state = (HtpState *)p->flow->alstate;
     json_t *js = NULL;
     if (htp_state != NULL) {
@@ -88,7 +94,7 @@ static json_t *LogFileMetaGetUri(Packet *p, File *ff) {
     return json_string("<unknown>");
 }
 
-static json_t *LogFileMetaGetHost(Packet *p, File *ff) {
+static json_t *LogFileMetaGetHost(const Packet *p, const File *ff) {
     HtpState *htp_state = (HtpState *)p->flow->alstate;
     json_t *js = NULL;
     if (htp_state != NULL) {
@@ -106,7 +112,7 @@ static json_t *LogFileMetaGetHost(Packet *p, File *ff) {
     return json_string("<unknown>");
 }
 
-static json_t *LogFileMetaGetReferer(Packet *p, File *ff) {
+static json_t *LogFileMetaGetReferer(const Packet *p, const File *ff) {
     HtpState *htp_state = (HtpState *)p->flow->alstate;
     json_t *js = NULL;
     if (htp_state != NULL) {
@@ -129,7 +135,7 @@ static json_t *LogFileMetaGetReferer(Packet *p, File *ff) {
     return json_string("<unknown>");
 }
 
-static json_t *LogFileMetaGetUserAgent(Packet *p, File *ff) {
+static json_t *LogFileMetaGetUserAgent(const Packet *p, const File *ff) {
     HtpState *htp_state = (HtpState *)p->flow->alstate;
     json_t *js = NULL;
     if (htp_state != NULL) {
@@ -156,9 +162,9 @@ static json_t *LogFileMetaGetUserAgent(Packet *p, File *ff) {
  *  \internal
  *  \brief Write meta data on a single line json record
  */
-static void LogFileWriteJsonRecord(AlertJsonThread /*LogFileLogThread*/ *aft, Packet *p, File *ff, int ipver) {
+static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff) {
     MemBuffer *buffer = (MemBuffer *)aft->buffer;
-    json_t *js = CreateJSONHeader(p, 0);
+    json_t *js = CreateJSONHeader((Packet *)p, 0); //TODO const
     if (unlikely(js == NULL))
         return;
 
@@ -216,122 +222,91 @@ static void LogFileWriteJsonRecord(AlertJsonThread /*LogFileLogThread*/ *aft, Pa
     json_object_set_new(fjs, "size", json_integer(ff->size));
 
     json_object_set_new(js, "file", fjs);
-    OutputJSON(js, aft, &aft->files_cnt);
+    OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, buffer);
     json_object_del(js, "file");
 
     json_object_clear(js);
     json_decref(js);
 }
 
-static TmEcode OutputFileLogWrap(ThreadVars *tv, Packet *p, void *data, int ipver)
+static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff)
 {
     SCEnter();
-    AlertJsonThread *aft = (AlertJsonThread *)data;
-    uint8_t flags = 0;
-    uint8_t direction = 0;
-
-    /* no flow, no htp state */
-    if (p->flow == NULL) {
-        SCReturnInt(TM_ECODE_OK);
-    }
-
-    if (p->flowflags & FLOW_PKT_TOCLIENT)
-        flags |= (direction = STREAM_TOCLIENT);
-    else
-        flags |= (direction = STREAM_TOSERVER);
-
-    int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
-    int file_trunc = 0;
-
-    FLOWLOCK_WRLOCK(p->flow);
-    file_trunc = StreamTcpReassembleDepthReached(p);
-
-    Flow *f = p->flow;
-    FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
-    SCLogDebug("ffc %p", ffc);
-    if (ffc != NULL) {
-        File *ff;
-        for (ff = ffc->head; ff != NULL; ff = ff->next) {
-            if (ff->flags & FILE_LOGGED)
-                continue;
-
-            if (FileForceMagic() && ff->magic == NULL) {
-                FilemagicGlobalLookup(ff);
-            }
+    JsonFileLogThread *aft = (JsonFileLogThread *)thread_data;
 
-            SCLogDebug("ff %p", ff);
+    BUG_ON(ff->flags & FILE_LOGGED);
 
-            if (file_trunc && ff->state < FILE_STATE_CLOSED)
-                ff->state = FILE_STATE_TRUNCATED;
+    SCLogDebug("ff %p", ff);
 
-            if (ff->state == FILE_STATE_CLOSED ||
-                    ff->state == FILE_STATE_TRUNCATED || ff->state == FILE_STATE_ERROR ||
-                    (file_close == 1 && ff->state < FILE_STATE_CLOSED))
-            {
-                LogFileWriteJsonRecord(aft, p, ff, ipver);
+    FileWriteJsonRecord(aft, p, ff);
+    return 0;
+}
 
-                ff->flags |= FILE_LOGGED;
-            }
-        }
 
-        FilePrune(ffc);
+#define OUTPUT_BUFFER_SIZE 65535
+static TmEcode JsonFileLogThreadInit(ThreadVars *t, void *initdata, void **data)
+{
+    JsonFileLogThread *aft = SCMalloc(sizeof(JsonFileLogThread));
+    if (unlikely(aft == NULL))
+        return TM_ECODE_FAILED;
+    memset(aft, 0, sizeof(JsonFileLogThread));
+
+    if(initdata == NULL)
+    {
+        SCLogDebug("Error getting context for HTTPLog.  \"initdata\" argument NULL");
+        SCFree(aft);
+        return TM_ECODE_FAILED;
     }
 
-    FLOWLOCK_UNLOCK(p->flow);
-    SCReturnInt(TM_ECODE_OK);
-}
+    /* Use the Ouptut Context (file pointer and mutex) */
+    aft->filelog_ctx = ((OutputCtx *)initdata)->data;
 
-TmEcode OutputFileLogIPv4(ThreadVars *tv, Packet *p, void *data)
-{
-    return OutputFileLogWrap(tv, p, data, AF_INET);
-}
+    aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
+    if (aft->buffer == NULL) {
+        SCFree(aft);
+        return TM_ECODE_FAILED;
+    }
 
-TmEcode OutputFileLogIPv6(ThreadVars *tv, Packet *p, void *data)
-{
-    return OutputFileLogWrap(tv, p, data, AF_INET6);
+    *data = (void *)aft;
+    return TM_ECODE_OK;
 }
 
-TmEcode OutputFileLog (ThreadVars *tv, Packet *p, void *data)
+static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)
 {
-    SCEnter();
-    int r = TM_ECODE_OK;
-
-    /* no flow, no htp state */
-    if (p->flow == NULL) {
-        SCReturnInt(TM_ECODE_OK);
-    }
-
-    if (!(PKT_IS_TCP(p))) {
-        SCReturnInt(TM_ECODE_OK);
+    JsonFileLogThread *aft = (JsonFileLogThread *)data;
+    if (aft == NULL) {
+        return TM_ECODE_OK;
     }
 
-    SCLogDebug("p->pcap_cnt %"PRIu64, p->pcap_cnt);
-
-    if (PKT_IS_IPV4(p)) {
-        r = OutputFileLogIPv4(tv, p, data);
-    } else if (PKT_IS_IPV6(p)) {
-        r = OutputFileLogIPv6(tv, p, data);
-    }
+    MemBufferFree(aft->buffer);
+    /* clear memory */
+    memset(aft, 0, sizeof(JsonFileLogThread));
 
-    SCReturnInt(r);
+    SCFree(aft);
+    return TM_ECODE_OK;
 }
 
+
 /** \brief Create a new http log LogFileCtx.
  *  \param conf Pointer to ConfNode containing this loggers configuration.
  *  \return NULL if failure, LogFileCtx* to the file_ctx if succesful
  * */
-OutputCtx *OutputFileLogInit(ConfNode *conf)
+OutputCtx *OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
 {
-    OutputFileCtx *file_ctx = SCMalloc(sizeof(OutputFileCtx));
-    if (unlikely(file_ctx == NULL))
+    AlertJsonThread *ajt = parent_ctx->data;
+
+    OutputFileCtx *output_file_ctx = SCMalloc(sizeof(OutputFileCtx));
+    if (unlikely(output_file_ctx == NULL))
         return NULL;
 
     OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
     if (unlikely(output_ctx == NULL)) {
-        SCFree(file_ctx);
+        SCFree(output_file_ctx);
         return NULL;
     }
 
+    output_file_ctx->file_ctx = ajt->file_ctx;
+
     if (conf) {
         const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
         if (force_magic != NULL && ConfValIsTrue(force_magic)) {
@@ -350,8 +325,34 @@ OutputCtx *OutputFileLogInit(ConfNode *conf)
         }
     }
 
+    output_ctx->data = output_file_ctx;
+
     FileForceTrackingEnable();
     return output_ctx;
 }
 
+void TmModuleJsonFileLogRegister (void) {
+    tmm_modules[TMM_JSONFILELOG].name = "JsonFileLog";
+    tmm_modules[TMM_JSONFILELOG].ThreadInit = JsonFileLogThreadInit;
+    tmm_modules[TMM_JSONFILELOG].ThreadDeinit = JsonFileLogThreadDeinit;
+
+    /* register as child of eve-log */
+    OutputRegisterFileSubModule("eve-log", "JsonFileLog", "eve-log.files",
+            OutputFileLogInitSub, JsonFileLogger);
+}
+
+#else
+
+static TmEcode OutputJsonThreadInit(ThreadVars *t, void *initdata, void **data)
+{
+    SCLogInfo("Can't init JSON output - JSON support was disabled during build.");
+    return TM_ECODE_FAILED;
+}
+
+void TmModuleJsonFileLogRegister (void)
+{
+    tmm_modules[TMM_JSONFILELOG].name = "JsonFileLog";
+    tmm_modules[TMM_JSONFILELOG].ThreadInit = OutputJsonThreadInit;
+}
+
 #endif

diff --git a/src/output-json-file.h b/src/output-json-file.h
index a97c0d54ae11785d67f2705c3c0914fcf74a9b4b..cb39618157bd79b4a6c4bb8ffc7185d7ee0bd48f 100644 (file)
--- a/src/output-json-file.h
+++ b/src/output-json-file.h
@@ -26,5 +26,6 @@
 
 TmEcode OutputFileLog (ThreadVars *tv, Packet *p, void *data);
 OutputCtx *OutputFileLogInit(ConfNode *);
+void TmModuleJsonFileLogRegister (void);
 
 #endif /* __OUTPUT_FILELOG_H__ */

diff --git a/src/output-json.c b/src/output-json.c
index 5a88a7c92492c0121589edf2504bf4d0dda88950..226cc4d35ccea701355aab3adcc96e5aceb7b337 100644 (file)
--- a/src/output-json.c
+++ b/src/output-json.c
@@ -163,8 +163,6 @@ static enum JsonOutput json_out = ALERT_FILE;
 #define OUTPUT_HTTP   (1<<4)
 #define OUTPUT_TLS    (1<<5)
 
-static uint32_t output_flags = 0;
-
 enum JsonFormat { COMPACT, INDENT };
 static enum JsonFormat format = COMPACT;
 
@@ -357,10 +355,6 @@ TmEcode OutputJSON(json_t *js, void *data, uint64_t *count)
 
 TmEcode OutputJson (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
 {
-    if (output_flags & OUTPUT_FILES) {
-        OutputFileLog(tv, p, data);
-    }
-
     return TM_ECODE_OK;
 }
 
@@ -527,13 +521,6 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf)
              * registration capability
              */
             TAILQ_FOREACH(output, &outputs->head, next) {
-                if (strcmp(output->val, "files") == 0) {
-                    SCLogDebug("Enabling files output");
-                    ConfNode *child = ConfNodeLookupChild(output, "files");
-                    json_ctx->files_ctx = OutputFileLogInit(child);
-                    output_flags |= OUTPUT_FILES;
-                    continue;
-                }
             }
         }
     }

diff --git a/src/suricata.c b/src/suricata.c
index 1a20728dc23702971400c183a028d6403e008bca..2b2c88a133d05a6bfcbe4f7227a35553bed55bda 100644 (file)
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -90,6 +90,7 @@
 #include "output-tlslog.h"
 #include "log-pcap.h"
 #include "log-file.h"
+#include "output-json-file.h"
 #include "log-filestore.h"
 
 #include "output-json.h"
@@ -811,6 +812,7 @@ void RegisterAllModules()
     TmModulePcapLogRegister();
     /* file log */
     TmModuleLogFileLogRegister();
+    TmModuleJsonFileLogRegister();
     TmModuleLogFilestoreRegister();
     /* dns log */
     TmModuleLogDnsLogRegister();

diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h
index 09b8e6b25fbd9c8dede1bf1a417a5d63ed905f6c..1336d77c0b675fcb34b340492408cef31bc074d3 100644 (file)
--- a/src/tm-threads-common.h
+++ b/src/tm-threads-common.h
@@ -89,6 +89,7 @@ typedef enum {
     TMM_JSONHTTPLOG,
     TMM_JSONDNSLOG,
     TMM_JSONTLSLOG,
+    TMM_JSONFILELOG,
     TMM_SIZE,
 } TmmId;