output: introduce concept of sub-modules

To support the 'eve-log' idea, we need to be able to force all log
modules to be enabled by the master eve-log module, and need to be
able to make all logs go into a single file. This didn't fit the
API so far, so added the sub-module concept.

A sub-module is a regular module, that registers itself as a sub-
module of another module:

    OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http",
            OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger);

The first argument is the name of the parent. The 4th argument is
the OutputCtx init function. It differs slightly from the non-sub
one. The different is that in addition to it's ConfNode, it gets
the OutputCtx from the parent. This way it can set the parents
LogFileCtx in it's own OutputCtx.

The runmode setup code will take care of all the extra setup. It's
possible to register a module both as a normal module and as a sub-
module, which can operate at the same time.

Only the TxLogger API is handled in this patch, the rest will be
updated later.

diff --git a/src/output-dnslog.c b/src/output-dnslog.c
index b9bab570502d92ef59e097bc04a6c154788f2960..677edaba6f68507e99f45463eccc91961791ceed 100644 (file)
--- a/src/output-dnslog.c
+++ b/src/output-dnslog.c
@@ -299,6 +299,35 @@ static void LogDnsLogDeInitCtx(OutputCtx *output_ctx)
     SCFree(output_ctx);
 }
 
+static OutputCtx *JsonDnsLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
+{
+    AlertJsonThread *ajt = parent_ctx->data;
+
+    LogDnsFileCtx *dnslog_ctx = SCMalloc(sizeof(LogDnsFileCtx));
+    if (unlikely(dnslog_ctx == NULL)) {
+        return NULL;
+    }
+    memset(dnslog_ctx, 0x00, sizeof(LogDnsFileCtx));
+
+    dnslog_ctx->file_ctx = ajt->file_ctx;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL)) {
+        SCFree(dnslog_ctx);
+        return NULL;
+    }
+
+    output_ctx->data = dnslog_ctx;
+    output_ctx->DeInit = LogDnsLogDeInitCtx;
+
+    SCLogDebug("DNS log sub-module initialized");
+
+    AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_DNS);
+    AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_DNS);
+
+    return output_ctx;
+}
+
 #define DEFAULT_LOG_FILENAME "dns.json"
 /** \brief Create a new dns log LogFileCtx.
  *  \param conf Pointer to ConfNode containing this loggers configuration.
@@ -356,6 +385,8 @@ void TmModuleJsonDnsLogRegister (void) {
 
     OutputRegisterTxModule(MODULE_NAME, "dns-json-log", JsonDnsLogInitCtx,
             ALPROTO_DNS, JsonDnsLogger);
+    OutputRegisterTxSubModule("eve-log", MODULE_NAME, "dns", JsonDnsLogInitCtxSub,
+            ALPROTO_DNS, JsonDnsLogger);
 }
 
 #else

diff --git a/src/output-httplog.c b/src/output-httplog.c
index 607052ba3a8795d33980c31842b2178ecc772311..492eda53f972053641c1655eb28eceb84ab6dbcb 100644 (file)
--- a/src/output-httplog.c
+++ b/src/output-httplog.c
@@ -274,6 +274,36 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf)
     return output_ctx;
 }
 
+OutputCtx *OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
+{
+    AlertJsonThread *ajt = parent_ctx->data;
+
+    LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx));
+    if (unlikely(http_ctx == NULL))
+        return NULL;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL))
+        return NULL;
+
+    http_ctx->file_ctx = ajt->file_ctx;
+    http_ctx->flags = LOG_HTTP_DEFAULT;
+
+    if (conf) {
+        const char *extended = ConfNodeLookupChildValue(conf, "extended");
+
+        if (extended != NULL) {
+            if (ConfValIsTrue(extended)) {
+                http_ctx->flags = LOG_HTTP_EXTENDED;
+            }
+        }
+    }
+    output_ctx->data = http_ctx;
+    output_ctx->DeInit = NULL;
+
+    return output_ctx;
+}
+
 #define OUTPUT_BUFFER_SIZE 65535
 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, void *initdata, void **data)
 {
@@ -324,8 +354,13 @@ void TmModuleJsonHttpLogRegister (void) {
     tmm_modules[TMM_JSONHTTPLOG].RegisterTests = NULL;
     tmm_modules[TMM_JSONHTTPLOG].cap_flags = 0;
 
+    /* register as separate module */
     OutputRegisterTxModule("JsonHttpLog", "http-json-log", OutputHttpLogInit,
             ALPROTO_HTTP, JsonHttpLogger);
+
+    /* also register as child of eve-log */
+    OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http", OutputHttpLogInitSub,
+            ALPROTO_HTTP, JsonHttpLogger);
 }
 
 #else

diff --git a/src/output.c b/src/output.c
index 286310ac92a0c302c0168fed841ff416985087b3..b56eb316e143ffa4e39a04156cad08e97f851f9d 100644 (file)
--- a/src/output.c
+++ b/src/output.c
@@ -148,6 +148,41 @@ error:
     exit(EXIT_FAILURE);
 }
 
+void
+OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto,
+    TxLogger TxLogFunc)
+{
+    if (unlikely(TxLogFunc == NULL)) {
+        goto error;
+    }
+
+    OutputModule *module = SCCalloc(1, sizeof(*module));
+    if (unlikely(module == NULL)) {
+        goto error;
+    }
+
+    module->name = SCStrdup(name);
+    if (unlikely(module->name == NULL))
+        goto error;
+    module->conf_name = SCStrdup(conf_name);
+    if (unlikely(module->conf_name == NULL))
+        goto error;
+    module->parent_name = SCStrdup(parent_name);
+    if (unlikely(module->parent_name == NULL))
+        goto error;
+    module->InitSubFunc = InitFunc;
+    module->TxLogFunc = TxLogFunc;
+    module->alproto = alproto;
+    TAILQ_INSERT_TAIL(&output_modules, module, entries);
+
+    SCLogDebug("Tx logger \"%s\" registered.", name);
+    return;
+error:
+    SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting...");
+    exit(EXIT_FAILURE);
+}
+
 /**
  * \brief Register a file output module.
  *

diff --git a/src/output.h b/src/output.h
index 78b16df8f0b01c8f3826a7288f39ea2823655a49..6d668238a90474553ef9a4df5e21ac49f595ed05 100644 (file)
--- a/src/output.h
+++ b/src/output.h
@@ -38,7 +38,9 @@
 typedef struct OutputModule_ {
     char *name;
     char *conf_name;
+    char *parent_name;
     OutputCtx *(*InitFunc)(ConfNode *);
+    OutputCtx *(*InitSubFunc)(ConfNode *, OutputCtx *parent_ctx);
 
     PacketLogger PacketLogFunc;
     PacketLogCondition PacketConditionFunc;
@@ -58,6 +60,9 @@ void OutputRegisterPacketModule(char *name, char *conf_name,
 void OutputRegisterTxModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), uint16_t alproto,
     TxLogger TxLogFunc);
+void OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto,
+    TxLogger TxLogFunc);
 void OutputRegisterFileModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc);
 void OutputRegisterFiledataModule(char *name, char *conf_name,

diff --git a/src/runmodes.c b/src/runmodes.c
index e47894356867ea3de9d6451021fab85162f4d5c8..2916deff7b19e36836bb6f2d434b1f7737557f3a 100644 (file)
--- a/src/runmodes.c
+++ b/src/runmodes.c
@@ -410,6 +410,121 @@ void RunModeShutDown(void)
     }
 }
 
+static TmModule *pkt_logger_module = NULL;
+static TmModule *tx_logger_module = NULL;
+static TmModule *file_logger_module = NULL;
+static TmModule *filedata_logger_module = NULL;
+
+/** \brief Turn output into thread module */
+static void SetupOutput(const char *name, OutputModule *module, OutputCtx *output_ctx)
+{
+    TmModule *tm_module = TmModuleGetByName(module->name);
+    if (tm_module == NULL) {
+        SCLogError(SC_ERR_INVALID_ARGUMENT,
+                "TmModuleGetByName for %s failed", module->name);
+        exit(EXIT_FAILURE);
+    }
+    if (strcmp(tmm_modules[TMM_ALERTDEBUGLOG].name, tm_module->name) == 0)
+        debuglog_enabled = 1;
+
+    if (module->PacketLogFunc) {
+        SCLogDebug("%s is a packet logger", module->name);
+        OutputRegisterPacketLogger(module->name, module->PacketLogFunc,
+                module->PacketConditionFunc, output_ctx);
+
+        /* need one instance of the packet logger module */
+        if (pkt_logger_module == NULL) {
+            pkt_logger_module = TmModuleGetByName("__packet_logger__");
+            if (pkt_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __packet_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = pkt_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__packet_logger__ added");
+        }
+    } else if (module->TxLogFunc) {
+        SCLogDebug("%s is a tx logger", module->name);
+        OutputRegisterTxLogger(module->name, module->alproto,
+                module->TxLogFunc, output_ctx);
+
+        /* need one instance of the tx logger module */
+        if (tx_logger_module == NULL) {
+            tx_logger_module = TmModuleGetByName("__tx_logger__");
+            if (tx_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __tx_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = tx_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__tx_logger__ added");
+        }
+    } else if (module->FileLogFunc) {
+        SCLogDebug("%s is a file logger", module->name);
+        OutputRegisterFileLogger(module->name, module->FileLogFunc, output_ctx);
+
+        /* need one instance of the tx logger module */
+        if (file_logger_module == NULL) {
+            file_logger_module = TmModuleGetByName("__file_logger__");
+            if (file_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __file_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = file_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__file_logger__ added");
+        }
+    } else if (module->FiledataLogFunc) {
+        SCLogDebug("%s is a filedata logger", module->name);
+        OutputRegisterFiledataLogger(module->name, module->FiledataLogFunc, output_ctx);
+
+        /* need one instance of the tx logger module */
+        if (filedata_logger_module == NULL) {
+            filedata_logger_module = TmModuleGetByName("__filedata_logger__");
+            if (filedata_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __filedata_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = filedata_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__filedata_logger__ added");
+        }
+    } else {
+        SCLogDebug("%s is a regular logger", module->name);
+
+        RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+        if (unlikely(runmode_output == NULL))
+            return;
+        runmode_output->tm_module = tm_module;
+        runmode_output->output_ctx = output_ctx;
+        TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+    }
+}
+
 /**
  * Initialize the output modules.
  */
@@ -422,11 +537,6 @@ void RunModeInitializeOutputs(void)
     }
 
     ConfNode *output, *output_config;
-    TmModule *tm_module;
-    TmModule *pkt_logger_module = NULL;
-    TmModule *tx_logger_module = NULL;
-    TmModule *file_logger_module = NULL;
-    TmModule *filedata_logger_module = NULL;
     const char *enabled;
 
     TAILQ_FOREACH(output, &outputs->head, next) {
@@ -470,6 +580,7 @@ void RunModeInitializeOutputs(void)
                 "No output module named %s, ignoring", output->val);
             continue;
         }
+
         OutputCtx *output_ctx = NULL;
         if (module->InitFunc != NULL) {
             output_ctx = module->InitFunc(output_config);
@@ -478,111 +589,52 @@ void RunModeInitializeOutputs(void)
                  * error. Maybe we should exit on init errors? */
                 continue;
             }
+        } else if (module->InitSubFunc != NULL) {
+            SCLogInfo("skipping submodule");
+            continue;
         }
-        tm_module = TmModuleGetByName(module->name);
-        if (tm_module == NULL) {
-            SCLogError(SC_ERR_INVALID_ARGUMENT,
-                    "TmModuleGetByName for %s failed", module->name);
-            exit(EXIT_FAILURE);
-        }
-        if (strcmp(tmm_modules[TMM_ALERTDEBUGLOG].name, tm_module->name) == 0)
-            debuglog_enabled = 1;
-
-        if (module->PacketLogFunc) {
-            SCLogDebug("%s is a packet logger", module->name);
-            OutputRegisterPacketLogger(module->name, module->PacketLogFunc,
-                    module->PacketConditionFunc, output_ctx);
-
-            /* need one instance of the packet logger module */
-            if (pkt_logger_module == NULL) {
-                pkt_logger_module = TmModuleGetByName("__packet_logger__");
-                if (pkt_logger_module == NULL) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __packet_logger__ failed");
-                    exit(EXIT_FAILURE);
-                }
 
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-                if (unlikely(runmode_output == NULL))
-                    return;
-                runmode_output->tm_module = pkt_logger_module;
-                runmode_output->output_ctx = NULL;
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
-                SCLogDebug("__packet_logger__ added");
-            }
-        } else if (module->TxLogFunc) {
-            SCLogDebug("%s is a tx logger", module->name);
-            OutputRegisterTxLogger(module->name, module->alproto,
-                    module->TxLogFunc, output_ctx);
-
-            /* need one instance of the tx logger module */
-            if (tx_logger_module == NULL) {
-                tx_logger_module = TmModuleGetByName("__tx_logger__");
-                if (tx_logger_module == NULL) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __tx_logger__ failed");
-                    exit(EXIT_FAILURE);
-                }
-
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-                if (unlikely(runmode_output == NULL))
-                    return;
-                runmode_output->tm_module = tx_logger_module;
-                runmode_output->output_ctx = NULL;
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
-                SCLogDebug("__tx_logger__ added");
-            }
-        } else if (module->FileLogFunc) {
-            SCLogDebug("%s is a file logger", module->name);
-            OutputRegisterFileLogger(module->name, module->FileLogFunc, output_ctx);
-
-            /* need one instance of the tx logger module */
-            if (file_logger_module == NULL) {
-                file_logger_module = TmModuleGetByName("__file_logger__");
-                if (file_logger_module == NULL) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __file_logger__ failed");
-                    exit(EXIT_FAILURE);
+        // TODO if module == parent, find it's children
+        if (strcmp(output->val, "eve-log") == 0) {
+            ConfNode *types = ConfNodeLookupChild(output_config, "types");
+            SCLogInfo("types %p", types);
+            if (types != NULL) {
+                ConfNode *type = NULL;
+                TAILQ_FOREACH(type, &types->head, next) {
+                    SCLogInfo("type %s", type->val);
+
+                    OutputModule *sub_module = OutputGetModuleByConfName(type->val);
+                    if (sub_module == NULL) {
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                                "No output module named %s, ignoring", type->val);
+                        continue;
+                    }
+                    if (sub_module->parent_name == NULL ||
+                            strcmp(sub_module->parent_name,output->val) != 0) {
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                                "bad parent for %s, ignoring", type->val);
+                        continue;
+                    }
+                    if (sub_module->InitSubFunc == NULL) {
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                                "bad sub-module for %s, ignoring", type->val);
+                        continue;
+                    }
+                    ConfNode *sub_output_config = ConfNodeLookupChild(type, type->val);
+                    // sub_output_config may be NULL if no config
+
+                    /* pass on parent output_ctx */
+                    OutputCtx *sub_output_ctx =
+                        sub_module->InitSubFunc(sub_output_config, output_ctx);
+                    if (sub_output_ctx == NULL) {
+                        continue;
+                    }
+
+                    SetupOutput(sub_module->name, sub_module, sub_output_ctx);
                 }
-
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-                if (unlikely(runmode_output == NULL))
-                    return;
-                runmode_output->tm_module = file_logger_module;
-                runmode_output->output_ctx = NULL;
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
-                SCLogDebug("__file_logger__ added");
-            }
-        } else if (module->FiledataLogFunc) {
-            SCLogDebug("%s is a filedata logger", module->name);
-            OutputRegisterFiledataLogger(module->name, module->FiledataLogFunc, output_ctx);
-
-            /* need one instance of the tx logger module */
-            if (filedata_logger_module == NULL) {
-                filedata_logger_module = TmModuleGetByName("__filedata_logger__");
-                if (filedata_logger_module == NULL) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __filedata_logger__ failed");
-                    exit(EXIT_FAILURE);
-                }
-
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-                if (unlikely(runmode_output == NULL))
-                    return;
-                runmode_output->tm_module = filedata_logger_module;
-                runmode_output->output_ctx = NULL;
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
-                SCLogDebug("__filedata_logger__ added");
             }
         } else {
-            SCLogDebug("%s is a regular logger", module->name);
-
-            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-            if (unlikely(runmode_output == NULL))
-                return;
-            runmode_output->tm_module = tm_module;
-            runmode_output->output_ctx = output_ctx;
-            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SetupOutput(module->name, module, output_ctx);
         }
     }
 }