email-json: add custom fields support

This patch adds a way to specify which MIME fields to log via
the custom keyword in the EVE configuration. it also adds an
extended logging where some fields are added. The logging support
mono value fields as well as multivalue fields via the use of
JSON array.

diff --git a/src/output-json-email-common.c b/src/output-json-email-common.c
index f7553460e3483246b7306e33eeff69c299d864ab..acbf78d08fe71292726f4fb81565fd07ab2c115b 100644 (file)
--- a/src/output-json-email-common.c
+++ b/src/output-json-email-common.c
@@ -55,6 +55,79 @@
 #ifdef HAVE_LIBJANSSON
 #include <jansson.h>
 
+#define LOG_EMAIL_DEFAULT    0
+#define LOG_EMAIL_EXTENDED   1
+#define LOG_EMAIL_ARRAY      2 /* require array handling */
+
+struct {
+    char *config_field;
+    char *email_field;
+    uint32_t flags;
+} email_fields[] =  {
+    { "reply_to", "reply-to", LOG_EMAIL_DEFAULT },
+    { "bcc", "bcc", LOG_EMAIL_DEFAULT },
+    { "message_id", "message-id", LOG_EMAIL_EXTENDED },
+    { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED },
+    { "user_agent", "user-agent", LOG_EMAIL_EXTENDED },
+    { "received", "received", LOG_EMAIL_ARRAY },
+    { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT },
+    { "relays", "relays", LOG_EMAIL_ARRAY },
+    { NULL, NULL, LOG_EMAIL_DEFAULT},
+};
+
+static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
+{
+    json_t *ajs = data;
+
+    if (ajs == NULL)
+        return 0;
+    char *value = BytesToString((uint8_t *)val, len);
+    json_array_append_new(ajs, json_string(value));
+    SCFree(value);
+    return 1;
+}
+
+static void JsonEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx)
+{
+    int f = 0;
+    MimeDecField *field;
+    MimeDecEntity *entity = tx->msg_tail;
+    if (entity == NULL) {
+        return;
+    }
+
+    while(email_fields[f].config_field) {
+        if (((email_ctx->fields & (1ULL<<f)) != 0)
+              ||
+              ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
+           ) {
+            if (email_fields[f].flags & LOG_EMAIL_ARRAY) {
+                json_t *ajs = json_array();
+                if (ajs) {
+                    int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, ajs);
+                    if (found > 0) {
+                        json_object_set_new(js, email_fields[f].config_field, ajs);
+                    } else {
+                        json_decref(ajs);
+                    }
+                }
+            } else {
+                field = MimeDecFindField(entity, email_fields[f].email_field);
+                if (field != NULL) {
+                    char *s = BytesToString((uint8_t *)field->value,
+                            (size_t)field->value_len);
+                    if (likely(s != NULL)) {
+                        json_object_set_new(js, email_fields[f].config_field, json_string(s));
+                        SCFree(s);
+                    }
+                }
+            }
+
+        }
+        f++;
+    }
+}
+
 /* JSON format logging */
 json_t *JsonEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id)
 {
@@ -258,6 +331,11 @@ json_t *JsonEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_
 TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
 {
     json_t *sjs = JsonEmailLogJsonData(f, state, vtx, tx_id);
+    OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx;
+    SMTPTransaction *tx = (SMTPTransaction *) vtx;
+
+    if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0))
+        JsonEmailLogJSONCustom(email_ctx, sjs, tx);
 
     if (sjs) {
         json_object_set_new(js, "email", sjs);
@@ -282,4 +360,40 @@ json_t *JsonEmailAddMetadata(const Flow *f)
 }
 
 
+void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx)
+{
+    if (conf) {
+        const char *extended = ConfNodeLookupChildValue(conf, "extended");
+
+        if (extended != NULL) {
+            if (ConfValIsTrue(extended)) {
+                email_ctx->flags = LOG_EMAIL_EXTENDED;
+            }
+        }
+
+        ConfNode *custom;
+        if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
+            ConfNode *field;
+            TAILQ_FOREACH(field, &custom->head, next) {
+                if (field != NULL) {
+                    int f = 0;
+                    while(email_fields[f].config_field) {
+                        if ((strcmp(email_fields[f].config_field,
+                                   field->val) == 0) ||
+                            (strcasecmp(email_fields[f].email_field,
+                                        field->val) == 0))
+                        {
+                            email_ctx->fields |= (1ULL<<f);
+                            break;
+                        }
+                        f++;
+                    }
+                }
+            }
+        }
+    }
+    return;
+}
+
+
 #endif

diff --git a/src/output-json-email-common.h b/src/output-json-email-common.h
index 4553e8572ce487637020d684b301059dfff833c1..afb3b91bfc16ecdee1fb018c6c4b8d1fc9e4d7cf 100644 (file)
--- a/src/output-json-email-common.h
+++ b/src/output-json-email-common.h
@@ -27,6 +27,7 @@
 typedef struct OutputJsonEmailCtx_ {
     LogFileCtx *file_ctx;
     uint32_t flags; /** Store mode */
+    uint64_t fields;/** Store fields */
 } OutputJsonEmailCtx;
 
 
@@ -40,4 +41,6 @@ TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, F
 json_t *JsonEmailAddMetadata(const Flow *f);
 #endif
 
+void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx);
+
 #endif /* __OUTPUT_JSON_EMAIL_COMMON_H__ */

diff --git a/src/output-json-smtp.c b/src/output-json-smtp.c
index 255209571c4a4dc152582d14dbe7226866f751b4..2d5860da97cdaf963ce54203ee273b8c4216717f 100644 (file)
--- a/src/output-json-smtp.c
+++ b/src/output-json-smtp.c
@@ -206,6 +206,8 @@ static OutputCtx *OutputSmtpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
 
     email_ctx->file_ctx = ojc->file_ctx;
 
+    OutputEmailInitConf(conf, email_ctx);
+
     output_ctx->data = email_ctx;
     output_ctx->DeInit = OutputSmtpLogDeInitCtxSub;