From: Maurizio Abba <mabba@lastline.com>
Date: Mon, 15 Jan 2018 15:59:28 +0000 (+0000)
Subject: xff: Use XFF configuration in eve and filestore
X-Git-Tag: suricata-4.1.0-rc1~93
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2543930d7498d23810e5784bb187442d65ad2c53;p=thirdparty%2Fsuricata.git

xff: Use XFF configuration in eve and filestore

XFF configuration is already set in app-layer-htp-xff, and in
output-json-alert. Extending XFF configuration to files and HTTP allow
to get the same behavior as for alerts.

Extend the configuration of filestore json to let filestore metafile
dump be aware of xff. This is available only if write-fileinfo is set
to yes and file-store version is 2.
---

diff --git a/src/alert-unified2-alert.c b/src/alert-unified2-alert.c
index 2abcd59528..e1523a3ef2 100644
--- a/src/alert-unified2-alert.c
+++ b/src/alert-unified2-alert.c
@@ -329,7 +329,7 @@ int Unified2Logger(ThreadVars *t, void *data, const Packet *p)
         int have_xff_ip = 0;
 
         if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
-            have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+            have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
         }
 
         if (have_xff_ip) {
@@ -916,9 +916,9 @@ static int Unified2IPv6TypeAlert(ThreadVars *t, const Packet *p, void *data)
 
             if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
                 if (pa->flags & PACKET_ALERT_FLAG_TX) {
-                    have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
                 } else {
-                    have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
                 }
             }
 
@@ -1104,9 +1104,9 @@ static int Unified2IPv4TypeAlert (ThreadVars *tv, const Packet *p, void *data)
 
             if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
                 if (pa->flags & PACKET_ALERT_FLAG_TX) {
-                    have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
                 } else {
-                    have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
                 }
             }
 
diff --git a/src/app-layer-htp-xff.c b/src/app-layer-htp-xff.c
index aaf7196a2c..fd48b7e93b 100644
--- a/src/app-layer-htp-xff.c
+++ b/src/app-layer-htp-xff.c
@@ -110,7 +110,7 @@ static int ParseXFFString(char *input, char *output, int output_size)
  * \retval 1 if the IP has been found and returned in dstbuf
  * \retval 0 if the IP has not being found or error
  */
-int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg,
+int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg,
         char *dstbuf, int dstbuflen)
 {
     uint8_t xff_chain[XFF_CHAIN_MAXLEN];
@@ -119,18 +119,18 @@ int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg,
     uint64_t total_txs = 0;
     uint8_t *p_xff = NULL;
 
-    htp_state = (HtpState *)FlowGetAppState(p->flow);
+    htp_state = (HtpState *)FlowGetAppState(f);
 
     if (htp_state == NULL) {
         SCLogDebug("no http state, XFF IP cannot be retrieved");
         return 0;
     }
 
-    total_txs = AppLayerParserGetTxCnt(p->flow, htp_state);
+    total_txs = AppLayerParserGetTxCnt(f, htp_state);
     if (tx_id >= total_txs)
         return 0;
 
-    tx = AppLayerParserGetTx(p->flow->proto, ALPROTO_HTTP, htp_state, tx_id);
+    tx = AppLayerParserGetTx(f->proto, ALPROTO_HTTP, htp_state, tx_id);
     if (tx == NULL) {
         SCLogDebug("tx is NULL, XFF cannot be retrieved");
         return 0;
@@ -174,21 +174,21 @@ int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg,
  *  \retval 1 if the IP has been found and returned in dstbuf
  *  \retval 0 if the IP has not being found or error
  */
-int HttpXFFGetIP(const Packet *p, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
+int HttpXFFGetIP(const Flow *f, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
 {
     HtpState *htp_state = NULL;
     uint64_t tx_id = 0;
     uint64_t total_txs = 0;
 
-    htp_state = (HtpState *)FlowGetAppState(p->flow);
+    htp_state = (HtpState *)FlowGetAppState(f);
     if (htp_state == NULL) {
         SCLogDebug("no http state, XFF IP cannot be retrieved");
         goto end;
     }
 
-    total_txs = AppLayerParserGetTxCnt(p->flow, htp_state);
+    total_txs = AppLayerParserGetTxCnt(f, htp_state);
     for (; tx_id < total_txs; tx_id++) {
-        if (HttpXFFGetIPFromTx(p, tx_id, xff_cfg, dstbuf, dstbuflen) == 1)
+        if (HttpXFFGetIPFromTx(f, tx_id, xff_cfg, dstbuf, dstbuflen) == 1)
             return 1;
     }
 
diff --git a/src/app-layer-htp-xff.h b/src/app-layer-htp-xff.h
index b02d95de2a..03671d3fbb 100644
--- a/src/app-layer-htp-xff.h
+++ b/src/app-layer-htp-xff.h
@@ -45,9 +45,9 @@ typedef struct HttpXFFCfg_ {
 
 void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result);
 
-int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
+int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
 
-int HttpXFFGetIP(const Packet *p, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
+int HttpXFFGetIP(const Flow *f, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
 
 void HTPXFFParserRegisterTests(void);
 
diff --git a/src/output-filestore.c b/src/output-filestore.c
index 8adb01aefe..6f4d8f3f68 100644
--- a/src/output-filestore.c
+++ b/src/output-filestore.c
@@ -19,6 +19,7 @@
 
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
+#include "app-layer-htp-xff.h"
 #include "app-layer-smtp.h"
 
 #include "output.h"
@@ -49,6 +50,7 @@ typedef struct OutputFilestoreCtx_ {
     char prefix[FILESTORE_PREFIX_MAX];
     char tmpdir[FILESTORE_PREFIX_MAX];
     bool fileinfo;
+    HttpXFFCfg *xff_cfg;
 } OutputFilestoreCtx;
 
 typedef struct OutputFilestoreLogThread_ {
@@ -162,7 +164,8 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv,
         snprintf(js_metadata_filename, sizeof(js_metadata_filename),
                 "%s.%"PRIuMAX".%u.json", final_filename,
                 (uintmax_t)p->ts.tv_sec, ff->file_store_id);
-        json_t *js_fileinfo = JsonBuildFileInfoRecord(p, ff, true, dir);
+        json_t *js_fileinfo = JsonBuildFileInfoRecord(p, ff, true, dir,
+                ctx->xff_cfg);
         if (likely(js_fileinfo != NULL)) {
             json_dump_file(js_fileinfo, js_metadata_filename, 0);
             json_decref(js_fileinfo);
@@ -307,6 +310,9 @@ static TmEcode OutputFilestoreLogThreadDeinit(ThreadVars *t, void *data)
 static void OutputFilestoreLogDeInitCtx(OutputCtx *output_ctx)
 {
     OutputFilestoreCtx *ctx = (OutputFilestoreCtx *)output_ctx->data;
+    if (ctx->xff_cfg != NULL) {
+        SCFree(ctx->xff_cfg);
+    }
     SCFree(ctx);
     SCFree(output_ctx);
 }
@@ -404,6 +410,11 @@ static OutputInitResult OutputFilestoreLogInitCtx(ConfNode *conf)
     strlcpy(ctx->prefix, log_directory, sizeof(ctx->prefix));
     snprintf(ctx->tmpdir, sizeof(ctx->tmpdir) - 1, "%s/tmp", log_directory);
 
+    ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, ctx->xff_cfg);
+    }
+
     OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
     if (unlikely(output_ctx == NULL)) {
         SCFree(ctx);
diff --git a/src/output-json-alert.c b/src/output-json-alert.c
index f8b36097c5..45a19a3ba1 100644
--- a/src/output-json-alert.c
+++ b/src/output-json-alert.c
@@ -598,9 +598,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
 
             if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
                 if (pa->flags & PACKET_ALERT_FLAG_TX) {
-                    have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
                 } else {
-                    have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
                 }
             }
 
diff --git a/src/output-json-file.c b/src/output-json-file.c
index f652a432af..c7a62b3417 100644
--- a/src/output-json-file.c
+++ b/src/output-json-file.c
@@ -65,6 +65,7 @@
 #include "output-json-smb.h"
 
 #include "app-layer-htp.h"
+#include "app-layer-htp-xff.h"
 #include "util-memcmp.h"
 #include "stream-tcp-reassemble.h"
 
@@ -73,6 +74,7 @@
 typedef struct OutputFileCtx_ {
     LogFileCtx *file_ctx;
     uint32_t file_cnt;
+    HttpXFFCfg *xff_cfg;
 } OutputFileCtx;
 
 typedef struct JsonFileLogThread_ {
@@ -81,7 +83,7 @@ typedef struct JsonFileLogThread_ {
 } JsonFileLogThread;
 
 json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
-        const bool stored, uint8_t dir)
+        const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
 {
     json_t *hjs = NULL;
     enum OutputJsonLogDirection fdir = LOG_DIR_FLOW;
@@ -205,6 +207,29 @@ json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
     json_object_set_new(fjs, "size", json_integer(FileTrackedSize(ff)));
     json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
 
+    /* xff header */
+    if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
+        int have_xff_ip = 0;
+        char buffer[XFF_MAXLEN];
+
+        if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
+            have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, buffer, XFF_MAXLEN);
+        }
+
+        if (have_xff_ip) {
+            if (xff_cfg->flags & XFF_EXTRADATA) {
+                json_object_set_new(js, "xff", json_string(buffer));
+            }
+            else if (xff_cfg->flags & XFF_OVERWRITE) {
+                if (p->flowflags & FLOW_PKT_TOCLIENT) {
+                    json_object_set(js, "dest_ip", json_string(buffer));
+                } else {
+                    json_object_set(js, "src_ip", json_string(buffer));
+                }
+            }
+        }
+    }
+
     /* originally just 'file', but due to bug 1127 naming it fileinfo */
     json_object_set_new(js, "fileinfo", fjs);
 
@@ -218,8 +243,9 @@ json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
 static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p,
                                 const File *ff, uint32_t dir)
 {
+    HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg;
     json_t *js = JsonBuildFileInfoRecord(p, ff,
-            ff->flags & FILE_STORED ? true : false, dir);
+            ff->flags & FILE_STORED ? true : false, dir, xff_cfg);
     if (unlikely(js == NULL)) {
         return;
     }
@@ -290,6 +316,9 @@ static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)
 static void OutputFileLogDeinitSub(OutputCtx *output_ctx)
 {
     OutputFileCtx *ff_ctx = output_ctx->data;
+    if (ff_ctx->xff_cfg != NULL) {
+        SCFree(ff_ctx->xff_cfg);
+    }
     SCFree(ff_ctx);
     SCFree(output_ctx);
 }
@@ -330,6 +359,10 @@ static OutputInitResult OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_c
 
         FileForceHashParseCfg(conf);
     }
+    output_file_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (output_file_ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, output_file_ctx->xff_cfg);
+    }
 
     output_ctx->data = output_file_ctx;
     output_ctx->DeInit = OutputFileLogDeinitSub;
diff --git a/src/output-json-file.h b/src/output-json-file.h
index 9b7657d216..3d1b723902 100644
--- a/src/output-json-file.h
+++ b/src/output-json-file.h
@@ -27,8 +27,10 @@
 void JsonFileLogRegister(void);
 
 #ifdef HAVE_LIBJANSSON
+#include "app-layer-htp-xff.h"
+
 json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
-        const bool stored, uint8_t dir);
+        const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg);
 #endif
 
 #endif /* __OUTPUT_JSON_FILE_H__ */
diff --git a/src/output-json-http.c b/src/output-json-http.c
index 7110e6b44e..47add0a0fe 100644
--- a/src/output-json-http.c
+++ b/src/output-json-http.c
@@ -40,6 +40,7 @@
 
 #include "output.h"
 #include "app-layer-htp.h"
+#include "app-layer-htp-xff.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
 #include "util-privs.h"
@@ -59,6 +60,7 @@ typedef struct LogHttpFileCtx_ {
     uint32_t flags; /** Store mode */
     uint64_t fields;/** Store fields */
     bool include_metadata;
+    HttpXFFCfg *xff_cfg;
 } LogHttpFileCtx;
 
 typedef struct JsonHttpLogThread_ {
@@ -466,6 +468,28 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
     MemBufferReset(jhl->buffer);
 
     JsonHttpLogJSON(jhl, js, tx, tx_id);
+    HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg;
+
+    /* xff header */
+    if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
+        int have_xff_ip = 0;
+        char buffer[XFF_MAXLEN];
+
+        have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN);
+
+        if (have_xff_ip) {
+            if (xff_cfg->flags & XFF_EXTRADATA) {
+                json_object_set_new(js, "xff", json_string(buffer));
+            }
+            else if (xff_cfg->flags & XFF_OVERWRITE) {
+                if (p->flowflags & FLOW_PKT_TOCLIENT) {
+                    json_object_set(js, "dest_ip", json_string(buffer));
+                } else {
+                    json_object_set(js, "src_ip", json_string(buffer));
+                }
+            }
+        }
+    }
 
     OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer);
     json_object_del(js, "http");
@@ -501,6 +525,9 @@ static void OutputHttpLogDeinit(OutputCtx *output_ctx)
     LogHttpFileCtx *http_ctx = output_ctx->data;
     LogFileCtx *logfile_ctx = http_ctx->file_ctx;
     LogFileFreeCtx(logfile_ctx);
+    if (http_ctx->xff_cfg) {
+        SCFree(http_ctx->xff_cfg);
+    }
     SCFree(http_ctx);
     SCFree(output_ctx);
 }
@@ -545,6 +572,11 @@ static OutputInitResult OutputHttpLogInit(ConfNode *conf)
             }
         }
     }
+    http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (http_ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, http_ctx->xff_cfg);
+    }
+
     output_ctx->data = http_ctx;
     output_ctx->DeInit = OutputHttpLogDeinit;
 
@@ -559,6 +591,9 @@ static OutputInitResult OutputHttpLogInit(ConfNode *conf)
 static void OutputHttpLogDeinitSub(OutputCtx *output_ctx)
 {
     LogHttpFileCtx *http_ctx = output_ctx->data;
+    if (http_ctx->xff_cfg) {
+        SCFree(http_ctx->xff_cfg);
+    }
     SCFree(http_ctx);
     SCFree(output_ctx);
 }
@@ -615,6 +650,11 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c
             }
         }
     }
+    http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (http_ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, http_ctx->xff_cfg);
+    }
+
     output_ctx->data = http_ctx;
     output_ctx->DeInit = OutputHttpLogDeinitSub;
 
diff --git a/suricata.yaml.in b/suricata.yaml.in
index 634d153390..d09c8aeac3 100644
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -208,6 +208,23 @@ outputs:
             # custom allows additional http fields to be included in eve-log
             # the example below adds three additional fields when uncommented
             #custom: [Accept-Encoding, Accept-Language, Authorization]
+            # HTTP X-Forwarded-For support by adding an extra field or overwriting
+            # the source or destination IP address (depending on flow direction)
+            # with the one reported in the X-Forwarded-For HTTP header. This is
+            # helpful when reviewing alerts for traffic that is being reverse
+            # or forward proxied.
+            xff:
+              enabled: no
+              # Two operation modes are available, "extra-data" and "overwrite".
+              mode: extra-data
+              # Two proxy deployments are supported, "reverse" and "forward". In
+              # a "reverse" deployment the IP address used is the last one, in a
+              # "forward" deployment the first IP address is used.
+              deployment: reverse
+              # Header name where the actual IP address will be reported, if more
+              # than one IP address is present, the last IP address will be the
+              # one taken into consideration.
+              header: X-Forwarded-For
         - dns:
             # This configuration uses the new DNS logging format,
             # the old configuration is still available:
@@ -251,6 +268,23 @@ outputs:
             # force logging of checksums, available hash functions are md5,
             # sha1 and sha256
             #force-hash: [md5]
+            # HTTP X-Forwarded-For support by adding an extra field or overwriting
+            # the source or destination IP address (depending on flow direction)
+            # with the one reported in the X-Forwarded-For HTTP header. This is
+            # helpful when reviewing alerts for traffic that is being reverse
+            # or forward proxied.
+            xff:
+              enabled: no
+              # Two operation modes are available, "extra-data" and "overwrite".
+              mode: extra-data
+              # Two proxy deployments are supported, "reverse" and "forward". In
+              # a "reverse" deployment the IP address used is the last one, in a
+              # "forward" deployment the first IP address is used.
+              deployment: reverse
+              # Header name where the actual IP address will be reported, if more
+              # than one IP address is present, the last IP address will be the
+              # one taken into consideration.
+              header: X-Forwarded-For
         #- drop:
         #    alerts: yes      # log alerts that caused drops
         #    flows: all       # start or all: 'start' logs only a single drop
@@ -515,6 +549,24 @@ outputs:
       # the use of this output module as it uses the SHA256 as the
       # file naming scheme.
       #force-hash: [sha1, md5]
+      # NOTE: X-Forwarded configuration is ignored if write-fileinfo is disabled
+      # HTTP X-Forwarded-For support by adding an extra field or overwriting
+      # the source or destination IP address (depending on flow direction)
+      # with the one reported in the X-Forwarded-For HTTP header. This is
+      # helpful when reviewing alerts for traffic that is being reverse
+      # or forward proxied.
+      xff:
+        enabled: no
+        # Two operation modes are available, "extra-data" and "overwrite".
+        mode: extra-data
+        # Two proxy deployments are supported, "reverse" and "forward". In
+        # a "reverse" deployment the IP address used is the last one, in a
+        # "forward" deployment the first IP address is used.
+        deployment: reverse
+        # Header name where the actual IP address will be reported, if more
+        # than one IP address is present, the last IP address will be the
+        # one taken into consideration.
+        header: X-Forwarded-For
 
   # output module to store extracted files to disk (old style, deprecated)
   #