unified2: fix xff extra-data output (Bug #2305)

In extra-data mode, suricata does not output xff data without
undocumented conditions (including enabling packet output). This
behaviour has been fixed to remove the hidden requirements. Fix
included removing previous xff data output implementation and adding a
new function for outputting xff that is called after outputting each
event.

IPv6 XFF entries were also being recorded incorrectly as if they were
IPv4 and this has been fixed.

diff --git a/src/alert-unified2-alert.c b/src/alert-unified2-alert.c
index 608a2f5b44c6c326e8eaac372c4d1d4be734b704..01af6560849fccc04257646129ffe9bee505e3da 100644 (file)
--- a/src/alert-unified2-alert.c
+++ b/src/alert-unified2-alert.c
@@ -224,6 +224,7 @@ TmEcode Unified2AlertThreadDeinit(ThreadVars *, void *);
 static int Unified2IPv4TypeAlert(ThreadVars *, const Packet *, void *);
 static int Unified2IPv6TypeAlert(ThreadVars *, const Packet *, void *);
 static int Unified2PacketTypeAlert(Unified2AlertThread *, const Packet *, uint32_t, int);
+static int Unified2XFFTypeAlert(Unified2AlertThread *, const Packet *, uint32_t);
 void Unified2RegisterTests(void);
 static int Unified2AlertOpenFileCtx(LogFileCtx *, const char *, bool);
 static void Unified2AlertDeInitCtx(OutputCtx *);
@@ -439,66 +440,10 @@ static int Unified2PrintStreamSegmentCallback(const Packet *p, void *data, const
     Unified2AlertThread *aun = (Unified2AlertThread *)data;
     Unified2AlertFileHeader *hdr = (Unified2AlertFileHeader*)(aun->data);
     Unified2Packet *phdr = (Unified2Packet *)(hdr + 1);
-    /** Prepare the pointers to extra data structures should they be required.
-     * If they are required we will shift the *hdr and the *phdr */
-    Unified2AlertFileHeader *eu2hdr =  (Unified2AlertFileHeader*)(aun->data);
-    Unified2ExtraDataHdr *ehdr = (Unified2ExtraDataHdr *)(eu2hdr + 1);
-    Unified2ExtraData *dhdr = (Unified2ExtraData *) (ehdr + 1);
-    uint32_t *edxff = (uint32_t *) (dhdr + 1);
 
     aun->length = 0;
     aun->offset = 0;
 
-    // If XFF is in extra data mode...
-    if (aun->xff_flags & XFF_EXTRADATA) {
-        memset(dhdr, 0, sizeof(Unified2ExtraData));
-
-        if (aun->xff_flags & UNIFIED2_ALERT_XFF_IPV4) {
-            eu2hdr->type = htonl (UNIFIED2_IDS_EVENT_EXTRADATA_TYPE);
-            eu2hdr->length = htonl(sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + sizeof(uint32_t));
-            ehdr->event_type = htonl(UNIFIED2_EXTRADATA_TYPE_EXTRA_DATA);
-            ehdr->event_length = htonl(sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + sizeof(uint32_t));
-            dhdr->sensor_id = 0;
-            dhdr->event_id = aun->event_id;
-            dhdr->event_second = htonl(p->ts.tv_sec);
-            dhdr->data_type = htonl(UNIFIED2_EXTRADATA_TYPE_BLOB);
-            dhdr->type = htonl(UNIFIED2_EXTRADATA_CLIENT_IPV4_TYPE);
-            dhdr->blob_length = htonl(3 * sizeof(uint32_t));
-            aun->length += sizeof(Unified2AlertFileHeader) + sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + sizeof(uint32_t);
-            aun->offset += sizeof(Unified2AlertFileHeader) + sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + sizeof(uint32_t);
-            *edxff=aun->xff_ip[0];
-            /** Shift the *hdr and *phdr pointers */
-            hdr = (Unified2AlertFileHeader*)(edxff + 1);
-            phdr = (Unified2Packet *)(hdr + 1);
-        }
-        else if (aun->xff_flags & UNIFIED2_ALERT_XFF_IPV6) {
-            eu2hdr->type = htonl(UNIFIED2_IDS_EVENT_EXTRADATA_TYPE);
-            eu2hdr->length = htonl(sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + 4 * sizeof(uint32_t));
-            ehdr->event_type = htonl(UNIFIED2_EXTRADATA_TYPE_EXTRA_DATA);
-            ehdr->event_length = htonl(sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + 4 * sizeof(uint32_t));
-            dhdr->sensor_id = 0;
-            dhdr->event_id = aun->event_id;
-            dhdr->event_second = htonl(p->ts.tv_sec);
-            dhdr->data_type = htonl(UNIFIED2_EXTRADATA_TYPE_BLOB);
-            dhdr->type = htonl(UNIFIED2_EXTRADATA_CLIENT_IPV6_TYPE);
-            dhdr->blob_length = htonl(6 * sizeof(uint32_t));
-            aun->length += sizeof(Unified2AlertFileHeader) + sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + 4 * sizeof(uint32_t);
-            aun->offset += sizeof(Unified2AlertFileHeader) + sizeof (Unified2ExtraDataHdr)
-                    + sizeof (Unified2ExtraData) + 4 * sizeof(uint32_t);
-            memcpy(edxff, aun->xff_ip, 4 * sizeof(uint32_t));
-            /** Shift the *hdr and *phdr pointers */
-            hdr = (Unified2AlertFileHeader*)(edxff + 4);
-            phdr = (Unified2Packet *)(hdr + 1);
-        }
-    }
-
     int ethh_offset = 0;
     EthernetHdr ethhdr = { {0,0,0,0,0,0}, {0,0,0,0,0,0}, htons(ETHERNET_TYPE_IPV6) };
     uint32_t hdr_length = 0;
@@ -774,6 +719,89 @@ static int Unified2PacketTypeAlert(Unified2AlertThread *aun, const Packet *p, ui
     return 1;
 }
 
+
+/**
+ *  \brief Function to fill unified2 xff extra data into the file.
+ *
+ *  No need to lock here, since it's already locked.
+ *  Will clear thread local data prior to writing, resetting length and offset
+ *
+ *  \param aun thread local data
+ *  \param p Packet
+ *  \param event_id unique event id
+ *
+ *  \retval 0 on succces
+ *  \retval -1 on failure
+ */
+static int Unified2XFFTypeAlert(Unified2AlertThread *aun, const Packet *p, uint32_t event_id)
+{
+    /* Return immediately if XFF extra-data mode is not enabled */
+    if ( !(aun->xff_flags & XFF_EXTRADATA) ) {
+        return 0;
+    }
+
+    /* Determine length info for various data fields*/
+    size_t addr_size;
+    unsigned int blob_length;
+    unsigned int dhdr_type;
+
+    if ( aun->xff_flags & UNIFIED2_ALERT_XFF_IPV4 ) {
+        addr_size = sizeof(uint32_t);
+        blob_length = 3 * sizeof(uint32_t);
+        dhdr_type = UNIFIED2_EXTRADATA_CLIENT_IPV4_TYPE;
+    } else if ( aun->xff_flags & UNIFIED2_ALERT_XFF_IPV6 ) {
+        addr_size = 4*sizeof(uint32_t);
+        blob_length = 6 * sizeof(uint32_t);
+        dhdr_type = UNIFIED2_EXTRADATA_CLIENT_IPV6_TYPE;
+    } else {
+        return -1;
+    }
+
+    int hdr_length = sizeof(Unified2ExtraDataHdr)
+                   + sizeof(Unified2ExtraData)
+                   + addr_size;
+    int total_len = sizeof(Unified2AlertFileHeader) + hdr_length;
+
+    /* Clear aun's data and set new length */
+    memset( aun->data, 0, aun->datalen );
+    if ( total_len > aun->datalen ) {
+        SCLogError( SC_ERR_INVALID_VALUE, "len too big for thread data: %d, %d",
+                total_len, aun->datalen );
+        return -1;
+    }
+
+    aun->length = total_len;
+    aun->offset = total_len;
+
+    /* Prepare pointers to file header, extra data header, extra data record,
+    * and the extra data itself */
+    Unified2AlertFileHeader *hdr = (Unified2AlertFileHeader *)(aun->data);
+    Unified2ExtraDataHdr *ehdr = (Unified2ExtraDataHdr *)(hdr + 1);
+    Unified2ExtraData *dhdr = (Unified2ExtraData *)(ehdr + 1);
+    uint32_t *xff = (uint32_t *) (dhdr + 1);
+
+    /* Fill in all data structures and write */
+    hdr->type = htonl( UNIFIED2_IDS_EVENT_EXTRADATA_TYPE );
+    hdr->length = htonl( hdr_length );
+
+    ehdr->event_type = htonl( UNIFIED2_EXTRADATA_TYPE_EXTRA_DATA );
+    ehdr->event_length = hdr->length;
+
+    dhdr->sensor_id = 0;
+    dhdr->event_id = event_id;
+    dhdr->event_second = htonl( p->ts.tv_sec );
+    dhdr->data_type = htonl( UNIFIED2_EXTRADATA_TYPE_BLOB );
+    dhdr->type = htonl( dhdr_type );
+    dhdr->blob_length = htonl( blob_length );
+
+    memcpy( xff, aun->xff_ip, addr_size );
+
+    Unified2Write(aun);
+
+    return 0;
+}
+
+
 /**
  *  \brief Function to fill unified2 ipv6 ids type format into the file.
  *
@@ -943,6 +971,19 @@ static int Unified2IPv6TypeAlert(ThreadVars *t, const Packet *p, void *data)
         aun->length = 0;
         aun->offset = 0;
 
+        /* Write the extra data if any (it doesn't lock inside, since we
+         * already locked here for rotation check) */
+        ret = Unified2XFFTypeAlert(aun, p, phdr->event_id);
+        if (ret != 0) {
+            SCLogError(SC_ERR_FWRITE, "Error: fwrite failed: %s", strerror(errno));
+            SCMutexUnlock(&file_ctx->fp_mutex);
+            return -1;
+        }
+
+        memset(aun->data, 0, aun->length);
+        aun->length = 0;
+        aun->offset = 0;
+
         /* stream flag based on state match, but only for TCP */
         int stream = (gphdr.protocol == IPPROTO_TCP) ?
             (pa->flags & (PACKET_ALERT_FLAG_STATE_MATCH|PACKET_ALERT_FLAG_STREAM_MATCH) ? 1 : 0) : 0;
@@ -1119,6 +1160,18 @@ static int Unified2IPv4TypeAlert (ThreadVars *tv, const Packet *p, void *data)
         aun->length = 0;
         aun->offset = 0;
 
+        /* Write the extra data if any (it doesn't lock inside, since we
+         * already locked here for rotation check) */
+        ret = Unified2XFFTypeAlert(aun, p, event_id);
+        if (ret != 0) {
+            SCMutexUnlock(&file_ctx->fp_mutex);
+            return -1;
+        }
+
+        memset(aun->data, 0, aun->length);
+        aun->length = 0;
+        aun->offset = 0;
+
         /* Write the alert (it doesn't lock inside, since we
          * already locked here for rotation check)
          */

diff --git a/src/alert-unified2-alert.h b/src/alert-unified2-alert.h
index f6cf0e3d870d203a37a533144ebe8ff8e317d3dc..de8345769356b4c41ee04ba76be628cc7d28847c 100644 (file)
--- a/src/alert-unified2-alert.h
+++ b/src/alert-unified2-alert.h
@@ -39,7 +39,7 @@
 #define UNIFIED2_IDS_EVENT_IPV6_MPLS_TYPE 100
 #define UNIFIED2_IDS_EVENT_EXTRADATA_TYPE 110
 #define UNIFIED2_EXTRADATA_CLIENT_IPV4_TYPE 1
-#define UNIFIED2_EXTRADATA_CLIENT_IPV6_TYPE 1
+#define UNIFIED2_EXTRADATA_CLIENT_IPV6_TYPE 2
 #define UNIFIED2_EXTRADATA_TYPE_BLOB 1
 #define UNIFIED2_EXTRADATA_TYPE_EXTRA_DATA 4