stream: conditionally add packet header to segment

This patch optionally adds packet header to the TCP segment
and update the for each segment function by changing the
callback.

This patch is based on the work by Scott Jordan <scottfgjordan@gmail.com>

diff --git a/src/alert-debuglog.c b/src/alert-debuglog.c
index a272740dd02e779faf29b45c8f16bbe969261efe..9dbb193d88bc748f5f5db6230095a7de227e7a0d 100644 (file)
--- a/src/alert-debuglog.c
+++ b/src/alert-debuglog.c
@@ -137,7 +137,8 @@ static void AlertDebugLogPktVars(AlertDebugLogThread *aft, const Packet *p)
 
 /** \todo doc
  * assume we have aft lock */
-static int AlertDebugPrintStreamSegmentCallback(const Packet *p, void *data, const uint8_t *buf, uint32_t buflen)
+static int AlertDebugPrintStreamSegmentCallback(
+        const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
 {
     AlertDebugLogThread *aft = (AlertDebugLogThread *)data;
 
@@ -293,9 +294,9 @@ static TmEcode AlertDebugLogger(ThreadVars *tv, const Packet *p, void *thread_da
             /* IDS mode reverse the data */
             /** \todo improve the order selection policy */
             if (p->flowflags & FLOW_PKT_TOSERVER) {
-                flag = FLOW_PKT_TOCLIENT;
+                flag = STREAM_DUMP_TOCLIENT;
             } else {
-                flag = FLOW_PKT_TOSERVER;
+                flag = STREAM_DUMP_TOSERVER;
             }
             ret = StreamSegmentForEach((const Packet *)p, flag,
                                  AlertDebugPrintStreamSegmentCallback,

diff --git a/src/output-json-alert.c b/src/output-json-alert.c
index 3f32692d964a45f3d36c76091cb3f9cd105320f1..6e90f8ab06980bd86cf4a51e94f482be169dcd2a 100644 (file)
--- a/src/output-json-alert.c
+++ b/src/output-json-alert.c
@@ -123,7 +123,8 @@ typedef struct JsonAlertLogThread_ {
 
 /* Callback function to pack payload contents from a stream into a buffer
  * so we can report them in JSON output. */
-static int AlertJsonDumpStreamSegmentCallback(const Packet *p, void *data, const uint8_t *buf, uint32_t buflen)
+static int AlertJsonDumpStreamSegmentCallback(
+        const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
 {
     MemBuffer *payload = (MemBuffer *)data;
     MemBufferWriteRaw(payload, buf, buflen);
@@ -731,9 +732,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
                 MemBufferReset(payload);
 
                 if (p->flowflags & FLOW_PKT_TOSERVER) {
-                    flag = FLOW_PKT_TOCLIENT;
+                    flag = STREAM_DUMP_TOCLIENT;
                 } else {
-                    flag = FLOW_PKT_TOSERVER;
+                    flag = STREAM_DUMP_TOSERVER;
                 }
 
                 StreamSegmentForEach((const Packet *)p, flag,

diff --git a/src/stream-tcp-list.c b/src/stream-tcp-list.c
index eb23b1db3f589308019813654396db272f7b656e..c02a235213eb2508077d69213eed4bc8b9777cc5 100644 (file)
--- a/src/stream-tcp-list.c
+++ b/src/stream-tcp-list.c
@@ -562,6 +562,58 @@ static int DoHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
     return 0;
 }
 
+/**
+ * \brief Adds the following information to the TcpSegment from the current
+ *  packet being processed: time values, packet length, and the
+ *  header data of the packet. This information is added to the TcpSegment so
+ *  that it can be used in pcap capturing (log-pcap-stream) to dump the tcp
+ *  session at the beginning of the pcap capture.
+ * \param seg TcpSegment where information is being stored.
+ * \param p Packet being processed.
+ * \param tv Thread-specific variables.
+ * \param ra_ctx TcpReassembly thread-specific variables
+ */
+static void StreamTcpSegmentAddPacketData(
+        TcpSegment *seg, Packet *p, ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx)
+{
+    if (seg->pcap_hdr_storage == NULL || seg->pcap_hdr_storage->pkt_hdr == NULL) {
+        return;
+    }
+
+    /* FIXME we need to address pseudo packet */
+
+    if (GET_PKT_DATA(p) != NULL && GET_PKT_LEN(p) > p->payload_len) {
+        seg->pcap_hdr_storage->ts.tv_sec = p->ts.tv_sec;
+        seg->pcap_hdr_storage->ts.tv_usec = p->ts.tv_usec;
+        seg->pcap_hdr_storage->pktlen = GET_PKT_LEN(p) - p->payload_len;
+        /*
+         * pkt_hdr members are initially allocated 64 bytes of memory. Thus,
+         * need to check that this is sufficient and allocate more memory if
+         * not.
+         */
+        if (GET_PKT_LEN(p) - p->payload_len > seg->pcap_hdr_storage->alloclen) {
+            uint8_t *tmp_pkt_hdr =
+                    SCRealloc(seg->pcap_hdr_storage->pkt_hdr, GET_PKT_LEN(p) - p->payload_len);
+            if (tmp_pkt_hdr == NULL) {
+                SCLogDebug("Failed to realloc");
+                seg->pcap_hdr_storage->ts.tv_sec = 0;
+                seg->pcap_hdr_storage->ts.tv_usec = 0;
+                seg->pcap_hdr_storage->pktlen = 0;
+                return;
+            } else {
+                seg->pcap_hdr_storage->pkt_hdr = tmp_pkt_hdr;
+                seg->pcap_hdr_storage->alloclen = GET_PKT_LEN(p) - p->payload_len;
+            }
+        }
+        memcpy(seg->pcap_hdr_storage->pkt_hdr, GET_PKT_DATA(p),
+                (size_t)GET_PKT_LEN(p) - p->payload_len);
+    } else {
+        seg->pcap_hdr_storage->ts.tv_sec = 0;
+        seg->pcap_hdr_storage->ts.tv_usec = 0;
+        seg->pcap_hdr_storage->pktlen = 0;
+    }
+}
+
 /**
  *  \return 0 ok
  *  \return -1 segment not inserted due to memcap issue
@@ -586,6 +638,9 @@ int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_
         StreamTcpSegmentReturntoPool(seg);
         SCReturnInt(-1);
     }
+    if (IsTcpSessionDumpingEnabled()) {
+        StreamTcpSegmentAddPacketData(seg, p, tv, ra_ctx);
+    }
 
     if (likely(r == 0)) {
         /* no overlap, straight data insert */

diff --git a/src/stream-tcp-private.h b/src/stream-tcp-private.h
index b229edfae800a2df37b7eebc88b7b5136088ab1d..8f35c8264bd5074053f63b94e62f87db0ce56d74 100644 (file)
--- a/src/stream-tcp-private.h
+++ b/src/stream-tcp-private.h
@@ -58,12 +58,27 @@ int TcpSackCompare(struct StreamTcpSackRecord *a, struct StreamTcpSackRecord *b)
 RB_HEAD(TCPSACK, StreamTcpSackRecord);
 RB_PROTOTYPE(TCPSACK, StreamTcpSackRecord, rb, TcpSackCompare);
 
+#define TCPSEG_PKT_HDR_DEFAULT_SIZE 64
+
+/*
+ * Struct to add the additional information required to use TcpSegments to dump
+ * a packet capture to file with the stream-pcap-log output option. This is only
+ * used if the session-dump option is enabled.
+ */
+typedef struct TcpSegmentPcapHdrStorage_ {
+    struct timeval ts;
+    uint32_t pktlen;
+    uint32_t alloclen;
+    uint8_t *pkt_hdr;
+} TcpSegmentPcapHdrStorage;
+
 typedef struct TcpSegment {
     PoolThreadReserved res;
     uint16_t payload_len;       /**< actual size of the payload */
     uint32_t seq;
     RB_ENTRY(TcpSegment) __attribute__((__packed__)) rb;
     StreamingBufferSegment sbseg;
+    TcpSegmentPcapHdrStorage *pcap_hdr_storage;
 } __attribute__((__packed__)) TcpSegment;
 
 /** \brief compare function for the Segment tree

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index 37d5670047ece00763188885f3b973090be15276..ae8f7a80ec6b13c8294a95ed70491594fe6f1d86 100644 (file)
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -81,6 +81,18 @@ static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
 /* Memory use counter */
 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
 
+static int g_tcp_session_dump_enabled = 0;
+
+inline bool IsTcpSessionDumpingEnabled(void)
+{
+    return g_tcp_session_dump_enabled == 1;
+}
+
+void EnableTcpSessionDumping(void)
+{
+    g_tcp_session_dump_enabled = 1;
+}
+
 /* prototypes */
 TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *);
 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
@@ -251,19 +263,64 @@ static void *TcpSegmentPoolAlloc(void)
     seg = SCMalloc(sizeof (TcpSegment));
     if (unlikely(seg == NULL))
         return NULL;
+
+    if (IsTcpSessionDumpingEnabled()) {
+        uint32_t memuse =
+                sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
+        if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
+            SCFree(seg);
+            return NULL;
+        }
+
+        seg->pcap_hdr_storage = SCCalloc(1, sizeof(TcpSegmentPcapHdrStorage));
+        if (seg->pcap_hdr_storage == NULL) {
+            SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate memory for "
+                                         "TcpSegmentPcapHdrStorage");
+            SCFree(seg);
+            return NULL;
+        } else {
+            seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
+            seg->pcap_hdr_storage->pkt_hdr =
+                    SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
+            if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
+                SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate memory for "
+                                             "packet header data within "
+                                             "TcpSegmentPcapHdrStorage");
+                SCFree(seg->pcap_hdr_storage);
+                SCFree(seg);
+                return NULL;
+            }
+        }
+    } else {
+        seg->pcap_hdr_storage = NULL;
+    }
+
     return seg;
 }
 
 static int TcpSegmentPoolInit(void *data, void *initdata)
 {
     TcpSegment *seg = (TcpSegment *) data;
+    TcpSegmentPcapHdrStorage *pcap_hdr;
+
+    pcap_hdr = seg->pcap_hdr_storage;
 
     /* do this before the can bail, so TcpSegmentPoolCleanup
      * won't have uninitialized memory to consider. */
     memset(seg, 0, sizeof (TcpSegment));
 
-    if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
-        return 0;
+    if (IsTcpSessionDumpingEnabled()) {
+        uint32_t memuse =
+                sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
+        seg->pcap_hdr_storage = pcap_hdr;
+        if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
+            return 0;
+        }
+        StreamTcpReassembleIncrMemuse(memuse);
+    } else {
+        if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
+            return 0;
+        }
     }
 
 #ifdef DEBUG
@@ -284,6 +341,17 @@ static void TcpSegmentPoolCleanup(void *ptr)
     if (ptr == NULL)
         return;
 
+    TcpSegment *seg = (TcpSegment *)ptr;
+    if (seg && seg->pcap_hdr_storage) {
+        if (seg->pcap_hdr_storage->pkt_hdr) {
+            SCFree(seg->pcap_hdr_storage->pkt_hdr);
+            StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen);
+        }
+        SCFree(seg->pcap_hdr_storage);
+        seg->pcap_hdr_storage = NULL;
+        StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage));
+    }
+
     StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
 
 #ifdef DEBUG
@@ -305,6 +373,10 @@ void StreamTcpSegmentReturntoPool(TcpSegment *seg)
     if (seg == NULL)
         return;
 
+    if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
+        seg->pcap_hdr_storage->pktlen = 0;
+    }
+
     PoolThreadReturn(segment_thread_pool, seg);
 }
 

diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h
index 02911bd820f3919c3c6912a7715cded884976070..d62e27b045b22f3d619a1c6001da4c66b19a804d 100644 (file)
--- a/src/stream-tcp-reassemble.h
+++ b/src/stream-tcp-reassemble.h
@@ -130,6 +130,9 @@ int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *);
 bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p);
 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth);
 
+bool IsTcpSessionDumpingEnabled(void);
+void EnableTcpSessionDumping(void);
+
 static inline bool STREAM_LASTACK_GT_BASESEQ(const TcpStream *stream)
 {
     /* last ack not yet initialized */

diff --git a/src/stream-tcp.c b/src/stream-tcp.c
index 3a91cb9e6fb3501b4863b086f13b182bf5a4c0fc..42c120fd9b56e0fa6ef2cbe0ea77f88e1162e46e 100644 (file)
--- a/src/stream-tcp.c
+++ b/src/stream-tcp.c
@@ -6319,7 +6319,7 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback
         return 0;
     }
 
-    if (flag & FLOW_PKT_TOSERVER) {
+    if (flag & STREAM_DUMP_TOSERVER) {
         stream = &(ssn->server);
     } else {
         stream = &(ssn->client);
@@ -6343,7 +6343,7 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback
         uint32_t seg_datalen;
         StreamingBufferSegmentGetData(&stream->sb, &seg->sbseg, &seg_data, &seg_datalen);
 
-        int ret = CallbackFunc(p, data, seg_data, seg_datalen);
+        int ret = CallbackFunc(p, seg, data, seg_data, seg_datalen);
         if (ret != 1) {
             SCLogDebug("Callback function has failed");
             return -1;

diff --git a/src/stream.h b/src/stream.h
index 78fbfcf1fec535f6be2975cec872f2c61c578359..e8f2cad7e0d1e6ff514029d66511d022aae8fe54 100644 (file)
--- a/src/stream.h
+++ b/src/stream.h
@@ -25,10 +25,16 @@
 #define __STREAM_H__
 
 #include "flow.h"
+#include "stream-tcp-private.h"
 
 #define STREAM_FLAGS_FOR_PACKET(p) PKT_IS_TOSERVER((p)) ? STREAM_TOSERVER : STREAM_TOCLIENT
 
-typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t);
+#define STREAM_DUMP_TOCLIENT BIT_U8(1)
+#define STREAM_DUMP_TOSERVER BIT_U8(2)
+#define STREAM_DUMP_HEADERS  BIT_U8(3)
+
+typedef int (*StreamSegmentCallback)(
+        const Packet *, TcpSegment *, void *, const uint8_t *, uint32_t);
 int StreamSegmentForEach(const Packet *p, uint8_t flag,
                          StreamSegmentCallback CallbackFunc,
                          void *data);