file inspection: improve logging when stream.depth limit is reached. #493.

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index b74e68e339d88d508d68040b7a3aee55c41b73b3..1e29dc797e52fb3c7e1d8b1fd00f013ef48796f9 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1077,7 +1077,7 @@ static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud) {
                 htud->boundary_len = (uint8_t)boundary_len;
                 memcpy(htud->boundary, boundary, boundary_len);
 
-                htud->flags |= HTP_BOUNDARY_SET;
+                htud->tsflags |= HTP_BOUNDARY_SET;
             } else {
                 SCLogDebug("invalid boundary");
                 return -1;
@@ -1306,8 +1306,8 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
             expected_boundary_end, expected_boundary_end_len);
 
     /* if we're in the file storage process, deal with that now */
-    if (htud->flags & HTP_FILENAME_SET) {
-        if (header_start != NULL || form_end != NULL || htud->flags & HTP_REQ_BODY_COMPLETE) {
+    if (htud->tsflags & HTP_FILENAME_SET) {
+        if (header_start != NULL || form_end != NULL || htud->tsflags & HTP_REQ_BODY_COMPLETE) {
             SCLogDebug("reached the end of the file");
 
             uint8_t *filedata = chunks_buffer;
@@ -1320,7 +1320,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                 filedata_len = form_end - filedata;
             } else if (form_end != NULL && form_end == header_start) {
                 filedata_len = form_end - filedata - 2; /* 0d 0a */
-            } else if (htud->flags & HTP_REQ_BODY_COMPLETE) {
+            } else if (htud->tsflags & HTP_REQ_BODY_COMPLETE) {
                 filedata_len = chunks_buffer_len;
                 flags = FILE_TRUNCATED;
             }
@@ -1335,7 +1335,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
             PrintRawDataFp(stdout, filedata, filedata_len);
             printf("FILEDATA (final chunk) END: \n");
 #endif
-            if (!(htud->flags & HTP_DONTSTORE)) {
+            if (!(htud->tsflags & HTP_DONTSTORE)) {
                 if (HTPFileClose(hstate, filedata, filedata_len, flags,
                             STREAM_TOSERVER) == -1)
                 {
@@ -1343,7 +1343,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                 }
             }
 
-            htud->flags &=~ HTP_FILENAME_SET;
+            htud->tsflags &=~ HTP_FILENAME_SET;
 
             /* fall through */
         } else {
@@ -1358,14 +1358,14 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                 printf("FILEDATA (part) END: \n");
 #endif
 
-                if (!(htud->flags & HTP_DONTSTORE)) {
+                if (!(htud->tsflags & HTP_DONTSTORE)) {
                     result = HTPFileStoreChunk(hstate, filedata,
                             filedata_len, STREAM_TOSERVER);
                     if (result == -1) {
                         goto end;
                     } else if (result == -2) {
                         /* we know for sure we're not storing the file */
-                        htud->flags |= HTP_DONTSTORE;
+                        htud->tsflags |= HTP_DONTSTORE;
                     }
                 }
 
@@ -1410,8 +1410,8 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
 
             SCLogDebug("we have a filename");
 
-            htud->flags |= HTP_FILENAME_SET;
-            htud->flags &= ~HTP_DONTSTORE;
+            htud->tsflags |= HTP_FILENAME_SET;
+            htud->tsflags &= ~HTP_DONTSTORE;
 
             SCLogDebug("header_end %p", header_end);
             SCLogDebug("form_end %p", form_end);
@@ -1457,7 +1457,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                 if (result == -1) {
                     goto end;
                 } else if (result == -2) {
-                    htud->flags |= HTP_DONTSTORE;
+                    htud->tsflags |= HTP_DONTSTORE;
                 } else {
                     if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
                         goto end;
@@ -1465,7 +1465,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                 }
 
                 htud->request_body.body_parsed += (header_end - chunks_buffer);
-                htud->flags &= ~HTP_FILENAME_SET;
+                htud->tsflags &= ~HTP_FILENAME_SET;
             } else {
                 SCLogDebug("chunk doesn't contain form end");
 
@@ -1505,7 +1505,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                     if (result == -1) {
                         goto end;
                     } else if (result == -2) {
-                        htud->flags |= HTP_DONTSTORE;
+                        htud->tsflags |= HTP_DONTSTORE;
                     }
                 } else {
                     filedata_len = header_next - filedata - 2;
@@ -1517,14 +1517,14 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                     if (result == -1) {
                         goto end;
                     } else if (result == -2) {
-                        htud->flags |= HTP_DONTSTORE;
+                        htud->tsflags |= HTP_DONTSTORE;
                     } else {
                         if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
                             goto end;
                         }
                     }
 
-                    htud->flags &= ~HTP_FILENAME_SET;
+                    htud->tsflags &= ~HTP_FILENAME_SET;
                     htud->request_body.body_parsed += (header_end - chunks_buffer);
                 }
             }
@@ -1580,7 +1580,7 @@ static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
     int result = 0;
 
     /* see if we need to open the file */
-    if (!(htud->flags & HTP_FILENAME_SET))
+    if (!(htud->tsflags & HTP_FILENAME_SET))
     {
         uint8_t *filename = NULL;
         size_t filename_len = 0;
@@ -1597,10 +1597,10 @@ static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
             if (result == -1) {
                 goto end;
             } else if (result == -2) {
-                htud->flags |= HTP_DONTSTORE;
+                htud->tsflags |= HTP_DONTSTORE;
             } else {
-                htud->flags |= HTP_FILENAME_SET;
-                htud->flags &= ~HTP_DONTSTORE;
+                htud->tsflags |= HTP_FILENAME_SET;
+                htud->tsflags &= ~HTP_DONTSTORE;
             }
         }
     }
@@ -1608,13 +1608,13 @@ static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
     {
         /* otherwise, just store the data */
 
-        if (!(htud->flags & HTP_DONTSTORE)) {
+        if (!(htud->tsflags & HTP_DONTSTORE)) {
             result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
             if (result == -1) {
                 goto end;
             } else if (result == -2) {
                 /* we know for sure we're not storing the file */
-                htud->flags |= HTP_DONTSTORE;
+                htud->tsflags |= HTP_DONTSTORE;
             }
         }
     }
@@ -1633,7 +1633,7 @@ static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
     int result = 0;
 
     /* see if we need to open the file */
-    if (!(htud->flags & HTP_FILENAME_SET))
+    if (!(htud->tsflags & HTP_FILENAME_SET))
     {
         uint8_t *filename = NULL;
         size_t filename_len = 0;
@@ -1650,10 +1650,10 @@ static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
             if (result == -1) {
                 goto end;
             } else if (result == -2) {
-                htud->flags |= HTP_DONTSTORE;
+                htud->tsflags |= HTP_DONTSTORE;
             } else {
-                htud->flags |= HTP_FILENAME_SET;
-                htud->flags &= ~HTP_DONTSTORE;
+                htud->tsflags |= HTP_FILENAME_SET;
+                htud->tsflags &= ~HTP_DONTSTORE;
             }
         }
     }
@@ -1661,13 +1661,13 @@ static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
     {
         /* otherwise, just store the data */
 
-        if (!(htud->flags & HTP_DONTSTORE)) {
+        if (!(htud->tsflags & HTP_DONTSTORE)) {
             result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
             if (result == -1) {
                 goto end;
             } else if (result == -2) {
                 /* we know for sure we're not storing the file */
-                htud->flags |= HTP_DONTSTORE;
+                htud->tsflags |= HTP_DONTSTORE;
             }
         }
     }
@@ -1685,7 +1685,7 @@ int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
     int result = 0;
 
     /* see if we need to open the file */
-    if (!(htud->flags & HTP_FILENAME_SET))
+    if (!(htud->tcflags & HTP_FILENAME_SET))
     {
         SCLogDebug("setting up file name");
 
@@ -1717,10 +1717,10 @@ int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
             if (result == -1) {
                 goto end;
             } else if (result == -2) {
-                htud->flags |= HTP_DONTSTORE;
+                htud->tcflags |= HTP_DONTSTORE;
             } else {
-                htud->flags |= HTP_FILENAME_SET;
-                htud->flags &= ~HTP_DONTSTORE;
+                htud->tcflags |= HTP_FILENAME_SET;
+                htud->tcflags &= ~HTP_DONTSTORE;
             }
         }
     }
@@ -1728,14 +1728,14 @@ int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
     {
         /* otherwise, just store the data */
 
-        if (!(htud->flags & HTP_DONTSTORE)) {
+        if (!(htud->tcflags & HTP_DONTSTORE)) {
             result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
             SCLogDebug("result %d", result);
             if (result == -1) {
                 goto end;
             } else if (result == -2) {
                 /* we know for sure we're not storing the file */
-                htud->flags |= HTP_DONTSTORE;
+                htud->tcflags |= HTP_DONTSTORE;
             }
         }
     }
@@ -1814,13 +1814,13 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
         int r = HtpBodyAppendChunk(htud, &htud->request_body, (uint8_t *)d->data, len);
         if (r < 0) {
-            htud->flags |= HTP_REQ_BODY_COMPLETE;
+            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
         } else if (hstate->request_body_limit > 0 &&
             htud->request_body.content_len_so_far >= hstate->request_body_limit)
         {
-            htud->flags |= HTP_REQ_BODY_COMPLETE;
+            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
         } else if (htud->request_body.content_len_so_far == htud->request_body.content_len) {
-            htud->flags |= HTP_REQ_BODY_COMPLETE;
+            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
         }
 
         uint8_t *chunks_buffer = NULL;
@@ -1828,7 +1828,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
         if (htud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
             /* multi-part body handling starts here */
-            if (!(htud->flags & HTP_BOUNDARY_SET)) {
+            if (!(htud->tsflags & HTP_BOUNDARY_SET)) {
                 goto end;
             }
 
@@ -1917,13 +1917,13 @@ int HTPCallbackResponseBodyData(htp_tx_data_t *d)
 
         int r = HtpBodyAppendChunk(htud, &htud->response_body, (uint8_t *)d->data, len);
         if (r < 0) {
-            htud->flags |= HTP_RES_BODY_COMPLETE;
+            htud->tcflags |= HTP_RES_BODY_COMPLETE;
         } else if (hstate->response_body_limit > 0 &&
             htud->response_body.content_len_so_far >= hstate->response_body_limit)
         {
-            htud->flags |= HTP_RES_BODY_COMPLETE;
+            htud->tcflags |= HTP_RES_BODY_COMPLETE;
         } else if (htud->response_body.content_len_so_far == htud->response_body.content_len) {
-            htud->flags |= HTP_RES_BODY_COMPLETE;
+            htud->tcflags |= HTP_RES_BODY_COMPLETE;
         }
 
         HtpResponseBodyHandle(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
@@ -1994,10 +1994,10 @@ static int HTPCallbackRequest(htp_connp_t *connp) {
     if (connp->in_tx != NULL) {
         HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(connp->in_tx);
         if (htud != NULL) {
-            if (htud->flags & HTP_FILENAME_SET) {
+            if (htud->tsflags & HTP_FILENAME_SET) {
                 SCLogDebug("closing file that was being stored");
                 (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
-                htud->flags &= ~HTP_FILENAME_SET;
+                htud->tsflags &= ~HTP_FILENAME_SET;
             }
         }
     }
@@ -2031,10 +2031,10 @@ static int HTPCallbackResponse(htp_connp_t *connp) {
     if (connp->out_tx != NULL) {
         HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(connp->out_tx);
         if (htud != NULL) {
-            if (htud->flags & HTP_FILENAME_SET) {
+            if (htud->tcflags & HTP_FILENAME_SET) {
                 SCLogDebug("closing file that was being stored");
                 (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
-                htud->flags &= ~HTP_FILENAME_SET;
+                htud->tcflags &= ~HTP_FILENAME_SET;
             }
         }
     }

diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index 24f3d4d631c8e9a3a11eb04a1668d66749dc6c67..52bdab623d16652d1e3b9d334d45b5d1f77f0174 100644 (file)
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -178,7 +178,8 @@ typedef struct HtpTxUserData_ {
     uint8_t *boundary;
     uint8_t boundary_len;
 
-    uint8_t flags;
+    uint8_t tsflags;
+    uint8_t tcflags;
 
     int16_t operation;
 

diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index 75106135d22b5b04fa8b58fba1b097bf354c5458..916e5dd4fc22fba93402be847f387b9e9e9033d5 100644 (file)
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -165,18 +165,18 @@ static void DetectEngineBufferHttpClientBodies(DetectEngineCtx *de_ctx,
             if ((htud->request_body.content_len_so_far > 0) &&
                     tx->progress != TX_PROGRESS_REQ_BODY) {
                 /* final length of the body */
-                htud->flags |= HTP_REQ_BODY_COMPLETE;
+                htud->tsflags |= HTP_REQ_BODY_COMPLETE;
             }
         }
 
         if (flags & STREAM_EOF) {
-            htud->flags |= HTP_REQ_BODY_COMPLETE;
+            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
         }
 
         /* inspect the body if the transfer is complete or we have hit
          * our body size limit */
         if (htud->request_body.content_len_so_far < BODY_MINIMAL_SIZE &&
-                !(htud->flags & HTP_REQ_BODY_COMPLETE)) {
+                !(htud->tsflags & HTP_REQ_BODY_COMPLETE)) {
             SCLogDebug("we still haven't seen the entire request body.  "
                     "Let's defer body inspection till we see the "
                     "entire body.");

diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 0e2670ac33bd4931894fa9b359463aad8e9ad91e..356743d2096f00c0e4b3ed7b403ad2e0d0bc084d 100644 (file)
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -159,18 +159,18 @@ static void DetectEngineBufferHttpServerBodies(DetectEngineCtx *de_ctx,
             if ((htud->response_body.content_len_so_far > 0) &&
                     tx->progress != TX_PROGRESS_REQ_BODY) {
                 /* final length of the body */
-                htud->flags |= HTP_RES_BODY_COMPLETE;
+                htud->tcflags |= HTP_RES_BODY_COMPLETE;
             }
         }
 
         if (flags & STREAM_EOF) {
-            htud->flags |= HTP_RES_BODY_COMPLETE;
+            htud->tcflags |= HTP_RES_BODY_COMPLETE;
         }
 
         /* inspect the body if the transfer is complete or we have hit
          * our body size limit */
         if (htud->response_body.content_len_so_far < BODY_MINIMAL_SIZE &&
-                !(htud->flags & HTP_RES_BODY_COMPLETE)) {
+                !(htud->tcflags & HTP_RES_BODY_COMPLETE)) {
             SCLogDebug("we still haven't seen the entire response body.  "
                     "Let's defer body inspection till we see the "
                     "entire body.");

diff --git a/src/log-file.c b/src/log-file.c
index 2037fd91463e6b3dfb6b8f80ef23536589a477d3..52485fa0ebd81b49350914b192f321f7d430a3fa 100644 (file)
--- a/src/log-file.c
+++ b/src/log-file.c
@@ -55,6 +55,7 @@
 
 #include "app-layer-htp.h"
 #include "util-memcmp.h"
+#include "stream-tcp-reassemble.h"
 
 #define MODULE_NAME "LogFileLog"
 
@@ -286,8 +287,10 @@ static TmEcode LogFileLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue
         flags |= STREAM_TOSERVER;
 
     int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
+    int file_trunc = 0;
 
     FLOWLOCK_WRLOCK(p->flow);
+    file_trunc = StreamTcpReassembleDepthReached(p);
 
     FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow, flags);
     SCLogDebug("ffc %p", ffc);
@@ -303,6 +306,9 @@ static TmEcode LogFileLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue
 
             SCLogDebug("ff %p", ff);
 
+            if (file_trunc && ff->state < FILE_STATE_CLOSED)
+                ff->state = FILE_STATE_TRUNCATED;
+
             if (ff->state == FILE_STATE_CLOSED ||
                     ff->state == FILE_STATE_TRUNCATED || ff->state == FILE_STATE_ERROR ||
                     (file_close == 1 && ff->state < FILE_STATE_CLOSED))

diff --git a/src/log-filestore.c b/src/log-filestore.c
index d07f68f1de80676a6edc86ca27ca04dc3a272f19..2649c31273786ed14fce6076a01919cb93c9560b 100644 (file)
--- a/src/log-filestore.c
+++ b/src/log-filestore.c
@@ -53,6 +53,7 @@
 
 #include "app-layer-htp.h"
 #include "util-memcmp.h"
+#include "stream-tcp-reassemble.h"
 
 #define MODULE_NAME "LogFilestoreLog"
 
@@ -282,8 +283,10 @@ static TmEcode LogFilestoreLogWrap(ThreadVars *tv, Packet *p, void *data, Packet
         flags |= STREAM_TOSERVER;
 
     int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
+    int file_trunc = 0;
 
     FLOWLOCK_WRLOCK(p->flow);
+    file_trunc = StreamTcpReassembleDepthReached(p);
 
     FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow, flags);
     SCLogDebug("ffc %p", ffc);
@@ -359,6 +362,9 @@ static TmEcode LogFilestoreLogWrap(ThreadVars *tv, Packet *p, void *data, Packet
 
                 close(file_fd);
 
+                if (file_trunc && ff->state < FILE_STATE_CLOSED)
+                    ff->state = FILE_STATE_TRUNCATED;
+
                 if (ff->state == FILE_STATE_CLOSED ||
                     ff->state == FILE_STATE_TRUNCATED ||
                     ff->state == FILE_STATE_ERROR ||

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index abc10de27d942c000ebceb13cff07047bb5c5001..01b5697e93da4c37e3a5b125e3053943624596e6 100644 (file)
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -1497,6 +1497,30 @@ static int HandleSegmentStartsAfterListSegment(ThreadVars *tv, TcpReassemblyThre
     SCReturnInt(0);
 }
 
+/**
+ *  \brief check if stream in pkt direction has depth reached
+ *
+ *  \param p packet with *LOCKED* flow
+ *
+ *  \retval 1 stream has depth reached
+ *  \retval 0 stream does not have depth reached
+ */
+int StreamTcpReassembleDepthReached(Packet *p) {
+    if (p->flow != NULL) {
+        TcpSession *ssn = p->flow->protoctx;
+        TcpStream *stream;
+        if (p->flowflags & FLOW_PKT_TOSERVER) {
+            stream = &ssn->client;
+        } else {
+            stream = &ssn->server;
+        }
+
+        return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
+    }
+
+    return 0;
+}
+
 /**
  *  \internal
  *  \brief Function to Check the reassembly depth valuer against the

diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h
index e5371d992ae637e6c2cfd1b07d338f80e6f10cfa..d3e54b5cb56998eea88ec78d4bf70ba8faa66879 100644 (file)
--- a/src/stream-tcp-reassemble.h
+++ b/src/stream-tcp-reassemble.h
@@ -92,6 +92,7 @@ void StreamTcpSegmentReturntoPool(TcpSegment *);
 void StreamTcpReassembleTriggerRawReassembly(TcpSession *);
 
 void StreamTcpPruneSession(Flow *, uint8_t);
+int StreamTcpReassembleDepthReached(Packet *p);
 
 #endif /* __STREAM_TCP_REASSEMBLE_H__ */