app-layer-htp: use stream depth with filestore

This permits to use stream-depth value set for file-store.

Currently if a file is being stored and hits a limit,
such as request or response body, it will be truncated
although file-store.stream-depth is enabled but the file should be
closed and not truncated.

Two unit tests have been added to verify that:
- a file is stored correctly
- chunk's length computation doesn’t cause an underflow

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 141c69afe3de3283f508e30919f03e03681a61f0..6f837d5df255950227000befe36c183f25055b1f 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -489,6 +489,49 @@ void AppLayerHtpNeedFileInspection(void)
     SCReturn;
 }
 
+static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
+{
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
+    if (tx_ud) {
+        if (flags & STREAM_TOCLIENT) {
+            tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
+        } else {
+            tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
+        }
+    }
+}
+
+static bool AppLayerHtpCheckStreamDepth(uint64_t content_len_so_far, uint8_t flags)
+{
+    if (flags & HTP_STREAM_DEPTH_SET) {
+        uint32_t stream_depth = FileReassemblyDepth();
+        if (content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
+                                              uint32_t stream_depth, uint8_t flags, uint32_t data_len)
+{
+    uint32_t chunk_len = 0;
+    if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
+        (content_len_so_far < (uint64_t)body_limit) &&
+        (content_len_so_far + (uint64_t)data_len) > body_limit)
+    {
+        chunk_len = body_limit - content_len_so_far;
+    } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
+               (content_len_so_far < (uint64_t)stream_depth) &&
+               (content_len_so_far + (uint64_t)data_len) > stream_depth)
+    {
+        chunk_len = stream_depth - content_len_so_far;
+    }
+    SCLogDebug("len %u", chunk_len);
+    return (chunk_len == 0 ? data_len : chunk_len);
+}
+
 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
 struct {
     const char *msg;
@@ -1780,17 +1823,17 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
     SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
 
     /* within limits, add the body chunk to the state. */
-    if (hstate->cfg->request.body_limit == 0 || tx_ud->request_body.content_len_so_far < hstate->cfg->request.body_limit)
+    if ((!(tx_ud->tsflags & HTP_STREAM_DEPTH_SET) &&
+        (hstate->cfg->request.body_limit == 0 || tx_ud->request_body.content_len_so_far < hstate->cfg->request.body_limit)) ||
+        AppLayerHtpCheckStreamDepth(tx_ud->request_body.content_len_so_far, tx_ud->tsflags))
     {
-        uint32_t len = (uint32_t)d->len;
-
-        if (hstate->cfg->request.body_limit > 0 &&
-                (tx_ud->request_body.content_len_so_far + len) > hstate->cfg->request.body_limit)
-        {
-            len = hstate->cfg->request.body_limit - tx_ud->request_body.content_len_so_far;
-            BUG_ON(len > (uint32_t)d->len);
-        }
-        SCLogDebug("len %u", len);
+        uint32_t stream_depth = FileReassemblyDepth();
+        uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
+                                                     hstate->cfg->request.body_limit,
+                                                     stream_depth,
+                                                     tx_ud->tsflags,
+                                                     (uint32_t)d->len);
+        BUG_ON(len > (uint32_t)d->len);
 
         HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
 
@@ -1904,17 +1947,17 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
     SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
 
     /* within limits, add the body chunk to the state. */
-    if (hstate->cfg->response.body_limit == 0 || tx_ud->response_body.content_len_so_far < hstate->cfg->response.body_limit)
+    if ((!(tx_ud->tcflags & HTP_STREAM_DEPTH_SET) &&
+        (hstate->cfg->response.body_limit == 0 || tx_ud->response_body.content_len_so_far < hstate->cfg->response.body_limit)) ||
+        AppLayerHtpCheckStreamDepth(tx_ud->response_body.content_len_so_far, tx_ud->tcflags))
     {
-        uint32_t len = (uint32_t)d->len;
-
-        if (hstate->cfg->response.body_limit > 0 &&
-                (tx_ud->response_body.content_len_so_far + len) > hstate->cfg->response.body_limit)
-        {
-            len = hstate->cfg->response.body_limit - tx_ud->response_body.content_len_so_far;
-            BUG_ON(len > (uint32_t)d->len);
-        }
-        SCLogDebug("len %u", len);
+        uint32_t stream_depth = FileReassemblyDepth();
+        uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
+                                                     hstate->cfg->response.body_limit,
+                                                     stream_depth,
+                                                     tx_ud->tcflags,
+                                                     (uint32_t)d->len);
+        BUG_ON(len > (uint32_t)d->len);
 
         HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
 
@@ -3084,6 +3127,9 @@ void RegisterHTPParsers(void)
         AppLayerParserRegisterDetectFlagsFuncs(IPPROTO_TCP, ALPROTO_HTTP,
                                                HTPGetTxDetectFlags, HTPSetTxDetectFlags);
 
+        AppLayerParserRegisterSetStreamDepthFlag(IPPROTO_TCP, ALPROTO_HTTP,
+                                                 AppLayerHtpSetStreamDepthFlag);
+
         AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_HTTP, STREAM_TOSERVER,
                                      HTPHandleRequestData);
         AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_HTTP, STREAM_TOCLIENT,
@@ -7143,6 +7189,167 @@ static int HTPParserTest25(void)
     PASS;
 }
 
+static int HTPParserTest26(void)
+{
+    char input[] = "\
+%YAML 1.1\n\
+---\n\
+libhtp:\n\
+\n\
+  default-config:\n\
+    personality: IDS\n\
+    request-body-limit: 1\n\
+    response-body-limit: 1\n\
+";
+    ConfCreateContextBackup();
+    ConfInit();
+    HtpConfigCreateBackup();
+    ConfYamlLoadString(input, strlen(input));
+    HTPConfigure();
+
+    Packet *p1 = NULL;
+    Packet *p2 = NULL;
+    ThreadVars th_v;
+    DetectEngineCtx *de_ctx = NULL;
+    DetectEngineThreadCtx *det_ctx = NULL;
+    Flow f;
+    uint8_t httpbuf1[] = "GET /alice.txt HTTP/1.1\r\n\r\n";
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\n"
+                         "Content-Type: text/plain\r\n"
+                         "Content-Length: 228\r\n\r\n"
+                         "Alice was beginning to get very tired of sitting by her sister on the bank."
+                         "Alice was beginning to get very tired of sitting by her sister on the bank.";
+    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
+    uint8_t httpbuf3[] = "Alice was beginning to get very tired of sitting by her sister on the bank.\r\n\r\n";
+    uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
+    TcpSession ssn;
+    HtpState *http_state = NULL;
+    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
+    FAIL_IF_NULL(alp_tctx);
+
+    memset(&th_v, 0, sizeof(th_v));
+    memset(&f, 0, sizeof(f));
+    memset(&ssn, 0, sizeof(ssn));
+
+    p1 = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
+    p2 = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
+
+    FLOW_INITIALIZE(&f);
+    f.protoctx = (void *)&ssn;
+    f.proto = IPPROTO_TCP;
+    f.flags |= FLOW_IPV4;
+
+    p1->flow = &f;
+    p1->flowflags |= FLOW_PKT_TOSERVER;
+    p1->flowflags |= FLOW_PKT_ESTABLISHED;
+    p1->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
+    p2->flow = &f;
+    p2->flowflags |= FLOW_PKT_TOCLIENT;
+    p2->flowflags |= FLOW_PKT_ESTABLISHED;
+    p2->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
+    f.alproto = ALPROTO_HTTP;
+
+    StreamTcpInitConfig(TRUE);
+
+    de_ctx = DetectEngineCtxInit();
+    FAIL_IF_NULL(de_ctx);
+
+    de_ctx->flags |= DE_QUIET;
+
+    de_ctx->sig_list = SigInit(de_ctx,"alert http any any -> any any "
+                               "(filestore; sid:1; rev:1;)");
+    FAIL_IF_NULL(de_ctx->sig_list);
+
+    SigGroupBuild(de_ctx);
+    DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
+
+
+    int r = AppLayerParserParse(&th_v, alp_tctx, &f, ALPROTO_HTTP,
+                                STREAM_TOSERVER, httpbuf1,
+                                httplen1);
+    FAIL_IF(r != 0);
+
+    http_state = f.alstate;
+    FAIL_IF_NULL(http_state);
+
+    /* do detect */
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p1);
+
+    FAIL_IF((PacketAlertCheck(p1, 1)));
+
+    /* do detect */
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p1);
+
+    FAIL_IF((PacketAlertCheck(p1, 1)));
+
+    r = AppLayerParserParse(&th_v, alp_tctx, &f, ALPROTO_HTTP,
+                            STREAM_TOCLIENT, httpbuf2,
+                            httplen2);
+    FAIL_IF(r != 0);
+
+    http_state = f.alstate;
+    FAIL_IF_NULL(http_state);
+
+    /* do detect */
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p2);
+
+    FAIL_IF(!(PacketAlertCheck(p2, 1)));
+
+    r = AppLayerParserParse(&th_v, alp_tctx, &f, ALPROTO_HTTP,
+                            STREAM_TOCLIENT, httpbuf3,
+                            httplen3);
+    FAIL_IF(r != 0);
+
+    http_state = f.alstate;
+    FAIL_IF_NULL(http_state);
+
+    FileContainer *ffc = HTPStateGetFiles(http_state, STREAM_TOCLIENT);
+    FAIL_IF_NULL(ffc);
+
+    File *ptr = ffc->head;
+    FAIL_IF(ptr->state != FILE_STATE_CLOSED);
+
+    AppLayerParserThreadCtxFree(alp_tctx);
+    DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
+    DetectEngineCtxFree(de_ctx);
+    StreamTcpFreeConfig(TRUE);
+
+    HTPFreeConfig();
+    FLOW_DESTROY(&f);
+    UTHFreePackets(&p1, 1);
+    UTHFreePackets(&p2, 1);
+    ConfDeInit();
+    ConfRestoreContextBackup();
+    HtpConfigRestoreBackup();
+    PASS;
+}
+
+static int HTPParserTest27(void)
+{
+    FileReassemblyDepthEnable(2000);
+
+    uint32_t len = 1000;
+
+    HtpTxUserData *tx_ud = SCMalloc(sizeof(HtpTxUserData));
+    FAIL_IF_NULL(tx_ud);
+
+    tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
+    tx_ud->request_body.content_len_so_far = 2500;
+
+    FAIL_IF(AppLayerHtpCheckStreamDepth(tx_ud->request_body.content_len_so_far, tx_ud->tsflags));
+
+    len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
+                                        0,
+                                        FileReassemblyDepth(),
+                                        tx_ud->tsflags,
+                                        len);
+    FAIL_IF(len != 1000);
+
+    SCFree(tx_ud);
+
+    PASS;
+}
 #endif /* UNITTESTS */
 
 /**
@@ -7200,6 +7407,8 @@ void HTPParserRegisterTests(void)
     UtRegisterTest("HTPParserTest23", HTPParserTest23);
     UtRegisterTest("HTPParserTest24", HTPParserTest24);
     UtRegisterTest("HTPParserTest25", HTPParserTest25);
+    UtRegisterTest("HTPParserTest26", HTPParserTest26);
+    UtRegisterTest("HTPParserTest27", HTPParserTest27);
 
     HTPFileParserRegisterTests();
     HTPXFFParserRegisterTests();

diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index 45fe4017bd4e39807e2107502ce80a1642755719..51ae1ad32ab921435a502a511bd7a9139f5fbaa5 100644 (file)
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -188,11 +188,12 @@ typedef struct HtpBody_ {
     uint64_t body_inspected;
 } HtpBody;
 
-#define HTP_CONTENTTYPE_SET     0x01    /**< We have the content type */
-#define HTP_BOUNDARY_SET        0x02    /**< We have a boundary string */
-#define HTP_BOUNDARY_OPEN       0x04    /**< We have a boundary string */
-#define HTP_FILENAME_SET        0x08   /**< filename is registered in the flow */
-#define HTP_DONTSTORE           0x10    /**< not storing this file */
+#define HTP_CONTENTTYPE_SET     BIT_U8(0)    /**< We have the content type */
+#define HTP_BOUNDARY_SET        BIT_U8(1)    /**< We have a boundary string */
+#define HTP_BOUNDARY_OPEN       BIT_U8(2)    /**< We have a boundary string */
+#define HTP_FILENAME_SET        BIT_U8(3)    /**< filename is registered in the flow */
+#define HTP_DONTSTORE           BIT_U8(4)    /**< not storing this file */
+#define HTP_STREAM_DEPTH_SET    BIT_U8(5)    /**< stream-depth is set */
 
 /** Now the Body Chunks will be stored per transaction, at
   * the tx user data */