Introduce a saner way to validate the completion of request and

response bodies.

Also don't change app state for http from inside inspection.

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 6c2773e2766d7117e22005e96cf8111e010619f7..10934e5631d69926a8f68a142ca3565ff9c75c76 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1154,13 +1154,14 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud,
 }
 
 int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
-        uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
+                                  void *tx, uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
 {
     int result = 0;
     uint8_t *expected_boundary = NULL;
     uint8_t *expected_boundary_end = NULL;
     uint8_t expected_boundary_len = 0;
     uint8_t expected_boundary_end_len = 0;
+    int tx_progress = 0;
 
 #ifdef PRINT
     printf("CHUNK START: \n");
@@ -1187,9 +1188,12 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
     SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
             header_end, form_end);
 
+    /* we currently only handle multipart for ts.  When we support it for tc,
+     * we will need to supply right direction */
+    tx_progress = AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0);
     /* if we're in the file storage process, deal with that now */
     if (htud->tsflags & HTP_FILENAME_SET) {
-        if (header_start != NULL || form_end != NULL || (htud->tsflags & HTP_REQ_BODY_COMPLETE)) {
+        if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
             SCLogDebug("reached the end of the file");
 
             uint8_t *filedata = chunks_buffer;
@@ -1202,7 +1206,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                 filedata_len = form_end - filedata;
             } else if (form_end != NULL && form_end == header_start) {
                 filedata_len = form_end - filedata - 2; /* 0d 0a */
-            } else if (htud->tsflags & HTP_REQ_BODY_COMPLETE) {
+            } else if (tx_progress > HTP_RESPONSE_BODY) {
                 filedata_len = chunks_buffer_len;
                 flags = FILE_TRUNCATED;
             }
@@ -1700,16 +1704,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
         }
         SCLogDebug("len %u", len);
 
-        int r = HtpBodyAppendChunk(tx_ud, &tx_ud->request_body, (uint8_t *)d->data, len);
-        if (r < 0) {
-            tx_ud->tsflags |= HTP_REQ_BODY_COMPLETE;
-        } else if (hstate->cfg->request_body_limit > 0 &&
-            tx_ud->request_body.content_len_so_far >= hstate->cfg->request_body_limit)
-        {
-            tx_ud->tsflags |= HTP_REQ_BODY_COMPLETE;
-        } else if (tx_ud->request_body.content_len_so_far == tx_ud->request_body.content_len) {
-            tx_ud->tsflags |= HTP_REQ_BODY_COMPLETE;
-        }
+        HtpBodyAppendChunk(tx_ud, &tx_ud->request_body, (uint8_t *)d->data, len);
 
         uint8_t *chunks_buffer = NULL;
         uint32_t chunks_buffer_len = 0;
@@ -1730,7 +1725,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
             printf("REASSCHUNK END: \n");
 #endif
 
-            HtpRequestBodyHandleMultipart(hstate, tx_ud, chunks_buffer, chunks_buffer_len);
+            HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
 
             if (chunks_buffer != NULL) {
                 SCFree(chunks_buffer);
@@ -1813,16 +1808,7 @@ int HTPCallbackResponseBodyData(htp_tx_data_t *d)
         }
         SCLogDebug("len %u", len);
 
-        int r = HtpBodyAppendChunk(tx_ud, &tx_ud->response_body, (uint8_t *)d->data, len);
-        if (r < 0) {
-            tx_ud->tcflags |= HTP_RES_BODY_COMPLETE;
-        } else if (hstate->cfg->response_body_limit > 0 &&
-            tx_ud->response_body.content_len_so_far >= hstate->cfg->response_body_limit)
-        {
-            tx_ud->tcflags |= HTP_RES_BODY_COMPLETE;
-        } else if (tx_ud->response_body.content_len_so_far == tx_ud->response_body.content_len) {
-            tx_ud->tcflags |= HTP_RES_BODY_COMPLETE;
-        }
+        HtpBodyAppendChunk(tx_ud, &tx_ud->response_body, (uint8_t *)d->data, len);
 
         HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
     }
@@ -4705,6 +4691,8 @@ static int HTPBodyReassemblyTest01(void)
     memset(&flow, 0x00, sizeof(flow));
     AppLayerParserStateStore parser;
     memset(&parser, 0x00, sizeof(parser));
+    htp_tx_t tx;
+    memset(&tx, 0, sizeof(tx));
 
     hstate.f = &flow;
     flow.alparser = &parser;
@@ -4730,7 +4718,7 @@ static int HTPBodyReassemblyTest01(void)
     printf("REASSCHUNK END: \n");
 #endif
 
-    HtpRequestBodyHandleMultipart(&hstate, &htud, chunks_buffer, chunks_buffer_len);
+    HtpRequestBodyHandleMultipart(&hstate, &htud, &tx, chunks_buffer, chunks_buffer_len);
 
     if (htud.request_body.content_len_so_far != 669) {
         printf("htud.request_body.content_len_so_far %"PRIu64": ", htud.request_body.content_len_so_far);

diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index 9bfcce96f6b21366a56236fcd5fdb68de63f9dcd..126e98d247a9fc136e055860f48ebb3ab7e0da71 100644 (file)
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -167,14 +167,11 @@ typedef struct HtpBody_ {
     uint64_t body_inspected;
 } HtpBody;
 
-#define HTP_REQ_BODY_COMPLETE   0x01    /**< body is complete or limit is reached,
-                                             either way, this is it. */
-#define HTP_RES_BODY_COMPLETE   0x02
-#define HTP_CONTENTTYPE_SET     0x04    /**< We have the content type */
-#define HTP_BOUNDARY_SET        0x08    /**< We have a boundary string */
-#define HTP_BOUNDARY_OPEN       0x10    /**< We have a boundary string */
-#define HTP_FILENAME_SET        0x20    /**< filename is registered in the flow */
-#define HTP_DONTSTORE           0x40    /**< not storing this file */
+#define HTP_CONTENTTYPE_SET     0x01    /**< We have the content type */
+#define HTP_BOUNDARY_SET        0x02    /**< We have a boundary string */
+#define HTP_BOUNDARY_OPEN       0x04    /**< We have a boundary string */
+#define HTP_FILENAME_SET        0x08   /**< filename is registered in the flow */
+#define HTP_DONTSTORE           0x10    /**< not storing this file */
 
 #define HTP_TX_HAS_FILE             0x01
 #define HTP_TX_HAS_FILENAME         0x02    /**< filename is known at this time */

diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index 532b5cb2efc25542b56ba40f76f03d5255a49b45..86857b8544e44e8b32bf6caefffee46eee10236c 100644 (file)
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -142,40 +142,11 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
         goto end;
     }
 
-    /* in case of chunked transfer encoding, we don't have the length
-     * of the request body until we see a chunk with length 0.  This
-     * doesn't let us use the request body callback function to
-     * figure out the end of request body.  Instead we do it here.  If
-     * the length is 0, and we have already seen content, it indicates
-     * chunked transfer.  We also check if the parser has truly seen
-     * the last chunk by checking the progress state for the
-     * transaction.  If we are done parsing all the chunks, we would
-     * have it set to something other than TX_PROGRESS_REQ_BODY.
-     * Either ways we should be moving away from buffering in the end
-     * and running content validation on this buffer type of architecture
-     * to a stateful inspection, where we can inspect body chunks as and
-     * when they come */
-    if (htud->request_body.content_len == 0) {
-        if ((htud->request_body.content_len_so_far > 0) &&
-            tx->request_progress != HTP_REQUEST_BODY) {
-            /* final length of the body */
-            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
-        }
-    } else {
-        if (htud->request_body.content_len == (uint64_t)tx->request_entity_len) {
-             SCLogDebug("content_len reached");
-             htud->tsflags |= HTP_RES_BODY_COMPLETE;
-        }
-    }
-
-    if (flags & STREAM_EOF) {
-        htud->tsflags |= HTP_REQ_BODY_COMPLETE;
-    }
-
     /* inspect the body if the transfer is complete or we have hit
      * our body size limit */
     if (htud->request_body.content_len_so_far < htp_state->cfg->request_inspect_min_size &&
-        !(htud->tsflags & HTP_REQ_BODY_COMPLETE)) {
+        !(AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_BODY) &&
+        !(flags & STREAM_EOF)) {
         SCLogDebug("we still haven't seen the entire request body.  "
                    "Let's defer body inspection till we see the "
                    "entire body.");

diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 35d1bc53047d2c9d41f42cca56afa609778ea39b..843abac2018fc28ebbfc55137e4d78948b5fdc31 100644 (file)
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -141,40 +141,11 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
         goto end;
     }
 
-    /* in case of chunked transfer encoding, we don't have the length
-     * of the response body until we see a chunk with length 0.  This
-     * doesn't let us use the response body callback function to
-     * figure out the end of response body.  Instead we do it here.  If
-     * the length is 0, and we have already seen content, it indicates
-     * chunked transfer.  We also check if the parser has truly seen
-     * the last chunk by checking the progress state for the
-     * transaction.  If we are done parsing all the chunks, we would
-     * have it set to something other than TX_PROGRESS_REQ_BODY.
-     * Either ways we should be moving away from buffering in the end
-     * and running content validation on this buffer type of architecture
-     * to a stateful inspection, where we can inspect body chunks as and
-     * when they come */
-    if (htud->response_body.content_len == 0) {
-        if ((htud->response_body.content_len_so_far > 0) &&
-            tx->response_progress != HTP_RESPONSE_BODY) {
-            /* final length of the body */
-            htud->tcflags |= HTP_RES_BODY_COMPLETE;
-        }
-    } else {
-        if (htud->response_body.content_len == (uint64_t)tx->response_entity_len) {
-             SCLogDebug("content_len reached");
-             htud->tcflags |= HTP_RES_BODY_COMPLETE;
-        }
-    }
-
-    if (flags & STREAM_EOF) {
-        htud->tcflags |= HTP_RES_BODY_COMPLETE;
-    }
-
     /* inspect the body if the transfer is complete or we have hit
      * our body size limit */
     if (htud->response_body.content_len_so_far < htp_state->cfg->response_inspect_min_size &&
-        !(htud->tcflags & HTP_RES_BODY_COMPLETE)) {
+        !(AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) > HTP_RESPONSE_BODY) &&
+        !(flags & STREAM_EOF)) {
         SCLogDebug("we still haven't seen the entire response body.  "
                    "Let's defer body inspection till we see the "
                    "entire body.");
@@ -2326,6 +2297,7 @@ libhtp:\n\
     result = 1;
 
 end:
+    HTPFreeConfig();
     HtpConfigRestoreBackup();
     ConfRestoreContextBackup();
 
@@ -2497,6 +2469,7 @@ libhtp:\n\
     result = 1;
 
 end:
+    HTPFreeConfig();
     HtpConfigRestoreBackup();
     ConfRestoreContextBackup();