From: Victor Julien <victor@inliniac.net>
Date: Thu, 5 Apr 2012 16:33:03 +0000 (+0200)
Subject: http: body inspection improvement
X-Git-Tag: suricata-1.3beta2~84
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d378b76c04d1946156aeb75bb8dba6f1ae4cdf6d;p=thirdparty%2Fsuricata.git

http: body inspection improvement

Improve http_client_body and file_data performance when request and
response body limits are set to high values.
---

diff --git a/src/app-layer-htp-body.c b/src/app-layer-htp-body.c
index c15fcb1e2c..970fc79faf 100644
--- a/src/app-layer-htp-body.c
+++ b/src/app-layer-htp-body.c
@@ -84,7 +84,7 @@ int HtpBodyAppendChunk(HtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32
         SCReturnInt(0);
     }
 
-    if (body->nchunks == 0) {
+    if (body->first == NULL) {
         /* New chunk */
         bd = (HtpBodyChunk *)SCMalloc(sizeof(HtpBodyChunk));
         if (bd == NULL)
@@ -93,7 +93,6 @@ int HtpBodyAppendChunk(HtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32
         bd->len = len;
         bd->stream_offset = 0;
         bd->next = NULL;
-        bd->id = 0;
 
         bd->data = SCMalloc(len);
         if (bd->data == NULL) {
@@ -102,7 +101,6 @@ int HtpBodyAppendChunk(HtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32
         memcpy(bd->data, data, len);
 
         body->first = body->last = bd;
-        body->nchunks++;
 
         body->content_len_so_far = len;
     } else {
@@ -113,7 +111,6 @@ int HtpBodyAppendChunk(HtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32
         bd->len = len;
         bd->stream_offset = body->content_len_so_far;
         bd->next = NULL;
-        bd->id = body->nchunks + 1;
 
         bd->data = SCMalloc(len);
         if (bd->data == NULL) {
@@ -123,12 +120,10 @@ int HtpBodyAppendChunk(HtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32
 
         body->last->next = bd;
         body->last = bd;
-        body->nchunks++;
 
         body->content_len_so_far += len;
     }
-    SCLogDebug("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"", body,
-                bd->id, bd->data, (uint32_t)bd->len);
+    SCLogDebug("Body %p; data %p, len %"PRIu32, body, bd->data, (uint32_t)bd->len);
 
     SCReturnInt(0);
 
@@ -152,17 +147,15 @@ void HtpBodyPrint(HtpBody *body)
     if (SCLogDebugEnabled()||1) {
         SCEnter();
 
-        if (body->nchunks == 0)
+        if (body->first == NULL)
             return;
 
         HtpBodyChunk *cur = NULL;
         SCLogDebug("--- Start body chunks at %p ---", body);
         printf("--- Start body chunks at %p ---\n", body);
         for (cur = body->first; cur != NULL; cur = cur->next) {
-            SCLogDebug("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"\n",
-                        body, cur->id, cur->data, (uint32_t)cur->len);
-            printf("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"\n",
-                        body, cur->id, cur->data, (uint32_t)cur->len);
+            SCLogDebug("Body %p; data %p, len %"PRIu32, body, cur->data, (uint32_t)cur->len);
+            printf("Body %p; data %p, len %"PRIu32"\n", body, cur->data, (uint32_t)cur->len);
             PrintRawDataFp(stdout, (uint8_t*)cur->data, cur->len);
         }
         SCLogDebug("--- End body chunks at %p ---", body);
@@ -178,13 +171,11 @@ void HtpBodyFree(HtpBody *body)
 {
     SCEnter();
 
-    if (body->nchunks == 0)
+    if (body->first == NULL)
         return;
 
-    SCLogDebug("Removing chunks of Body %p; Last Chunk id: %"PRIu32", data %p,"
-               " len %"PRIu32, body, body->last->id, body->last->data,
-                (uint32_t)body->last->len);
-    body->nchunks = 0;
+    SCLogDebug("Removing chunks of Body %p; data %p, len %"PRIu32, body,
+            body->last->data, (uint32_t)body->last->len);
 
     HtpBodyChunk *cur = NULL;
     HtpBodyChunk *prev = NULL;
@@ -211,7 +202,7 @@ void HtpBodyPrune(HtpBody *body)
 {
     SCEnter();
 
-    if (body == NULL || body->nchunks == 0) {
+    if (body == NULL || body->first == NULL) {
         SCReturn;
     }
 
@@ -219,9 +210,8 @@ void HtpBodyPrune(HtpBody *body)
         SCReturn;
     }
 
-    SCLogDebug("Pruning chunks of Body %p; Last Chunk id: %"PRIu32", data %p,"
-               " len %"PRIu32, body, body->last->id, body->last->data,
-                (uint32_t)body->last->len);
+    SCLogDebug("Pruning chunks of Body %p; data %p, len %"PRIu32, body,
+            body->last->data, (uint32_t)body->last->len);
 
     HtpBodyChunk *cur = body->first;
     while (cur != NULL) {
@@ -231,7 +221,7 @@ void HtpBodyPrune(HtpBody *body)
                 "body->body_parsed %"PRIu64, cur->stream_offset, cur->len,
                 cur->stream_offset + cur->len, body->body_parsed);
 
-        if ((cur->stream_offset + cur->len) >= body->body_parsed) {
+        if (cur->stream_offset >= body->body_inspected) {
             break;
         }
 
@@ -240,9 +230,6 @@ void HtpBodyPrune(HtpBody *body)
             body->last = next;
         }
 
-        if (body->nchunks > 0)
-            body->nchunks--;
-
         if (cur->data != NULL) {
             SCFree(cur->data);
         }
diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 298c662b93..6b9da90051 100644
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1284,7 +1284,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
 
     /* if we're in the file storage process, deal with that now */
     if (htud->flags & HTP_FILENAME_SET) {
-        if (header_start != NULL || form_end != NULL || htud->flags & HTP_BODY_COMPLETE) {
+        if (header_start != NULL || form_end != NULL || htud->flags & HTP_REQ_BODY_COMPLETE) {
             SCLogDebug("reached the end of the file");
 
             uint8_t *filedata = chunks_buffer;
@@ -1297,7 +1297,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                 filedata_len = form_end - filedata;
             } else if (form_end != NULL && form_end == header_start) {
                 filedata_len = form_end - filedata - 2; /* 0d 0a */
-            } else if (htud->flags & HTP_BODY_COMPLETE) {
+            } else if (htud->flags & HTP_REQ_BODY_COMPLETE) {
                 filedata_len = chunks_buffer_len;
                 flags = FILE_TRUNCATED;
             }
@@ -1758,14 +1758,14 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
             SCLogDebug("POST");
             int r = HtpRequestBodySetupMultipart(d, htud);
             if (r == 1) {
-                htud->request_body.type = HTP_BODY_REQUEST_MULTIPART;
+                htud->request_body_type = HTP_BODY_REQUEST_MULTIPART;
             } else if (r == 0) {
-                htud->request_body.type = HTP_BODY_REQUEST_POST;
+                htud->request_body_type = HTP_BODY_REQUEST_POST;
                 SCLogDebug("not multipart");
             }
         } else if (d->tx->request_method_number == M_PUT) {
             if (HtpRequestBodySetupPUT(d, htud) == 0) {
-                htud->request_body.type = HTP_BODY_REQUEST_PUT;
+                htud->request_body_type = HTP_BODY_REQUEST_PUT;
             }
         }
 
@@ -1791,19 +1791,19 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
         int r = HtpBodyAppendChunk(htud, &htud->request_body, (uint8_t *)d->data, len);
         if (r < 0) {
-            htud->flags |= HTP_BODY_COMPLETE;
+            htud->flags |= HTP_REQ_BODY_COMPLETE;
         } else if (hstate->request_body_limit > 0 &&
             htud->request_body.content_len_so_far >= hstate->request_body_limit)
         {
-            htud->flags |= HTP_BODY_COMPLETE;
+            htud->flags |= HTP_REQ_BODY_COMPLETE;
         } else if (htud->request_body.content_len_so_far == htud->request_body.content_len) {
-            htud->flags |= HTP_BODY_COMPLETE;
+            htud->flags |= HTP_REQ_BODY_COMPLETE;
         }
 
         uint8_t *chunks_buffer = NULL;
         uint32_t chunks_buffer_len = 0;
 
-        if (htud->request_body.type == HTP_BODY_REQUEST_MULTIPART) {
+        if (htud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
             /* multi-part body handling starts here */
             if (!(htud->flags & HTP_BOUNDARY_SET)) {
                 goto end;
@@ -1824,9 +1824,9 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
             if (chunks_buffer != NULL) {
                 SCFree(chunks_buffer);
             }
-        } else if (htud->request_body.type == HTP_BODY_REQUEST_POST) {
+        } else if (htud->request_body_type == HTP_BODY_REQUEST_POST) {
             HtpRequestBodyHandlePOST(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
-        } else if (htud->request_body.type == HTP_BODY_REQUEST_PUT) {
+        } else if (htud->request_body_type == HTP_BODY_REQUEST_PUT) {
             HtpRequestBodyHandlePUT(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
         }
 
@@ -1868,6 +1868,10 @@ int HTPCallbackResponseBodyData(htp_tx_data_t *d)
         memset(htud, 0, sizeof(HtpTxUserData));
         htud->operation = HTP_BODY_RESPONSE;
 
+        htp_header_t *cl = table_getc(d->tx->response_headers, "content-length");
+        if (cl != NULL)
+            htud->response_body.content_len = htp_parse_content_length(cl->value);
+
         /* Set the user data for handling body chunks on this transaction */
         htp_tx_set_user_data(d->tx, htud);
     }
@@ -1890,13 +1894,13 @@ int HTPCallbackResponseBodyData(htp_tx_data_t *d)
 
         int r = HtpBodyAppendChunk(htud, &htud->response_body, (uint8_t *)d->data, len);
         if (r < 0) {
-            htud->flags |= HTP_BODY_COMPLETE;
+            htud->flags |= HTP_RES_BODY_COMPLETE;
         } else if (hstate->response_body_limit > 0 &&
             htud->response_body.content_len_so_far >= hstate->response_body_limit)
         {
-            htud->flags |= HTP_BODY_COMPLETE;
+            htud->flags |= HTP_RES_BODY_COMPLETE;
         } else if (htud->response_body.content_len_so_far == htud->response_body.content_len) {
-            htud->flags |= HTP_BODY_COMPLETE;
+            htud->flags |= HTP_RES_BODY_COMPLETE;
         }
 
         HtpResponseBodyHandle(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index 89937aa232..2999495fc3 100644
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -121,38 +121,37 @@ enum {
                                              matched on some rule */
 
 /** Struct used to hold chunks of a body on a request */
-typedef struct HtpBodyChunk_ {
+struct HtpBodyChunk_ {
     uint8_t *data;              /**< Pointer to the data of the chunk */
-    uint32_t len;               /**< Length of the chunk */
-    uint32_t id;                /**< number of chunk of the current body */
     struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */
     uint64_t stream_offset;
-} HtpBodyChunk;
+    uint32_t len;               /**< Length of the chunk */
+} __attribute__((__packed__));
+typedef struct HtpBodyChunk_ HtpBodyChunk;
 
 /** Struct used to hold all the chunks of a body on a request */
 typedef struct HtpBody_ {
     HtpBodyChunk *first; /**< Pointer to the first chunk */
     HtpBodyChunk *last;  /**< Pointer to the last chunk */
-    uint32_t nchunks;    /**< Number of chunks in the current operation */
-    uint8_t type;
 
     /* Holds the length of the htp request body */
     uint64_t content_len;
     /* Holds the length of the htp request body seen so far */
     uint64_t content_len_so_far;
-
+    /* parser tracker */
     uint64_t body_parsed;
-
-    /* pahole: padding: 3 */
+    /* inspection tracker */
+    uint64_t body_inspected;
 } HtpBody;
 
-#define HTP_BODY_COMPLETE       0x01    /**< body is complete or limit is reached,
+#define HTP_REQ_BODY_COMPLETE   0x01    /**< body is complete or limit is reached,
                                              either way, this is it. */
-#define HTP_CONTENTTYPE_SET     0x02    /**< We have the content type */
-#define HTP_BOUNDARY_SET        0x04    /**< We have a boundary string */
-#define HTP_BOUNDARY_OPEN       0x08    /**< We have a boundary string */
-#define HTP_FILENAME_SET        0x10    /**< filename is registered in the flow */
-#define HTP_DONTSTORE           0x20    /**< not storing this file */
+#define HTP_RES_BODY_COMPLETE   0x02
+#define HTP_CONTENTTYPE_SET     0x04    /**< We have the content type */
+#define HTP_BOUNDARY_SET        0x08    /**< We have a boundary string */
+#define HTP_BOUNDARY_OPEN       0x10    /**< We have a boundary string */
+#define HTP_FILENAME_SET        0x20    /**< filename is registered in the flow */
+#define HTP_DONTSTORE           0x40    /**< not storing this file */
 
 #define HTP_TX_HAS_FILE             0x01
 #define HTP_TX_HAS_FILENAME         0x02    /**< filename is known at this time */
@@ -180,6 +179,10 @@ typedef struct HtpTxUserData_ {
     uint8_t flags;
 
     int16_t operation;
+
+    uint8_t request_body_type;
+    uint8_t response_body_type;
+
 } HtpTxUserData;
 
 typedef struct HtpState_ {
diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index e2813b49f9..6dae89ed81 100644
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2012 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -55,6 +55,9 @@
 #include "app-layer-htp.h"
 #include "app-layer-protos.h"
 
+#define BODY_SCAN_WINDOW 4096
+#define BODY_MINIMAL_SIZE 32768
+
 /**
  * \brief Helps buffer request bodies for different transactions and stores them
  *        away in detection code.
@@ -73,10 +76,6 @@ static void DetectEngineBufferHttpClientBodies(DetectEngineCtx *de_ctx,
     htp_tx_t *tx = NULL;
     int i = 0;
 
-    if (det_ctx->hcbd_buffers_list_len > 0) {
-        SCReturn;
-    }
-
     if (htp_state == NULL) {
         SCLogDebug("no HTTP state");
         goto end;
@@ -95,23 +94,20 @@ static void DetectEngineBufferHttpClientBodies(DetectEngineCtx *de_ctx,
 
     /* let's get the transaction count.  We need this to hold the client body
      * buffer for each transaction */
-    det_ctx->hcbd_buffers_list_len = list_size(htp_state->connp->conn->transactions) - tmp_idx;
+    size_t txs = list_size(htp_state->connp->conn->transactions) - tmp_idx;
     /* no transactions?!  cool.  get out of here */
-    if (det_ctx->hcbd_buffers_list_len == 0)
+    if (txs == 0) {
         goto end;
+    } else if (txs > det_ctx->hcbd_buffers_list_len) {
+        det_ctx->hcbd = SCRealloc(det_ctx->hcbd, txs * sizeof(HttpReassembledBody));
+        if (det_ctx->hcbd == NULL) {
+            goto end;
+        }
 
-    /* assign space to hold buffers.  Each per transaction */
-    det_ctx->hcbd_buffers = SCMalloc(det_ctx->hcbd_buffers_list_len * sizeof(uint8_t *));
-    if (det_ctx->hcbd_buffers == NULL) {
-        goto end;
-    }
-    memset(det_ctx->hcbd_buffers, 0, det_ctx->hcbd_buffers_list_len * sizeof(uint8_t *));
-
-    det_ctx->hcbd_buffers_len = SCMalloc(det_ctx->hcbd_buffers_list_len * sizeof(uint32_t));
-    if (det_ctx->hcbd_buffers_len == NULL) {
-        goto end;
+        memset(det_ctx->hcbd + det_ctx->hcbd_buffers_list_len, 0,
+                (txs - det_ctx->hcbd_buffers_list_len) * sizeof(HttpReassembledBody));
+        det_ctx->hcbd_buffers_list_len = txs;
     }
-    memset(det_ctx->hcbd_buffers_len, 0, det_ctx->hcbd_buffers_list_len * sizeof(uint32_t));
 
     idx = AppLayerTransactionGetInspectId(f);
     if (idx == -1) {
@@ -120,73 +116,106 @@ static void DetectEngineBufferHttpClientBodies(DetectEngineCtx *de_ctx,
 
     int size = (int)list_size(htp_state->connp->conn->transactions);
     for (; idx < size; idx++, i++) {
+        /* already set up */
+        if (det_ctx->hcbd[i].buffer_len > 0) {
+            SCLogDebug("set up already");
+            continue;
+        }
 
         tx = list_get(htp_state->connp->conn->transactions, idx);
-        if (tx == NULL)
+        if (tx == NULL) {
+            SCLogDebug("no tx");
             continue;
+        }
 
         HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
-        if (htud == NULL)
+        if (htud == NULL) {
+            SCLogDebug("no htud");
             continue;
+        }
 
-        HtpBodyChunk *cur = htud->request_body.first;
+        /* no new data */
+        if (htud->request_body.body_inspected == htud->request_body.content_len_so_far) {
+            SCLogDebug("no new data");
+            continue;
+        }
 
-        if (htud->request_body.nchunks == 0) {
+        HtpBodyChunk *cur = htud->request_body.first;
+        if (cur == NULL) {
             SCLogDebug("No http chunks to inspect for this transacation");
             continue;
-        } else {
-            /* no chunks?!! move on to the next transaction */
-            if (cur == NULL) {
-                SCLogDebug("No http chunks to inspect");
-                continue;
-            }
+        }
 
-            /* in case of chunked transfer encoding, we don't have the length
-             * of the request body until we see a chunk with length 0.  This
-             * doesn't let us use the request body callback function to
-             * figure out the end of request body.  Instead we do it here.  If
-             * the length is 0, and we have already seen content, it indicates
-             * chunked transfer.  We also check if the parser has truly seen
-             * the last chunk by checking the progress state for the
-             * transaction.  If we are done parsing all the chunks, we would
-             * have it set to something other than TX_PROGRESS_REQ_BODY.
-             * Either ways we should be moving away from buffering in the end
-             * and running content validation on this buffer type of architecture
-             * to a stateful inspection, where we can inspect body chunks as and
-             * when they come */
-            if (htud->request_body.content_len == 0) {
-                if ((htud->request_body.content_len_so_far > 0) &&
+        /* in case of chunked transfer encoding, we don't have the length
+         * of the request body until we see a chunk with length 0.  This
+         * doesn't let us use the request body callback function to
+         * figure out the end of request body.  Instead we do it here.  If
+         * the length is 0, and we have already seen content, it indicates
+         * chunked transfer.  We also check if the parser has truly seen
+         * the last chunk by checking the progress state for the
+         * transaction.  If we are done parsing all the chunks, we would
+         * have it set to something other than TX_PROGRESS_REQ_BODY.
+         * Either ways we should be moving away from buffering in the end
+         * and running content validation on this buffer type of architecture
+         * to a stateful inspection, where we can inspect body chunks as and
+         * when they come */
+        if (htud->request_body.content_len == 0) {
+            if ((htud->request_body.content_len_so_far > 0) &&
                     tx->progress != TX_PROGRESS_REQ_BODY) {
-                    /* final length of the body */
-                    htud->flags |= HTP_BODY_COMPLETE;
+                /* final length of the body */
+                htud->flags |= HTP_REQ_BODY_COMPLETE;
+            }
+        }
+
+        /* inspect the body if the transfer is complete or we have hit
+         * our body size limit */
+        if (htud->request_body.content_len_so_far < BODY_MINIMAL_SIZE &&
+                !(htud->flags & HTP_REQ_BODY_COMPLETE)) {
+            SCLogDebug("we still haven't seen the entire request body.  "
+                    "Let's defer body inspection till we see the "
+                    "entire body.");
+            continue;
+        }
+
+        int first = 1;
+        while (cur != NULL) {
+            /* see if we can filter out chunks */
+            if (htud->request_body.body_inspected > 0) {
+                if (cur->stream_offset < htud->request_body.body_inspected) {
+                    if (htud->request_body.body_inspected - cur->stream_offset > BODY_SCAN_WINDOW) {
+                        cur = cur->next;
+                        continue;
+                    } else {
+                        /* include this one */
+                    }
+                } else {
+                    /* include this one */
                 }
             }
 
-            /* inspect the body if the transfer is complete or we have hit
-             * our body size limit */
-            if (!(htud->flags & HTP_BODY_COMPLETE)) {
-                SCLogDebug("we still haven't seen the entire request body.  "
-                        "Let's defer body inspection till we see the "
-                        "entire body.");
-                continue;
+            if (first) {
+                det_ctx->hcbd[i].offset = cur->stream_offset;
+                first = 0;
             }
 
-            uint8_t *chunks_buffer = NULL;
-            int32_t chunks_buffer_len = 0;
-            while (cur != NULL) {
-                chunks_buffer_len += cur->len;
-                if ( (chunks_buffer = SCRealloc(chunks_buffer, chunks_buffer_len)) == NULL) {
+            /* see if we need to grow the buffer */
+            if (det_ctx->hcbd[i].buffer == NULL || det_ctx->hcbd[i].buffer_len + cur->len > det_ctx->hcbd[i].buffer_size) {
+                det_ctx->hcbd[i].buffer_size += cur->len * 2;
+
+                if ((det_ctx->hcbd[i].buffer = SCRealloc(det_ctx->hcbd[i].buffer, det_ctx->hcbd[i].buffer_size)) == NULL) {
                     goto end;
                 }
-
-                memcpy(chunks_buffer + chunks_buffer_len - cur->len, cur->data, cur->len);
-                cur = cur->next;
             }
-            /* store the buffers.  We will need it for further inspection */
-            det_ctx->hcbd_buffers[i] = chunks_buffer;
-            det_ctx->hcbd_buffers_len[i] = chunks_buffer_len;
+            memcpy(det_ctx->hcbd[i].buffer + det_ctx->hcbd[i].buffer_len, cur->data, cur->len);
+            det_ctx->hcbd[i].buffer_len += cur->len;
 
-        } /* else - if (htud->body.nchunks == 0) */
+            cur = cur->next;
+        }
+
+        /* update inspected tracker */
+        htud->request_body.body_inspected =
+            htud->request_body.last->stream_offset +
+            htud->request_body.last->len;
     } /* for (idx = AppLayerTransactionGetInspectId(f); .. */
 
 end:
@@ -200,17 +229,17 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx,
     int i;
     uint32_t cnt = 0;
 
-    /* bail before locking if we have nothing to do */
-    if (det_ctx->hcbd_buffers_list_len == 0) {
-        FLOWLOCK_WRLOCK(f);
-        DetectEngineBufferHttpClientBodies(de_ctx, det_ctx, f, htp_state);
-        FLOWLOCK_UNLOCK(f);
-    }
+    FLOWLOCK_WRLOCK(f);
+    DetectEngineBufferHttpClientBodies(de_ctx, det_ctx, f, htp_state);
+    FLOWLOCK_UNLOCK(f);
 
     for (i = 0; i < det_ctx->hcbd_buffers_list_len; i++) {
+        if (det_ctx->hcbd[i].buffer_len == 0)
+            continue;
+
         cnt += HttpClientBodyPatternSearch(det_ctx,
-                                           det_ctx->hcbd_buffers[i],
-                                           det_ctx->hcbd_buffers_len[i],
+                                           det_ctx->hcbd[i].buffer,
+                                           det_ctx->hcbd[i].buffer_len,
                                            flags);
     }
 
@@ -239,18 +268,15 @@ int DetectEngineInspectHttpClientBody(DetectEngineCtx *de_ctx,
     int r = 0;
     int i = 0;
 
-    /* bail before locking if we have nothing to do */
-    if (det_ctx->hcbd_buffers_list_len == 0) {
-        FLOWLOCK_WRLOCK(f);
-        DetectEngineBufferHttpClientBodies(de_ctx, det_ctx, f, alstate);
-        FLOWLOCK_UNLOCK(f);
-    }
+    FLOWLOCK_WRLOCK(f);
+    DetectEngineBufferHttpClientBodies(de_ctx, det_ctx, f, alstate);
+    FLOWLOCK_UNLOCK(f);
 
     for (i = 0; i < det_ctx->hcbd_buffers_list_len; i++) {
-        uint8_t *hcbd_buffer = det_ctx->hcbd_buffers[i];
-        uint32_t hcbd_buffer_len = det_ctx->hcbd_buffers_len[i];
+        uint8_t *hcbd_buffer = det_ctx->hcbd[i].buffer;
+        uint32_t hcbd_buffer_len = det_ctx->hcbd[i].buffer_len;
 
-        if (hcbd_buffer == NULL)
+        if (hcbd_buffer == NULL || hcbd_buffer_len == 0)
             continue;
 
         det_ctx->buffer_offset = 0;
@@ -262,8 +288,6 @@ int DetectEngineInspectHttpClientBody(DetectEngineCtx *de_ctx,
                                           hcbd_buffer,
                                           hcbd_buffer_len,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HCBD, NULL);
-        //r = DoInspectHttpClientBody(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HCBDMATCH],
-        //hcbd_buffer, hcbd_buffer_len);
         if (r == 1) {
             break;
         }
@@ -279,23 +303,10 @@ int DetectEngineInspectHttpClientBody(DetectEngineCtx *de_ctx,
  */
 void DetectEngineCleanHCBDBuffers(DetectEngineThreadCtx *det_ctx)
 {
-    if (det_ctx->hcbd_buffers_list_len != 0) {
-        int i;
-        for (i = 0; i < det_ctx->hcbd_buffers_list_len; i++) {
-            if (det_ctx->hcbd_buffers[i] != NULL)
-                SCFree(det_ctx->hcbd_buffers[i]);
-        }
-        if (det_ctx->hcbd_buffers != NULL) {
-            SCFree(det_ctx->hcbd_buffers);
-            det_ctx->hcbd_buffers = NULL;
-        }
-        if (det_ctx->hcbd_buffers_len != NULL) {
-            SCFree(det_ctx->hcbd_buffers_len);
-            det_ctx->hcbd_buffers_len = NULL;
-        }
-        det_ctx->hcbd_buffers_list_len = 0;
+    int i;
+    for (i = 0; i < det_ctx->hcbd_buffers_list_len; i++) {
+        det_ctx->hcbd[i].buffer_len = 0;
     }
-
     return;
 }
 
@@ -1051,14 +1062,12 @@ static int DetectEngineHttpClientBodyTest07(void)
     int r = AppLayerParse(NULL, &f, ALPROTO_HTTP, STREAM_TOSERVER, http1_buf, http1_len);
     if (r != 0) {
         printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
-        result = 0;
         goto end;
     }
 
     http_state = f.alstate;
     if (http_state == NULL) {
         printf("no http state: \n");
-        result = 0;
         goto end;
     }
 
@@ -1073,7 +1082,6 @@ static int DetectEngineHttpClientBodyTest07(void)
     r = AppLayerParse(NULL, &f, ALPROTO_HTTP, STREAM_TOSERVER, http2_buf, http2_len);
     if (r != 0) {
         printf("toserver chunk 1 returned %" PRId32 ", expected 0: \n", r);
-        result = 0;
         goto end;
     }
 
diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 619d6ec450..d7a2c708ce 100644
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -56,6 +56,9 @@
 #include "app-layer-htp.h"
 #include "app-layer-protos.h"
 
+#define BODY_SCAN_WINDOW 4096
+#define BODY_MINIMAL_SIZE 32768
+
 /**
  * \brief Helps buffer response bodies for different transactions and stores them
  *        away in detection code.
@@ -74,10 +77,6 @@ static void DetectEngineBufferHttpServerBodies(DetectEngineCtx *de_ctx,
     htp_tx_t *tx = NULL;
     int i = 0;
 
-    if (det_ctx->hsbd_buffers_list_len > 0) {
-        SCReturn;
-    }
-
     if (htp_state == NULL) {
         SCLogDebug("no HTTP state");
         goto end;
@@ -96,23 +95,20 @@ static void DetectEngineBufferHttpServerBodies(DetectEngineCtx *de_ctx,
 
     /* let's get the transaction count.  We need this to hold the server body
      * buffer for each transaction */
-    det_ctx->hsbd_buffers_list_len = list_size(htp_state->connp->conn->transactions) - tmp_idx;
+    size_t txs = list_size(htp_state->connp->conn->transactions) - tmp_idx;
     /* no transactions?!  cool.  get out of here */
-    if (det_ctx->hsbd_buffers_list_len == 0)
-        goto end;
-
-    /* assign space to hold buffers.  Each per transaction */
-    det_ctx->hsbd_buffers = SCMalloc(det_ctx->hsbd_buffers_list_len * sizeof(uint8_t *));
-    if (det_ctx->hsbd_buffers == NULL) {
+    if (txs == 0) {
         goto end;
-    }
-    memset(det_ctx->hsbd_buffers, 0, det_ctx->hsbd_buffers_list_len * sizeof(uint8_t *));
+    } else if (txs > det_ctx->hsbd_buffers_list_len) {
+        det_ctx->hsbd = SCRealloc(det_ctx->hsbd, txs * sizeof(HttpReassembledBody));
+        if (det_ctx->hsbd == NULL) {
+            goto end;
+        }
 
-    det_ctx->hsbd_buffers_len = SCMalloc(det_ctx->hsbd_buffers_list_len * sizeof(uint32_t));
-    if (det_ctx->hsbd_buffers_len == NULL) {
-        goto end;
+        memset(det_ctx->hsbd + det_ctx->hsbd_buffers_list_len, 0,
+                (txs - det_ctx->hsbd_buffers_list_len) * sizeof(HttpReassembledBody));
+        det_ctx->hsbd_buffers_list_len = txs;
     }
-    memset(det_ctx->hsbd_buffers_len, 0, det_ctx->hsbd_buffers_list_len * sizeof(uint32_t));
 
     idx = AppLayerTransactionGetInspectId(f);
     if (idx == -1) {
@@ -121,6 +117,9 @@ static void DetectEngineBufferHttpServerBodies(DetectEngineCtx *de_ctx,
 
     int size = (int)list_size(htp_state->connp->conn->transactions);
     for (; idx < size; idx++, i++) {
+        /* already set up */
+        if (det_ctx->hsbd[i].buffer_len > 0)
+            continue;
 
         tx = list_get(htp_state->connp->conn->transactions, idx);
         if (tx == NULL)
@@ -130,64 +129,89 @@ static void DetectEngineBufferHttpServerBodies(DetectEngineCtx *de_ctx,
         if (htud == NULL)
             continue;
 
-        HtpBodyChunk *cur = htud->response_body.first;
+        /* no new data */
+        if (htud->response_body.body_inspected == htud->response_body.content_len_so_far) {
+            continue;
+        }
 
-        if (htud->response_body.nchunks == 0) {
+        HtpBodyChunk *cur = htud->response_body.first;
+        if (cur == NULL) {
             SCLogDebug("No http chunks to inspect for this transacation");
             continue;
-        } else {
-            /* no chunks?!! move on to the next transaction */
-            if (cur == NULL) {
-                SCLogDebug("No http chunks to inspect");
-                continue;
-            }
+        }
 
-            /* in case of chunked transfer encoding, we don't have the length
-             * of the response body until we see a chunk with length 0.  This
-             * doesn't let us use the response body callback function to
-             * figure out the end of response body.  Instead we do it here.  If
-             * the length is 0, and we have already seen content, it indicates
-             * chunked transfer.  We also check if the parser has truly seen
-             * the last chunk by checking the progress state for the
-             * transaction.  If we are done parsing all the chunks, we would
-             * have it set to something other than TX_PROGRESS_REQ_BODY.
-             * Either ways we should be moving away from buffering in the end
-             * and running content validation on this buffer type of architecture
-             * to a stateful inspection, where we can inspect body chunks as and
-             * when they come */
-            if (htud->response_body.content_len == 0) {
-                if ((htud->response_body.content_len_so_far > 0) &&
+        /* in case of chunked transfer encoding, we don't have the length
+         * of the response body until we see a chunk with length 0.  This
+         * doesn't let us use the response body callback function to
+         * figure out the end of response body.  Instead we do it here.  If
+         * the length is 0, and we have already seen content, it indicates
+         * chunked transfer.  We also check if the parser has truly seen
+         * the last chunk by checking the progress state for the
+         * transaction.  If we are done parsing all the chunks, we would
+         * have it set to something other than TX_PROGRESS_REQ_BODY.
+         * Either ways we should be moving away from buffering in the end
+         * and running content validation on this buffer type of architecture
+         * to a stateful inspection, where we can inspect body chunks as and
+         * when they come */
+        if (htud->response_body.content_len == 0) {
+            if ((htud->response_body.content_len_so_far > 0) &&
                     tx->progress != TX_PROGRESS_REQ_BODY) {
-                    /* final length of the body */
-                    htud->flags |= HTP_BODY_COMPLETE;
+                /* final length of the body */
+                htud->flags |= HTP_RES_BODY_COMPLETE;
+            }
+        }
+
+        /* inspect the body if the transfer is complete or we have hit
+         * our body size limit */
+        if (htud->response_body.content_len_so_far < BODY_MINIMAL_SIZE &&
+                !(htud->flags & HTP_RES_BODY_COMPLETE)) {
+            SCLogDebug("we still haven't seen the entire response body.  "
+                    "Let's defer body inspection till we see the "
+                    "entire body.");
+            continue;
+        }
+
+        //SCLogInfo("now we inspect! %"PRIu64, htud->response_body.content_len_so_far);
+
+        int first = 1;
+        while (cur != NULL) {
+            /* see if we can filter out chunks */
+            if (htud->response_body.body_inspected > 0) {
+                if (cur->stream_offset < htud->response_body.body_inspected) {
+                    if (htud->response_body.body_inspected - cur->stream_offset > BODY_SCAN_WINDOW) {
+                        cur = cur->next;
+                        continue;
+                    } else {
+                        /* include this one */
+                    }
+                } else {
+                    /* include this one */
                 }
             }
 
-            /* inspect the body if the transfer is complete or we have hit
-             * our body size limit */
-            if (!(htud->flags & HTP_BODY_COMPLETE)) {
-                SCLogDebug("we still haven't seen the entire response body.  "
-                        "Let's defer body inspection till we see the "
-                        "entire body.");
-                continue;
+            if (first) {
+                det_ctx->hsbd[i].offset = cur->stream_offset;
+                first = 0;
             }
 
-            uint8_t *chunks_buffer = NULL;
-            int32_t chunks_buffer_len = 0;
-            while (cur != NULL) {
-                chunks_buffer_len += cur->len;
-                if ( (chunks_buffer = SCRealloc(chunks_buffer, chunks_buffer_len)) == NULL) {
+            /* see if we need to grow the buffer */
+            if (det_ctx->hsbd[i].buffer == NULL || det_ctx->hsbd[i].buffer_len + cur->len > det_ctx->hsbd[i].buffer_size) {
+                det_ctx->hsbd[i].buffer_size += cur->len * 2;
+
+                if ((det_ctx->hsbd[i].buffer = SCRealloc(det_ctx->hsbd[i].buffer, det_ctx->hsbd[i].buffer_size)) == NULL) {
                     goto end;
                 }
-
-                memcpy(chunks_buffer + chunks_buffer_len - cur->len, cur->data, cur->len);
-                cur = cur->next;
             }
-            /* store the buffers.  We will need it for further inspection */
-            det_ctx->hsbd_buffers[i] = chunks_buffer;
-            det_ctx->hsbd_buffers_len[i] = chunks_buffer_len;
+            memcpy(det_ctx->hsbd[i].buffer + det_ctx->hsbd[i].buffer_len, cur->data, cur->len);
+            det_ctx->hsbd[i].buffer_len += cur->len;
 
-        } /* else - if (htud->body.nchunks == 0) */
+            cur = cur->next;
+        }
+
+        /* update inspected tracker */
+        htud->response_body.body_inspected =
+            htud->response_body.last->stream_offset +
+            htud->response_body.last->len;
     } /* for (idx = AppLayerTransactionGetInspectId(f); .. */
 
 end:
@@ -201,17 +225,17 @@ int DetectEngineRunHttpServerBodyMpm(DetectEngineCtx *de_ctx,
     int i;
     uint32_t cnt = 0;
 
-    /* bail before locking if we have nothing to do */
-    if (det_ctx->hsbd_buffers_list_len == 0) {
-        FLOWLOCK_WRLOCK(f);
-        DetectEngineBufferHttpServerBodies(de_ctx, det_ctx, f, htp_state);
-        FLOWLOCK_UNLOCK(f);
-    }
+    FLOWLOCK_WRLOCK(f);
+    DetectEngineBufferHttpServerBodies(de_ctx, det_ctx, f, htp_state);
+    FLOWLOCK_UNLOCK(f);
 
     for (i = 0; i < det_ctx->hsbd_buffers_list_len; i++) {
+        if (det_ctx->hsbd[i].buffer_len == 0)
+            continue;
+
         cnt += HttpServerBodyPatternSearch(det_ctx,
-                                           det_ctx->hsbd_buffers[i],
-                                           det_ctx->hsbd_buffers_len[i],
+                                           det_ctx->hsbd[i].buffer,
+                                           det_ctx->hsbd[i].buffer_len,
                                            flags);
     }
 
@@ -240,18 +264,15 @@ int DetectEngineInspectHttpServerBody(DetectEngineCtx *de_ctx,
     int r = 0;
     int i = 0;
 
-    /* bail before locking if we have nothing to do */
-    if (det_ctx->hsbd_buffers_list_len == 0) {
-        FLOWLOCK_WRLOCK(f);
-        DetectEngineBufferHttpServerBodies(de_ctx, det_ctx, f, alstate);
-        FLOWLOCK_UNLOCK(f);
-    }
+    FLOWLOCK_WRLOCK(f);
+    DetectEngineBufferHttpServerBodies(de_ctx, det_ctx, f, alstate);
+    FLOWLOCK_UNLOCK(f);
 
     for (i = 0; i < det_ctx->hsbd_buffers_list_len; i++) {
-        uint8_t *hsbd_buffer = det_ctx->hsbd_buffers[i];
-        uint32_t hsbd_buffer_len = det_ctx->hsbd_buffers_len[i];
+        uint8_t *hsbd_buffer = det_ctx->hsbd[i].buffer;
+        uint32_t hsbd_buffer_len = det_ctx->hsbd[i].buffer_len;
 
-        if (hsbd_buffer == NULL)
+        if (hsbd_buffer == NULL || hsbd_buffer_len == 0)
             continue;
 
         det_ctx->buffer_offset = 0;
@@ -263,8 +284,6 @@ int DetectEngineInspectHttpServerBody(DetectEngineCtx *de_ctx,
                                           hsbd_buffer,
                                           hsbd_buffer_len,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HSBD, NULL);
-        //r = DoInspectHttpServerBody(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HSBDMATCH],
-        //hsbd_buffer, hsbd_buffer_len);
         if (r == 1) {
             break;
         }
@@ -280,23 +299,10 @@ int DetectEngineInspectHttpServerBody(DetectEngineCtx *de_ctx,
  */
 void DetectEngineCleanHSBDBuffers(DetectEngineThreadCtx *det_ctx)
 {
-    if (det_ctx->hsbd_buffers_list_len != 0) {
-        int i;
-        for (i = 0; i < det_ctx->hsbd_buffers_list_len; i++) {
-            if (det_ctx->hsbd_buffers[i] != NULL)
-                SCFree(det_ctx->hsbd_buffers[i]);
-        }
-        if (det_ctx->hsbd_buffers != NULL) {
-            SCFree(det_ctx->hsbd_buffers);
-            det_ctx->hsbd_buffers = NULL;
-        }
-        if (det_ctx->hsbd_buffers_len != NULL) {
-            SCFree(det_ctx->hsbd_buffers_len);
-            det_ctx->hsbd_buffers_len = NULL;
-        }
-        det_ctx->hsbd_buffers_list_len = 0;
+    int i;
+    for (i = 0; i < det_ctx->hsbd_buffers_list_len; i++) {
+        det_ctx->hsbd[i].buffer_len = 0;
     }
-
     return;
 }
 
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 8b150e9145..cdc69fa2ce 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -484,6 +484,25 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data) {
     if (det_ctx->bj_values != NULL)
         SCFree(det_ctx->bj_values);
 
+    if (det_ctx->hsbd != NULL) {
+        SCLogDebug("det_ctx hsbd %u", det_ctx->hsbd_buffers_list_len);
+        for (i = 0; i < det_ctx->hsbd_buffers_list_len; i++) {
+            if (det_ctx->hsbd[i].buffer != NULL)
+                SCFree(det_ctx->hsbd[i].buffer);
+        }
+        SCFree(det_ctx->hsbd);
+    }
+
+    if (det_ctx->hcbd != NULL) {
+        SCLogDebug("det_ctx hcbd %u", det_ctx->hcbd_buffers_list_len);
+        for (i = 0; i < det_ctx->hcbd_buffers_list_len; i++) {
+            if (det_ctx->hcbd[i].buffer != NULL)
+                SCFree(det_ctx->hcbd[i].buffer);
+            SCLogDebug("det_ctx->hcbd[i].buffer_size %u", det_ctx->hcbd[i].buffer_size);
+        }
+        SCFree(det_ctx->hcbd);
+    }
+
     SCFree(det_ctx);
 
     return TM_ECODE_OK;
diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c
index 90546a5619..a73238b736 100644
--- a/src/detect-http-client-body.c
+++ b/src/detect-http-client-body.c
@@ -1609,7 +1609,7 @@ static int DetectHttpClientBodyTest15(void) {
     HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(t1);
 
     HtpBodyChunk *cur = htud->request_body.first;
-    if (htud->request_body.nchunks == 0) {
+    if (htud->request_body.first == NULL) {
         SCLogDebug("No body data in t1 (it should be removed only when the tx is destroyed): ");
         goto end;
     }
@@ -1622,7 +1622,7 @@ static int DetectHttpClientBodyTest15(void) {
     htud = (HtpTxUserData *) htp_tx_get_user_data(t2);
 
     cur = htud->request_body.first;
-    if (htud->request_body.nchunks == 0) {
+    if (htud->request_body.first == NULL) {
         SCLogDebug("No body data in t1 (it should be removed only when the tx is destroyed): ");
         goto end;
     }
diff --git a/src/detect-http-server-body.c b/src/detect-http-server-body.c
index 5b0691f8d8..0b049c83e9 100644
--- a/src/detect-http-server-body.c
+++ b/src/detect-http-server-body.c
@@ -562,14 +562,12 @@ static int DetectHttpServerBodyTest07(void)
     int r = AppLayerParse(NULL, &f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, http_buf1, http_len1);
     if (r != 0) {
         printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
-        result = 0;
         goto end;
     }
 
     r = AppLayerParse(NULL, &f, ALPROTO_HTTP, STREAM_TOCLIENT|STREAM_START, http_buf2, http_len2);
     if (r != 0) {
         printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r);
-        result = 0;
         goto end;
     }
 
@@ -582,8 +580,8 @@ static int DetectHttpServerBodyTest07(void)
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p1);
 
-    if (!(PacketAlertCheck(p1, 1))) {
-        printf("sid 1 didn't match on p1 but should have: ");
+    if ((PacketAlertCheck(p1, 1))) {
+        printf("sid 1 matched on chunk2 but should have: ");
         goto end;
     }
 
@@ -595,8 +593,8 @@ static int DetectHttpServerBodyTest07(void)
 
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p2);
-    if (PacketAlertCheck(p2, 1)) {
-        printf("sid 1 matched on p2 but shouldn't have: ");
+    if (!(PacketAlertCheck(p2, 1))) {
+        printf("sid 1 didn't match on p2 (chunk3) but should have: ");
         goto end;
     }
 
@@ -2805,8 +2803,8 @@ static int DetectHttpServerBodyFileDataTest02(void)
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p1);
 
-    if (!(PacketAlertCheck(p1, 1))) {
-        printf("sid 1 didn't match on p1 but should have: ");
+    if (PacketAlertCheck(p1, 1)) {
+        printf("sid 1 matched on p1 but should have: ");
         goto end;
     }
 
@@ -2818,8 +2816,8 @@ static int DetectHttpServerBodyFileDataTest02(void)
 
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p2);
-    if (PacketAlertCheck(p2, 1)) {
-        printf("sid 1 matched on p2 but shouldn't have: ");
+    if (!(PacketAlertCheck(p2, 1))) {
+        printf("sid 1 didn't match on p2 but should have: ");
         goto end;
     }
 
diff --git a/src/detect-pcre.c b/src/detect-pcre.c
index 1e88c35a58..d8dcf4230f 100644
--- a/src/detect-pcre.c
+++ b/src/detect-pcre.c
@@ -3452,7 +3452,7 @@ static int DetectPcreTxBodyChunksTest01(void) {
     }
 
     HtpBodyChunk *cur = htud->request_body.first;
-    if (htud->request_body.nchunks == 0) {
+    if (htud->request_body.first == NULL) {
         SCLogDebug("No body data in t1 (it should be removed only when the tx is destroyed): ");
         goto end;
     }
@@ -3465,7 +3465,7 @@ static int DetectPcreTxBodyChunksTest01(void) {
     htud = (HtpTxUserData *) htp_tx_get_user_data(t2);
 
     cur = htud->request_body.first;
-    if (htud->request_body.nchunks == 0) {
+    if (htud->request_body.first == NULL) {
         SCLogDebug("No body data in t1 (it should be removed only when the tx is destroyed): ");
         goto end;
     }
@@ -3669,7 +3669,7 @@ static int DetectPcreTxBodyChunksTest02(void) {
     HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(t1);
 
     HtpBodyChunk *cur = htud->request_body.first;
-    if (htud->request_body.nchunks == 0) {
+    if (htud->request_body.first == NULL) {
         SCLogDebug("No body data in t1 (it should be removed only when the tx is destroyed): ");
         goto end;
     }
@@ -3682,7 +3682,7 @@ static int DetectPcreTxBodyChunksTest02(void) {
     htud = (HtpTxUserData *) htp_tx_get_user_data(t2);
 
     cur = htud->request_body.first;
-    if (htud->request_body.nchunks == 0) {
+    if (htud->request_body.first == NULL) {
         SCLogDebug("No body data in t1 (it should be removed only when the tx is destroyed): ");
         goto end;
     }
diff --git a/src/detect-uricontent.c b/src/detect-uricontent.c
index e480522fd2..5fa4040faf 100644
--- a/src/detect-uricontent.c
+++ b/src/detect-uricontent.c
@@ -276,14 +276,6 @@ static inline int DoDetectAppLayerUricontentMatch (DetectEngineThreadCtx *det_ct
                 "%" PRIu32 ")", det_ctx->sgh, det_ctx->sgh->
                 mpm_uricontent_maxlen, det_ctx->sgh->sig_cnt);
 
-        det_ctx->uris++;
-
-        if (det_ctx->sgh->mpm_uricontent_maxlen == 1) det_ctx->pkts_uri_searched1++;
-        else if (det_ctx->sgh->mpm_uricontent_maxlen == 2) det_ctx->pkts_uri_searched2++;
-        else if (det_ctx->sgh->mpm_uricontent_maxlen == 3) det_ctx->pkts_uri_searched3++;
-        else if (det_ctx->sgh->mpm_uricontent_maxlen == 4) det_ctx->pkts_uri_searched4++;
-        else det_ctx->pkts_uri_searched++;
-
         ret += UriPatternSearch(det_ctx, uri, uri_len, flags);
 
         SCLogDebug("post search: cnt %" PRIu32, ret);
diff --git a/src/detect.c b/src/detect.c
index df2bdca661..f2c0e18002 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -1285,7 +1285,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt);
 
     p->alerts.cnt = 0;
-    det_ctx->pkts++;
     det_ctx->filestore_cnt = 0;
 
     /* No need to perform any detection on this packet, if the the given flag is set.*/
@@ -8649,9 +8648,6 @@ int SigTest40NoPayloadInspection02(void) {
     else
         result &= 1;
 
-    if (det_ctx->pkts_searched == 1)
-        result &= 0;
-
     SigGroupCleanup(de_ctx);
     SigCleanSignatures(de_ctx);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
diff --git a/src/detect.h b/src/detect.h
index a89df367e6..4d1066914a 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -667,6 +667,13 @@ enum {
     ENGINE_SGH_MPM_FACTORY_CONTEXT_AUTO
 };
 
+typedef struct HttpReassembledBody_ {
+    uint8_t *buffer;
+    uint32_t buffer_size;   /**< size of the buffer itself */
+    uint32_t buffer_len;    /**< data len in the buffer */
+    uint64_t offset;        /**< data offset */
+} HttpReassembledBody;
+
 #define DETECT_FILESTORE_MAX 15
 
 /**
@@ -684,17 +691,16 @@ typedef struct DetectionEngineThreadCtx_ {
     /* used by pcre match function alone */
     uint32_t pcre_match_start_offset;
 
-    uint8_t **hcbd_buffers;
-    uint32_t *hcbd_buffers_len;
-    uint16_t hcbd_buffers_list_len;
-
     /* counter for the filestore array below -- up here for cache reasons. */
     uint16_t filestore_cnt;
 
     uint16_t hhd_buffers_list_len;
+
+    uint16_t hcbd_buffers_list_len;
     uint16_t hsbd_buffers_list_len;
-    uint8_t **hsbd_buffers;
-    uint32_t *hsbd_buffers_len;
+
+    HttpReassembledBody *hsbd;
+    HttpReassembledBody *hcbd;
 
     uint8_t **hhd_buffers;
     uint32_t *hhd_buffers_len;
@@ -706,6 +712,9 @@ typedef struct DetectionEngineThreadCtx_ {
     uint16_t discontinue_matching;
     uint16_t flags;
 
+    /** ID of the transaction currently being inspected. */
+    uint16_t tx_id;
+
     /* holds the current recursion depth on content inspection */
     int inspection_recursion_counter;
 
@@ -732,24 +741,6 @@ typedef struct DetectionEngineThreadCtx_ {
     PatternMatcherQueue pmq;
     PatternMatcherQueue smsg_pmq[256];
 
-    /** ID of the transaction currently being inspected. */
-    uint16_t tx_id;
-
-    /* counters */
-    uint32_t pkts;
-    uint32_t pkts_searched;
-    uint32_t pkts_searched1;
-    uint32_t pkts_searched2;
-    uint32_t pkts_searched3;
-    uint32_t pkts_searched4;
-
-    uint32_t uris;
-    uint32_t pkts_uri_searched;
-    uint32_t pkts_uri_searched1;
-    uint32_t pkts_uri_searched2;
-    uint32_t pkts_uri_searched3;
-    uint32_t pkts_uri_searched4;
-
     /** ip only rules ctx */
     DetectEngineIPOnlyThreadCtx io_ctx;