From: Victor Julien <victor@inliniac.net>
Date: Wed, 11 Nov 2015 23:19:52 +0000 (+0100)
Subject: http body handling: use streaming buffer API
X-Git-Tag: suricata-3.1RC1~120
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=46e55f1e346101ed1998ad45681e288a857a7766;p=thirdparty%2Fsuricata.git

http body handling: use streaming buffer API

Convert HTTP body handling to use the Streaming Buffer API. This means
the HtpBodyChunks no longer maintain their own data segments, but
instead add their data to the StreamingBuffer instance in the HtpBody
structure.

In case the HtpBodyChunk needs to access it's data it can do so still
through the Streaming Buffer API.

Updates & simplifies the various users of the reassembled bodies:
multipart parsing and the detection engine.
---

diff --git a/src/app-layer-htp-body.c b/src/app-layer-htp-body.c
index d3720f8d39..53194307b4 100644
--- a/src/app-layer-htp-body.c
+++ b/src/app-layer-htp-body.c
@@ -62,6 +62,9 @@
 
 #include "util-memcmp.h"
 
+static StreamingBufferConfig default_cfg = {
+    0, 0, 3072, HTPMalloc, HTPCalloc, HTPRealloc, HTPFree };
+
 /**
  * \brief Append a chunk of body to the HtpBody struct
  *
@@ -82,20 +85,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len)
         SCReturnInt(0);
     }
 
+    if (body->sb == NULL) {
+        body->sb = StreamingBufferInit(&default_cfg);
+        if (body->sb == NULL)
+            SCReturnInt(-1);
+    }
+
     if (body->first == NULL) {
         /* New chunk */
         bd = (HtpBodyChunk *)HTPCalloc(1, sizeof(HtpBodyChunk));
         if (bd == NULL)
             goto error;
 
-        bd->len = len;
-        bd->stream_offset = 0;
-
-        bd->data = HTPCalloc(1, len);
-        if (bd->data == NULL) {
-            goto error;
-        }
-        memcpy(bd->data, data, len);
+        StreamingBufferAppend(body->sb, &bd->sbseg, data, len);
 
         body->first = body->last = bd;
 
@@ -105,29 +107,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len)
         if (bd == NULL)
             goto error;
 
-        bd->len = len;
-        bd->stream_offset = body->content_len_so_far;
-
-        bd->data = HTPCalloc(1, len);
-        if (bd->data == NULL) {
-            goto error;
-        }
-        memcpy(bd->data, data, len);
+        StreamingBufferAppend(body->sb, &bd->sbseg, data, len);
 
         body->last->next = bd;
         body->last = bd;
 
         body->content_len_so_far += len;
     }
-    SCLogDebug("Body %p; data %p, len %"PRIu32, body, bd->data, (uint32_t)bd->len);
+    SCLogDebug("body %p", body);
 
     SCReturnInt(0);
 
 error:
     if (bd != NULL) {
-        if (bd->data != NULL) {
-            HTPFree(bd->data, bd->len);
-        }
         HTPFree(bd, sizeof(HtpBodyChunk));
     }
     SCReturnInt(-1);
@@ -150,9 +142,12 @@ void HtpBodyPrint(HtpBody *body)
         SCLogDebug("--- Start body chunks at %p ---", body);
         printf("--- Start body chunks at %p ---\n", body);
         for (cur = body->first; cur != NULL; cur = cur->next) {
-            SCLogDebug("Body %p; data %p, len %"PRIu32, body, cur->data, (uint32_t)cur->len);
-            printf("Body %p; data %p, len %"PRIu32"\n", body, cur->data, (uint32_t)cur->len);
-            PrintRawDataFp(stdout, (uint8_t*)cur->data, cur->len);
+            const uint8_t *data = NULL;
+            uint32_t data_len = 0;
+            StreamingBufferSegmentGetData(body->sb, &cur->sbseg, &data, &data_len);
+            SCLogDebug("Body %p; data %p, len %"PRIu32, body, data, data_len);
+            printf("Body %p; data %p, len %"PRIu32"\n", body, data, data_len);
+            PrintRawDataFp(stdout, data, data_len);
         }
         SCLogDebug("--- End body chunks at %p ---", body);
     }
@@ -170,8 +165,7 @@ void HtpBodyFree(HtpBody *body)
     if (body->first == NULL)
         return;
 
-    SCLogDebug("Removing chunks of Body %p; data %p, len %"PRIu32, body,
-            body->last->data, (uint32_t)body->last->len);
+    SCLogDebug("removing chunks of body %p", body);
 
     HtpBodyChunk *cur = NULL;
     HtpBodyChunk *prev = NULL;
@@ -179,12 +173,12 @@ void HtpBodyFree(HtpBody *body)
     prev = body->first;
     while (prev != NULL) {
         cur = prev->next;
-        if (prev->data != NULL)
-            HTPFree(prev->data, prev->len);
         HTPFree(prev, sizeof(HtpBodyChunk));
         prev = cur;
     }
     body->first = body->last = NULL;
+
+    StreamingBufferFree(body->sb);
 }
 
 /**
@@ -230,24 +224,26 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction)
         SCReturn;
     }
 
-    SCLogDebug("Pruning chunks of Body %p; data %p, len %"PRIu32, body,
-            body->last->data, (uint32_t)body->last->len);
+    uint64_t left_edge = body->body_inspected;
+    if (left_edge <= min_size || left_edge <= window)
+        left_edge = 0;
+    if (left_edge)
+        left_edge -= window;
+
+    if (left_edge) {
+        SCLogDebug("sliding body to offset %"PRIu64, left_edge);
+        StreamingBufferSlideToOffset(body->sb, left_edge);
+    }
+
+    SCLogDebug("pruning chunks of body %p", body);
 
     HtpBodyChunk *cur = body->first;
     while (cur != NULL) {
         HtpBodyChunk *next = cur->next;
+        SCLogDebug("cur %p", cur);
 
-        SCLogDebug("cur->stream_offset %"PRIu64" + cur->len %u = %"PRIu64", "
-                "body->body_parsed %"PRIu64, cur->stream_offset, cur->len,
-                cur->stream_offset + cur->len, body->body_parsed);
-
-        uint64_t left_edge = body->body_inspected;
-        if (left_edge <= min_size || left_edge <= window)
-            left_edge = 0;
-        if (left_edge)
-            left_edge -= window;
-
-        if (cur->stream_offset + cur->len > left_edge) {
+        if (!StreamingBufferSegmentIsBeforeWindow(body->sb, &cur->sbseg)) {
+            SCLogDebug("not removed");
             break;
         }
 
@@ -256,12 +252,10 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction)
             body->last = next;
         }
 
-        if (cur->data != NULL) {
-            HTPFree(cur->data, cur->len);
-        }
         HTPFree(cur, sizeof(HtpBodyChunk));
 
         cur = next;
+        SCLogDebug("removed");
     }
 
     SCReturn;
diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c
index 9e68f09708..33b747f13e 100644
--- a/src/app-layer-htp-file.c
+++ b/src/app-layer-htp-file.c
@@ -629,8 +629,11 @@ static int HTPFileParserTest03(void)
         goto end;
     }
 
-    if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
+    if (http_state->files_ts->head->chunks_head == NULL ||
+        http_state->files_ts->head->chunks_head->len != 11)
+    {
+        if (http_state->files_ts->head->chunks_head != NULL)
+            printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
         goto end;
     }
 
diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 45c14d7d81..f8069c757b 100644
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1170,60 +1170,11 @@ static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
  *  \param chunks_buffer_len pointer to pass back the buffer length to the caller
  */
 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
-        uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
+        const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
 {
-    uint8_t *buf = NULL;
-    uint8_t *pbuf = NULL;
-    uint32_t buf_len = 0;
-    HtpBodyChunk *cur = htud->request_body.first;
-
-    for ( ; cur != NULL; cur = cur->next) {
-        SCLogDebug("chunk %p", cur);
-
-        /* skip body chunks entirely before what we parsed already */
-        if ((uint64_t )cur->stream_offset + cur->len <= htud->request_body.body_parsed) {
-            SCLogDebug("skipping chunk");
-            continue;
-        }
-
-        SCLogDebug("cur->stream_offset %"PRIu64", cur->len %"PRIu32", body_parsed %"PRIu64,
-            cur->stream_offset, cur->len, htud->request_body.body_parsed);
-
-        if (cur->stream_offset < htud->request_body.body_parsed &&
-                cur->stream_offset + cur->len >= htud->request_body.body_parsed) {
-            SCLogDebug("use part");
-
-            uint32_t toff = htud->request_body.body_parsed - cur->stream_offset;
-            uint32_t tlen = (cur->stream_offset + cur->len) - htud->request_body.body_parsed;
-            uint8_t *pbuf = NULL;
-
-            buf_len += tlen;
-            if ((pbuf = HTPRealloc(buf, buf_len - tlen, buf_len)) == NULL) {
-                HTPFree(buf, buf_len - tlen);
-                buf = NULL;
-                buf_len = 0;
-                break;
-            }
-            buf = pbuf;
-            memcpy(buf + buf_len - tlen, cur->data + toff, tlen);
-
-        } else {
-            SCLogDebug("use entire chunk");
-
-            buf_len += cur->len;
-            if ((pbuf = HTPRealloc(buf, buf_len - cur->len, buf_len)) == NULL) {
-                HTPFree(buf, buf_len - cur->len);
-                buf = NULL;
-                buf_len = 0;
-                break;
-            }
-            buf = pbuf;
-            memcpy(buf + buf_len - cur->len, cur->data, cur->len);
-        }
-    }
-
-    *chunks_buffer = buf;
-    *chunks_buffer_len = buf_len;
+    StreamingBufferGetDataAtOffset(htud->request_body.sb,
+            chunks_buffer, chunks_buffer_len,
+            htud->request_body.body_parsed);
 }
 
 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
@@ -1249,8 +1200,8 @@ static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
     memcpy(boundary + 2, htud->boundary, htud->boundary_len);
 }
 
-int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
-                                  void *tx, uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
+int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
+        const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
 {
     int result = 0;
     uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
@@ -1288,7 +1239,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
         if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
             SCLogDebug("reached the end of the file");
 
-            uint8_t *filedata = chunks_buffer;
+            const uint8_t *filedata = chunks_buffer;
             uint32_t filedata_len = 0;
             uint8_t flags = 0;
 
@@ -1328,7 +1279,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
             SCLogDebug("not yet at the end of the file");
 
             if (chunks_buffer_len > expected_boundary_end_len) {
-                uint8_t *filedata = chunks_buffer;
+                const uint8_t *filedata = chunks_buffer;
                 uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
 #ifdef PRINT
                 printf("FILEDATA (part) START: \n");
@@ -1816,7 +1767,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
         HtpBodyAppendChunk(&tx_ud->request_body, d->data, len);
 
-        uint8_t *chunks_buffer = NULL;
+        const uint8_t *chunks_buffer = NULL;
         uint32_t chunks_buffer_len = 0;
 
         if (tx_ud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
@@ -1837,9 +1788,6 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
             HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
 
-            if (chunks_buffer != NULL) {
-                HTPFree(chunks_buffer, chunks_buffer_len);
-            }
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
             HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
@@ -2189,6 +2137,10 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
 #endif
     cfg_prec->randomize_range = HTP_CONFIG_DEFAULT_RANDOMIZE_RANGE;
 
+    cfg_prec->sbcfg.flags = 0;
+    cfg_prec->sbcfg.buf_size = cfg_prec->request_inspect_window;
+    cfg_prec->sbcfg.buf_slide = 0;
+
     htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
     htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
     htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
@@ -5666,7 +5618,7 @@ static int HTPBodyReassemblyTest01(void)
     r = HtpBodyAppendChunk(&htud.request_body, chunk2, sizeof(chunk2)-1);
     BUG_ON(r != 0);
 
-    uint8_t *chunks_buffer = NULL;
+    const uint8_t *chunks_buffer = NULL;
     uint32_t chunks_buffer_len = 0;
 
     HtpRequestBodyReassemble(&htud, &chunks_buffer, &chunks_buffer_len);
diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index d7aab93993..3c7ae66cbf 100644
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -37,6 +37,7 @@
 #include "util-file.h"
 #include "app-layer-htp-mem.h"
 #include "detect-engine-state.h"
+#include "util-streaming-buffer.h"
 
 #include <htp/htp.h>
 
@@ -158,14 +159,14 @@ typedef struct HTPCfgRec_ {
     int                 randomize;
     int                 randomize_range;
     int                 http_body_inline;
+
+    StreamingBufferConfig sbcfg;
 } HTPCfgRec;
 
 /** Struct used to hold chunks of a body on a request */
 struct HtpBodyChunk_ {
-    uint8_t *data;              /**< Pointer to the data of the chunk */
     struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */
-    uint64_t stream_offset;
-    uint32_t len;               /**< Length of the chunk */
+    StreamingBufferSegment sbseg;
     int logged;
 } __attribute__((__packed__));
 typedef struct HtpBodyChunk_ HtpBodyChunk;
@@ -175,6 +176,8 @@ typedef struct HtpBody_ {
     HtpBodyChunk *first; /**< Pointer to the first chunk */
     HtpBodyChunk *last;  /**< Pointer to the last chunk */
 
+    StreamingBuffer *sb;
+
     /* Holds the length of the htp request body seen so far */
     uint64_t content_len_so_far;
     /* parser tracker */
diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index 5fc48c5fc0..1b4b1534a3 100644
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -97,7 +97,7 @@ static inline int HCBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size)
 
 /**
  */
-static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
+static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
@@ -106,7 +106,7 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                uint32_t *stream_start_offset)
 {
     int index = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
     *buffer_len = 0;
     *stream_start_offset = 0;
 
@@ -170,49 +170,12 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
         goto end;
     }
 
-    int first = 1;
-    while (cur != NULL) {
-        /* see if we can filter out chunks */
-        if (htud->request_body.body_inspected > 0) {
-            if (cur->stream_offset < htud->request_body.body_inspected) {
-                if ((htud->request_body.body_inspected - cur->stream_offset) > htp_state->cfg->request_inspect_min_size) {
-                    cur = cur->next;
-                    continue;
-                } else {
-                    /* include this one */
-                }
-            } else {
-                /* include this one */
-            }
-        }
-
-        if (first) {
-            det_ctx->hcbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* see if we need to grow the buffer */
-        if (det_ctx->hcbd[index].buffer == NULL || (det_ctx->hcbd[index].buffer_len + cur->len) > det_ctx->hcbd[index].buffer_size) {
-            void *ptmp;
-            det_ctx->hcbd[index].buffer_size += cur->len * 2;
-
-            if ((ptmp = SCRealloc(det_ctx->hcbd[index].buffer, det_ctx->hcbd[index].buffer_size)) == NULL) {
-                SCFree(det_ctx->hcbd[index].buffer);
-                det_ctx->hcbd[index].buffer = NULL;
-                det_ctx->hcbd[index].buffer_size = 0;
-                det_ctx->hcbd[index].buffer_len = 0;
-                goto end;
-            }
-            det_ctx->hcbd[index].buffer = ptmp;
-        }
-        memcpy(det_ctx->hcbd[index].buffer + det_ctx->hcbd[index].buffer_len, cur->data, cur->len);
-        det_ctx->hcbd[index].buffer_len += cur->len;
-
-        cur = cur->next;
-    }
+    StreamingBufferGetData(htud->request_body.sb,
+            &det_ctx->hcbd[index].buffer, &det_ctx->hcbd[index].buffer_len,
+            &det_ctx->hcbd[index].offset);
 
     /* update inspected tracker */
-    htud->request_body.body_inspected = htud->request_body.last->stream_offset + htud->request_body.last->len;
+    htud->request_body.body_inspected = htud->request_body.last->sbseg.stream_offset + htud->request_body.last->sbseg.segment_len;
 
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
@@ -258,7 +221,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx,
     uint32_t cnt = 0;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx,
+    const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -267,7 +230,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx,
     if (buffer_len == 0)
         goto end;
 
-    cnt = HttpClientBodyPatternSearch(det_ctx, buffer, buffer_len, flags);
+    cnt = HttpClientBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
 
  end:
     return cnt;
@@ -282,7 +245,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv,
     HtpState *htp_state = (HtpState *)alstate;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id,
+    const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -296,7 +259,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HCBDMATCH],
                                           f,
-                                          buffer,
+                                          (uint8_t *)buffer,
                                           buffer_len,
                                           stream_start_offset,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HCBD, NULL);
diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 48648e3f4e..3918c6e094 100644
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -96,134 +96,7 @@ static inline int HSBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size)
     return 0;
 }
 
-static void HSBDGetBufferForTXInIDSMode(DetectEngineThreadCtx *det_ctx,
-                                        HtpState *htp_state, HtpBodyChunk *cur,
-                                        HtpTxUserData *htud, int index)
-{
-    int first = 1;
-    while (cur != NULL) {
-        /* see if we can filter out chunks */
-        if (htud->response_body.body_inspected > 0) {
-            if (cur->stream_offset < htud->response_body.body_inspected) {
-                if ((htud->response_body.body_inspected - cur->stream_offset) > htp_state->cfg->response_inspect_window) {
-                    cur = cur->next;
-                    continue;
-                } else {
-                    /* include this one */
-                }
-            } else {
-                /* include this one */
-            }
-        }
-
-        if (first) {
-            det_ctx->hsbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* see if we need to grow the buffer */
-        if (det_ctx->hsbd[index].buffer == NULL || (det_ctx->hsbd[index].buffer_len + cur->len) > det_ctx->hsbd[index].buffer_size) {
-            void *ptmp;
-            uint32_t newsize = det_ctx->hsbd[index].buffer_size + (cur->len * 2);
-
-            if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, newsize)) == NULL) {
-                HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size);
-                det_ctx->hsbd[index].buffer = NULL;
-                det_ctx->hsbd[index].buffer_size = 0;
-                det_ctx->hsbd[index].buffer_len = 0;
-                return;
-            }
-            det_ctx->hsbd[index].buffer = ptmp;
-            det_ctx->hsbd[index].buffer_size = newsize;
-        }
-        memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data, cur->len);
-        det_ctx->hsbd[index].buffer_len += cur->len;
-
-        cur = cur->next;
-    }
-
-    /* update inspected tracker */
-    htud->response_body.body_inspected = htud->response_body.last->stream_offset + htud->response_body.last->len;
-}
-
-#define MAX_WINDOW 10*1024*1024
-static void HSBDGetBufferForTXInIPSMode(DetectEngineThreadCtx *det_ctx,
-                                        HtpState *htp_state, HtpBodyChunk *cur,
-                                        HtpTxUserData *htud, int index)
-{
-    uint32_t window_size = 0;
-
-    /* how much from before body_inspected will we consider? */
-    uint32_t cfg_win =
-        htud->response_body.body_inspected >= htp_state->cfg->response_inspect_min_size ?
-            htp_state->cfg->response_inspect_window :
-            htp_state->cfg->response_inspect_min_size;
-
-    /* but less if we don't have that much before body_inspected */
-    if ((htud->response_body.body_inspected - htud->response_body.first->stream_offset) < cfg_win) {
-        cfg_win = htud->response_body.body_inspected - htud->response_body.first->stream_offset;
-    }
-    window_size = (htud->response_body.content_len_so_far - htud->response_body.body_inspected) + cfg_win;
-    if (window_size > MAX_WINDOW) {
-        SCLogDebug("weird: body size is %uk", window_size/1024);
-        window_size = MAX_WINDOW;
-    }
-
-    if (det_ctx->hsbd[index].buffer == NULL || window_size > det_ctx->hsbd[index].buffer_size) {
-        void *ptmp;
-
-        if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, window_size)) == NULL) {
-            HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size);
-            det_ctx->hsbd[index].buffer = NULL;
-            det_ctx->hsbd[index].buffer_size = 0;
-            det_ctx->hsbd[index].buffer_len = 0;
-            return;
-        }
-        det_ctx->hsbd[index].buffer = ptmp;
-        det_ctx->hsbd[index].buffer_size = window_size;
-    }
-
-    uint32_t left_edge = htud->response_body.body_inspected - cfg_win;
-
-    int first = 1;
-    while (cur != NULL) {
-        if (first) {
-            det_ctx->hsbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* entirely before our window */
-        if ((cur->stream_offset + cur->len) <= left_edge) {
-            cur = cur->next;
-            continue;
-        } else {
-            uint32_t offset = 0;
-            if (cur->stream_offset < left_edge && (cur->stream_offset + cur->len) > left_edge) {
-                offset = left_edge - cur->stream_offset;
-                BUG_ON(offset > cur->len);
-            }
-
-            /* unusual: if window isn't big enough, we just give up */
-            if (det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size) {
-                htud->response_body.body_inspected = cur->stream_offset;
-                SCReturn;
-            }
-
-            BUG_ON(det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size);
-
-            memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data + offset, cur->len - offset);
-            det_ctx->hsbd[index].buffer_len += (cur->len - offset);
-            det_ctx->hsbd[index].offset -= offset;
-        }
-
-        cur = cur->next;
-    }
-
-    /* update inspected tracker to point before the current window */
-    htud->response_body.body_inspected = htud->response_body.content_len_so_far;
-}
-
-static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
+static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
@@ -232,7 +105,7 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                uint32_t *stream_start_offset)
 {
     int index = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
     *buffer_len = 0;
     *stream_start_offset = 0;
 
@@ -303,14 +176,25 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                        "entire body.");
             goto end;
         }
-        HSBDGetBufferForTXInIDSMode(det_ctx, htp_state, cur, htud, index);
+    }
+
+    StreamingBufferGetData(htud->response_body.sb,
+            &det_ctx->hsbd[index].buffer, &det_ctx->hsbd[index].buffer_len,
+            &det_ctx->hsbd[index].offset);
+
+    /* update inspected tracker */
+    if (!htp_state->cfg->http_body_inline) {
+        htud->response_body.body_inspected = htud->response_body.last->sbseg.stream_offset +
+                                             htud->response_body.last->sbseg.segment_len;
     } else {
-        HSBDGetBufferForTXInIPSMode(det_ctx, htp_state, cur, htud, index);
+        htud->response_body.body_inspected = htud->response_body.content_len_so_far;
     }
+    SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected);
 
     buffer = det_ctx->hsbd[index].buffer;
     *buffer_len = det_ctx->hsbd[index].buffer_len;
     *stream_start_offset = det_ctx->hsbd[index].offset;
+
  end:
     return buffer;
 }
@@ -361,7 +245,7 @@ int DetectEngineRunHttpServerBodyMpm(DetectEngineCtx *de_ctx,
     if (buffer_len == 0)
         goto end;
 
-    cnt = HttpServerBodyPatternSearch(det_ctx, buffer, buffer_len, flags);
+    cnt = HttpServerBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
 
  end:
     return cnt;
@@ -377,7 +261,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv,
     HtpState *htp_state = (HtpState *)alstate;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id,
+    const uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -391,7 +275,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_FILEDATA],
                                           f,
-                                          buffer,
+                                          (uint8_t *)buffer,
                                           buffer_len,
                                           stream_start_offset,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HSBD, NULL);
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 34f1565202..f591056848 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -1687,22 +1687,12 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     /* HSBD */
     if (det_ctx->hsbd != NULL) {
         SCLogDebug("det_ctx hsbd %u", det_ctx->hsbd_buffers_size);
-        for (i = 0; i < det_ctx->hsbd_buffers_size; i++) {
-            if (det_ctx->hsbd[i].buffer != NULL) {
-                HTPFree(det_ctx->hsbd[i].buffer, det_ctx->hsbd[i].buffer_size);
-            }
-        }
         SCFree(det_ctx->hsbd);
     }
 
     /* HSCB */
     if (det_ctx->hcbd != NULL) {
         SCLogDebug("det_ctx hcbd %u", det_ctx->hcbd_buffers_size);
-        for (i = 0; i < det_ctx->hcbd_buffers_size; i++) {
-            if (det_ctx->hcbd[i].buffer != NULL)
-                SCFree(det_ctx->hcbd[i].buffer);
-            SCLogDebug("det_ctx->hcbd[i].buffer_size %u", det_ctx->hcbd[i].buffer_size);
-        }
         SCFree(det_ctx->hcbd);
     }
 
diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c
index 729629a098..2813d4ca99 100644
--- a/src/detect-http-client-body.c
+++ b/src/detect-http-client-body.c
@@ -1641,7 +1641,9 @@ static int DetectHttpClientBodyTest15(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg,
+                (uint8_t *)"Body one!!", 10) != 1)
+    {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -1654,7 +1656,9 @@ static int DetectHttpClientBodyTest15(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg,
+                (uint8_t *)"Body two!!", 10) != 1)
+    {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
diff --git a/src/detect-pcre.c b/src/detect-pcre.c
index b4303744f4..72692ee073 100644
--- a/src/detect-pcre.c
+++ b/src/detect-pcre.c
@@ -3116,7 +3116,7 @@ static int DetectPcreTxBodyChunksTest01(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3129,7 +3129,7 @@ static int DetectPcreTxBodyChunksTest01(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3358,7 +3358,7 @@ static int DetectPcreTxBodyChunksTest02(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3371,7 +3371,7 @@ static int DetectPcreTxBodyChunksTest02(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
diff --git a/src/detect.h b/src/detect.h
index 90c6502984..8a57b08161 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -697,7 +697,7 @@ enum {
 };
 
 typedef struct HttpReassembledBody_ {
-    uint8_t *buffer;
+    const uint8_t *buffer;
     uint32_t buffer_size;   /**< size of the buffer itself */
     uint32_t buffer_len;    /**< data len in the buffer */
     uint64_t offset;        /**< data offset */
diff --git a/src/output-streaming.c b/src/output-streaming.c
index 4ae7c58c2e..234fbee70e 100644
--- a/src/output-streaming.c
+++ b/src/output-streaming.c
@@ -205,7 +205,7 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags)
                         }
 
                         uint8_t flags = iflags | OUTPUT_STREAMING_FLAG_TRANSACTION;
-                        if (chunk->stream_offset == 0)
+                        if (chunk->sbseg.stream_offset == 0)
                             flags |= OUTPUT_STREAMING_FLAG_OPEN;
                         /* if we need to close and we're at the last segment in the list
                          * we add the 'close' flag so the logger can close up. */
@@ -213,9 +213,13 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags)
                             flags |= OUTPUT_STREAMING_FLAG_CLOSE;
                         }
 
+                        const uint8_t *data = NULL;
+                        uint32_t data_len = 0;
+                        StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len);
+
                         // invoke Streamer
-                        Streamer(cbdata, f, chunk->data, (uint32_t)chunk->len, tx_id, flags);
-                        //PrintRawDataFp(stdout, chunk->data, chunk->len);
+                        Streamer(cbdata, f, data, data_len, tx_id, flags);
+                        //PrintRawDataFp(stdout, data, data_len);
                         chunk->logged = 1;
                         tx_logged = 1;
                     }
diff --git a/src/util-lua-http.c b/src/util-lua-http.c
index 3d97b0f640..1207f7f9ef 100644
--- a/src/util-lua-http.c
+++ b/src/util-lua-http.c
@@ -285,7 +285,12 @@ static int HttpGetBody(lua_State *luastate, int dir)
     lua_newtable(luastate);
     while (chunk != NULL) {
         lua_pushinteger(luastate, index);
-        LuaPushStringBuffer(luastate, chunk->data, chunk->len);
+
+        const uint8_t *data = NULL;
+        uint32_t data_len = 0;
+        StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len);
+        LuaPushStringBuffer(luastate, data, data_len);
+
         lua_settable(luastate, -3);
 
         chunk = chunk->next;
@@ -293,8 +298,8 @@ static int HttpGetBody(lua_State *luastate, int dir)
     }
 
     if (body->first && body->last) {
-        lua_pushinteger(luastate, body->first->stream_offset);
-        lua_pushinteger(luastate, body->last->stream_offset + body->last->len);
+        lua_pushinteger(luastate, body->first->sbseg.stream_offset);
+        lua_pushinteger(luastate, body->last->sbseg.stream_offset + body->last->sbseg.segment_len);
         return 3;
     } else {
         return 1;